1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * Platform specific implementation code 27 * Currently only suspend to RAM is supported (ACPI S3) 28 */ 29 30 #define SUNDDI_IMPL 31 32 #include <sys/types.h> 33 #include <sys/promif.h> 34 #include <sys/prom_isa.h> 35 #include <sys/prom_plat.h> 36 #include <sys/cpuvar.h> 37 #include <sys/pte.h> 38 #include <vm/hat.h> 39 #include <vm/page.h> 40 #include <vm/as.h> 41 #include <sys/cpr.h> 42 #include <sys/kmem.h> 43 #include <sys/clock.h> 44 #include <sys/kmem.h> 45 #include <sys/panic.h> 46 #include <vm/seg_kmem.h> 47 #include <sys/cpu_module.h> 48 #include <sys/callb.h> 49 #include <sys/machsystm.h> 50 #include <sys/vmsystm.h> 51 #include <sys/systm.h> 52 #include <sys/archsystm.h> 53 #include <sys/stack.h> 54 #include <sys/fs/ufs_fs.h> 55 #include <sys/memlist.h> 56 #include <sys/bootconf.h> 57 #include <sys/thread.h> 58 #include <sys/x_call.h> 59 #include <sys/smp_impldefs.h> 60 #include <vm/vm_dep.h> 61 #include <sys/psm.h> 62 #include <sys/epm.h> 63 #include <sys/cpr_wakecode.h> 64 #include <sys/x86_archext.h> 65 #include <sys/reboot.h> 66 #include <sys/acpi/acpi.h> 67 #include <sys/acpica.h> 68 #include <sys/fp.h> 69 #include <sys/sysmacros.h> 70 71 #define AFMT "%lx" 72 73 extern int flushes_require_xcalls; 74 extern cpuset_t cpu_ready_set; 75 76 #if defined(__amd64) 77 extern void *wc_long_mode_64(void); 78 #endif /* __amd64 */ 79 extern int tsc_gethrtime_enable; 80 extern void i_cpr_start_cpu(void); 81 82 ushort_t cpr_mach_type = CPR_MACHTYPE_X86; 83 void (*cpr_start_cpu_func)(void) = i_cpr_start_cpu; 84 85 static wc_cpu_t *wc_other_cpus = NULL; 86 static cpuset_t procset; 87 88 static void 89 init_real_mode_platter(int cpun, uint32_t offset, uint_t cr4, wc_desctbr_t gdt); 90 91 static int i_cpr_platform_alloc(psm_state_request_t *req); 92 static void i_cpr_platform_free(psm_state_request_t *req); 93 static int i_cpr_save_apic(psm_state_request_t *req); 94 static int i_cpr_restore_apic(psm_state_request_t *req); 95 static int wait_for_set(cpuset_t *set, int who); 96 97 static void i_cpr_save_stack(kthread_t *t, wc_cpu_t *wc_cpu); 98 void i_cpr_restore_stack(kthread_t *t, greg_t *save_stack); 99 100 #ifdef STACK_GROWTH_DOWN 101 #define CPR_GET_STACK_START(t) ((t)->t_stkbase) 102 #define CPR_GET_STACK_END(t) ((t)->t_stk) 103 #else 104 #define CPR_GET_STACK_START(t) ((t)->t_stk) 105 #define CPR_GET_STACK_END(t) ((t)->t_stkbase) 106 #endif /* STACK_GROWTH_DOWN */ 107 108 /* 109 * restart paused slave cpus 110 */ 111 void 112 i_cpr_machdep_setup(void) 113 { 114 if (ncpus > 1) { 115 CPR_DEBUG(CPR_DEBUG1, ("MP restarted...\n")); 116 mutex_enter(&cpu_lock); 117 start_cpus(); 118 mutex_exit(&cpu_lock); 119 } 120 } 121 122 123 /* 124 * Stop all interrupt activities in the system 125 */ 126 void 127 i_cpr_stop_intr(void) 128 { 129 (void) spl7(); 130 } 131 132 /* 133 * Set machine up to take interrupts 134 */ 135 void 136 i_cpr_enable_intr(void) 137 { 138 (void) spl0(); 139 } 140 141 /* 142 * Save miscellaneous information which needs to be written to the 143 * state file. This information is required to re-initialize 144 * kernel/prom handshaking. 145 */ 146 void 147 i_cpr_save_machdep_info(void) 148 { 149 int notcalled = 0; 150 ASSERT(notcalled); 151 } 152 153 154 void 155 i_cpr_set_tbr(void) 156 { 157 } 158 159 160 processorid_t 161 i_cpr_bootcpuid(void) 162 { 163 return (0); 164 } 165 166 /* 167 * cpu0 should contain bootcpu info 168 */ 169 cpu_t * 170 i_cpr_bootcpu(void) 171 { 172 ASSERT(MUTEX_HELD(&cpu_lock)); 173 174 return (cpu_get(i_cpr_bootcpuid())); 175 } 176 177 /* 178 * Save context for the specified CPU 179 */ 180 void * 181 i_cpr_save_context(void *arg) 182 { 183 long index = (long)arg; 184 psm_state_request_t *papic_state; 185 int resuming; 186 int ret; 187 wc_cpu_t *wc_cpu = wc_other_cpus + index; 188 189 PMD(PMD_SX, ("i_cpr_save_context() index = %ld\n", index)) 190 191 ASSERT(index < NCPU); 192 193 papic_state = &(wc_cpu)->wc_apic_state; 194 195 ret = i_cpr_platform_alloc(papic_state); 196 ASSERT(ret == 0); 197 198 ret = i_cpr_save_apic(papic_state); 199 ASSERT(ret == 0); 200 201 i_cpr_save_stack(curthread, wc_cpu); 202 203 /* 204 * wc_save_context returns twice, once when susending and 205 * once when resuming, wc_save_context() returns 0 when 206 * suspending and non-zero upon resume 207 */ 208 resuming = (wc_save_context(wc_cpu) == 0); 209 210 /* 211 * do NOT call any functions after this point, because doing so 212 * will modify the stack that we are running on 213 */ 214 215 if (resuming) { 216 217 ret = i_cpr_restore_apic(papic_state); 218 ASSERT(ret == 0); 219 220 i_cpr_platform_free(papic_state); 221 222 /* 223 * Enable interrupts on this cpu. 224 * Do not bind interrupts to this CPU's local APIC until 225 * the CPU is ready to receive interrupts. 226 */ 227 ASSERT(CPU->cpu_id != i_cpr_bootcpuid()); 228 mutex_enter(&cpu_lock); 229 cpu_enable_intr(CPU); 230 mutex_exit(&cpu_lock); 231 232 /* 233 * Setting the bit in cpu_ready_set must be the last operation 234 * in processor initialization; the boot CPU will continue to 235 * boot once it sees this bit set for all active CPUs. 236 */ 237 CPUSET_ATOMIC_ADD(cpu_ready_set, CPU->cpu_id); 238 239 PMD(PMD_SX, 240 ("i_cpr_save_context() resuming cpu %d in cpu_ready_set\n", 241 CPU->cpu_id)) 242 } else { 243 /* 244 * Disable interrupts on this CPU so that PSM knows not to bind 245 * interrupts here on resume until the CPU has executed 246 * cpu_enable_intr() (above) in the resume path. 247 * We explicitly do not grab cpu_lock here because at this point 248 * in the suspend process, the boot cpu owns cpu_lock and all 249 * other cpus are also executing in the pause thread (only 250 * modifying their respective CPU structure). 251 */ 252 (void) cpu_disable_intr(CPU); 253 } 254 255 PMD(PMD_SX, ("i_cpr_save_context: wc_save_context returns %d\n", 256 resuming)) 257 258 return (NULL); 259 } 260 261 static ushort_t *warm_reset_vector = NULL; 262 263 static ushort_t * 264 map_warm_reset_vector() 265 { 266 /*LINTED*/ 267 if (!(warm_reset_vector = (ushort_t *)psm_map_phys(WARM_RESET_VECTOR, 268 sizeof (ushort_t *), PROT_READ|PROT_WRITE))) 269 return (NULL); 270 271 /* 272 * setup secondary cpu bios boot up vector 273 */ 274 *warm_reset_vector = (ushort_t)((caddr_t) 275 /*LINTED*/ 276 ((struct rm_platter *)rm_platter_va)->rm_code - rm_platter_va 277 + ((ulong_t)rm_platter_va & 0xf)); 278 warm_reset_vector++; 279 *warm_reset_vector = (ushort_t)(rm_platter_pa >> 4); 280 281 --warm_reset_vector; 282 return (warm_reset_vector); 283 } 284 285 void 286 i_cpr_pre_resume_cpus() 287 { 288 /* 289 * this is a cut down version of start_other_cpus() 290 * just do the initialization to wake the other cpus 291 */ 292 unsigned who; 293 int boot_cpuid = i_cpr_bootcpuid(); 294 uint32_t code_length = 0; 295 caddr_t wakevirt = rm_platter_va; 296 /*LINTED*/ 297 wakecode_t *wp = (wakecode_t *)wakevirt; 298 char *str = "i_cpr_pre_resume_cpus"; 299 extern int get_tsc_ready(); 300 int err; 301 302 /*LINTED*/ 303 rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va; 304 305 /* 306 * If startup wasn't able to find a page under 1M, we cannot 307 * proceed. 308 */ 309 if (rm_platter_va == 0) { 310 cmn_err(CE_WARN, "Cannot suspend the system because no " 311 "memory below 1M could be found for processor startup"); 312 return; 313 } 314 315 /* 316 * Copy the real mode code at "real_mode_start" to the 317 * page at rm_platter_va. 318 */ 319 warm_reset_vector = map_warm_reset_vector(); 320 if (warm_reset_vector == NULL) { 321 PMD(PMD_SX, ("i_cpr_pre_resume_cpus() returning #2\n")) 322 return; 323 } 324 325 flushes_require_xcalls = 1; 326 327 /* 328 * We lock our affinity to the master CPU to ensure that all slave CPUs 329 * do their TSC syncs with the same CPU. 330 */ 331 332 affinity_set(CPU_CURRENT); 333 334 /* 335 * Mark the boot cpu as being ready and in the procset, since we are 336 * running on that cpu. 337 */ 338 CPUSET_ONLY(cpu_ready_set, boot_cpuid); 339 CPUSET_ONLY(procset, boot_cpuid); 340 341 for (who = 0; who < max_ncpus; who++) { 342 343 wc_cpu_t *cpup = wc_other_cpus + who; 344 wc_desctbr_t gdt; 345 346 if (who == boot_cpuid) 347 continue; 348 349 if (!CPU_IN_SET(mp_cpus, who)) 350 continue; 351 352 PMD(PMD_SX, ("%s() waking up %d cpu\n", str, who)) 353 354 bcopy(cpup, &(wp->wc_cpu), sizeof (wc_cpu_t)); 355 356 gdt.base = cpup->wc_gdt_base; 357 gdt.limit = cpup->wc_gdt_limit; 358 359 #if defined(__amd64) 360 code_length = (uint32_t)((uintptr_t)wc_long_mode_64 - 361 (uintptr_t)wc_rm_start); 362 #else 363 code_length = 0; 364 #endif 365 366 init_real_mode_platter(who, code_length, cpup->wc_cr4, gdt); 367 368 mutex_enter(&cpu_lock); 369 err = mach_cpuid_start(who, rm_platter_va); 370 mutex_exit(&cpu_lock); 371 if (err != 0) { 372 cmn_err(CE_WARN, "cpu%d: failed to start during " 373 "suspend/resume error %d", who, err); 374 continue; 375 } 376 377 PMD(PMD_SX, ("%s() #1 waiting for %d in procset\n", str, who)) 378 379 if (!wait_for_set(&procset, who)) 380 continue; 381 382 PMD(PMD_SX, ("%s() %d cpu started\n", str, who)) 383 384 PMD(PMD_SX, ("%s() tsc_ready = %d\n", str, get_tsc_ready())) 385 386 if (tsc_gethrtime_enable) { 387 PMD(PMD_SX, ("%s() calling tsc_sync_master\n", str)) 388 tsc_sync_master(who); 389 } 390 391 PMD(PMD_SX, ("%s() waiting for %d in cpu_ready_set\n", str, 392 who)) 393 /* 394 * Wait for cpu to declare that it is ready, we want the 395 * cpus to start serially instead of in parallel, so that 396 * they do not contend with each other in wc_rm_start() 397 */ 398 if (!wait_for_set(&cpu_ready_set, who)) 399 continue; 400 401 /* 402 * do not need to re-initialize dtrace using dtrace_cpu_init 403 * function 404 */ 405 PMD(PMD_SX, ("%s() cpu %d now ready\n", str, who)) 406 } 407 408 affinity_clear(); 409 410 PMD(PMD_SX, ("%s() all cpus now ready\n", str)) 411 412 } 413 414 static void 415 unmap_warm_reset_vector(ushort_t *warm_reset_vector) 416 { 417 psm_unmap_phys((caddr_t)warm_reset_vector, sizeof (ushort_t *)); 418 } 419 420 /* 421 * We need to setup a 1:1 (virtual to physical) mapping for the 422 * page containing the wakeup code. 423 */ 424 static struct as *save_as; /* when switching to kas */ 425 426 static void 427 unmap_wakeaddr_1to1(uint64_t wakephys) 428 { 429 uintptr_t wp = (uintptr_t)wakephys; 430 hat_setup(save_as->a_hat, 0); /* switch back from kernel hat */ 431 hat_unload(kas.a_hat, (caddr_t)wp, PAGESIZE, HAT_UNLOAD); 432 } 433 434 void 435 i_cpr_post_resume_cpus() 436 { 437 uint64_t wakephys = rm_platter_pa; 438 439 if (warm_reset_vector != NULL) 440 unmap_warm_reset_vector(warm_reset_vector); 441 442 hat_unload(kas.a_hat, (caddr_t)(uintptr_t)rm_platter_pa, MMU_PAGESIZE, 443 HAT_UNLOAD); 444 445 /* 446 * cmi_post_mpstartup() is only required upon boot not upon 447 * resume from RAM 448 */ 449 450 PT(PT_UNDO1to1); 451 /* Tear down 1:1 mapping for wakeup code */ 452 unmap_wakeaddr_1to1(wakephys); 453 } 454 455 /* ARGSUSED */ 456 void 457 i_cpr_handle_xc(int flag) 458 { 459 } 460 461 int 462 i_cpr_reusable_supported(void) 463 { 464 return (0); 465 } 466 static void 467 map_wakeaddr_1to1(uint64_t wakephys) 468 { 469 uintptr_t wp = (uintptr_t)wakephys; 470 hat_devload(kas.a_hat, (caddr_t)wp, PAGESIZE, btop(wakephys), 471 (PROT_READ|PROT_WRITE|PROT_EXEC|HAT_STORECACHING_OK|HAT_NOSYNC), 472 HAT_LOAD); 473 save_as = curthread->t_procp->p_as; 474 hat_setup(kas.a_hat, 0); /* switch to kernel-only hat */ 475 } 476 477 478 void 479 prt_other_cpus() 480 { 481 int who; 482 483 if (ncpus == 1) { 484 PMD(PMD_SX, ("prt_other_cpus() other cpu table empty for " 485 "uniprocessor machine\n")) 486 return; 487 } 488 489 for (who = 0; who < max_ncpus; who++) { 490 491 wc_cpu_t *cpup = wc_other_cpus + who; 492 493 if (!CPU_IN_SET(mp_cpus, who)) 494 continue; 495 496 PMD(PMD_SX, ("prt_other_cpus() who = %d, gdt=%p:%x, " 497 "idt=%p:%x, ldt=%lx, tr=%lx, kgsbase=" 498 AFMT ", sp=%lx\n", who, 499 (void *)cpup->wc_gdt_base, cpup->wc_gdt_limit, 500 (void *)cpup->wc_idt_base, cpup->wc_idt_limit, 501 (long)cpup->wc_ldt, (long)cpup->wc_tr, 502 (long)cpup->wc_kgsbase, (long)cpup->wc_rsp)) 503 } 504 } 505 506 /* 507 * Power down the system. 508 */ 509 int 510 i_cpr_power_down(int sleeptype) 511 { 512 caddr_t wakevirt = rm_platter_va; 513 uint64_t wakephys = rm_platter_pa; 514 ulong_t saved_intr; 515 uint32_t code_length = 0; 516 wc_desctbr_t gdt; 517 /*LINTED*/ 518 wakecode_t *wp = (wakecode_t *)wakevirt; 519 /*LINTED*/ 520 rm_platter_t *wcpp = (rm_platter_t *)wakevirt; 521 wc_cpu_t *cpup = &(wp->wc_cpu); 522 dev_info_t *ppm; 523 int ret = 0; 524 power_req_t power_req; 525 char *str = "i_cpr_power_down"; 526 #if defined(__amd64) 527 /*LINTED*/ 528 rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va; 529 #endif 530 extern int cpr_suspend_succeeded; 531 extern void kernel_wc_code(); 532 533 ASSERT(sleeptype == CPR_TORAM); 534 ASSERT(CPU->cpu_id == 0); 535 536 if ((ppm = PPM(ddi_root_node())) == NULL) { 537 PMD(PMD_SX, ("%s: root node not claimed\n", str)) 538 return (ENOTTY); 539 } 540 541 PMD(PMD_SX, ("Entering %s()\n", str)) 542 543 PT(PT_IC); 544 saved_intr = intr_clear(); 545 546 PT(PT_1to1); 547 /* Setup 1:1 mapping for wakeup code */ 548 map_wakeaddr_1to1(wakephys); 549 550 PMD(PMD_SX, ("ncpus=%d\n", ncpus)) 551 552 PMD(PMD_SX, ("wc_rm_end - wc_rm_start=%lx WC_CODESIZE=%x\n", 553 ((size_t)((uintptr_t)wc_rm_end - (uintptr_t)wc_rm_start)), 554 WC_CODESIZE)) 555 556 PMD(PMD_SX, ("wakevirt=%p, wakephys=%x\n", 557 (void *)wakevirt, (uint_t)wakephys)) 558 559 ASSERT(((size_t)((uintptr_t)wc_rm_end - (uintptr_t)wc_rm_start)) < 560 WC_CODESIZE); 561 562 bzero(wakevirt, PAGESIZE); 563 564 /* Copy code to rm_platter */ 565 bcopy((caddr_t)wc_rm_start, wakevirt, 566 (size_t)((uintptr_t)wc_rm_end - (uintptr_t)wc_rm_start)); 567 568 prt_other_cpus(); 569 570 #if defined(__amd64) 571 572 PMD(PMD_SX, ("real_mode_platter->rm_cr4=%lx, getcr4()=%lx\n", 573 (ulong_t)real_mode_platter->rm_cr4, (ulong_t)getcr4())) 574 575 PMD(PMD_SX, ("real_mode_platter->rm_pdbr=%lx, getcr3()=%lx\n", 576 (ulong_t)real_mode_platter->rm_pdbr, getcr3())) 577 578 real_mode_platter->rm_cr4 = getcr4(); 579 real_mode_platter->rm_pdbr = getcr3(); 580 581 rmp_gdt_init(real_mode_platter); 582 583 /* 584 * Since the CPU needs to jump to protected mode using an identity 585 * mapped address, we need to calculate it here. 586 */ 587 real_mode_platter->rm_longmode64_addr = rm_platter_pa + 588 (uint32_t)((uintptr_t)wc_long_mode_64 - (uintptr_t)wc_rm_start); 589 590 PMD(PMD_SX, ("real_mode_platter->rm_cr4=%lx, getcr4()=%lx\n", 591 (ulong_t)real_mode_platter->rm_cr4, getcr4())) 592 PMD(PMD_SX, ("real_mode_platter->rm_pdbr=%lx, getcr3()=%lx\n", 593 (ulong_t)real_mode_platter->rm_pdbr, getcr3())) 594 595 PMD(PMD_SX, ("real_mode_platter->rm_longmode64_addr=%lx\n", 596 (ulong_t)real_mode_platter->rm_longmode64_addr)) 597 598 #endif 599 600 PT(PT_SC); 601 if (wc_save_context(cpup)) { 602 603 ret = i_cpr_platform_alloc(&(wc_other_cpus->wc_apic_state)); 604 if (ret != 0) 605 return (ret); 606 607 ret = i_cpr_save_apic(&(wc_other_cpus->wc_apic_state)); 608 PMD(PMD_SX, ("%s: i_cpr_save_apic() returned %d\n", str, ret)) 609 if (ret != 0) 610 return (ret); 611 612 PMD(PMD_SX, ("wakephys=%x, kernel_wc_code=%p\n", 613 (uint_t)wakephys, (void *)&kernel_wc_code)) 614 PMD(PMD_SX, ("virtaddr=%lx, retaddr=%lx\n", 615 (long)cpup->wc_virtaddr, (long)cpup->wc_retaddr)) 616 PMD(PMD_SX, ("ebx=%x, edi=%x, esi=%x, ebp=%x, esp=%x\n", 617 cpup->wc_ebx, cpup->wc_edi, cpup->wc_esi, cpup->wc_ebp, 618 cpup->wc_esp)) 619 PMD(PMD_SX, ("cr0=%lx, cr3=%lx, cr4=%lx\n", 620 (long)cpup->wc_cr0, (long)cpup->wc_cr3, 621 (long)cpup->wc_cr4)) 622 PMD(PMD_SX, ("cs=%x, ds=%x, es=%x, ss=%x, fs=%lx, gs=%lx, " 623 "flgs=%lx\n", cpup->wc_cs, cpup->wc_ds, cpup->wc_es, 624 cpup->wc_ss, (long)cpup->wc_fs, (long)cpup->wc_gs, 625 (long)cpup->wc_eflags)) 626 627 PMD(PMD_SX, ("gdt=%p:%x, idt=%p:%x, ldt=%lx, tr=%lx, " 628 "kgbase=%lx\n", (void *)cpup->wc_gdt_base, 629 cpup->wc_gdt_limit, (void *)cpup->wc_idt_base, 630 cpup->wc_idt_limit, (long)cpup->wc_ldt, 631 (long)cpup->wc_tr, (long)cpup->wc_kgsbase)) 632 633 gdt.base = cpup->wc_gdt_base; 634 gdt.limit = cpup->wc_gdt_limit; 635 636 #if defined(__amd64) 637 code_length = (uint32_t)((uintptr_t)wc_long_mode_64 - 638 (uintptr_t)wc_rm_start); 639 #else 640 code_length = 0; 641 #endif 642 643 init_real_mode_platter(0, code_length, cpup->wc_cr4, gdt); 644 645 #if defined(__amd64) 646 PMD(PMD_SX, ("real_mode_platter->rm_cr4=%lx, getcr4()=%lx\n", 647 (ulong_t)wcpp->rm_cr4, getcr4())) 648 649 PMD(PMD_SX, ("real_mode_platter->rm_pdbr=%lx, getcr3()=%lx\n", 650 (ulong_t)wcpp->rm_pdbr, getcr3())) 651 652 PMD(PMD_SX, ("real_mode_platter->rm_longmode64_addr=%lx\n", 653 (ulong_t)wcpp->rm_longmode64_addr)) 654 655 PMD(PMD_SX, 656 ("real_mode_platter->rm_temp_gdt[TEMPGDT_KCODE64]=%lx\n", 657 (ulong_t)wcpp->rm_temp_gdt[TEMPGDT_KCODE64])) 658 #endif 659 660 PMD(PMD_SX, ("gdt=%p:%x, idt=%p:%x, ldt=%lx, tr=%lx, " 661 "kgsbase=%lx\n", (void *)wcpp->rm_gdt_base, 662 wcpp->rm_gdt_lim, (void *)wcpp->rm_idt_base, 663 wcpp->rm_idt_lim, (long)cpup->wc_ldt, (long)cpup->wc_tr, 664 (long)cpup->wc_kgsbase)) 665 666 power_req.request_type = PMR_PPM_ENTER_SX; 667 power_req.req.ppm_power_enter_sx_req.sx_state = S3; 668 power_req.req.ppm_power_enter_sx_req.test_point = 669 cpr_test_point; 670 power_req.req.ppm_power_enter_sx_req.wakephys = wakephys; 671 672 PMD(PMD_SX, ("%s: pm_ctlops PMR_PPM_ENTER_SX\n", str)) 673 PT(PT_PPMCTLOP); 674 (void) pm_ctlops(ppm, ddi_root_node(), DDI_CTLOPS_POWER, 675 &power_req, &ret); 676 PMD(PMD_SX, ("%s: returns %d\n", str, ret)) 677 678 /* 679 * If it works, we get control back to the else branch below 680 * If we get control back here, it didn't work. 681 * XXX return EINVAL here? 682 */ 683 684 unmap_wakeaddr_1to1(wakephys); 685 intr_restore(saved_intr); 686 687 return (ret); 688 } else { 689 cpr_suspend_succeeded = 1; 690 691 power_req.request_type = PMR_PPM_EXIT_SX; 692 power_req.req.ppm_power_enter_sx_req.sx_state = S3; 693 694 PMD(PMD_SX, ("%s: pm_ctlops PMR_PPM_EXIT_SX\n", str)) 695 PT(PT_PPMCTLOP); 696 (void) pm_ctlops(ppm, ddi_root_node(), DDI_CTLOPS_POWER, 697 &power_req, &ret); 698 PMD(PMD_SX, ("%s: returns %d\n", str, ret)) 699 700 ret = i_cpr_restore_apic(&(wc_other_cpus->wc_apic_state)); 701 /* 702 * the restore should never fail, if the saved suceeded 703 */ 704 ASSERT(ret == 0); 705 706 i_cpr_platform_free(&(wc_other_cpus->wc_apic_state)); 707 708 /* 709 * Enable interrupts on boot cpu. 710 */ 711 ASSERT(CPU->cpu_id == i_cpr_bootcpuid()); 712 mutex_enter(&cpu_lock); 713 cpu_enable_intr(CPU); 714 mutex_exit(&cpu_lock); 715 716 PT(PT_INTRRESTORE); 717 intr_restore(saved_intr); 718 PT(PT_CPU); 719 720 return (ret); 721 } 722 } 723 724 /* 725 * Stop all other cpu's before halting or rebooting. We pause the cpu's 726 * instead of sending a cross call. 727 * Stolen from sun4/os/mp_states.c 728 */ 729 730 static int cpu_are_paused; /* sic */ 731 732 void 733 i_cpr_stop_other_cpus(void) 734 { 735 mutex_enter(&cpu_lock); 736 if (cpu_are_paused) { 737 mutex_exit(&cpu_lock); 738 return; 739 } 740 pause_cpus(NULL, NULL); 741 cpu_are_paused = 1; 742 743 mutex_exit(&cpu_lock); 744 } 745 746 int 747 i_cpr_is_supported(int sleeptype) 748 { 749 extern int cpr_supported_override; 750 extern int cpr_platform_enable; 751 extern int pm_S3_enabled; 752 753 if (sleeptype != CPR_TORAM) 754 return (0); 755 756 /* 757 * The next statement tests if a specific platform has turned off 758 * cpr support. 759 */ 760 if (cpr_supported_override) 761 return (0); 762 763 /* 764 * If a platform has specifically turned on cpr support ... 765 */ 766 if (cpr_platform_enable) 767 return (1); 768 769 return (pm_S3_enabled); 770 } 771 772 void 773 i_cpr_bitmap_cleanup(void) 774 { 775 } 776 777 void 778 i_cpr_free_memory_resources(void) 779 { 780 } 781 782 /* 783 * Needed only for S3 so far 784 */ 785 static int 786 i_cpr_platform_alloc(psm_state_request_t *req) 787 { 788 #ifdef DEBUG 789 char *str = "i_cpr_platform_alloc"; 790 #endif 791 792 PMD(PMD_SX, ("cpu = %d, %s(%p) \n", CPU->cpu_id, str, (void *)req)) 793 794 if (psm_state == NULL) { 795 PMD(PMD_SX, ("%s() : psm_state == NULL\n", str)) 796 return (0); 797 } 798 799 req->psr_cmd = PSM_STATE_ALLOC; 800 return ((*psm_state)(req)); 801 } 802 803 /* 804 * Needed only for S3 so far 805 */ 806 static void 807 i_cpr_platform_free(psm_state_request_t *req) 808 { 809 #ifdef DEBUG 810 char *str = "i_cpr_platform_free"; 811 #endif 812 813 PMD(PMD_SX, ("cpu = %d, %s(%p) \n", CPU->cpu_id, str, (void *)req)) 814 815 if (psm_state == NULL) { 816 PMD(PMD_SX, ("%s() : psm_state == NULL\n", str)) 817 return; 818 } 819 820 req->psr_cmd = PSM_STATE_FREE; 821 (void) (*psm_state)(req); 822 } 823 824 static int 825 i_cpr_save_apic(psm_state_request_t *req) 826 { 827 #ifdef DEBUG 828 char *str = "i_cpr_save_apic"; 829 #endif 830 831 if (psm_state == NULL) { 832 PMD(PMD_SX, ("%s() : psm_state == NULL\n", str)) 833 return (0); 834 } 835 836 req->psr_cmd = PSM_STATE_SAVE; 837 return ((*psm_state)(req)); 838 } 839 840 static int 841 i_cpr_restore_apic(psm_state_request_t *req) 842 { 843 #ifdef DEBUG 844 char *str = "i_cpr_restore_apic"; 845 #endif 846 847 if (psm_state == NULL) { 848 PMD(PMD_SX, ("%s() : psm_state == NULL\n", str)) 849 return (0); 850 } 851 852 req->psr_cmd = PSM_STATE_RESTORE; 853 return ((*psm_state)(req)); 854 } 855 856 857 /* stop lint complaining about offset not being used in 32bit mode */ 858 #if !defined(__amd64) 859 /*ARGSUSED*/ 860 #endif 861 static void 862 init_real_mode_platter(int cpun, uint32_t offset, uint_t cr4, wc_desctbr_t gdt) 863 { 864 /*LINTED*/ 865 rm_platter_t *real_mode_platter = (rm_platter_t *)rm_platter_va; 866 867 /* 868 * Fill up the real mode platter to make it easy for real mode code to 869 * kick it off. This area should really be one passed by boot to kernel 870 * and guaranteed to be below 1MB and aligned to 16 bytes. Should also 871 * have identical physical and virtual address in paged mode. 872 */ 873 874 real_mode_platter->rm_pdbr = getcr3(); 875 real_mode_platter->rm_cpu = cpun; 876 real_mode_platter->rm_cr4 = cr4; 877 878 real_mode_platter->rm_gdt_base = gdt.base; 879 real_mode_platter->rm_gdt_lim = gdt.limit; 880 881 #if defined(__amd64) 882 if (getcr3() > 0xffffffffUL) 883 panic("Cannot initialize CPUs; kernel's 64-bit page tables\n" 884 "located above 4G in physical memory (@ 0x%llx).", 885 (unsigned long long)getcr3()); 886 887 /* 888 * Setup pseudo-descriptors for temporary GDT and IDT for use ONLY 889 * by code in real_mode_start(): 890 * 891 * GDT[0]: NULL selector 892 * GDT[1]: 64-bit CS: Long = 1, Present = 1, bits 12, 11 = 1 893 * 894 * Clear the IDT as interrupts will be off and a limit of 0 will cause 895 * the CPU to triple fault and reset on an NMI, seemingly as reasonable 896 * a course of action as any other, though it may cause the entire 897 * platform to reset in some cases... 898 */ 899 real_mode_platter->rm_temp_gdt[0] = 0ULL; 900 real_mode_platter->rm_temp_gdt[TEMPGDT_KCODE64] = 0x20980000000000ULL; 901 902 real_mode_platter->rm_temp_gdt_lim = (ushort_t) 903 (sizeof (real_mode_platter->rm_temp_gdt) - 1); 904 real_mode_platter->rm_temp_gdt_base = rm_platter_pa + 905 offsetof(rm_platter_t, rm_temp_gdt); 906 907 real_mode_platter->rm_temp_idt_lim = 0; 908 real_mode_platter->rm_temp_idt_base = 0; 909 910 /* 911 * Since the CPU needs to jump to protected mode using an identity 912 * mapped address, we need to calculate it here. 913 */ 914 real_mode_platter->rm_longmode64_addr = rm_platter_pa + offset; 915 #endif /* __amd64 */ 916 917 /* return; */ 918 } 919 920 void 921 i_cpr_start_cpu(void) 922 { 923 924 struct cpu *cp = CPU; 925 926 char *str = "i_cpr_start_cpu"; 927 extern void init_cpu_syscall(struct cpu *cp); 928 929 PMD(PMD_SX, ("%s() called\n", str)) 930 931 PMD(PMD_SX, ("%s() #0 cp->cpu_base_spl %d\n", str, 932 cp->cpu_base_spl)) 933 934 mutex_enter(&cpu_lock); 935 if (cp == i_cpr_bootcpu()) { 936 mutex_exit(&cpu_lock); 937 PMD(PMD_SX, 938 ("%s() called on bootcpu nothing to do!\n", str)) 939 return; 940 } 941 mutex_exit(&cpu_lock); 942 943 /* 944 * We need to Sync PAT with cpu0's PAT. We have to do 945 * this with interrupts disabled. 946 */ 947 if (is_x86_feature(x86_featureset, X86FSET_PAT)) 948 pat_sync(); 949 950 /* 951 * If we use XSAVE, we need to restore XFEATURE_ENABLE_MASK register. 952 */ 953 if (fp_save_mech == FP_XSAVE) { 954 setup_xfem(); 955 } 956 957 /* 958 * Initialize this CPU's syscall handlers 959 */ 960 init_cpu_syscall(cp); 961 962 PMD(PMD_SX, ("%s() #1 cp->cpu_base_spl %d\n", str, cp->cpu_base_spl)) 963 964 /* 965 * Do not need to call cpuid_pass2(), cpuid_pass3(), cpuid_pass4() or 966 * init_cpu_info(), since the work that they do is only needed to 967 * be done once at boot time 968 */ 969 970 971 mutex_enter(&cpu_lock); 972 CPUSET_ADD(procset, cp->cpu_id); 973 mutex_exit(&cpu_lock); 974 975 PMD(PMD_SX, ("%s() #2 cp->cpu_base_spl %d\n", str, 976 cp->cpu_base_spl)) 977 978 if (tsc_gethrtime_enable) { 979 PMD(PMD_SX, ("%s() calling tsc_sync_slave\n", str)) 980 tsc_sync_slave(); 981 } 982 983 PMD(PMD_SX, ("%s() cp->cpu_id %d, cp->cpu_intr_actv %d\n", str, 984 cp->cpu_id, cp->cpu_intr_actv)) 985 PMD(PMD_SX, ("%s() #3 cp->cpu_base_spl %d\n", str, 986 cp->cpu_base_spl)) 987 988 (void) spl0(); /* enable interrupts */ 989 990 PMD(PMD_SX, ("%s() #4 cp->cpu_base_spl %d\n", str, 991 cp->cpu_base_spl)) 992 993 /* 994 * Set up the CPU module for this CPU. This can't be done before 995 * this CPU is made CPU_READY, because we may (in heterogeneous systems) 996 * need to go load another CPU module. The act of attempting to load 997 * a module may trigger a cross-call, which will ASSERT unless this 998 * cpu is CPU_READY. 999 */ 1000 1001 /* 1002 * cmi already been init'd (during boot), so do not need to do it again 1003 */ 1004 #ifdef PM_REINITMCAONRESUME 1005 if (is_x86_feature(x86_featureset, X86FSET_MCA)) 1006 cmi_mca_init(); 1007 #endif 1008 1009 PMD(PMD_SX, ("%s() returning\n", str)) 1010 1011 /* return; */ 1012 } 1013 1014 void 1015 i_cpr_alloc_cpus(void) 1016 { 1017 char *str = "i_cpr_alloc_cpus"; 1018 1019 PMD(PMD_SX, ("%s() CPU->cpu_id %d\n", str, CPU->cpu_id)) 1020 /* 1021 * we allocate this only when we actually need it to save on 1022 * kernel memory 1023 */ 1024 1025 if (wc_other_cpus == NULL) { 1026 wc_other_cpus = kmem_zalloc(max_ncpus * sizeof (wc_cpu_t), 1027 KM_SLEEP); 1028 } 1029 1030 } 1031 1032 void 1033 i_cpr_free_cpus(void) 1034 { 1035 int index; 1036 wc_cpu_t *wc_cpu; 1037 1038 if (wc_other_cpus != NULL) { 1039 for (index = 0; index < max_ncpus; index++) { 1040 wc_cpu = wc_other_cpus + index; 1041 if (wc_cpu->wc_saved_stack != NULL) { 1042 kmem_free(wc_cpu->wc_saved_stack, 1043 wc_cpu->wc_saved_stack_size); 1044 } 1045 } 1046 1047 kmem_free((void *) wc_other_cpus, 1048 max_ncpus * sizeof (wc_cpu_t)); 1049 wc_other_cpus = NULL; 1050 } 1051 } 1052 1053 /* 1054 * wrapper for acpica_ddi_save_resources() 1055 */ 1056 void 1057 i_cpr_save_configuration(dev_info_t *dip) 1058 { 1059 acpica_ddi_save_resources(dip); 1060 } 1061 1062 /* 1063 * wrapper for acpica_ddi_restore_resources() 1064 */ 1065 void 1066 i_cpr_restore_configuration(dev_info_t *dip) 1067 { 1068 acpica_ddi_restore_resources(dip); 1069 } 1070 1071 static int 1072 wait_for_set(cpuset_t *set, int who) 1073 { 1074 int delays; 1075 char *str = "wait_for_set"; 1076 1077 for (delays = 0; !CPU_IN_SET(*set, who); delays++) { 1078 if (delays == 500) { 1079 /* 1080 * After five seconds, things are probably 1081 * looking a bit bleak - explain the hang. 1082 */ 1083 cmn_err(CE_NOTE, "cpu%d: started, " 1084 "but not running in the kernel yet", who); 1085 PMD(PMD_SX, ("%s() %d cpu started " 1086 "but not running in the kernel yet\n", 1087 str, who)) 1088 } else if (delays > 2000) { 1089 /* 1090 * We waited at least 20 seconds, bail .. 1091 */ 1092 cmn_err(CE_WARN, "cpu%d: timed out", who); 1093 PMD(PMD_SX, ("%s() %d cpu timed out\n", 1094 str, who)) 1095 return (0); 1096 } 1097 1098 /* 1099 * wait at least 10ms, then check again.. 1100 */ 1101 drv_usecwait(10000); 1102 } 1103 1104 return (1); 1105 } 1106 1107 static void 1108 i_cpr_save_stack(kthread_t *t, wc_cpu_t *wc_cpu) 1109 { 1110 size_t stack_size; /* size of stack */ 1111 caddr_t start = CPR_GET_STACK_START(t); /* stack start */ 1112 caddr_t end = CPR_GET_STACK_END(t); /* stack end */ 1113 1114 stack_size = (size_t)end - (size_t)start; 1115 1116 if (wc_cpu->wc_saved_stack_size < stack_size) { 1117 if (wc_cpu->wc_saved_stack != NULL) { 1118 kmem_free(wc_cpu->wc_saved_stack, 1119 wc_cpu->wc_saved_stack_size); 1120 } 1121 wc_cpu->wc_saved_stack = kmem_zalloc(stack_size, KM_SLEEP); 1122 wc_cpu->wc_saved_stack_size = stack_size; 1123 } 1124 1125 bcopy(start, wc_cpu->wc_saved_stack, stack_size); 1126 } 1127 1128 void 1129 i_cpr_restore_stack(kthread_t *t, greg_t *save_stack) 1130 { 1131 size_t stack_size; /* size of stack */ 1132 caddr_t start = CPR_GET_STACK_START(t); /* stack start */ 1133 caddr_t end = CPR_GET_STACK_END(t); /* stack end */ 1134 1135 stack_size = (size_t)end - (size_t)start; 1136 1137 bcopy(save_stack, start, stack_size); 1138 }