1/* 2 * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. 3 * 4 * @APPLE_LICENSE_HEADER_START@ 5 * 6 * The contents of this file constitute Original Code as defined in and 7 * are subject to the Apple Public Source License Version 1.1 (the 8 * "License"). You may not use this file except in compliance with the 9 * License. Please obtain a copy of the License at 10 * http://www.apple.com/publicsource and read it before using this file. 11 * 12 * This Original Code and all software distributed under the License are 13 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER 14 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, 15 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the 17 * License for the specific language governing rights and limitations 18 * under the License. 19 * 20 * @APPLE_LICENSE_HEADER_END@ 21 */ 22/* 23 * @OSF_COPYRIGHT@ 24 */ 25/* 26 * Mach Operating System 27 * Copyright (c) 1989 Carnegie-Mellon University 28 * All rights reserved. The CMU software License Agreement specifies 29 * the terms and conditions for use and redistribution. 30 */ 31 32#include <mach_rt.h> 33#include <platforms.h> 34#include <mach_ldebug.h> 35#include <i386/asm.h> 36 37#include "assym.s" 38 39#define PAUSE rep; nop 40 41/* 42 * When performance isn't the only concern, it's 43 * nice to build stack frames... 44 */ 45#define BUILD_STACK_FRAMES (GPROF || \ 46 ((MACH_LDEBUG || ETAP_LOCK_TRACE) && MACH_KDB)) 47 48#if BUILD_STACK_FRAMES 49 50/* STack-frame-relative: */ 51#define L_PC B_PC 52#define L_ARG0 B_ARG0 53#define L_ARG1 B_ARG1 54 55#define LEAF_ENTRY(name) \ 56 Entry(name); \ 57 FRAME; \ 58 MCOUNT 59 60#define LEAF_ENTRY2(n1,n2) \ 61 Entry(n1); \ 62 Entry(n2); \ 63 FRAME; \ 64 MCOUNT 65 66#define LEAF_RET \ 67 EMARF; \ 68 ret 69 70#else /* BUILD_STACK_FRAMES */ 71 72/* Stack-pointer-relative: */ 73#define L_PC S_PC 74#define L_ARG0 S_ARG0 75#define L_ARG1 S_ARG1 76 77#define LEAF_ENTRY(name) \ 78 Entry(name) 79 80#define LEAF_ENTRY2(n1,n2) \ 81 Entry(n1); \ 82 Entry(n2) 83 84#define LEAF_RET \ 85 ret 86 87#endif /* BUILD_STACK_FRAMES */ 88 89 90/* Non-leaf routines always have a stack frame: */ 91 92#define NONLEAF_ENTRY(name) \ 93 Entry(name); \ 94 FRAME; \ 95 MCOUNT 96 97#define NONLEAF_ENTRY2(n1,n2) \ 98 Entry(n1); \ 99 Entry(n2); \ 100 FRAME; \ 101 MCOUNT 102 103#define NONLEAF_RET \ 104 EMARF; \ 105 ret 106 107 108#define M_ILK (%edx) 109#define M_LOCKED MUTEX_LOCKED(%edx) 110#define M_WAITERS MUTEX_WAITERS(%edx) 111#define M_PROMOTED_PRI MUTEX_PROMOTED_PRI(%edx) 112#define M_ITAG MUTEX_ITAG(%edx) 113#define M_PTR MUTEX_PTR(%edx) 114#if MACH_LDEBUG 115#define M_TYPE MUTEX_TYPE(%edx) 116#define M_PC MUTEX_PC(%edx) 117#define M_THREAD MUTEX_THREAD(%edx) 118#endif /* MACH_LDEBUG */ 119 120#include <i386/mp.h> 121#define CX(addr,reg) addr(,reg,4) 122 123#if MACH_LDEBUG 124/* 125 * Routines for general lock debugging. 126 */ 127#define S_TYPE SLOCK_TYPE(%edx) 128#define S_PC SLOCK_PC(%edx) 129#define S_THREAD SLOCK_THREAD(%edx) 130#define S_DURATIONH SLOCK_DURATIONH(%edx) 131#define S_DURATIONL SLOCK_DURATIONL(%edx) 132 133/* 134 * Checks for expected lock types and calls "panic" on 135 * mismatch. Detects calls to Mutex functions with 136 * type simplelock and vice versa. 137 */ 138#define CHECK_MUTEX_TYPE() \ 139 cmpl $ MUTEX_TAG,M_TYPE ; \ 140 je 1f ; \ 141 pushl $2f ; \ 142 call EXT(panic) ; \ 143 hlt ; \ 144 .data ; \ 1452: String "not a mutex!" ; \ 146 .text ; \ 1471: 148 149#define CHECK_SIMPLE_LOCK_TYPE() \ 150 cmpl $ USLOCK_TAG,S_TYPE ; \ 151 je 1f ; \ 152 pushl $2f ; \ 153 call EXT(panic) ; \ 154 hlt ; \ 155 .data ; \ 1562: String "not a simple lock!" ; \ 157 .text ; \ 1581: 159 160/* 161 * If one or more simplelocks are currently held by a thread, 162 * an attempt to acquire a mutex will cause this check to fail 163 * (since a mutex lock may context switch, holding a simplelock 164 * is not a good thing). 165 */ 166#if MACH_RT 167#define CHECK_PREEMPTION_LEVEL() \ 168 cmpl $0,%gs:CPU_PREEMPTION_LEVEL ; \ 169 je 1f ; \ 170 pushl $2f ; \ 171 call EXT(panic) ; \ 172 hlt ; \ 173 .data ; \ 1742: String "preemption_level != 0!" ; \ 175 .text ; \ 1761: 177#else /* MACH_RT */ 178#define CHECK_PREEMPTION_LEVEL() 179#endif /* MACH_RT */ 180 181#define CHECK_NO_SIMPLELOCKS() \ 182 cmpl $0,%gs:CPU_SIMPLE_LOCK_COUNT ; \ 183 je 1f ; \ 184 pushl $2f ; \ 185 call EXT(panic) ; \ 186 hlt ; \ 187 .data ; \ 1882: String "simple_locks_held!" ; \ 189 .text ; \ 1901: 191 192/* 193 * Verifies return to the correct thread in "unlock" situations. 194 */ 195#define CHECK_THREAD(thd) \ 196 movl %gs:CPU_ACTIVE_THREAD,%ecx ; \ 197 testl %ecx,%ecx ; \ 198 je 1f ; \ 199 cmpl %ecx,thd ; \ 200 je 1f ; \ 201 pushl $2f ; \ 202 call EXT(panic) ; \ 203 hlt ; \ 204 .data ; \ 2052: String "wrong thread!" ; \ 206 .text ; \ 2071: 208 209#define CHECK_MYLOCK(thd) \ 210 movl %gs:CPU_ACTIVE_THREAD,%ecx ; \ 211 testl %ecx,%ecx ; \ 212 je 1f ; \ 213 cmpl %ecx,thd ; \ 214 jne 1f ; \ 215 pushl $2f ; \ 216 call EXT(panic) ; \ 217 hlt ; \ 218 .data ; \ 2192: String "mylock attempt!" ; \ 220 .text ; \ 2211: 222 223#define METER_SIMPLE_LOCK_LOCK(reg) \ 224 pushl reg ; \ 225 call EXT(meter_simple_lock) ; \ 226 popl reg 227 228#define METER_SIMPLE_LOCK_UNLOCK(reg) \ 229 pushl reg ; \ 230 call EXT(meter_simple_unlock) ; \ 231 popl reg 232 233#else /* MACH_LDEBUG */ 234#define CHECK_MUTEX_TYPE() 235#define CHECK_SIMPLE_LOCK_TYPE 236#define CHECK_THREAD(thd) 237#define CHECK_PREEMPTION_LEVEL() 238#define CHECK_NO_SIMPLELOCKS() 239#define CHECK_MYLOCK(thd) 240#define METER_SIMPLE_LOCK_LOCK(reg) 241#define METER_SIMPLE_LOCK_UNLOCK(reg) 242#endif /* MACH_LDEBUG */ 243 244 245/* 246 * void hw_lock_init(hw_lock_t) 247 * 248 * Initialize a hardware lock. 249 */ 250LEAF_ENTRY(hw_lock_init) 251 movl L_ARG0,%edx /* fetch lock pointer */ 252 movl $0,0(%edx) /* clear the lock */ 253 LEAF_RET 254 255/* 256 * void hw_lock_lock(hw_lock_t) 257 * 258 * Acquire lock, spinning until it becomes available. 259 * MACH_RT: also return with preemption disabled. 260 */ 261LEAF_ENTRY(hw_lock_lock) 262 movl L_ARG0,%edx /* fetch lock pointer */ 263 264 movl L_PC,%ecx 2651: DISABLE_PREEMPTION 266 movl 0(%edx), %eax 267 testl %eax,%eax /* lock locked? */ 268 jne 3f /* branch if so */ 269 lock; cmpxchgl %ecx,0(%edx) /* try to acquire the HW lock */ 270 jne 3f 271 movl $1,%eax /* In case this was a timeout call */ 272 LEAF_RET /* if yes, then nothing left to do */ 273 2743: ENABLE_PREEMPTION /* no reason we can't be preemptable */ 275 PAUSE /* pause for hyper-threading */ 276 jmp 1b /* try again */ 277 278/* 279 * unsigned int hw_lock_to(hw_lock_t, unsigned int) 280 * 281 * Acquire lock, spinning until it becomes available or timeout. 282 * MACH_RT: also return with preemption disabled. 283 */ 284LEAF_ENTRY(hw_lock_to) 2851: 286 movl L_ARG0,%edx /* fetch lock pointer */ 287 movl L_PC,%ecx 288 /* 289 * Attempt to grab the lock immediately 290 * - fastpath without timeout nonsense. 291 */ 292 DISABLE_PREEMPTION 293 movl 0(%edx), %eax 294 testl %eax,%eax /* lock locked? */ 295 jne 2f /* branch if so */ 296 lock; cmpxchgl %ecx,0(%edx) /* try to acquire the HW lock */ 297 jne 2f /* branch on failure */ 298 movl $1,%eax 299 LEAF_RET 300 3012: 302#define INNER_LOOP_COUNT 1000 303 /* 304 * Failed to get the lock so set the timeout 305 * and then spin re-checking the lock but pausing 306 * every so many (INNER_LOOP_COUNT) spins to check for timeout. 307 */ 308 movl L_ARG1,%ecx /* fetch timeout */ 309 push %edi 310 push %ebx 311 mov %edx,%edi 312 313 rdtsc /* read cyclecount into %edx:%eax */ 314 addl %ecx,%eax /* fetch and timeout */ 315 adcl $0,%edx /* add carry */ 316 mov %edx,%ecx 317 mov %eax,%ebx /* %ecx:%ebx is the timeout expiry */ 3183: 319 ENABLE_PREEMPTION /* no reason not to be preempted now */ 3204: 321 /* 322 * The inner-loop spin to look for the lock being freed. 323 */ 324 mov $(INNER_LOOP_COUNT),%edx 3255: 326 PAUSE /* pause for hyper-threading */ 327 movl 0(%edi),%eax /* spin checking lock value in cache */ 328 testl %eax,%eax 329 je 6f /* zero => unlocked, try to grab it */ 330 decl %edx /* decrement inner loop count */ 331 jnz 5b /* time to check for timeout? */ 332 333 /* 334 * Here after spinning INNER_LOOP_COUNT times, check for timeout 335 */ 336 rdtsc /* cyclecount into %edx:%eax */ 337 cmpl %ecx,%edx /* compare high-order 32-bits */ 338 jb 4b /* continue spinning if less, or */ 339 cmpl %ebx,%eax /* compare low-order 32-bits */ 340 jb 5b /* continue if less, else bail */ 341 xor %eax,%eax /* with 0 return value */ 342 pop %ebx 343 pop %edi 344 LEAF_RET 345 3466: 347 /* 348 * Here to try to grab the lock that now appears to be free 349 * after contention. 350 */ 351 movl 8+L_PC,%edx /* calling pc (8+ for pushed regs) */ 352 DISABLE_PREEMPTION 353 lock; cmpxchgl %edx,0(%edi) /* try to acquire the HW lock */ 354 jne 3b /* no - spin again */ 355 movl $1,%eax /* yes */ 356 pop %ebx 357 pop %edi 358 LEAF_RET 359 360/* 361 * void hw_lock_unlock(hw_lock_t) 362 * 363 * Unconditionally release lock. 364 * MACH_RT: release preemption level. 365 */ 366LEAF_ENTRY(hw_lock_unlock) 367 movl L_ARG0,%edx /* fetch lock pointer */ 368 movl $0,0(%edx) /* clear the lock */ 369 ENABLE_PREEMPTION 370 LEAF_RET 371 372/* 373 * unsigned int hw_lock_try(hw_lock_t) 374 * MACH_RT: returns with preemption disabled on success. 375 */ 376LEAF_ENTRY(hw_lock_try) 377 movl L_ARG0,%edx /* fetch lock pointer */ 378 379 movl L_PC,%ecx 380 DISABLE_PREEMPTION 381 movl 0(%edx),%eax 382 testl %eax,%eax 383 jne 1f 384 lock; cmpxchgl %ecx,0(%edx) /* try to acquire the HW lock */ 385 jne 1f 386 387 movl $1,%eax /* success */ 388 LEAF_RET 389 3901: ENABLE_PREEMPTION /* failure: release preemption... */ 391 xorl %eax,%eax /* ...and return failure */ 392 LEAF_RET 393 394/* 395 * unsigned int hw_lock_held(hw_lock_t) 396 * MACH_RT: doesn't change preemption state. 397 * N.B. Racy, of course. 398 */ 399LEAF_ENTRY(hw_lock_held) 400 movl L_ARG0,%edx /* fetch lock pointer */ 401 402 movl 0(%edx),%eax /* check lock value */ 403 testl %eax,%eax 404 movl $1,%ecx 405 cmovne %ecx,%eax /* 0 => unlocked, 1 => locked */ 406 LEAF_RET 407 408LEAF_ENTRY(mutex_init) 409 movl L_ARG0,%edx /* fetch lock pointer */ 410 xorl %eax,%eax 411 movl %eax,M_ILK /* clear interlock */ 412 movl %eax,M_LOCKED /* clear locked flag */ 413 movw %ax,M_WAITERS /* init waiter count */ 414 movw %ax,M_PROMOTED_PRI 415 416#if MACH_LDEBUG 417 movl $ MUTEX_TAG,M_TYPE /* set lock type */ 418 movl %eax,M_PC /* init caller pc */ 419 movl %eax,M_THREAD /* and owning thread */ 420#endif 421 422 LEAF_RET 423 424NONLEAF_ENTRY2(mutex_lock,_mutex_lock) 425 426 movl B_ARG0,%edx /* fetch lock pointer */ 427 428 CHECK_MUTEX_TYPE() 429 CHECK_NO_SIMPLELOCKS() 430 CHECK_PREEMPTION_LEVEL() 431 432 pushf /* save interrupt state */ 433 cli /* disable interrupts */ 434 435ml_retry: 436 movl B_PC,%ecx 437 438ml_get_hw: 439 movl M_ILK,%eax /* read interlock */ 440 testl %eax,%eax /* unlocked? */ 441 je 1f /* yes - attempt to lock it */ 442 PAUSE /* no - pause */ 443 jmp ml_get_hw /* try again */ 4441: 445 lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */ 446 jne ml_get_hw /* branch on failure to retry */ 447 448 movl M_LOCKED,%ecx /* get lock owner */ 449 testl %ecx,%ecx /* is the mutex locked? */ 450 jne ml_fail /* yes, we lose */ 451 movl %gs:CPU_ACTIVE_THREAD,%ecx 452 movl %ecx,M_LOCKED 453 454#if MACH_LDEBUG 455 movl %ecx,M_THREAD 456 movl B_PC,%ecx 457 movl %ecx,M_PC 458#endif 459 460 pushl %edx /* save mutex address */ 461 pushl %edx 462 call EXT(lck_mtx_lock_acquire) 463 addl $4,%esp 464 popl %edx /* restore mutex address */ 465 466 xorl %eax,%eax 467 movl %eax,M_ILK 468 469 popf /* restore interrupt state */ 470 471 NONLEAF_RET 472 473ml_fail: 474ml_block: 475 CHECK_MYLOCK(M_THREAD) 476 pushl M_LOCKED 477 pushl %edx /* push mutex address */ 478 call EXT(lck_mtx_lock_wait) /* wait for the lock */ 479 addl $8,%esp 480 movl B_ARG0,%edx /* refetch mutex address */ 481 jmp ml_retry /* and try again */ 482 483NONLEAF_ENTRY2(mutex_try,_mutex_try) 484 485 movl B_ARG0,%edx /* fetch lock pointer */ 486 487 CHECK_MUTEX_TYPE() 488 CHECK_NO_SIMPLELOCKS() 489 490 movl B_PC,%ecx 491 492 pushf /* save interrupt state */ 493 cli /* disable interrupts */ 494 495mt_get_hw: 496 movl M_ILK,%eax /* read interlock */ 497 testl %eax,%eax /* unlocked? */ 498 je 1f /* yes - attempt to lock it */ 499 PAUSE /* no - pause */ 500 jmp mt_get_hw /* try again */ 5011: 502 lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */ 503 jne mt_get_hw /* branch on failure to retry */ 504 505 movl M_LOCKED,%ecx /* get lock owner */ 506 testl %ecx,%ecx /* is the mutex locked? */ 507 jne mt_fail /* yes, we lose */ 508 movl %gs:CPU_ACTIVE_THREAD,%ecx 509 movl %ecx,M_LOCKED 510 511#if MACH_LDEBUG 512 movl %ecx,M_THREAD 513 movl B_PC,%ecx 514 movl %ecx,M_PC 515#endif 516 517 pushl %edx /* save mutex address */ 518 pushl %edx 519 call EXT(lck_mtx_lock_acquire) 520 addl $4,%esp 521 popl %edx /* restore mutex address */ 522 523 xorl %eax,%eax 524 movl %eax,M_ILK 525 526 popf /* restore interrupt state */ 527 528 movl $1,%eax 529 530 NONLEAF_RET 531 532mt_fail: 533 xorl %eax,%eax 534 movl %eax,M_ILK 535 536 popf /* restore interrupt state */ 537 538 xorl %eax,%eax 539 540 NONLEAF_RET 541 542NONLEAF_ENTRY(mutex_unlock) 543 movl B_ARG0,%edx /* fetch lock pointer */ 544 545 CHECK_MUTEX_TYPE() 546 CHECK_THREAD(M_THREAD) 547 548 movl B_PC,%ecx 549 550 pushf /* save interrupt state */ 551 cli /* disable interrupts */ 552 553mu_get_hw: 554 movl M_ILK,%eax /* read interlock */ 555 testl %eax,%eax /* unlocked? */ 556 je 1f /* yes - attempt to lock it */ 557 PAUSE /* no - pause */ 558 jmp mu_get_hw /* try again */ 5591: 560 lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */ 561 jne mu_get_hw /* branch on failure to retry */ 562 563 cmpw $0,M_WAITERS /* are there any waiters? */ 564 jne mu_wakeup /* yes, more work to do */ 565 566mu_doit: 567 568#if MACH_LDEBUG 569 movl $0,M_THREAD /* disown thread */ 570#endif 571 572 xorl %ecx,%ecx 573 movl %ecx,M_LOCKED /* unlock the mutex */ 574 575 movl %ecx,M_ILK 576 577 popf /* restore interrupt state */ 578 579 NONLEAF_RET 580 581mu_wakeup: 582 pushl M_LOCKED 583 pushl %edx /* push mutex address */ 584 call EXT(lck_mtx_unlock_wakeup)/* yes, wake a thread */ 585 addl $8,%esp 586 movl B_ARG0,%edx /* restore lock pointer */ 587 jmp mu_doit 588 589/* 590 * lck_mtx_lock() 591 * lck_mtx_try_lock() 592 * lck_mutex_unlock() 593 * 594 * These are variants of mutex_lock(), mutex_try() and mutex_unlock() without 595 * DEBUG checks (which require fields not present in lck_mtx_t's). 596 */ 597NONLEAF_ENTRY(lck_mtx_lock) 598 599 movl B_ARG0,%edx /* fetch lock pointer */ 600 cmpl $(MUTEX_IND),M_ITAG /* is this indirect? */ 601 cmove M_PTR,%edx /* yes - take indirection */ 602 603 CHECK_NO_SIMPLELOCKS() 604 CHECK_PREEMPTION_LEVEL() 605 606 pushf /* save interrupt state */ 607 cli /* disable interrupts */ 608 609lml_retry: 610 movl B_PC,%ecx 611 612lml_get_hw: 613 movl M_ILK,%eax /* read interlock */ 614 testl %eax,%eax /* unlocked? */ 615 je 1f /* yes - attempt to lock it */ 616 PAUSE /* no - pause */ 617 jmp lml_get_hw /* try again */ 6181: 619 lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */ 620 jne lml_get_hw /* branch on failure to retry */ 621 622 movl M_LOCKED,%ecx /* get lock owner */ 623 testl %ecx,%ecx /* is the mutex locked? */ 624 jne lml_fail /* yes, we lose */ 625 movl %gs:CPU_ACTIVE_THREAD,%ecx 626 movl %ecx,M_LOCKED 627 628 pushl %edx /* save mutex address */ 629 pushl %edx 630 call EXT(lck_mtx_lock_acquire) 631 addl $4,%esp 632 popl %edx /* restore mutex address */ 633 634 xorl %eax,%eax 635 movl %eax,M_ILK 636 637 popf /* restore interrupt state */ 638 639 NONLEAF_RET 640 641lml_fail: 642 CHECK_MYLOCK(M_THREAD) 643 pushl %edx /* save mutex address */ 644 pushl M_LOCKED 645 pushl %edx /* push mutex address */ 646 call EXT(lck_mtx_lock_wait) /* wait for the lock */ 647 addl $8,%esp 648 popl %edx /* restore mutex address */ 649 jmp lml_retry /* and try again */ 650 651NONLEAF_ENTRY(lck_mtx_try_lock) 652 653 movl B_ARG0,%edx /* fetch lock pointer */ 654 cmpl $(MUTEX_IND),M_ITAG /* is this indirect? */ 655 cmove M_PTR,%edx /* yes - take indirection */ 656 657 CHECK_NO_SIMPLELOCKS() 658 CHECK_PREEMPTION_LEVEL() 659 660 movl B_PC,%ecx 661 662 pushf /* save interrupt state */ 663 cli /* disable interrupts */ 664 665lmt_get_hw: 666 movl M_ILK,%eax /* read interlock */ 667 testl %eax,%eax /* unlocked? */ 668 je 1f /* yes - attempt to lock it */ 669 PAUSE /* no - pause */ 670 jmp lmt_get_hw /* try again */ 6711: 672 lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */ 673 jne lmt_get_hw /* branch on failure to retry */ 674 675 movl M_LOCKED,%ecx /* get lock owner */ 676 testl %ecx,%ecx /* is the mutex locked? */ 677 jne lmt_fail /* yes, we lose */ 678 movl %gs:CPU_ACTIVE_THREAD,%ecx 679 movl %ecx,M_LOCKED 680 681 pushl %edx /* save mutex address */ 682 pushl %edx 683 call EXT(lck_mtx_lock_acquire) 684 addl $4,%esp 685 popl %edx /* restore mutex address */ 686 687 xorl %eax,%eax 688 movl %eax,M_ILK 689 690 popf /* restore interrupt state */ 691 692 movl $1,%eax /* return success */ 693 NONLEAF_RET 694 695lmt_fail: 696 xorl %eax,%eax 697 movl %eax,M_ILK 698 699 popf /* restore interrupt state */ 700 701 xorl %eax,%eax /* return failure */ 702 NONLEAF_RET 703 704NONLEAF_ENTRY(lck_mtx_unlock) 705 706 movl B_ARG0,%edx /* fetch lock pointer */ 707 cmpl $(MUTEX_IND),M_ITAG /* is this indirect? */ 708 cmove M_PTR,%edx /* yes - take indirection */ 709 710 movl B_PC,%ecx 711 712 pushf /* save interrupt state */ 713 cli /* disable interrupts */ 714 715lmu_get_hw: 716 movl M_ILK,%eax /* read interlock */ 717 testl %eax,%eax /* unlocked? */ 718 je 1f /* yes - attempt to lock it */ 719 PAUSE /* no - pause */ 720 jmp lmu_get_hw /* try again */ 7211: 722 lock; cmpxchgl %ecx,M_ILK /* atomic compare and exchange */ 723 jne lmu_get_hw /* branch on failure to retry */ 724 725 cmpw $0,M_WAITERS /* are there any waiters? */ 726 jne lmu_wakeup /* yes, more work to do */ 727 728lmu_doit: 729 xorl %ecx,%ecx 730 movl %ecx,M_LOCKED /* unlock the mutex */ 731 732 movl %ecx,M_ILK 733 734 popf /* restore interrupt state */ 735 736 NONLEAF_RET 737 738lmu_wakeup: 739 pushl %edx /* save mutex address */ 740 pushl M_LOCKED 741 pushl %edx /* push mutex address */ 742 call EXT(lck_mtx_unlock_wakeup)/* yes, wake a thread */ 743 addl $8,%esp 744 popl %edx /* restore mutex pointer */ 745 jmp lmu_doit 746 747LEAF_ENTRY(lck_mtx_ilk_unlock) 748 movl L_ARG0,%edx /* no indirection here */ 749 750 xorl %eax,%eax 751 movl %eax,M_ILK 752 753 LEAF_RET 754 755LEAF_ENTRY(_disable_preemption) 756#if MACH_RT 757 _DISABLE_PREEMPTION 758#endif /* MACH_RT */ 759 LEAF_RET 760 761LEAF_ENTRY(_enable_preemption) 762#if MACH_RT 763#if MACH_ASSERT 764 cmpl $0,%gs:CPU_PREEMPTION_LEVEL 765 jg 1f 766 pushl %gs:CPU_PREEMPTION_LEVEL 767 pushl $2f 768 call EXT(panic) 769 hlt 770 .data 7712: String "_enable_preemption: preemption_level(%d) < 0!" 772 .text 7731: 774#endif /* MACH_ASSERT */ 775 _ENABLE_PREEMPTION 776#endif /* MACH_RT */ 777 LEAF_RET 778 779LEAF_ENTRY(_enable_preemption_no_check) 780#if MACH_RT 781#if MACH_ASSERT 782 cmpl $0,%gs:CPU_PREEMPTION_LEVEL 783 jg 1f 784 pushl $2f 785 call EXT(panic) 786 hlt 787 .data 7882: String "_enable_preemption_no_check: preemption_level <= 0!" 789 .text 7901: 791#endif /* MACH_ASSERT */ 792 _ENABLE_PREEMPTION_NO_CHECK 793#endif /* MACH_RT */ 794 LEAF_RET 795 796 797LEAF_ENTRY(_mp_disable_preemption) 798#if MACH_RT 799 _DISABLE_PREEMPTION 800#endif /* MACH_RT */ 801 LEAF_RET 802 803LEAF_ENTRY(_mp_enable_preemption) 804#if MACH_RT 805#if MACH_ASSERT 806 cmpl $0,%gs:CPU_PREEMPTION_LEVEL 807 jg 1f 808 pushl %gs:CPU_PREEMPTION_LEVEL 809 pushl $2f 810 call EXT(panic) 811 hlt 812 .data 8132: String "_mp_enable_preemption: preemption_level (%d) <= 0!" 814 .text 8151: 816#endif /* MACH_ASSERT */ 817 _ENABLE_PREEMPTION 818#endif /* MACH_RT */ 819 LEAF_RET 820 821LEAF_ENTRY(_mp_enable_preemption_no_check) 822#if MACH_RT 823#if MACH_ASSERT 824 cmpl $0,%gs:CPU_PREEMPTION_LEVEL 825 jg 1f 826 pushl $2f 827 call EXT(panic) 828 hlt 829 .data 8302: String "_mp_enable_preemption_no_check: preemption_level <= 0!" 831 .text 8321: 833#endif /* MACH_ASSERT */ 834 _ENABLE_PREEMPTION_NO_CHECK 835#endif /* MACH_RT */ 836 LEAF_RET 837 838 839LEAF_ENTRY(i_bit_set) 840 movl L_ARG0,%edx 841 movl L_ARG1,%eax 842 lock 843 bts %dl,(%eax) 844 LEAF_RET 845 846LEAF_ENTRY(i_bit_clear) 847 movl L_ARG0,%edx 848 movl L_ARG1,%eax 849 lock 850 btr %dl,(%eax) 851 LEAF_RET 852 853LEAF_ENTRY(bit_lock) 854 movl L_ARG0,%ecx 855 movl L_ARG1,%eax 8561: 857 lock 858 bts %ecx,(%eax) 859 jb 1b 860 LEAF_RET 861 862LEAF_ENTRY(bit_lock_try) 863 movl L_ARG0,%ecx 864 movl L_ARG1,%eax 865 lock 866 bts %ecx,(%eax) 867 jb bit_lock_failed 868 LEAF_RET /* %eax better not be null ! */ 869bit_lock_failed: 870 xorl %eax,%eax 871 LEAF_RET 872 873LEAF_ENTRY(bit_unlock) 874 movl L_ARG0,%ecx 875 movl L_ARG1,%eax 876 lock 877 btr %ecx,(%eax) 878 LEAF_RET 879