Rev 2131 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 2131 | Rev 2422 | ||
---|---|---|---|
Line 55... | Line 55... | ||
55 | #include <arch/mm/page.h> |
55 | #include <arch/mm/page.h> |
56 | #include <genarch/mm/page_pt.h> |
56 | #include <genarch/mm/page_pt.h> |
57 | #include <genarch/mm/page_ht.h> |
57 | #include <genarch/mm/page_ht.h> |
58 | #include <mm/asid.h> |
58 | #include <mm/asid.h> |
59 | #include <arch/mm/asid.h> |
59 | #include <arch/mm/asid.h> |
- | 60 | #include <preemption.h> |
|
60 | #include <synch/spinlock.h> |
61 | #include <synch/spinlock.h> |
61 | #include <synch/mutex.h> |
62 | #include <synch/mutex.h> |
62 | #include <adt/list.h> |
63 | #include <adt/list.h> |
63 | #include <adt/btree.h> |
64 | #include <adt/btree.h> |
64 | #include <proc/task.h> |
65 | #include <proc/task.h> |
Line 93... | Line 94... | ||
93 | */ |
94 | */ |
94 | static slab_cache_t *as_slab; |
95 | static slab_cache_t *as_slab; |
95 | #endif |
96 | #endif |
96 | 97 | ||
97 | /** |
98 | /** |
- | 99 | * This lock serializes access to the ASID subsystem. |
|
- | 100 | * It protects: |
|
98 | * This lock protects inactive_as_with_asid_head list. It must be acquired |
101 | * - inactive_as_with_asid_head list |
- | 102 | * - as->asid for each as of the as_t type |
|
99 | * before as_t mutex. |
103 | * - asids_allocated counter |
100 | */ |
104 | */ |
101 | SPINLOCK_INITIALIZE(inactive_as_with_asid_lock); |
105 | SPINLOCK_INITIALIZE(asidlock); |
102 | 106 | ||
103 | /** |
107 | /** |
104 | * This list contains address spaces that are not active on any |
108 | * This list contains address spaces that are not active on any |
105 | * processor and that have valid ASID. |
109 | * processor and that have valid ASID. |
106 | */ |
110 | */ |
Line 176... | Line 180... | ||
176 | if (flags & FLAG_AS_KERNEL) |
180 | if (flags & FLAG_AS_KERNEL) |
177 | as->asid = ASID_KERNEL; |
181 | as->asid = ASID_KERNEL; |
178 | else |
182 | else |
179 | as->asid = ASID_INVALID; |
183 | as->asid = ASID_INVALID; |
180 | 184 | ||
181 | as->refcount = 0; |
185 | atomic_set(&as->refcount, 0); |
182 | as->cpu_refcount = 0; |
186 | as->cpu_refcount = 0; |
183 | #ifdef AS_PAGE_TABLE |
187 | #ifdef AS_PAGE_TABLE |
184 | as->genarch.page_table = page_table_create(flags); |
188 | as->genarch.page_table = page_table_create(flags); |
185 | #else |
189 | #else |
186 | page_table_create(flags); |
190 | page_table_create(flags); |
Line 191... | Line 195... | ||
191 | 195 | ||
192 | /** Destroy adress space. |
196 | /** Destroy adress space. |
193 | * |
197 | * |
194 | * When there are no tasks referencing this address space (i.e. its refcount is |
198 | * When there are no tasks referencing this address space (i.e. its refcount is |
195 | * zero), the address space can be destroyed. |
199 | * zero), the address space can be destroyed. |
- | 200 | * |
|
- | 201 | * We know that we don't hold any spinlock. |
|
196 | */ |
202 | */ |
197 | void as_destroy(as_t *as) |
203 | void as_destroy(as_t *as) |
198 | { |
204 | { |
199 | ipl_t ipl; |
205 | ipl_t ipl; |
200 | bool cond; |
206 | bool cond; |
- | 207 | DEADLOCK_PROBE_INIT(p_asidlock); |
|
201 | 208 | ||
202 | ASSERT(as->refcount == 0); |
209 | ASSERT(atomic_get(&as->refcount) == 0); |
203 | 210 | ||
204 | /* |
211 | /* |
205 | * Since there is no reference to this area, |
212 | * Since there is no reference to this area, |
206 | * it is safe not to lock its mutex. |
213 | * it is safe not to lock its mutex. |
207 | */ |
214 | */ |
- | 215 | ||
- | 216 | /* |
|
- | 217 | * We need to avoid deadlock between TLB shootdown and asidlock. |
|
- | 218 | * We therefore try to take asid conditionally and if we don't succeed, |
|
- | 219 | * we enable interrupts and try again. This is done while preemption is |
|
- | 220 | * disabled to prevent nested context switches. We also depend on the |
|
- | 221 | * fact that so far no spinlocks are held. |
|
- | 222 | */ |
|
- | 223 | preemption_disable(); |
|
208 | ipl = interrupts_disable(); |
224 | ipl = interrupts_read(); |
- | 225 | retry: |
|
- | 226 | interrupts_disable(); |
|
209 | spinlock_lock(&inactive_as_with_asid_lock); |
227 | if (!spinlock_trylock(&asidlock)) { |
- | 228 | interrupts_enable(); |
|
- | 229 | DEADLOCK_PROBE(p_asidlock, DEADLOCK_THRESHOLD); |
|
- | 230 | goto retry; |
|
- | 231 | } |
|
- | 232 | preemption_enable(); /* Interrupts disabled, enable preemption */ |
|
210 | if (as->asid != ASID_INVALID && as != AS_KERNEL) { |
233 | if (as->asid != ASID_INVALID && as != AS_KERNEL) { |
211 | if (as != AS && as->cpu_refcount == 0) |
234 | if (as != AS && as->cpu_refcount == 0) |
212 | list_remove(&as->inactive_as_with_asid_link); |
235 | list_remove(&as->inactive_as_with_asid_link); |
213 | asid_put(as->asid); |
236 | asid_put(as->asid); |
214 | } |
237 | } |
215 | spinlock_unlock(&inactive_as_with_asid_lock); |
238 | spinlock_unlock(&asidlock); |
216 | 239 | ||
217 | /* |
240 | /* |
218 | * Destroy address space areas of the address space. |
241 | * Destroy address space areas of the address space. |
219 | * The B+tree must be walked carefully because it is |
242 | * The B+tree must be walked carefully because it is |
220 | * also being destroyed. |
243 | * also being destroyed. |
Line 466... | Line 489... | ||
466 | } |
489 | } |
467 | 490 | ||
468 | /* |
491 | /* |
469 | * Finish TLB shootdown sequence. |
492 | * Finish TLB shootdown sequence. |
470 | */ |
493 | */ |
- | 494 | ||
471 | tlb_invalidate_pages(as->asid, area->base + pages * PAGE_SIZE, |
495 | tlb_invalidate_pages(as->asid, area->base + pages * PAGE_SIZE, |
472 | area->pages - pages); |
496 | area->pages - pages); |
473 | tlb_shootdown_finalize(); |
- | |
474 | - | ||
475 | /* |
497 | /* |
476 | * Invalidate software translation caches (e.g. TSB on sparc64). |
498 | * Invalidate software translation caches (e.g. TSB on sparc64). |
477 | */ |
499 | */ |
478 | as_invalidate_translation_cache(as, area->base + |
500 | as_invalidate_translation_cache(as, area->base + |
479 | pages * PAGE_SIZE, area->pages - pages); |
501 | pages * PAGE_SIZE, area->pages - pages); |
- | 502 | tlb_shootdown_finalize(); |
|
- | 503 | ||
480 | } else { |
504 | } else { |
481 | /* |
505 | /* |
482 | * Growing the area. |
506 | * Growing the area. |
483 | * Check for overlaps with other address space areas. |
507 | * Check for overlaps with other address space areas. |
484 | */ |
508 | */ |
Line 562... | Line 586... | ||
562 | } |
586 | } |
563 | 587 | ||
564 | /* |
588 | /* |
565 | * Finish TLB shootdown sequence. |
589 | * Finish TLB shootdown sequence. |
566 | */ |
590 | */ |
567 | tlb_invalidate_pages(as->asid, area->base, area->pages); |
- | |
568 | tlb_shootdown_finalize(); |
- | |
569 | 591 | ||
- | 592 | tlb_invalidate_pages(as->asid, area->base, area->pages); |
|
570 | /* |
593 | /* |
571 | * Invalidate potential software translation caches (e.g. TSB on |
594 | * Invalidate potential software translation caches (e.g. TSB on |
572 | * sparc64). |
595 | * sparc64). |
573 | */ |
596 | */ |
574 | as_invalidate_translation_cache(as, area->base, area->pages); |
597 | as_invalidate_translation_cache(as, area->base, area->pages); |
- | 598 | tlb_shootdown_finalize(); |
|
575 | 599 | ||
576 | btree_destroy(&area->used_space); |
600 | btree_destroy(&area->used_space); |
577 | 601 | ||
578 | area->attributes |= AS_AREA_ATTR_PARTIAL; |
602 | area->attributes |= AS_AREA_ATTR_PARTIAL; |
579 | 603 | ||
Line 611... | Line 635... | ||
611 | * |
635 | * |
612 | * @return Zero on success or ENOENT if there is no such task or if there is no |
636 | * @return Zero on success or ENOENT if there is no such task or if there is no |
613 | * such address space area, EPERM if there was a problem in accepting the area |
637 | * such address space area, EPERM if there was a problem in accepting the area |
614 | * or ENOMEM if there was a problem in allocating destination address space |
638 | * or ENOMEM if there was a problem in allocating destination address space |
615 | * area. ENOTSUP is returned if the address space area backend does not support |
639 | * area. ENOTSUP is returned if the address space area backend does not support |
616 | * sharing or if the kernel detects an attempt to create an illegal address |
- | |
617 | * alias. |
640 | * sharing. |
618 | */ |
641 | */ |
619 | int as_area_share(as_t *src_as, uintptr_t src_base, size_t acc_size, |
642 | int as_area_share(as_t *src_as, uintptr_t src_base, size_t acc_size, |
620 | as_t *dst_as, uintptr_t dst_base, int dst_flags_mask) |
643 | as_t *dst_as, uintptr_t dst_base, int dst_flags_mask) |
621 | { |
644 | { |
622 | ipl_t ipl; |
645 | ipl_t ipl; |
Line 665... | Line 688... | ||
665 | mutex_unlock(&src_as->lock); |
688 | mutex_unlock(&src_as->lock); |
666 | interrupts_restore(ipl); |
689 | interrupts_restore(ipl); |
667 | return EPERM; |
690 | return EPERM; |
668 | } |
691 | } |
669 | 692 | ||
670 | #ifdef CONFIG_VIRT_IDX_DCACHE |
- | |
671 | if (!(dst_flags_mask & AS_AREA_EXEC)) { |
- | |
672 | if (PAGE_COLOR(src_area->base) != PAGE_COLOR(dst_base)) { |
- | |
673 | /* |
- | |
674 | * Refuse to create an illegal address alias. |
- | |
675 | */ |
- | |
676 | mutex_unlock(&src_area->lock); |
- | |
677 | mutex_unlock(&src_as->lock); |
- | |
678 | interrupts_restore(ipl); |
- | |
679 | return ENOTSUP; |
- | |
680 | } |
- | |
681 | } |
- | |
682 | #endif /* CONFIG_VIRT_IDX_DCACHE */ |
- | |
683 | - | ||
684 | /* |
693 | /* |
685 | * Now we are committed to sharing the area. |
694 | * Now we are committed to sharing the area. |
686 | * First, prepare the area for sharing. |
695 | * First, prepare the area for sharing. |
687 | * Then it will be safe to unlock it. |
696 | * Then it will be safe to unlock it. |
688 | */ |
697 | */ |
Line 873... | Line 882... | ||
873 | } |
882 | } |
874 | 883 | ||
875 | /** Switch address spaces. |
884 | /** Switch address spaces. |
876 | * |
885 | * |
877 | * Note that this function cannot sleep as it is essentially a part of |
886 | * Note that this function cannot sleep as it is essentially a part of |
878 | * scheduling. Sleeping here would lead to deadlock on wakeup. |
887 | * scheduling. Sleeping here would lead to deadlock on wakeup. Another |
- | 888 | * thing which is forbidden in this context is locking the address space. |
|
- | 889 | * |
|
- | 890 | * When this function is enetered, no spinlocks may be held. |
|
879 | * |
891 | * |
880 | * @param old Old address space or NULL. |
892 | * @param old Old address space or NULL. |
881 | * @param new New address space. |
893 | * @param new New address space. |
882 | */ |
894 | */ |
883 | void as_switch(as_t *old_as, as_t *new_as) |
895 | void as_switch(as_t *old_as, as_t *new_as) |
884 | { |
896 | { |
- | 897 | DEADLOCK_PROBE_INIT(p_asidlock); |
|
885 | ipl_t ipl; |
898 | preemption_disable(); |
- | 899 | retry: |
|
886 | bool needs_asid = false; |
900 | (void) interrupts_disable(); |
- | 901 | if (!spinlock_trylock(&asidlock)) { |
|
- | 902 | /* |
|
- | 903 | * Avoid deadlock with TLB shootdown. |
|
- | 904 | * We can enable interrupts here because |
|
- | 905 | * preemption is disabled. We should not be |
|
- | 906 | * holding any other lock. |
|
887 | 907 | */ |
|
888 | ipl = interrupts_disable(); |
908 | (void) interrupts_enable(); |
889 | spinlock_lock(&inactive_as_with_asid_lock); |
909 | DEADLOCK_PROBE(p_asidlock, DEADLOCK_THRESHOLD); |
- | 910 | goto retry; |
|
- | 911 | } |
|
- | 912 | preemption_enable(); |
|
890 | 913 | ||
891 | /* |
914 | /* |
892 | * First, take care of the old address space. |
915 | * First, take care of the old address space. |
893 | */ |
916 | */ |
894 | if (old_as) { |
917 | if (old_as) { |
895 | mutex_lock_active(&old_as->lock); |
- | |
896 | ASSERT(old_as->cpu_refcount); |
918 | ASSERT(old_as->cpu_refcount); |
897 | if((--old_as->cpu_refcount == 0) && (old_as != AS_KERNEL)) { |
919 | if((--old_as->cpu_refcount == 0) && (old_as != AS_KERNEL)) { |
898 | /* |
920 | /* |
899 | * The old address space is no longer active on |
921 | * The old address space is no longer active on |
900 | * any processor. It can be appended to the |
922 | * any processor. It can be appended to the |
Line 903... | Line 925... | ||
903 | */ |
925 | */ |
904 | ASSERT(old_as->asid != ASID_INVALID); |
926 | ASSERT(old_as->asid != ASID_INVALID); |
905 | list_append(&old_as->inactive_as_with_asid_link, |
927 | list_append(&old_as->inactive_as_with_asid_link, |
906 | &inactive_as_with_asid_head); |
928 | &inactive_as_with_asid_head); |
907 | } |
929 | } |
908 | mutex_unlock(&old_as->lock); |
- | |
909 | 930 | ||
910 | /* |
931 | /* |
911 | * Perform architecture-specific tasks when the address space |
932 | * Perform architecture-specific tasks when the address space |
912 | * is being removed from the CPU. |
933 | * is being removed from the CPU. |
913 | */ |
934 | */ |
Line 915... | Line 936... | ||
915 | } |
936 | } |
916 | 937 | ||
917 | /* |
938 | /* |
918 | * Second, prepare the new address space. |
939 | * Second, prepare the new address space. |
919 | */ |
940 | */ |
920 | mutex_lock_active(&new_as->lock); |
- | |
921 | if ((new_as->cpu_refcount++ == 0) && (new_as != AS_KERNEL)) { |
941 | if ((new_as->cpu_refcount++ == 0) && (new_as != AS_KERNEL)) { |
922 | if (new_as->asid != ASID_INVALID) { |
942 | if (new_as->asid != ASID_INVALID) |
923 | list_remove(&new_as->inactive_as_with_asid_link); |
943 | list_remove(&new_as->inactive_as_with_asid_link); |
924 | } else { |
944 | else |
925 | /* |
- | |
926 | * Defer call to asid_get() until new_as->lock is released. |
- | |
927 | */ |
- | |
928 | needs_asid = true; |
945 | new_as->asid = asid_get(); |
929 | } |
- | |
930 | } |
946 | } |
931 | #ifdef AS_PAGE_TABLE |
947 | #ifdef AS_PAGE_TABLE |
932 | SET_PTL0_ADDRESS(new_as->genarch.page_table); |
948 | SET_PTL0_ADDRESS(new_as->genarch.page_table); |
933 | #endif |
949 | #endif |
934 | mutex_unlock(&new_as->lock); |
- | |
935 | - | ||
936 | if (needs_asid) { |
- | |
937 | /* |
- | |
938 | * Allocation of new ASID was deferred |
- | |
939 | * until now in order to avoid deadlock. |
- | |
940 | */ |
- | |
941 | asid_t asid; |
- | |
942 | - | ||
943 | asid = asid_get(); |
- | |
944 | mutex_lock_active(&new_as->lock); |
- | |
945 | new_as->asid = asid; |
- | |
946 | mutex_unlock(&new_as->lock); |
- | |
947 | } |
- | |
948 | spinlock_unlock(&inactive_as_with_asid_lock); |
- | |
949 | interrupts_restore(ipl); |
- | |
950 | 950 | ||
951 | /* |
951 | /* |
952 | * Perform architecture-specific steps. |
952 | * Perform architecture-specific steps. |
953 | * (e.g. write ASID to hardware register etc.) |
953 | * (e.g. write ASID to hardware register etc.) |
954 | */ |
954 | */ |
955 | as_install_arch(new_as); |
955 | as_install_arch(new_as); |
956 | 956 | ||
- | 957 | spinlock_unlock(&asidlock); |
|
- | 958 | ||
957 | AS = new_as; |
959 | AS = new_as; |
958 | } |
960 | } |
959 | 961 | ||
960 | /** Convert address space area flags to page flags. |
962 | /** Convert address space area flags to page flags. |
961 | * |
963 | * |