Rev 3104 | Rev 3183 | Go to most recent revision | Show entire file | Regard whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 3104 | Rev 3180 | ||
---|---|---|---|
Line 237... | Line 237... | ||
237 | * |
237 | * |
238 | * @param slab If the caller knows directly slab of the object, otherwise NULL |
238 | * @param slab If the caller knows directly slab of the object, otherwise NULL |
239 | * |
239 | * |
240 | * @return Number of freed pages |
240 | * @return Number of freed pages |
241 | */ |
241 | */ |
242 | static count_t slab_obj_destroy(slab_cache_t *cache, void *obj, |
242 | static count_t slab_obj_destroy(slab_cache_t *cache, void *obj, slab_t *slab) |
243 | slab_t *slab) |
- | |
244 | { |
243 | { |
245 | int freed = 0; |
244 | int freed = 0; |
246 | 245 | ||
247 | if (!slab) |
246 | if (!slab) |
248 | slab = obj2slab(obj); |
247 | slab = obj2slab(obj); |
Line 299... | Line 298... | ||
299 | slab = slab_space_alloc(cache, flags); |
298 | slab = slab_space_alloc(cache, flags); |
300 | if (!slab) |
299 | if (!slab) |
301 | return NULL; |
300 | return NULL; |
302 | spinlock_lock(&cache->slablock); |
301 | spinlock_lock(&cache->slablock); |
303 | } else { |
302 | } else { |
304 | slab = list_get_instance(cache->partial_slabs.next, slab_t, link); |
303 | slab = list_get_instance(cache->partial_slabs.next, slab_t, |
- | 304 | link); |
|
305 | list_remove(&slab->link); |
305 | list_remove(&slab->link); |
306 | } |
306 | } |
307 | obj = slab->start + slab->nextavail * cache->size; |
307 | obj = slab->start + slab->nextavail * cache->size; |
308 | slab->nextavail = *((int *)obj); |
308 | slab->nextavail = *((int *)obj); |
309 | slab->available--; |
309 | slab->available--; |
Line 330... | Line 330... | ||
330 | * Finds a full magazine in cache, takes it from list |
330 | * Finds a full magazine in cache, takes it from list |
331 | * and returns it |
331 | * and returns it |
332 | * |
332 | * |
333 | * @param first If true, return first, else last mag |
333 | * @param first If true, return first, else last mag |
334 | */ |
334 | */ |
335 | static slab_magazine_t * get_mag_from_cache(slab_cache_t *cache, |
335 | static slab_magazine_t *get_mag_from_cache(slab_cache_t *cache, int first) |
336 | int first) |
- | |
337 | { |
336 | { |
338 | slab_magazine_t *mag = NULL; |
337 | slab_magazine_t *mag = NULL; |
339 | link_t *cur; |
338 | link_t *cur; |
340 | 339 | ||
341 | spinlock_lock(&cache->maglock); |
340 | spinlock_lock(&cache->maglock); |
Line 366... | Line 365... | ||
366 | /** |
365 | /** |
367 | * Free all objects in magazine and free memory associated with magazine |
366 | * Free all objects in magazine and free memory associated with magazine |
368 | * |
367 | * |
369 | * @return Number of freed pages |
368 | * @return Number of freed pages |
370 | */ |
369 | */ |
371 | static count_t magazine_destroy(slab_cache_t *cache, |
370 | static count_t magazine_destroy(slab_cache_t *cache, slab_magazine_t *mag) |
372 | slab_magazine_t *mag) |
- | |
373 | { |
371 | { |
374 | unsigned int i; |
372 | unsigned int i; |
375 | count_t frames = 0; |
373 | count_t frames = 0; |
376 | 374 | ||
377 | for (i = 0; i < mag->busy; i++) { |
375 | for (i = 0; i < mag->busy; i++) { |
Line 528... | Line 526... | ||
528 | 526 | ||
529 | /** Return number of objects that fit in certain cache size */ |
527 | /** Return number of objects that fit in certain cache size */ |
530 | static unsigned int comp_objects(slab_cache_t *cache) |
528 | static unsigned int comp_objects(slab_cache_t *cache) |
531 | { |
529 | { |
532 | if (cache->flags & SLAB_CACHE_SLINSIDE) |
530 | if (cache->flags & SLAB_CACHE_SLINSIDE) |
533 | return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / cache->size; |
531 | return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / |
- | 532 | cache->size; |
|
534 | else |
533 | else |
535 | return (PAGE_SIZE << cache->order) / cache->size; |
534 | return (PAGE_SIZE << cache->order) / cache->size; |
536 | } |
535 | } |
537 | 536 | ||
538 | /** Return wasted space in slab */ |
537 | /** Return wasted space in slab */ |
Line 555... | Line 554... | ||
555 | { |
554 | { |
556 | unsigned int i; |
555 | unsigned int i; |
557 | 556 | ||
558 | ASSERT(_slab_initialized >= 2); |
557 | ASSERT(_slab_initialized >= 2); |
559 | 558 | ||
560 | cache->mag_cache = malloc(sizeof(slab_mag_cache_t) * config.cpu_count,0); |
559 | cache->mag_cache = malloc(sizeof(slab_mag_cache_t) * config.cpu_count, |
- | 560 | 0); |
|
561 | for (i = 0; i < config.cpu_count; i++) { |
561 | for (i = 0; i < config.cpu_count; i++) { |
562 | memsetb(&cache->mag_cache[i], sizeof(cache->mag_cache[i]), 0); |
562 | memsetb(&cache->mag_cache[i], sizeof(cache->mag_cache[i]), 0); |
563 | spinlock_initialize(&cache->mag_cache[i].lock, "slab_maglock_cpu"); |
563 | spinlock_initialize(&cache->mag_cache[i].lock, |
- | 564 | "slab_maglock_cpu"); |
|
564 | } |
565 | } |
565 | } |
566 | } |
566 | 567 | ||
567 | /** Initialize allocated memory as a slab cache */ |
568 | /** Initialize allocated memory as a slab cache */ |
568 | static void |
569 | static void |
569 | _slab_cache_create(slab_cache_t *cache, |
570 | _slab_cache_create(slab_cache_t *cache, char *name, size_t size, size_t align, |
570 | char *name, |
- | |
571 | size_t size, |
- | |
572 | size_t align, |
- | |
573 | int (*constructor)(void *obj, int kmflag), |
571 | int (*constructor)(void *obj, int kmflag), int (*destructor)(void *obj), |
574 | int (*destructor)(void *obj), |
- | |
575 | int flags) |
572 | int flags) |
576 | { |
573 | { |
577 | int pages; |
574 | int pages; |
578 | ipl_t ipl; |
575 | ipl_t ipl; |
579 | 576 | ||
Line 627... | Line 624... | ||
627 | spinlock_unlock(&slab_cache_lock); |
624 | spinlock_unlock(&slab_cache_lock); |
628 | interrupts_restore(ipl); |
625 | interrupts_restore(ipl); |
629 | } |
626 | } |
630 | 627 | ||
631 | /** Create slab cache */ |
628 | /** Create slab cache */ |
632 | slab_cache_t * slab_cache_create(char *name, |
- | |
633 | size_t size, |
629 | slab_cache_t * |
634 | size_t align, |
630 | slab_cache_create(char *name, size_t size, size_t align, |
635 | int (*constructor)(void *obj, int kmflag), |
631 | int (*constructor)(void *obj, int kmflag), int (*destructor)(void *obj), |
636 | int (*destructor)(void *obj), |
- | |
637 | int flags) |
632 | int flags) |
638 | { |
633 | { |
639 | slab_cache_t *cache; |
634 | slab_cache_t *cache; |
640 | 635 | ||
641 | cache = slab_alloc(&slab_cache_cache, 0); |
636 | cache = slab_alloc(&slab_cache_cache, 0); |
Line 715... | Line 710... | ||
715 | 710 | ||
716 | /* Destroy all magazines */ |
711 | /* Destroy all magazines */ |
717 | _slab_reclaim(cache, SLAB_RECLAIM_ALL); |
712 | _slab_reclaim(cache, SLAB_RECLAIM_ALL); |
718 | 713 | ||
719 | /* All slabs must be empty */ |
714 | /* All slabs must be empty */ |
720 | if (!list_empty(&cache->full_slabs) \ |
715 | if (!list_empty(&cache->full_slabs) || |
721 | || !list_empty(&cache->partial_slabs)) |
716 | !list_empty(&cache->partial_slabs)) |
722 | panic("Destroying cache that is not empty."); |
717 | panic("Destroying cache that is not empty."); |
723 | 718 | ||
724 | if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) |
719 | if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) |
725 | free(cache->mag_cache); |
720 | free(cache->mag_cache); |
726 | slab_free(&slab_cache_cache, cache); |
721 | slab_free(&slab_cache_cache, cache); |
727 | } |
722 | } |
728 | 723 | ||
729 | /** Allocate new object from cache - if no flags given, always returns |
724 | /** Allocate new object from cache - if no flags given, always returns memory */ |
730 | memory */ |
- | |
731 | void * slab_alloc(slab_cache_t *cache, int flags) |
725 | void *slab_alloc(slab_cache_t *cache, int flags) |
732 | { |
726 | { |
733 | ipl_t ipl; |
727 | ipl_t ipl; |
734 | void *result = NULL; |
728 | void *result = NULL; |
735 | 729 | ||
Line 755... | Line 749... | ||
755 | { |
749 | { |
756 | ipl_t ipl; |
750 | ipl_t ipl; |
757 | 751 | ||
758 | ipl = interrupts_disable(); |
752 | ipl = interrupts_disable(); |
759 | 753 | ||
760 | if ((cache->flags & SLAB_CACHE_NOMAGAZINE) \ |
754 | if ((cache->flags & SLAB_CACHE_NOMAGAZINE) || |
761 | || magazine_obj_put(cache, obj)) { |
755 | magazine_obj_put(cache, obj)) { |
762 | - | ||
763 | slab_obj_destroy(cache, obj, slab); |
756 | slab_obj_destroy(cache, obj, slab); |
764 | 757 | ||
765 | } |
758 | } |
766 | interrupts_restore(ipl); |
759 | interrupts_restore(ipl); |
767 | atomic_dec(&cache->allocated_objs); |
760 | atomic_dec(&cache->allocated_objs); |
Line 784... | Line 777... | ||
784 | 777 | ||
785 | /* TODO: Add assert, that interrupts are disabled, otherwise |
778 | /* TODO: Add assert, that interrupts are disabled, otherwise |
786 | * memory allocation from interrupts can deadlock. |
779 | * memory allocation from interrupts can deadlock. |
787 | */ |
780 | */ |
788 | 781 | ||
789 | for (cur = slab_cache_list.next;cur!=&slab_cache_list; cur=cur->next) { |
782 | for (cur = slab_cache_list.next; cur != &slab_cache_list; |
- | 783 | cur = cur->next) { |
|
790 | cache = list_get_instance(cur, slab_cache_t, link); |
784 | cache = list_get_instance(cur, slab_cache_t, link); |
791 | frames += _slab_reclaim(cache, flags); |
785 | frames += _slab_reclaim(cache, flags); |
792 | } |
786 | } |
793 | 787 | ||
794 | spinlock_unlock(&slab_cache_lock); |
788 | spinlock_unlock(&slab_cache_lock); |
Line 804... | Line 798... | ||
804 | link_t *cur; |
798 | link_t *cur; |
805 | ipl_t ipl; |
799 | ipl_t ipl; |
806 | 800 | ||
807 | ipl = interrupts_disable(); |
801 | ipl = interrupts_disable(); |
808 | spinlock_lock(&slab_cache_lock); |
802 | spinlock_lock(&slab_cache_lock); |
809 | printf("slab name size pages obj/pg slabs cached allocated ctl\n"); |
803 | printf("slab name size pages obj/pg slabs cached allocated" |
- | 804 | " ctl\n"); |
|
810 | printf("---------------- -------- ------ ------ ------ ------ --------- ---\n"); |
805 | printf("---------------- -------- ------ ------ ------ ------ ---------" |
- | 806 | " ---\n"); |
|
811 | 807 | ||
812 | for (cur = slab_cache_list.next; cur != &slab_cache_list; cur = cur->next) { |
808 | for (cur = slab_cache_list.next; cur != &slab_cache_list; |
- | 809 | cur = cur->next) { |
|
813 | cache = list_get_instance(cur, slab_cache_t, link); |
810 | cache = list_get_instance(cur, slab_cache_t, link); |
814 | 811 | ||
815 | printf("%-16s %8" PRIs " %6d %6u %6ld %6ld %9ld %-3s\n", |
812 | printf("%-16s %8" PRIs " %6d %6u %6ld %6ld %9ld %-3s\n", |
816 | cache->name, cache->size, (1 << cache->order), cache->objects, |
813 | cache->name, cache->size, (1 << cache->order), |
817 | atomic_get(&cache->allocated_slabs), atomic_get(&cache->cached_objs), |
814 | cache->objects, atomic_get(&cache->allocated_slabs), |
- | 815 | atomic_get(&cache->cached_objs), |
|
- | 816 | atomic_get(&cache->allocated_objs), |
|
818 | atomic_get(&cache->allocated_objs), cache->flags & SLAB_CACHE_SLINSIDE ? "in" : "out"); |
817 | cache->flags & SLAB_CACHE_SLINSIDE ? "in" : "out"); |
819 | } |
818 | } |
820 | spinlock_unlock(&slab_cache_lock); |
819 | spinlock_unlock(&slab_cache_lock); |
821 | interrupts_restore(ipl); |
820 | interrupts_restore(ipl); |
822 | } |
821 | } |
823 | 822 | ||
824 | void slab_cache_init(void) |
823 | void slab_cache_init(void) |
825 | { |
824 | { |
826 | int i, size; |
825 | int i, size; |
827 | 826 | ||
828 | /* Initialize magazine cache */ |
827 | /* Initialize magazine cache */ |
829 | _slab_cache_create(&mag_cache, |
828 | _slab_cache_create(&mag_cache, "slab_magazine", |
830 | "slab_magazine", |
- | |
831 | sizeof(slab_magazine_t) + SLAB_MAG_SIZE * sizeof(void*), |
829 | sizeof(slab_magazine_t) + SLAB_MAG_SIZE * sizeof(void*), |
832 | sizeof(uintptr_t), |
830 | sizeof(uintptr_t), NULL, NULL, SLAB_CACHE_NOMAGAZINE | |
833 | NULL, NULL, |
- | |
834 | SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE); |
831 | SLAB_CACHE_SLINSIDE); |
835 | /* Initialize slab_cache cache */ |
832 | /* Initialize slab_cache cache */ |
836 | _slab_cache_create(&slab_cache_cache, |
833 | _slab_cache_create(&slab_cache_cache, "slab_cache", |
837 | "slab_cache", |
- | |
838 | sizeof(slab_cache_cache), |
834 | sizeof(slab_cache_cache), sizeof(uintptr_t), NULL, NULL, |
839 | sizeof(uintptr_t), |
- | |
840 | NULL, NULL, |
- | |
841 | SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE); |
835 | SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE); |
842 | /* Initialize external slab cache */ |
836 | /* Initialize external slab cache */ |
843 | slab_extern_cache = slab_cache_create("slab_extern", |
837 | slab_extern_cache = slab_cache_create("slab_extern", sizeof(slab_t), 0, |
844 | sizeof(slab_t), |
- | |
845 | 0, NULL, NULL, |
- | |
846 | SLAB_CACHE_SLINSIDE | SLAB_CACHE_MAGDEFERRED); |
838 | NULL, NULL, SLAB_CACHE_SLINSIDE | SLAB_CACHE_MAGDEFERRED); |
847 | 839 | ||
848 | /* Initialize structures for malloc */ |
840 | /* Initialize structures for malloc */ |
849 | for (i=0, size=(1 << SLAB_MIN_MALLOC_W); |
841 | for (i = 0, size = (1 << SLAB_MIN_MALLOC_W); |
850 | i < (SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1); |
842 | i < (SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1); |
851 | i++, size <<= 1) { |
843 | i++, size <<= 1) { |
852 | malloc_caches[i] = slab_cache_create(malloc_names[i], |
844 | malloc_caches[i] = slab_cache_create(malloc_names[i], size, 0, |
853 | size, 0, |
- | |
854 | NULL,NULL, SLAB_CACHE_MAGDEFERRED); |
845 | NULL, NULL, SLAB_CACHE_MAGDEFERRED); |
855 | } |
846 | } |
856 | #ifdef CONFIG_DEBUG |
847 | #ifdef CONFIG_DEBUG |
857 | _slab_initialized = 1; |
848 | _slab_initialized = 1; |
858 | #endif |
849 | #endif |
Line 874... | Line 865... | ||
874 | _slab_initialized = 2; |
865 | _slab_initialized = 2; |
875 | #endif |
866 | #endif |
876 | 867 | ||
877 | spinlock_lock(&slab_cache_lock); |
868 | spinlock_lock(&slab_cache_lock); |
878 | 869 | ||
879 | for (cur=slab_cache_list.next; cur != &slab_cache_list;cur=cur->next){ |
870 | for (cur = slab_cache_list.next; cur != &slab_cache_list; |
- | 871 | cur = cur->next){ |
|
880 | s = list_get_instance(cur, slab_cache_t, link); |
872 | s = list_get_instance(cur, slab_cache_t, link); |
881 | if ((s->flags & SLAB_CACHE_MAGDEFERRED) != SLAB_CACHE_MAGDEFERRED) |
873 | if ((s->flags & SLAB_CACHE_MAGDEFERRED) != |
- | 874 | SLAB_CACHE_MAGDEFERRED) |
|
882 | continue; |
875 | continue; |
883 | make_magcache(s); |
876 | make_magcache(s); |
884 | s->flags &= ~SLAB_CACHE_MAGDEFERRED; |
877 | s->flags &= ~SLAB_CACHE_MAGDEFERRED; |
885 | } |
878 | } |
886 | 879 |