167,7 → 167,7 |
* Allocate frames for slab space and initialize |
* |
*/ |
static slab_t * slab_space_alloc(slab_cache_t *cache, int flags) |
static slab_t *slab_space_alloc(slab_cache_t *cache, int flags) |
{ |
void *data; |
slab_t *slab; |
179,7 → 179,7 |
if (!data) { |
return NULL; |
} |
if (! (cache->flags & SLAB_CACHE_SLINSIDE)) { |
if (!(cache->flags & SLAB_CACHE_SLINSIDE)) { |
slab = slab_alloc(slab_extern_cache, flags); |
if (!slab) { |
frame_free(KA2PA(data)); |
200,7 → 200,7 |
slab->cache = cache; |
|
for (i = 0; i < cache->objects; i++) |
*((int *) (slab->start + i*cache->size)) = i+1; |
*((int *) (slab->start + i*cache->size)) = i + 1; |
|
atomic_inc(&cache->allocated_slabs); |
return slab; |
239,8 → 239,7 |
* |
* @return Number of freed pages |
*/ |
static count_t slab_obj_destroy(slab_cache_t *cache, void *obj, |
slab_t *slab) |
static count_t slab_obj_destroy(slab_cache_t *cache, void *obj, slab_t *slab) |
{ |
int freed = 0; |
|
256,7 → 255,7 |
ASSERT(slab->available < cache->objects); |
|
*((int *)obj) = slab->nextavail; |
slab->nextavail = (obj - slab->start)/cache->size; |
slab->nextavail = (obj - slab->start) / cache->size; |
slab->available++; |
|
/* Move it to correct list */ |
281,7 → 280,7 |
* |
* @return Object address or null |
*/ |
static void * slab_obj_create(slab_cache_t *cache, int flags) |
static void *slab_obj_create(slab_cache_t *cache, int flags) |
{ |
slab_t *slab; |
void *obj; |
301,7 → 300,8 |
return NULL; |
spinlock_lock(&cache->slablock); |
} else { |
slab = list_get_instance(cache->partial_slabs.next, slab_t, link); |
slab = list_get_instance(cache->partial_slabs.next, slab_t, |
link); |
list_remove(&slab->link); |
} |
obj = slab->start + slab->nextavail * cache->size; |
332,8 → 332,7 |
* |
* @param first If true, return first, else last mag |
*/ |
static slab_magazine_t * get_mag_from_cache(slab_cache_t *cache, |
int first) |
static slab_magazine_t *get_mag_from_cache(slab_cache_t *cache, int first) |
{ |
slab_magazine_t *mag = NULL; |
link_t *cur; |
368,8 → 367,7 |
* |
* @return Number of freed pages |
*/ |
static count_t magazine_destroy(slab_cache_t *cache, |
slab_magazine_t *mag) |
static count_t magazine_destroy(slab_cache_t *cache, slab_magazine_t *mag) |
{ |
unsigned int i; |
count_t frames = 0; |
389,7 → 387,7 |
* |
* Assume cpu_magazine lock is held |
*/ |
static slab_magazine_t * get_full_current_mag(slab_cache_t *cache) |
static slab_magazine_t *get_full_current_mag(slab_cache_t *cache) |
{ |
slab_magazine_t *cmag, *lastmag, *newmag; |
|
423,7 → 421,7 |
* |
* @return Pointer to object or NULL if not available |
*/ |
static void * magazine_obj_get(slab_cache_t *cache) |
static void *magazine_obj_get(slab_cache_t *cache) |
{ |
slab_magazine_t *mag; |
void *obj; |
458,7 → 456,7 |
* allocate new, exchange last & current |
* |
*/ |
static slab_magazine_t * make_empty_current_mag(slab_cache_t *cache) |
static slab_magazine_t *make_empty_current_mag(slab_cache_t *cache) |
{ |
slab_magazine_t *cmag,*lastmag,*newmag; |
|
530,7 → 528,8 |
static unsigned int comp_objects(slab_cache_t *cache) |
{ |
if (cache->flags & SLAB_CACHE_SLINSIDE) |
return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / cache->size; |
return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / |
cache->size; |
else |
return (PAGE_SIZE << cache->order) / cache->size; |
} |
557,22 → 556,20 |
|
ASSERT(_slab_initialized >= 2); |
|
cache->mag_cache = malloc(sizeof(slab_mag_cache_t) * config.cpu_count,0); |
cache->mag_cache = malloc(sizeof(slab_mag_cache_t) * config.cpu_count, |
0); |
for (i = 0; i < config.cpu_count; i++) { |
memsetb(&cache->mag_cache[i], sizeof(cache->mag_cache[i]), 0); |
spinlock_initialize(&cache->mag_cache[i].lock, "slab_maglock_cpu"); |
spinlock_initialize(&cache->mag_cache[i].lock, |
"slab_maglock_cpu"); |
} |
} |
|
/** Initialize allocated memory as a slab cache */ |
static void |
_slab_cache_create(slab_cache_t *cache, |
char *name, |
size_t size, |
size_t align, |
int (*constructor)(void *obj, int kmflag), |
int (*destructor)(void *obj), |
int flags) |
_slab_cache_create(slab_cache_t *cache, char *name, size_t size, size_t align, |
int (*constructor)(void *obj, int kmflag), int (*destructor)(void *obj), |
int flags) |
{ |
int pages; |
ipl_t ipl; |
595,7 → 592,7 |
list_initialize(&cache->magazines); |
spinlock_initialize(&cache->slablock, "slab_lock"); |
spinlock_initialize(&cache->maglock, "slab_maglock"); |
if (! (cache->flags & SLAB_CACHE_NOMAGAZINE)) |
if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) |
make_magcache(cache); |
|
/* Compute slab sizes, object counts in slabs etc. */ |
608,7 → 605,7 |
if (pages == 1) |
cache->order = 0; |
else |
cache->order = fnzb(pages-1)+1; |
cache->order = fnzb(pages - 1) + 1; |
|
while (badness(cache) > SLAB_MAX_BADNESS(cache)) { |
cache->order += 1; |
629,18 → 626,16 |
} |
|
/** Create slab cache */ |
slab_cache_t * slab_cache_create(char *name, |
size_t size, |
size_t align, |
int (*constructor)(void *obj, int kmflag), |
int (*destructor)(void *obj), |
int flags) |
slab_cache_t * |
slab_cache_create(char *name, size_t size, size_t align, |
int (*constructor)(void *obj, int kmflag), int (*destructor)(void *obj), |
int flags) |
{ |
slab_cache_t *cache; |
|
cache = slab_alloc(&slab_cache_cache, 0); |
_slab_cache_create(cache, name, size, align, constructor, destructor, |
flags); |
flags); |
return cache; |
} |
|
664,7 → 659,7 |
* endless loop |
*/ |
magcount = atomic_get(&cache->magazine_counter); |
while (magcount-- && (mag=get_mag_from_cache(cache,0))) { |
while (magcount-- && (mag=get_mag_from_cache(cache, 0))) { |
frames += magazine_destroy(cache,mag); |
if (!(flags & SLAB_RECLAIM_ALL) && frames) |
break; |
717,8 → 712,8 |
_slab_reclaim(cache, SLAB_RECLAIM_ALL); |
|
/* All slabs must be empty */ |
if (!list_empty(&cache->full_slabs) \ |
|| !list_empty(&cache->partial_slabs)) |
if (!list_empty(&cache->full_slabs) || |
!list_empty(&cache->partial_slabs)) |
panic("Destroying cache that is not empty."); |
|
if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) |
726,9 → 721,8 |
slab_free(&slab_cache_cache, cache); |
} |
|
/** Allocate new object from cache - if no flags given, always returns |
memory */ |
void * slab_alloc(slab_cache_t *cache, int flags) |
/** Allocate new object from cache - if no flags given, always returns memory */ |
void *slab_alloc(slab_cache_t *cache, int flags) |
{ |
ipl_t ipl; |
void *result = NULL; |
757,9 → 751,8 |
|
ipl = interrupts_disable(); |
|
if ((cache->flags & SLAB_CACHE_NOMAGAZINE) \ |
|| magazine_obj_put(cache, obj)) { |
|
if ((cache->flags & SLAB_CACHE_NOMAGAZINE) || |
magazine_obj_put(cache, obj)) { |
slab_obj_destroy(cache, obj, slab); |
|
} |
786,7 → 779,8 |
* memory allocation from interrupts can deadlock. |
*/ |
|
for (cur = slab_cache_list.next;cur!=&slab_cache_list; cur=cur->next) { |
for (cur = slab_cache_list.next; cur != &slab_cache_list; |
cur = cur->next) { |
cache = list_get_instance(cur, slab_cache_t, link); |
frames += _slab_reclaim(cache, flags); |
} |
800,25 → 794,77 |
/* Print list of slabs */ |
void slab_print_list(void) |
{ |
slab_cache_t *cache; |
link_t *cur; |
ipl_t ipl; |
|
ipl = interrupts_disable(); |
spinlock_lock(&slab_cache_lock); |
printf("slab name size pages obj/pg slabs cached allocated ctl\n"); |
printf("---------------- -------- ------ ------ ------ ------ --------- ---\n"); |
|
for (cur = slab_cache_list.next; cur != &slab_cache_list; cur = cur->next) { |
int skip = 0; |
|
printf("slab name size pages obj/pg slabs cached allocated" |
" ctl\n"); |
printf("---------------- -------- ------ ------ ------ ------ ---------" |
" ---\n"); |
|
while (true) { |
slab_cache_t *cache; |
link_t *cur; |
ipl_t ipl; |
int i; |
|
/* |
* We must not hold the slab_cache_lock spinlock when printing |
* the statistics. Otherwise we can easily deadlock if the print |
* needs to allocate memory. |
* |
* Therefore, we walk through the slab cache list, skipping some |
* amount of already processed caches during each iteration and |
* gathering statistics about the first unprocessed cache. For |
* the sake of printing the statistics, we realese the |
* slab_cache_lock and reacquire it afterwards. Then the walk |
* starts again. |
* |
* This limits both the efficiency and also accuracy of the |
* obtained statistics. The efficiency is decreased because the |
* time complexity of the algorithm is quadratic instead of |
* linear. The accuracy is impacted because we drop the lock |
* after processing one cache. If there is someone else |
* manipulating the cache list, we might omit an arbitrary |
* number of caches or process one cache multiple times. |
* However, we don't bleed for this algorithm for it is only |
* statistics. |
*/ |
|
ipl = interrupts_disable(); |
spinlock_lock(&slab_cache_lock); |
|
for (i = 0, cur = slab_cache_list.next; |
i < skip && cur != &slab_cache_list; |
i++, cur = cur->next) |
; |
|
if (cur == &slab_cache_list) { |
spinlock_unlock(&slab_cache_lock); |
interrupts_restore(ipl); |
break; |
} |
|
skip++; |
|
cache = list_get_instance(cur, slab_cache_t, link); |
|
char *name = cache->name; |
uint8_t order = cache->order; |
size_t size = cache->size; |
unsigned int objects = cache->objects; |
long allocated_slabs = atomic_get(&cache->allocated_slabs); |
long cached_objs = atomic_get(&cache->cached_objs); |
long allocated_objs = atomic_get(&cache->allocated_objs); |
int flags = cache->flags; |
|
spinlock_unlock(&slab_cache_lock); |
interrupts_restore(ipl); |
|
printf("%-16s %8" PRIs " %6d %6u %6ld %6ld %9ld %-3s\n", |
cache->name, cache->size, (1 << cache->order), cache->objects, |
atomic_get(&cache->allocated_slabs), atomic_get(&cache->cached_objs), |
atomic_get(&cache->allocated_objs), cache->flags & SLAB_CACHE_SLINSIDE ? "in" : "out"); |
name, size, (1 << order), objects, allocated_slabs, |
cached_objs, allocated_objs, |
flags & SLAB_CACHE_SLINSIDE ? "in" : "out"); |
} |
spinlock_unlock(&slab_cache_lock); |
interrupts_restore(ipl); |
} |
|
void slab_cache_init(void) |
826,32 → 872,24 |
int i, size; |
|
/* Initialize magazine cache */ |
_slab_cache_create(&mag_cache, |
"slab_magazine", |
sizeof(slab_magazine_t) + SLAB_MAG_SIZE * sizeof(void*), |
sizeof(uintptr_t), |
NULL, NULL, |
SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE); |
_slab_cache_create(&mag_cache, "slab_magazine", |
sizeof(slab_magazine_t) + SLAB_MAG_SIZE * sizeof(void*), |
sizeof(uintptr_t), NULL, NULL, SLAB_CACHE_NOMAGAZINE | |
SLAB_CACHE_SLINSIDE); |
/* Initialize slab_cache cache */ |
_slab_cache_create(&slab_cache_cache, |
"slab_cache", |
sizeof(slab_cache_cache), |
sizeof(uintptr_t), |
NULL, NULL, |
SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE); |
_slab_cache_create(&slab_cache_cache, "slab_cache", |
sizeof(slab_cache_cache), sizeof(uintptr_t), NULL, NULL, |
SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE); |
/* Initialize external slab cache */ |
slab_extern_cache = slab_cache_create("slab_extern", |
sizeof(slab_t), |
0, NULL, NULL, |
SLAB_CACHE_SLINSIDE | SLAB_CACHE_MAGDEFERRED); |
slab_extern_cache = slab_cache_create("slab_extern", sizeof(slab_t), 0, |
NULL, NULL, SLAB_CACHE_SLINSIDE | SLAB_CACHE_MAGDEFERRED); |
|
/* Initialize structures for malloc */ |
for (i=0, size=(1 << SLAB_MIN_MALLOC_W); |
i < (SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1); |
i++, size <<= 1) { |
malloc_caches[i] = slab_cache_create(malloc_names[i], |
size, 0, |
NULL,NULL, SLAB_CACHE_MAGDEFERRED); |
for (i = 0, size = (1 << SLAB_MIN_MALLOC_W); |
i < (SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1); |
i++, size <<= 1) { |
malloc_caches[i] = slab_cache_create(malloc_names[i], size, 0, |
NULL, NULL, SLAB_CACHE_MAGDEFERRED); |
} |
#ifdef CONFIG_DEBUG |
_slab_initialized = 1; |
876,9 → 914,11 |
|
spinlock_lock(&slab_cache_lock); |
|
for (cur=slab_cache_list.next; cur != &slab_cache_list;cur=cur->next){ |
for (cur = slab_cache_list.next; cur != &slab_cache_list; |
cur = cur->next){ |
s = list_get_instance(cur, slab_cache_t, link); |
if ((s->flags & SLAB_CACHE_MAGDEFERRED) != SLAB_CACHE_MAGDEFERRED) |
if ((s->flags & SLAB_CACHE_MAGDEFERRED) != |
SLAB_CACHE_MAGDEFERRED) |
continue; |
make_magcache(s); |
s->flags &= ~SLAB_CACHE_MAGDEFERRED; |
889,7 → 929,7 |
|
/**************************************/ |
/* kalloc/kfree functions */ |
void * malloc(unsigned int size, int flags) |
void *malloc(unsigned int size, int flags) |
{ |
ASSERT(_slab_initialized); |
ASSERT(size && size <= (1 << SLAB_MAX_MALLOC_W)); |
902,7 → 942,7 |
return slab_alloc(malloc_caches[idx], flags); |
} |
|
void * realloc(void *ptr, unsigned int size, int flags) |
void *realloc(void *ptr, unsigned int size, int flags) |
{ |
ASSERT(_slab_initialized); |
ASSERT(size <= (1 << SLAB_MAX_MALLOC_W)); |