WebSVN – HelenOS-historic – Diff – /kernel/trunk/generic/src/mm/slab.c


/*
 * Copyright (C) 2006 Ondrej Palkovsky
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
#include <synch/spinlock.h>
#include <mm/slab.h>
#include <list.h>
#include <memstr.h>
#include <align.h>
#include <mm/heap.h>
#include <mm/frame.h>
#include <config.h>
#include <print.h>
#include <arch.h>
#include <panic.h>
#include <debug.h>
 
SPINLOCK_INITIALIZE(slab_cache_lock);
LIST_INITIALIZE(slab_cache_list);
 
slab_cache_t mag_cache;
 
 
typedef struct {
    slab_cache_t *cache; /**< Pointer to parent cache */
    link_t link;       /* List of full/partial slabs */
    void *start;       /**< Start address of first available item */
    count_t available; /**< Count of available items in this slab */
    index_t nextavail; /**< The index of next available item */
}slab_t;
 
/**************************************/
/* SLAB allocation functions          */
 
/**
 * Allocate frames for slab space and initialize
 *
 * TODO: Change slab_t allocation to slab_alloc(????), malloc with flags!!
 */
static slab_t * slab_space_alloc(slab_cache_t *cache, int flags)
{
    void *data;
    slab_t *slab;
    size_t fsize;
    int i;
    zone_t *zone = NULL;
    int status;
    frame_t *frame;
 
    data = (void *)frame_alloc(FRAME_KA | flags, cache->order, &status, &zone);
    if (status != FRAME_OK) {
        return NULL;
    }
    if (! cache->flags & SLAB_CACHE_SLINSIDE) {
        slab = malloc(sizeof(*slab)); // , flags);
        if (!slab) {
            frame_free((__address)data);
            return NULL;
        }
    } else {
        fsize = (PAGE_SIZE << cache->order);
        slab = data + fsize - sizeof(*slab);
    }
       
    /* Fill in slab structures */
    /* TODO: some better way of accessing the frame */
    for (i=0; i < (1 << cache->order); i++) {
        frame = ADDR2FRAME(zone, KA2PA((__address)(data+i*PAGE_SIZE)));
        frame->parent = slab;
    }
 
    slab->start = data;
    slab->available = cache->objects;
    slab->nextavail = 0;
 
    for (i=0; i<cache->objects;i++)
        *((int *) (slab->start + i*cache->size)) = i+1;
 
    atomic_inc(&cache->allocated_slabs);
 
    return slab;
}
 
/**
 * Deallocate space associated with SLAB
 *
 * @return number of freed frames
 */
static count_t slab_space_free(slab_cache_t *cache, slab_t *slab)
{
    frame_free((__address)slab->start);
    if (! cache->flags & SLAB_CACHE_SLINSIDE)
        free(slab);
 
    atomic_dec(&cache->allocated_slabs);
   
    return 1 << cache->order;
}
 
/** Map object to slab structure */
static slab_t * obj2slab(void *obj)
{
    frame_t *frame;
 
    frame = frame_addr2frame((__address)obj);
    return (slab_t *)frame->parent;
}
 
/**************************************/
/* SLAB functions */
 
 
/**
 * Return object to slab and call a destructor
 *
 * Assume the cache->lock is held;
 *
 * @param slab If the caller knows directly slab of the object, otherwise NULL
 *
 * @return Number of freed pages
 */
static count_t slab_obj_destroy(slab_cache_t *cache, void *obj,
                slab_t *slab)
{
    count_t frames = 0;
 
    if (!slab)
        slab = obj2slab(obj);
 
 
 
    *((int *)obj) = slab->nextavail;
    slab->nextavail = (obj - slab->start)/cache->size;
    slab->available++;
 
    /* Move it to correct list */
    if (slab->available == 1) {
        /* It was in full, move to partial */
        list_remove(&slab->link);
        list_prepend(&slab->link, &cache->partial_slabs);
    }
    if (slab->available == cache->objects) {
        /* Free associated memory */
        list_remove(&slab->link);
        /* Avoid deadlock */
        spinlock_unlock(&cache->lock);
        frames = slab_space_free(cache, slab);
        spinlock_lock(&cache->lock);
    }
 
 
 
    return frames;
}
 
/**
 * Take new object from slab or create new if needed
 *
 * Assume cache->lock is held.
 *
 * @return Object address or null
 */
static void * slab_obj_create(slab_cache_t *cache, int flags)
{
    slab_t *slab;
    void *obj;
 
    if (list_empty(&cache->partial_slabs)) {
        /* Allow recursion and reclaiming
         * - this should work, as the SLAB control structures
         *   are small and do not need to allocte with anything
         *   other ten frame_alloc when they are allocating,
         *   that's why we should get recursion at most 1-level deep
         */
        spinlock_unlock(&cache->lock);
        slab = slab_space_alloc(cache, flags);
        spinlock_lock(&cache->lock);
        if (!slab) {
            return NULL;
        }
    } else {
        slab = list_get_instance(cache->partial_slabs.next,
                     slab_t,
                     link);
        list_remove(&slab->link);
    }
    obj = slab->start + slab->nextavail * cache->size;
    slab->nextavail = *((int *)obj);
    slab->available--;
    if (! slab->available)
        list_prepend(&slab->link, &cache->full_slabs);
    else
        list_prepend(&slab->link, &cache->partial_slabs);
    return obj;
}
 
/**************************************/
/* CPU-Cache slab functions */
 
/**
 * Free all objects in magazine and free memory associated with magazine
 *
 * Assume mag_cache[cpu].lock is locked
 *
 * @return Number of freed pages
 */
static count_t magazine_destroy(slab_cache_t *cache,
                slab_magazine_t *mag)
{
    int i;
    count_t frames = 0;
 
    for (i=0;i < mag->busy; i++)
        frames += slab_obj_destroy(cache, mag->objs[i], NULL);
   
    slab_free(&mag_cache, mag);
 
    return frames;
}
 
/**
 * Try to find object in CPU-cache magazines
 *
 * @return Pointer to object or NULL if not available
 */
static void * magazine_obj_get(slab_cache_t *cache)
{
    slab_magazine_t *mag;
 
    spinlock_lock(&cache->mag_cache[CPU->id].lock);
 
    mag = cache->mag_cache[CPU->id].current;
    if (!mag)
        goto out;
 
    if (!mag->busy) {
        /* If current is empty && last exists && not empty, exchange */
        if (cache->mag_cache[CPU->id].last \
            && cache->mag_cache[CPU->id].last->busy) {
            cache->mag_cache[CPU->id].current = cache->mag_cache[CPU->id].last;
            cache->mag_cache[CPU->id].last = mag;
            mag = cache->mag_cache[CPU->id].current;
            goto gotit;
        }
        /* If still not busy, exchange current with some from
         * other full magazines */
        spinlock_lock(&cache->lock);
        if (list_empty(&cache->magazines)) {
            spinlock_unlock(&cache->lock);
            goto out;
        }
        /* Free current magazine and take one from list */
        slab_free(&mag_cache, mag);
        mag = list_get_instance(cache->magazines.next,
                    slab_magazine_t,
                    link);
        list_remove(&mag->link);
       
        spinlock_unlock(&cache->lock);
    }
gotit:
    spinlock_unlock(&cache->mag_cache[CPU->id].lock);
    return mag->objs[--mag->busy];
out:   
    spinlock_unlock(&cache->mag_cache[CPU->id].lock);
    return NULL;
}
 
/**
 * Put object into CPU-cache magazine
 *
 * We have 2 magazines bound to processor.
 * First try the current.
 *  If full, try the last.
 *   If full, put to magazines list.
 *   allocate new, exchange last & current
 *
 * @return 0 - success, -1 - could not get memory
 */
static int magazine_obj_put(slab_cache_t *cache, void *obj)
{
    slab_magazine_t *mag;
 
    spinlock_lock(&cache->mag_cache[CPU->id].lock);
   
    mag = cache->mag_cache[CPU->id].current;
    if (!mag) {
        /* We do not want to sleep just because of caching */
        /* Especially we do not want reclaiming to start, as
         * this would deadlock */
        mag = slab_alloc(&mag_cache, FRAME_ATOMIC | FRAME_NO_RECLAIM);
        if (!mag) /* Allocation failed, give up on caching */
            goto errout;
 
        cache->mag_cache[CPU->id].current = mag;
        mag->size = SLAB_MAG_SIZE;
        mag->busy = 0;
    } else if (mag->busy == mag->size) {
        /* If the last is full | empty, allocate new */
        mag = cache->mag_cache[CPU->id].last;
        if (!mag || mag->size == mag->busy) {
            if (mag)
                list_prepend(&mag->link, &cache->magazines);
 
            mag = slab_alloc(&mag_cache, FRAME_ATOMIC | FRAME_NO_RECLAIM);
            if (!mag)
                goto errout;
           
            mag->size = SLAB_MAG_SIZE;
            mag->busy = 0;
            cache->mag_cache[CPU->id].last = mag;
        }
        /* Exchange the 2 */
        cache->mag_cache[CPU->id].last = cache->mag_cache[CPU->id].current;
        cache->mag_cache[CPU->id].current = mag;
    }
    mag->objs[mag->busy++] = obj;
 
    spinlock_unlock(&cache->mag_cache[CPU->id].lock);
    return 0;
errout:
    spinlock_unlock(&cache->mag_cache[CPU->id].lock);
    return -1;
}
 
 
/**************************************/
/* SLAB CACHE functions */
 
/** Return number of objects that fit in certain cache size */
static int comp_objects(slab_cache_t *cache)
{
    if (cache->flags & SLAB_CACHE_SLINSIDE)
        return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / cache->size;
    else
        return (PAGE_SIZE << cache->order) / cache->size;
}
 
/** Return wasted space in slab */
static int badness(slab_cache_t *cache)
{
    int objects;
    int ssize;
 
    objects = comp_objects(cache);
    ssize = PAGE_SIZE << cache->order;
    if (cache->flags & SLAB_CACHE_SLINSIDE)
        ssize -= sizeof(slab_t);
    return ssize - objects*cache->size;
}
 
/** Initialize allocated memory as a slab cache */
static void
_slab_cache_create(slab_cache_t *cache,
           char *name,
           size_t size,
           size_t align,
           int (*constructor)(void *obj, int kmflag),
           void (*destructor)(void *obj),
           int flags)
{
    int i;
 
    memsetb((__address)cache, sizeof(*cache), 0);
    cache->name = name;
 
    if (align < sizeof(__native))
        align = sizeof(__native);
    size = ALIGN_UP(size, align);
       
    cache->size = size;
 
    cache->constructor = constructor;
    cache->destructor = destructor;
    cache->flags = flags;
 
    list_initialize(&cache->full_slabs);
    list_initialize(&cache->partial_slabs);
    list_initialize(&cache->magazines);
    spinlock_initialize(&cache->lock, "cachelock");
    if (! cache->flags & SLAB_CACHE_NOMAGAZINE) {
        for (i=0; i< config.cpu_count; i++)
            spinlock_initialize(&cache->mag_cache[i].lock,
                        "cpucachelock");
    }
 
    /* Compute slab sizes, object counts in slabs etc. */
    if (cache->size < SLAB_INSIDE_SIZE)
        cache->flags |= SLAB_CACHE_SLINSIDE;
 
    /* Minimum slab order */
    cache->order = (cache->size-1) >> PAGE_WIDTH;
 
    while (badness(cache) > SLAB_MAX_BADNESS(cache)) {
        cache->order += 1;
    }
 
    cache->objects = comp_objects(cache);
    /* If info fits in, put it inside */
    if (badness(cache) > sizeof(slab_t))
        cache->flags |= SLAB_CACHE_SLINSIDE;
 
    spinlock_lock(&slab_cache_lock);
 
    list_append(&cache->link, &slab_cache_list);
 
    spinlock_unlock(&slab_cache_lock);
}
 
/** Create slab cache  */
slab_cache_t * slab_cache_create(char *name,
                 size_t size,
                 size_t align,
                 int (*constructor)(void *obj, int kmflag),
                 void (*destructor)(void *obj),
                 int flags)
{
    slab_cache_t *cache;
 
    cache = malloc(sizeof(*cache) + config.cpu_count*sizeof(cache->mag_cache[0]));
    _slab_cache_create(cache, name, size, align, constructor, destructor,
               flags);
    return cache;
}
 
/**
 * Reclaim space occupied by objects that are already free
 *
 * @param flags If contains SLAB_RECLAIM_ALL, do aggressive freeing
 * @return Number of freed pages
 *
 * TODO: Add light reclaim
 */
static count_t _slab_reclaim(slab_cache_t *cache, int flags)
{
    int i;
    slab_magazine_t *mag;
    link_t *cur;
    count_t frames = 0;
   
    if (cache->flags & SLAB_CACHE_NOMAGAZINE)
        return 0; /* Nothing to do */
   
    /* First lock all cpu caches, then the complete cache lock */
    for (i=0; i < config.cpu_count; i++)
        spinlock_lock(&cache->mag_cache[i].lock);
    spinlock_lock(&cache->lock);
   
    if (flags & SLAB_RECLAIM_ALL) {
        /* Aggressive memfree */
 
        /* Destroy CPU magazines */
        for (i=0; i<config.cpu_count; i++) {
            mag = cache->mag_cache[i].current;
            if (mag)
                frames += magazine_destroy(cache, mag);
            cache->mag_cache[i].current = NULL;
           
            mag = cache->mag_cache[i].last;
            if (mag)
                frames += magazine_destroy(cache, mag);
            cache->mag_cache[i].last = NULL;
        }
    }
    /* Destroy full magazines */
    cur=cache->magazines.prev;
    while (cur!=&cache->magazines) {
        mag = list_get_instance(cur, slab_magazine_t, link);
       
        cur = cur->prev;
        list_remove(cur->next);
        frames += magazine_destroy(cache,mag);
        /* If we do not do full reclaim, break
         * as soon as something is freed */
        if (!(flags & SLAB_RECLAIM_ALL) && frames)
            break;
    }
   
    spinlock_unlock(&cache->lock);
    for (i=0; i < config.cpu_count; i++)
        spinlock_unlock(&cache->mag_cache[i].lock);
   
    return frames;
}
 
/** Check that there are no slabs and remove cache from system  */
void slab_cache_destroy(slab_cache_t *cache)
{
    /* Do not lock anything, we assume the software is correct and
     * does not touch the cache when it decides to destroy it */
   
    /* Destroy all magazines */
    _slab_reclaim(cache, SLAB_RECLAIM_ALL);
 
    /* All slabs must be empty */
    if (!list_empty(&cache->full_slabs) \
        || !list_empty(&cache->partial_slabs))
        panic("Destroying cache that is not empty.");
 
    spinlock_lock(&slab_cache_lock);
    list_remove(&cache->link);
    spinlock_unlock(&slab_cache_lock);
 
    free(cache);
}
 
/** Allocate new object from cache - if no flags given, always returns
    memory */
void * slab_alloc(slab_cache_t *cache, int flags)
{
    ipl_t ipl;
    void *result = NULL;
 
    /* Disable interrupts to avoid deadlocks with interrupt handlers */
    ipl = interrupts_disable();
   
    if (!cache->flags & SLAB_CACHE_NOMAGAZINE)
        result = magazine_obj_get(cache);
 
    if (!result) {
        spinlock_lock(&cache->lock);
        result = slab_obj_create(cache, flags);
        spinlock_unlock(&cache->lock);
    }
 
    if (result)
        atomic_inc(&cache->allocated_objs);
 
    interrupts_restore(ipl);
 
 
    return result;
}
 
/** Return object to cache  */
void slab_free(slab_cache_t *cache, void *obj)
{
    ipl_t ipl;
 
    ipl = interrupts_disable();
 
    if ((cache->flags & SLAB_CACHE_NOMAGAZINE) \
        || magazine_obj_put(cache, obj)) {
       
        spinlock_lock(&cache->lock);
        slab_obj_destroy(cache, obj, NULL);
        spinlock_unlock(&cache->lock);
    }
    atomic_dec(&cache->allocated_objs);
    interrupts_restore(ipl);
}
 
/* Go through all caches and reclaim what is possible */
count_t slab_reclaim(int flags)
{
    slab_cache_t *cache;
    link_t *cur;
    count_t frames = 0;
 
    spinlock_lock(&slab_cache_lock);
 
    for (cur = slab_cache_list.next;cur!=&slab_cache_list; cur=cur->next) {
        cache = list_get_instance(cur, slab_cache_t, link);
        frames += _slab_reclaim(cache, flags);
    }
 
    spinlock_unlock(&slab_cache_lock);
 
    return frames;
}
 
 
/* Print list of slabs */
void slab_print_list(void)
{
    slab_cache_t *cache;
    link_t *cur;
 
    spinlock_lock(&slab_cache_lock);
    printf("SLAB name\tOsize\tPages\tOcnt\tSlabs\tAllocobjs\tCtl\n");
    for (cur = slab_cache_list.next;cur!=&slab_cache_list; cur=cur->next) {
        cache = list_get_instance(cur, slab_cache_t, link);
        printf("%s\t%d\t%d\t%d\t%d\t%d\t\t%s\n", cache->name, cache->size,
               (1 << cache->order), cache->objects,
               atomic_get(&cache->allocated_slabs),
               atomic_get(&cache->allocated_objs),
               cache->flags & SLAB_CACHE_SLINSIDE ? "In" : "Out");
    }
    spinlock_unlock(&slab_cache_lock);
}
 
void slab_cache_init(void)
{
    /* Initialize magazine cache */
    _slab_cache_create(&mag_cache,
               "slab_magazine",
               sizeof(slab_magazine_t)+SLAB_MAG_SIZE*sizeof(void*),
               sizeof(__address),
               NULL, NULL,
               SLAB_CACHE_NOMAGAZINE);
 
    /* Initialize structures for malloc */
}
 

Rev 765	Rev 766
1	/*	1	/*
2	* Copyright (C) 2006 Ondrej Palkovsky	2	* Copyright (C) 2006 Ondrej Palkovsky
3	* All rights reserved.	3	* All rights reserved.
4	*	4	*
5	* Redistribution and use in source and binary forms, with or without	5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions	6	* modification, are permitted provided that the following conditions
7	* are met:	7	* are met:
8	*	8	*
9	* - Redistributions of source code must retain the above copyright	9	* - Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.	10	* notice, this list of conditions and the following disclaimer.
11	* - Redistributions in binary form must reproduce the above copyright	11	* - Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the	12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.	13	* documentation and/or other materials provided with the distribution.
14	* - The name of the author may not be used to endorse or promote products	14	* - The name of the author may not be used to endorse or promote products
15	* derived from this software without specific prior written permission.	15	* derived from this software without specific prior written permission.
16	*	16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR	17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES	18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.	19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,	20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT	21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,	22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY	23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT	24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF	25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*/	27	*/
28		28
29	#include <synch/spinlock.h>	29	#include <synch/spinlock.h>
30	#include <mm/slab.h>	30	#include <mm/slab.h>
31	#include <list.h>	31	#include <list.h>
32	#include <memstr.h>	32	#include <memstr.h>
33	#include <align.h>	33	#include <align.h>
34	#include <mm/heap.h>	34	#include <mm/heap.h>
35	#include <mm/frame.h>	35	#include <mm/frame.h>
36	#include <config.h>	36	#include <config.h>
37	#include <print.h>	37	#include <print.h>
38	#include <arch.h>	38	#include <arch.h>
39	#include <panic.h>	39	#include <panic.h>
40	#include <debug.h>	40	#include <debug.h>
41		41
42	SPINLOCK_INITIALIZE(slab_cache_lock);	42	SPINLOCK_INITIALIZE(slab_cache_lock);
43	LIST_INITIALIZE(slab_cache_list);	43	LIST_INITIALIZE(slab_cache_list);
44		44
45	slab_cache_t mag_cache;	45	slab_cache_t mag_cache;
46		46
47		47
48	typedef struct {	48	typedef struct {
49	slab_cache_t cache; /< Pointer to parent cache /	49	slab_cache_t cache; /< Pointer to parent cache /
50	link_t link; /* List of full/partial slabs */	50	link_t link; /* List of full/partial slabs */
51	void start; /< Start address of first available item /	51	void start; /< Start address of first available item /
52	count_t available; /*< Count of available items in this slab /	52	count_t available; /*< Count of available items in this slab /
53	index_t nextavail; /*< The index of next available item /	53	index_t nextavail; /*< The index of next available item /
54	}slab_t;	54	}slab_t;
55		55
56	/**************************************/	56	/**************************************/
57	/* SLAB allocation functions */	57	/* SLAB allocation functions */
58		58
59	/**	59	/**
60	* Allocate frames for slab space and initialize	60	* Allocate frames for slab space and initialize
61	*	61	*
62	* TODO: Change slab_t allocation to slab_alloc(????), malloc with flags!!	62	* TODO: Change slab_t allocation to slab_alloc(????), malloc with flags!!
63	*/	63	*/
64	static slab_t * slab_space_alloc(slab_cache_t *cache, int flags)	64	static slab_t * slab_space_alloc(slab_cache_t *cache, int flags)
65	{	65	{
66	void *data;	66	void *data;
67	slab_t *slab;	67	slab_t *slab;
68	size_t fsize;	68	size_t fsize;
69	int i;	69	int i;
70	zone_t *zone = NULL;	70	zone_t *zone = NULL;
71	int status;	71	int status;
72	frame_t *frame;	72	frame_t *frame;
73		73
74	data = (void *)frame_alloc(FRAME_KA \| flags, cache->order, &status, &zone);	74	data = (void *)frame_alloc(FRAME_KA \| flags, cache->order, &status, &zone);
75	if (status != FRAME_OK) {	75	if (status != FRAME_OK) {
76	return NULL;	76	return NULL;
77	}	77	}
78	if (! cache->flags & SLAB_CACHE_SLINSIDE) {	78	if (! cache->flags & SLAB_CACHE_SLINSIDE) {
79	slab = malloc(sizeof(*slab)); // , flags);	79	slab = malloc(sizeof(*slab)); // , flags);
80	if (!slab) {	80	if (!slab) {
81	frame_free((__address)data);	81	frame_free((__address)data);
82	return NULL;	82	return NULL;
83	}	83	}
84	} else {	84	} else {
85	fsize = (PAGE_SIZE << cache->order);	85	fsize = (PAGE_SIZE << cache->order);
86	slab = data + fsize - sizeof(*slab);	86	slab = data + fsize - sizeof(*slab);
87	}	87	}
88		88
89	/* Fill in slab structures */	89	/* Fill in slab structures */
90	/* TODO: some better way of accessing the frame */	90	/* TODO: some better way of accessing the frame */
91	for (i=0; i< (1<<cache->order); i++) {	91	for (i=0; i < (1 << cache->order); i++) {
92	frame = ADDR2FRAME(zone, KA2PA((__address)(data+i*PAGE_SIZE)));	92	frame = ADDR2FRAME(zone, KA2PA((__address)(data+i*PAGE_SIZE)));
93	frame->parent = slab;	93	frame->parent = slab;
94	}	94	}
95		95
96	slab->start = data;	96	slab->start = data;
97	slab->available = cache->objects;	97	slab->available = cache->objects;
98	slab->nextavail = 0;	98	slab->nextavail = 0;
99		99
100	for (i=0; i<cache->objects;i++)	100	for (i=0; i<cache->objects;i++)
101	((int ) (slab->start + i*cache->size)) = i+1;	101	((int ) (slab->start + i*cache->size)) = i+1;
102		102
103	atomic_inc(&cache->allocated_slabs);	103	atomic_inc(&cache->allocated_slabs);
104		104
105	return slab;	105	return slab;
106	}	106	}
107		107
108	/**	108	/**
109	* Free space associated with SLAB	109	* Deallocate space associated with SLAB
110	*	110	*
111	* @return number of freed frames	111	* @return number of freed frames
112	*/	112	*/
113	static count_t slab_space_free(slab_cache_t cache, slab_t slab)	113	static count_t slab_space_free(slab_cache_t cache, slab_t slab)
114	{	114	{
115	frame_free((__address)slab->start);	115	frame_free((__address)slab->start);
116	if (! cache->flags & SLAB_CACHE_SLINSIDE)	116	if (! cache->flags & SLAB_CACHE_SLINSIDE)
117	free(slab);	117	free(slab);
118		118
119	atomic_dec(&cache->allocated_slabs);	119	atomic_dec(&cache->allocated_slabs);
120		120
121	return 1 << cache->order;	121	return 1 << cache->order;
122	}	122	}
123		123
124	/** Map object to slab structure */	124	/** Map object to slab structure */
125	static slab_t * obj2slab(void *obj)	125	static slab_t * obj2slab(void *obj)
126	{	126	{
127	frame_t *frame;	127	frame_t *frame;
128		128
129	frame = frame_addr2frame((__address)obj);	129	frame = frame_addr2frame((__address)obj);
130	return (slab_t *)frame->parent;	130	return (slab_t *)frame->parent;
131	}	131	}
132		132
133	/**************************************/	133	/**************************************/
134	/* SLAB functions */	134	/* SLAB functions */
135		135
136		136
137	/**	137	/**
138	* Return object to slab and call a destructor	138	* Return object to slab and call a destructor
139	*	139	*
140	* Assume the cache->lock is held;	140	* Assume the cache->lock is held;
141	*	141	*
142	* @param slab If the caller knows directly slab of the object, otherwise NULL	142	* @param slab If the caller knows directly slab of the object, otherwise NULL
143	*	143	*
144	* @return Number of freed pages	144	* @return Number of freed pages
145	*/	145	*/
146	static count_t slab_obj_destroy(slab_cache_t cache, void obj,	146	static count_t slab_obj_destroy(slab_cache_t cache, void obj,
147	slab_t *slab)	147	slab_t *slab)
148	{	148	{
149	count_t frames = 0;	149	count_t frames = 0;
150		150
151	if (!slab)	151	if (!slab)
152	slab = obj2slab(obj);	152	slab = obj2slab(obj);
153		153
154	spinlock_lock(&cache->lock);	-
155		-
156	((int )obj) = slab->nextavail;	154	((int )obj) = slab->nextavail;
157	slab->nextavail = (obj - slab->start)/cache->size;	155	slab->nextavail = (obj - slab->start)/cache->size;
158	slab->available++;	156	slab->available++;
159		157
160	/* Move it to correct list */	158	/* Move it to correct list */
161	if (slab->available == 1) {	159	if (slab->available == 1) {
162	/* It was in full, move to partial */	160	/* It was in full, move to partial */
163	list_remove(&slab->link);	161	list_remove(&slab->link);
164	list_prepend(&slab->link, &cache->partial_slabs);	162	list_prepend(&slab->link, &cache->partial_slabs);
165	}	163	}
166	if (slab->available == cache->objects) {	164	if (slab->available == cache->objects) {
167	/* Free associated memory */	165	/* Free associated memory */
168	list_remove(&slab->link);	166	list_remove(&slab->link);
169	/* Avoid deadlock */	167	/* Avoid deadlock */
170	spinlock_unlock(&cache->lock);	168	spinlock_unlock(&cache->lock);
171	frames = slab_space_free(cache, slab);	169	frames = slab_space_free(cache, slab);
172	spinlock_lock(&cache->lock);	170	spinlock_lock(&cache->lock);
173	}	171	}
174		172
175	spinlock_unlock(&cache->lock);	-
176		-
177	return frames;	173	return frames;
178	}	174	}
179		175
180	/**	176	/**
181	* Take new object from slab or create new if needed	177	* Take new object from slab or create new if needed
182	*	178	*
183	* Assume cache->lock is held.	179	* Assume cache->lock is held.
184	*	180	*
185	* @return Object address or null	181	* @return Object address or null
186	*/	182	*/
187	static void * slab_obj_create(slab_cache_t *cache, int flags)	183	static void * slab_obj_create(slab_cache_t *cache, int flags)
188	{	184	{
189	slab_t *slab;	185	slab_t *slab;
190	void *obj;	186	void *obj;
191		187
192	if (list_empty(&cache->partial_slabs)) {	188	if (list_empty(&cache->partial_slabs)) {
193	/* Allow recursion and reclaiming	189	/* Allow recursion and reclaiming
194	* - this should work, as the SLAB control structures	190	* - this should work, as the SLAB control structures
195	* are small and do not need to allocte with anything	191	* are small and do not need to allocte with anything
196	* other ten frame_alloc when they are allocating,	192	* other ten frame_alloc when they are allocating,
197	* that's why we should get recursion at most 1-level deep	193	* that's why we should get recursion at most 1-level deep
198	*/	194	*/
199	spinlock_unlock(&cache->lock);	195	spinlock_unlock(&cache->lock);
200	slab = slab_space_alloc(cache, flags);	196	slab = slab_space_alloc(cache, flags);
201	spinlock_lock(&cache->lock);	197	spinlock_lock(&cache->lock);
202	if (!slab) {	198	if (!slab) {
203	return NULL;	199	return NULL;
204	}	200	}
205	} else {	201	} else {
206	slab = list_get_instance(cache->partial_slabs.next,	202	slab = list_get_instance(cache->partial_slabs.next,
207	slab_t,	203	slab_t,
208	link);	204	link);
209	list_remove(&slab->link);	205	list_remove(&slab->link);
210	}	206	}
211	obj = slab->start + slab->nextavail * cache->size;	207	obj = slab->start + slab->nextavail * cache->size;
212	slab->nextavail = ((int )obj);	208	slab->nextavail = ((int )obj);
213	slab->available--;	209	slab->available--;
214	if (! slab->available)	210	if (! slab->available)
215	list_prepend(&slab->link, &cache->full_slabs);	211	list_prepend(&slab->link, &cache->full_slabs);
216	else	212	else
217	list_prepend(&slab->link, &cache->partial_slabs);	213	list_prepend(&slab->link, &cache->partial_slabs);
218	return obj;	214	return obj;
219	}	215	}
220		216
221	/**************************************/	217	/**************************************/
222	/* CPU-Cache slab functions */	218	/* CPU-Cache slab functions */
223		219
224	/**	220	/**
225	* Free all objects in magazine and free memory associated with magazine	221	* Free all objects in magazine and free memory associated with magazine
226	*	222	*
227	* Assume mag_cache[cpu].lock is locked	223	* Assume mag_cache[cpu].lock is locked
228	*	224	*
229	* @return Number of freed pages	225	* @return Number of freed pages
230	*/	226	*/
231	static count_t magazine_destroy(slab_cache_t *cache,	227	static count_t magazine_destroy(slab_cache_t *cache,
232	slab_magazine_t *mag)	228	slab_magazine_t *mag)
233	{	229	{
234	int i;	230	int i;
235	count_t frames = 0;	231	count_t frames = 0;
236		232
237	for (i=0;i < mag->busy; i++)	233	for (i=0;i < mag->busy; i++)
238	frames += slab_obj_destroy(cache, mag->objs[i], NULL);	234	frames += slab_obj_destroy(cache, mag->objs[i], NULL);
239		235
240	slab_free(&mag_cache, mag);	236	slab_free(&mag_cache, mag);
241		237
242	return frames;	238	return frames;
243	}	239	}
244		240
245	/**	241	/**
246	* Try to find object in CPU-cache magazines	242	* Try to find object in CPU-cache magazines
247	*	243	*
248	* @return Pointer to object or NULL if not available	244	* @return Pointer to object or NULL if not available
249	*/	245	*/
250	static void * magazine_obj_get(slab_cache_t *cache)	246	static void * magazine_obj_get(slab_cache_t *cache)
251	{	247	{
252	slab_magazine_t *mag;	248	slab_magazine_t *mag;
253		249
254	spinlock_lock(&cache->mag_cache[CPU->id].lock);	250	spinlock_lock(&cache->mag_cache[CPU->id].lock);
255		251
256	mag = cache->mag_cache[CPU->id].current;	252	mag = cache->mag_cache[CPU->id].current;
257	if (!mag)	253	if (!mag)
258	goto out;	254	goto out;
259		255
260	if (!mag->busy) {	256	if (!mag->busy) {
261	/* If current is empty && last exists && not empty, exchange */	257	/* If current is empty && last exists && not empty, exchange */
262	if (cache->mag_cache[CPU->id].last \	258	if (cache->mag_cache[CPU->id].last \
263	&& cache->mag_cache[CPU->id].last->busy) {	259	&& cache->mag_cache[CPU->id].last->busy) {
264	cache->mag_cache[CPU->id].current = cache->mag_cache[CPU->id].last;	260	cache->mag_cache[CPU->id].current = cache->mag_cache[CPU->id].last;
265	cache->mag_cache[CPU->id].last = mag;	261	cache->mag_cache[CPU->id].last = mag;
266	mag = cache->mag_cache[CPU->id].current;	262	mag = cache->mag_cache[CPU->id].current;
267	goto gotit;	263	goto gotit;
268	}	264	}
269	/* If still not busy, exchange current with some from	265	/* If still not busy, exchange current with some from
270	* other full magazines */	266	* other full magazines */
271	spinlock_lock(&cache->lock);	267	spinlock_lock(&cache->lock);
272	if (list_empty(&cache->magazines)) {	268	if (list_empty(&cache->magazines)) {
273	spinlock_unlock(&cache->lock);	269	spinlock_unlock(&cache->lock);
274	goto out;	270	goto out;
275	}	271	}
276	/* Free current magazine and take one from list */	272	/* Free current magazine and take one from list */
277	slab_free(&mag_cache, mag);	273	slab_free(&mag_cache, mag);
278	mag = list_get_instance(cache->magazines.next,	274	mag = list_get_instance(cache->magazines.next,
279	slab_magazine_t,	275	slab_magazine_t,
280	link);	276	link);
281	list_remove(&mag->link);	277	list_remove(&mag->link);
282		278
283	spinlock_unlock(&cache->lock);	279	spinlock_unlock(&cache->lock);
284	}	280	}
285	gotit:	281	gotit:
286	spinlock_unlock(&cache->mag_cache[CPU->id].lock);	282	spinlock_unlock(&cache->mag_cache[CPU->id].lock);
287	return mag->objs[--mag->busy];	283	return mag->objs[--mag->busy];
288	out:	284	out:
289	spinlock_unlock(&cache->mag_cache[CPU->id].lock);	285	spinlock_unlock(&cache->mag_cache[CPU->id].lock);
290	return NULL;	286	return NULL;
291	}	287	}
292		288
293	/**	289	/**
294	* Put object into CPU-cache magazine	290	* Put object into CPU-cache magazine
295	*	291	*
296	* We have 2 magazines bound to processor.	292	* We have 2 magazines bound to processor.
297	* First try the current.	293	* First try the current.
298	* If full, try the last.	294	* If full, try the last.
299	* If full, put to magazines list.	295	* If full, put to magazines list.
300	* allocate new, exchange last & current	296	* allocate new, exchange last & current
301	*	297	*
302	* @return 0 - success, -1 - could not get memory	298	* @return 0 - success, -1 - could not get memory
303	*/	299	*/
304	static int magazine_obj_put(slab_cache_t cache, void obj)	300	static int magazine_obj_put(slab_cache_t cache, void obj)
305	{	301	{
306	slab_magazine_t *mag;	302	slab_magazine_t *mag;
307		303
308	spinlock_lock(&cache->mag_cache[CPU->id].lock);	304	spinlock_lock(&cache->mag_cache[CPU->id].lock);
309		305
310	mag = cache->mag_cache[CPU->id].current;	306	mag = cache->mag_cache[CPU->id].current;
311	if (!mag) {	307	if (!mag) {
312	/* We do not want to sleep just because of caching */	308	/* We do not want to sleep just because of caching */
313	/* Especially we do not want reclaiming to start, as	309	/* Especially we do not want reclaiming to start, as
314	* this would deadlock */	310	* this would deadlock */
315	mag = slab_alloc(&mag_cache, FRAME_ATOMIC \| FRAME_NO_RECLAIM);	311	mag = slab_alloc(&mag_cache, FRAME_ATOMIC \| FRAME_NO_RECLAIM);
316	if (!mag) /* Allocation failed, give up on caching */	312	if (!mag) /* Allocation failed, give up on caching */
317	goto errout;	313	goto errout;
318		314
319	cache->mag_cache[CPU->id].current = mag;	315	cache->mag_cache[CPU->id].current = mag;
320	mag->size = SLAB_MAG_SIZE;	316	mag->size = SLAB_MAG_SIZE;
321	mag->busy = 0;	317	mag->busy = 0;
322	} else if (mag->busy == mag->size) {	318	} else if (mag->busy == mag->size) {
323	/* If the last is full \| empty, allocate new */	319	/* If the last is full \| empty, allocate new */
324	mag = cache->mag_cache[CPU->id].last;	320	mag = cache->mag_cache[CPU->id].last;
325	if (!mag \|\| mag->size == mag->busy) {	321	if (!mag \|\| mag->size == mag->busy) {
326	if (mag)	322	if (mag)
327	list_prepend(&mag->link, &cache->magazines);	323	list_prepend(&mag->link, &cache->magazines);
328		324
329	mag = slab_alloc(&mag_cache, FRAME_ATOMIC \| FRAME_NO_RECLAIM);	325	mag = slab_alloc(&mag_cache, FRAME_ATOMIC \| FRAME_NO_RECLAIM);
330	if (!mag)	326	if (!mag)
331	goto errout;	327	goto errout;
332		328
333	mag->size = SLAB_MAG_SIZE;	329	mag->size = SLAB_MAG_SIZE;
334	mag->busy = 0;	330	mag->busy = 0;
335	cache->mag_cache[CPU->id].last = mag;	331	cache->mag_cache[CPU->id].last = mag;
336	}	332	}
337	/* Exchange the 2 */	333	/* Exchange the 2 */
338	cache->mag_cache[CPU->id].last = cache->mag_cache[CPU->id].current;	334	cache->mag_cache[CPU->id].last = cache->mag_cache[CPU->id].current;
339	cache->mag_cache[CPU->id].current = mag;	335	cache->mag_cache[CPU->id].current = mag;
340	}	336	}
341	mag->objs[mag->busy++] = obj;	337	mag->objs[mag->busy++] = obj;
342		338
343	spinlock_unlock(&cache->mag_cache[CPU->id].lock);	339	spinlock_unlock(&cache->mag_cache[CPU->id].lock);
344	return 0;	340	return 0;
345	errout:	341	errout:
346	spinlock_unlock(&cache->mag_cache[CPU->id].lock);	342	spinlock_unlock(&cache->mag_cache[CPU->id].lock);
347	return -1;	343	return -1;
348	}	344	}
349		345
350		346
351	/**************************************/	347	/**************************************/
352	/* SLAB CACHE functions */	348	/* SLAB CACHE functions */
353		349
354	/** Return number of objects that fit in certain cache size */	350	/** Return number of objects that fit in certain cache size */
355	static int comp_objects(slab_cache_t *cache)	351	static int comp_objects(slab_cache_t *cache)
356	{	352	{
357	if (cache->flags & SLAB_CACHE_SLINSIDE)	353	if (cache->flags & SLAB_CACHE_SLINSIDE)
358	return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / cache->size;	354	return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / cache->size;
359	else	355	else
360	return (PAGE_SIZE << cache->order) / cache->size;	356	return (PAGE_SIZE << cache->order) / cache->size;
361	}	357	}
362		358
363	/** Return wasted space in slab */	359	/** Return wasted space in slab */
364	static int badness(slab_cache_t *cache)	360	static int badness(slab_cache_t *cache)
365	{	361	{
366	int objects;	362	int objects;
367	int ssize;	363	int ssize;
368		364
369	objects = comp_objects(cache);	365	objects = comp_objects(cache);
370	ssize = PAGE_SIZE << cache->order;	366	ssize = PAGE_SIZE << cache->order;
371	if (cache->flags & SLAB_CACHE_SLINSIDE)	367	if (cache->flags & SLAB_CACHE_SLINSIDE)
372	ssize -= sizeof(slab_t);	368	ssize -= sizeof(slab_t);
373	return ssize - objects*cache->size;	369	return ssize - objects*cache->size;
374	}	370	}
375		371
376	/** Initialize allocated memory as a slab cache */	372	/** Initialize allocated memory as a slab cache */
377	static void	373	static void
378	_slab_cache_create(slab_cache_t *cache,	374	_slab_cache_create(slab_cache_t *cache,
379	char *name,	375	char *name,
380	size_t size,	376	size_t size,
381	size_t align,	377	size_t align,
382	int (constructor)(void obj, int kmflag),	378	int (constructor)(void obj, int kmflag),
383	void (destructor)(void obj),	379	void (destructor)(void obj),
384	int flags)	380	int flags)
385	{	381	{
386	int i;	382	int i;
387		383
388	memsetb((__address)cache, sizeof(*cache), 0);	384	memsetb((__address)cache, sizeof(*cache), 0);
389	cache->name = name;	385	cache->name = name;
390		386
391	if (align)	387	if (align < sizeof(__native))
-		388	align = sizeof(__native);
392	size = ALIGN_UP(size, align);	389	size = ALIGN_UP(size, align);
-		390
393	cache->size = size;	391	cache->size = size;
394		392
395	cache->constructor = constructor;	393	cache->constructor = constructor;
396	cache->destructor = destructor;	394	cache->destructor = destructor;
397	cache->flags = flags;	395	cache->flags = flags;
398		396
399	list_initialize(&cache->full_slabs);	397	list_initialize(&cache->full_slabs);
400	list_initialize(&cache->partial_slabs);	398	list_initialize(&cache->partial_slabs);
401	list_initialize(&cache->magazines);	399	list_initialize(&cache->magazines);
402	spinlock_initialize(&cache->lock, "cachelock");	400	spinlock_initialize(&cache->lock, "cachelock");
403	if (! cache->flags & SLAB_CACHE_NOMAGAZINE) {	401	if (! cache->flags & SLAB_CACHE_NOMAGAZINE) {
404	for (i=0; i< config.cpu_count; i++)	402	for (i=0; i< config.cpu_count; i++)
405	spinlock_initialize(&cache->mag_cache[i].lock,	403	spinlock_initialize(&cache->mag_cache[i].lock,
406	"cpucachelock");	404	"cpucachelock");
407	}	405	}
408		406
409	/* Compute slab sizes, object counts in slabs etc. */	407	/* Compute slab sizes, object counts in slabs etc. */
410	if (cache->size < SLAB_INSIDE_SIZE)	408	if (cache->size < SLAB_INSIDE_SIZE)
411	cache->flags \|= SLAB_CACHE_SLINSIDE;	409	cache->flags \|= SLAB_CACHE_SLINSIDE;
412		410
413	/* Minimum slab order */	411	/* Minimum slab order */
414	cache->order = (cache->size >> PAGE_WIDTH) + 1;	412	cache->order = (cache->size-1) >> PAGE_WIDTH;
415		413
416	while (badness(cache) > SLAB_MAX_BADNESS(cache)) {	414	while (badness(cache) > SLAB_MAX_BADNESS(cache)) {
417	cache->order += 1;	415	cache->order += 1;
418	}	416	}
419		-
420	cache->objects = comp_objects(cache);	417	cache->objects = comp_objects(cache);
-		418	/* If info fits in, put it inside */
-		419	if (badness(cache) > sizeof(slab_t))
-		420	cache->flags \|= SLAB_CACHE_SLINSIDE;
421		421
422	spinlock_lock(&slab_cache_lock);	422	spinlock_lock(&slab_cache_lock);
423		423
424	list_append(&cache->link, &slab_cache_list);	424	list_append(&cache->link, &slab_cache_list);
425		425
426	spinlock_unlock(&slab_cache_lock);	426	spinlock_unlock(&slab_cache_lock);
427	}	427	}
428		428
429	/** Create slab cache */	429	/** Create slab cache */
430	slab_cache_t * slab_cache_create(char *name,	430	slab_cache_t * slab_cache_create(char *name,
431	size_t size,	431	size_t size,
432	size_t align,	432	size_t align,
433	int (constructor)(void obj, int kmflag),	433	int (constructor)(void obj, int kmflag),
434	void (destructor)(void obj),	434	void (destructor)(void obj),
435	int flags)	435	int flags)
436	{	436	{
437	slab_cache_t *cache;	437	slab_cache_t *cache;
438		438
439	cache = malloc(sizeof(cache) + config.cpu_countsizeof(cache->mag_cache[0]));	439	cache = malloc(sizeof(cache) + config.cpu_countsizeof(cache->mag_cache[0]));
440	_slab_cache_create(cache, name, size, align, constructor, destructor,	440	_slab_cache_create(cache, name, size, align, constructor, destructor,
441	flags);	441	flags);
442	return cache;	442	return cache;
443	}	443	}
444		444
445	/**	445	/**
446	* Reclaim space occupied by objects that are already free	446	* Reclaim space occupied by objects that are already free
447	*	447	*
448	* @param flags If contains SLAB_RECLAIM_ALL, do aggressive freeing	448	* @param flags If contains SLAB_RECLAIM_ALL, do aggressive freeing
449	* @return Number of freed pages	449	* @return Number of freed pages
450	*	450	*
451	* TODO: Add light reclaim	451	* TODO: Add light reclaim
452	*/	452	*/
453	static count_t _slab_reclaim(slab_cache_t *cache, int flags)	453	static count_t _slab_reclaim(slab_cache_t *cache, int flags)
454	{	454	{
455	int i;	455	int i;
456	slab_magazine_t *mag;	456	slab_magazine_t *mag;
457	link_t *cur;	457	link_t *cur;
458	count_t frames = 0;	458	count_t frames = 0;
459		459
460	if (cache->flags & SLAB_CACHE_NOMAGAZINE)	460	if (cache->flags & SLAB_CACHE_NOMAGAZINE)
461	return 0; /* Nothing to do */	461	return 0; /* Nothing to do */
462		462
463	/* First lock all cpu caches, then the complete cache lock */	463	/* First lock all cpu caches, then the complete cache lock */
464	for (i=0; i < config.cpu_count; i++)	464	for (i=0; i < config.cpu_count; i++)
465	spinlock_lock(&cache->mag_cache[i].lock);	465	spinlock_lock(&cache->mag_cache[i].lock);
466	spinlock_lock(&cache->lock);	466	spinlock_lock(&cache->lock);
467		467
468	if (flags & SLAB_RECLAIM_ALL) {	468	if (flags & SLAB_RECLAIM_ALL) {
469	/* Aggressive memfree */	469	/* Aggressive memfree */
470		470
471	/* Destroy CPU magazines */	471	/* Destroy CPU magazines */
472	for (i=0; i<config.cpu_count; i++) {	472	for (i=0; i<config.cpu_count; i++) {
473	mag = cache->mag_cache[i].current;	473	mag = cache->mag_cache[i].current;
474	if (mag)	474	if (mag)
475	frames += magazine_destroy(cache, mag);	475	frames += magazine_destroy(cache, mag);
476	cache->mag_cache[i].current = NULL;	476	cache->mag_cache[i].current = NULL;
477		477
478	mag = cache->mag_cache[i].last;	478	mag = cache->mag_cache[i].last;
479	if (mag)	479	if (mag)
480	frames += magazine_destroy(cache, mag);	480	frames += magazine_destroy(cache, mag);
481	cache->mag_cache[i].last = NULL;	481	cache->mag_cache[i].last = NULL;
482	}	482	}
483	}	483	}
484	/* Destroy full magazines */	484	/* Destroy full magazines */
485	cur=cache->magazines.prev;	485	cur=cache->magazines.prev;
486	while (cur!=&cache->magazines) {	486	while (cur!=&cache->magazines) {
487	mag = list_get_instance(cur, slab_magazine_t, link);	487	mag = list_get_instance(cur, slab_magazine_t, link);
488		488
489	cur = cur->prev;	489	cur = cur->prev;
490	list_remove(cur->next);	490	list_remove(cur->next);
491	frames += magazine_destroy(cache,mag);	491	frames += magazine_destroy(cache,mag);
492	/* If we do not do full reclaim, break	492	/* If we do not do full reclaim, break
493	* as soon as something is freed */	493	* as soon as something is freed */
494	if (!(flags & SLAB_RECLAIM_ALL) && frames)	494	if (!(flags & SLAB_RECLAIM_ALL) && frames)
495	break;	495	break;

Subversion Repositories HelenOS-historic

(root)/kernel/trunk/generic/src/mm/slab.c – Rev 765 → 766