WebSVN – HelenOS-historic – Diff – /kernel/trunk/generic/src/mm/slab.c


/*
 * Copyright (C) 2006 Ondrej Palkovsky
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 *
 * - Redistributions of source code must retain the above copyright
 *   notice, this list of conditions and the following disclaimer.
 * - Redistributions in binary form must reproduce the above copyright
 *   notice, this list of conditions and the following disclaimer in the
 *   documentation and/or other materials provided with the distribution.
 * - The name of the author may not be used to endorse or promote products
 *   derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
 
#include <synch/spinlock.h>
#include <mm/slab.h>
#include <list.h>
#include <memstr.h>
#include <align.h>
#include <mm/heap.h>
#include <mm/frame.h>
#include <config.h>
#include <print.h>
#include <arch.h>
#include <panic.h>
#include <debug.h>
 
SPINLOCK_INITIALIZE(slab_cache_lock);
LIST_INITIALIZE(slab_cache_list);
 
slab_cache_t mag_cache;
 
 
typedef struct {
    slab_cache_t *cache; /**< Pointer to parent cache */
    link_t link;       /* List of full/partial slabs */
    void *start;       /**< Start address of first available item */
    count_t available; /**< Count of available items in this slab */
    index_t nextavail; /**< The index of next available item */
}slab_t;
 
/**************************************/
/* SLAB allocation functions          */
 
/**
 * Allocate frames for slab space and initialize
 *
 * TODO: Change slab_t allocation to slab_alloc(????), malloc with flags!!
 */
static slab_t * slab_space_alloc(slab_cache_t *cache, int flags)
{
    void *data;
    slab_t *slab;
    size_t fsize;
    int i;
    zone_t *zone = NULL;
    int status;
    frame_t *frame;
 
    data = (void *)frame_alloc(FRAME_KA | flags, cache->order, &status, &zone);
    if (status != FRAME_OK) {
        return NULL;
    }
    if (! cache->flags & SLAB_CACHE_SLINSIDE) {
        slab = malloc(sizeof(*slab)); // , flags);
        if (!slab) {
            frame_free((__address)data);
            return NULL;
        }
    } else {
        fsize = (PAGE_SIZE << cache->order);
        slab = data + fsize - sizeof(*slab);
    }
       
    /* Fill in slab structures */
    /* TODO: some better way of accessing the frame */
    for (i=0; i< (1<<cache->order); i++) {
        frame = ADDR2FRAME(zone, KA2PA((__address)(data+i*PAGE_SIZE)));
        frame->parent = slab;
    }
 
    slab->start = data;
    slab->available = cache->objects;
    slab->nextavail = 0;
 
    for (i=0; i<cache->objects;i++)
        *((int *) (slab->start + i*cache->size)) = i+1;
 
    atomic_inc(&cache->allocated_slabs);
 
    return slab;
}
 
/**
 * Free space associated with SLAB
 *
 * @return number of freed frames
 */
static count_t slab_space_free(slab_cache_t *cache, slab_t *slab)
{
    frame_free((__address)slab->start);
    if (! cache->flags & SLAB_CACHE_SLINSIDE)
        free(slab);
 
    atomic_dec(&cache->allocated_slabs);
   
    return 1 << cache->order;
}
 
/** Map object to slab structure */
static slab_t * obj2slab(void *obj)
{
    frame_t *frame;
 
    frame = frame_addr2frame((__address)obj);
    return (slab_t *)frame->parent;
}
 
/**************************************/
/* SLAB functions */
 
 
/**
 * Return object to slab and call a destructor
 *
 * Assume the cache->lock is held;
 *
 * @param slab If the caller knows directly slab of the object, otherwise NULL
 *
 * @return Number of freed pages
 */
static count_t slab_obj_destroy(slab_cache_t *cache, void *obj,
                slab_t *slab)
{
    count_t frames = 0;
 
    if (!slab)
        slab = obj2slab(obj);
 
    spinlock_lock(&cache->lock);
 
    *((int *)obj) = slab->nextavail;
    slab->nextavail = (obj - slab->start)/cache->size;
    slab->available++;
 
    /* Move it to correct list */
    if (slab->available == 1) {
        /* It was in full, move to partial */
        list_remove(&slab->link);
        list_prepend(&slab->link, &cache->partial_slabs);
    }
    if (slab->available == cache->objects) {
        /* Free associated memory */
        list_remove(&slab->link);
        /* Avoid deadlock */
        spinlock_unlock(&cache->lock);
        frames = slab_space_free(cache, slab);
        spinlock_lock(&cache->lock);
    }
 
    spinlock_unlock(&cache->lock);
 
    return frames;
}
 
/**
 * Take new object from slab or create new if needed
 *
 * Assume cache->lock is held.
 *
 * @return Object address or null
 */
static void * slab_obj_create(slab_cache_t *cache, int flags)
{
    slab_t *slab;
    void *obj;
 
    if (list_empty(&cache->partial_slabs)) {
        /* Allow recursion and reclaiming
         * - this should work, as the SLAB control structures
         *   are small and do not need to allocte with anything
         *   other ten frame_alloc when they are allocating,
         *   that's why we should get recursion at most 1-level deep
         */
        spinlock_unlock(&cache->lock);
        slab = slab_space_alloc(cache, flags);
        spinlock_lock(&cache->lock);
        if (!slab) {
            return NULL;
        }
    } else {
        slab = list_get_instance(cache->partial_slabs.next,
                     slab_t,
                     link);
        list_remove(&slab->link);
    }
    obj = slab->start + slab->nextavail * cache->size;
    slab->nextavail = *((int *)obj);
    slab->available--;
    if (! slab->available)
        list_prepend(&slab->link, &cache->full_slabs);
    else
        list_prepend(&slab->link, &cache->partial_slabs);
    return obj;
}
 
/**************************************/
/* CPU-Cache slab functions */
 
/**
 * Free all objects in magazine and free memory associated with magazine
 *
 * Assume mag_cache[cpu].lock is locked
 *
 * @return Number of freed pages
 */
static count_t magazine_destroy(slab_cache_t *cache,
                slab_magazine_t *mag)
{
    int i;
    count_t frames = 0;
 
    for (i=0;i < mag->busy; i++)
        frames += slab_obj_destroy(cache, mag->objs[i], NULL);
   
    slab_free(&mag_cache, mag);
 
    return frames;
}
 
/**
 * Try to find object in CPU-cache magazines
 *
 * @return Pointer to object or NULL if not available
 */
static void * magazine_obj_get(slab_cache_t *cache)
{
    slab_magazine_t *mag;
 
    spinlock_lock(&cache->mag_cache[CPU->id].lock);
 
    mag = cache->mag_cache[CPU->id].current;
    if (!mag)
        goto out;
 
    if (!mag->busy) {
        /* If current is empty && last exists && not empty, exchange */
        if (cache->mag_cache[CPU->id].last \
            && cache->mag_cache[CPU->id].last->busy) {
            cache->mag_cache[CPU->id].current = cache->mag_cache[CPU->id].last;
            cache->mag_cache[CPU->id].last = mag;
            mag = cache->mag_cache[CPU->id].current;
            goto gotit;
        }
        /* If still not busy, exchange current with some from
         * other full magazines */
        spinlock_lock(&cache->lock);
        if (list_empty(&cache->magazines)) {
            spinlock_unlock(&cache->lock);
            goto out;
        }
        /* Free current magazine and take one from list */
        slab_free(&mag_cache, mag);
        mag = list_get_instance(cache->magazines.next,
                    slab_magazine_t,
                    link);
        list_remove(&mag->link);
       
        spinlock_unlock(&cache->lock);
    }
gotit:
    spinlock_unlock(&cache->mag_cache[CPU->id].lock);
    return mag->objs[--mag->busy];
out:   
    spinlock_unlock(&cache->mag_cache[CPU->id].lock);
    return NULL;
}
 
/**
 * Put object into CPU-cache magazine
 *
 * We have 2 magazines bound to processor.
 * First try the current.
 *  If full, try the last.
 *   If full, put to magazines list.
 *   allocate new, exchange last & current
 *
 * @return 0 - success, -1 - could not get memory
 */
static int magazine_obj_put(slab_cache_t *cache, void *obj)
{
    slab_magazine_t *mag;
 
    spinlock_lock(&cache->mag_cache[CPU->id].lock);
   
    mag = cache->mag_cache[CPU->id].current;
    if (!mag) {
        /* We do not want to sleep just because of caching */
        /* Especially we do not want reclaiming to start, as
         * this would deadlock */
        mag = slab_alloc(&mag_cache, FRAME_ATOMIC | FRAME_NO_RECLAIM);
        if (!mag) /* Allocation failed, give up on caching */
            goto errout;
 
        cache->mag_cache[CPU->id].current = mag;
        mag->size = SLAB_MAG_SIZE;
        mag->busy = 0;
    } else if (mag->busy == mag->size) {
        /* If the last is full | empty, allocate new */
        mag = cache->mag_cache[CPU->id].last;
        if (!mag || mag->size == mag->busy) {
            if (mag)
                list_prepend(&mag->link, &cache->magazines);
 
            mag = slab_alloc(&mag_cache, FRAME_ATOMIC | FRAME_NO_RECLAIM);
            if (!mag)
                goto errout;
           
            mag->size = SLAB_MAG_SIZE;
            mag->busy = 0;
            cache->mag_cache[CPU->id].last = mag;
        }
        /* Exchange the 2 */
        cache->mag_cache[CPU->id].last = cache->mag_cache[CPU->id].current;
        cache->mag_cache[CPU->id].current = mag;
    }
    mag->objs[mag->busy++] = obj;
 
    spinlock_unlock(&cache->mag_cache[CPU->id].lock);
    return 0;
errout:
    spinlock_unlock(&cache->mag_cache[CPU->id].lock);
    return -1;
}
 
 
/**************************************/
/* SLAB CACHE functions */
 
/** Return number of objects that fit in certain cache size */
static int comp_objects(slab_cache_t *cache)
{
    if (cache->flags & SLAB_CACHE_SLINSIDE)
        return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / cache->size;
    else
        return (PAGE_SIZE << cache->order) / cache->size;
}
 
/** Return wasted space in slab */
static int badness(slab_cache_t *cache)
{
    int objects;
    int ssize;
 
    objects = comp_objects(cache);
    ssize = PAGE_SIZE << cache->order;
    if (cache->flags & SLAB_CACHE_SLINSIDE)
        ssize -= sizeof(slab_t);
    return ssize - objects*cache->size;
}
 
/** Initialize allocated memory as a slab cache */
static void
_slab_cache_create(slab_cache_t *cache,
           char *name,
           size_t size,
           size_t align,
           int (*constructor)(void *obj, int kmflag),
           void (*destructor)(void *obj),
           int flags)
{
    int i;
 
    memsetb((__address)cache, sizeof(*cache), 0);
    cache->name = name;
 
    if (align)
        size = ALIGN_UP(size, align);
    cache->size = size;
 
    cache->constructor = constructor;
    cache->destructor = destructor;
    cache->flags = flags;
 
    list_initialize(&cache->full_slabs);
    list_initialize(&cache->partial_slabs);
    list_initialize(&cache->magazines);
    spinlock_initialize(&cache->lock, "cachelock");
    if (! cache->flags & SLAB_CACHE_NOMAGAZINE) {
        for (i=0; i< config.cpu_count; i++)
            spinlock_initialize(&cache->mag_cache[i].lock,
                        "cpucachelock");
    }
 
    /* Compute slab sizes, object counts in slabs etc. */
    if (cache->size < SLAB_INSIDE_SIZE)
        cache->flags |= SLAB_CACHE_SLINSIDE;
 
    /* Minimum slab order */
    cache->order = (cache->size / PAGE_SIZE) + 1;
       
    while (badness(cache) > SLAB_MAX_BADNESS(cache)) {
        cache->order += 1;
    }
 
    cache->objects = comp_objects(cache);
 
    spinlock_lock(&slab_cache_lock);
 
    list_append(&cache->link, &slab_cache_list);
 
    spinlock_unlock(&slab_cache_lock);
}
 
/** Create slab cache  */
slab_cache_t * slab_cache_create(char *name,
                 size_t size,
                 size_t align,
                 int (*constructor)(void *obj, int kmflag),
                 void (*destructor)(void *obj),
                 int flags)
{
    slab_cache_t *cache;
 
    cache = malloc(sizeof(*cache) + config.cpu_count*sizeof(cache->mag_cache[0]));
    _slab_cache_create(cache, name, size, align, constructor, destructor,
               flags);
    return cache;
}
 
/**
 * Reclaim space occupied by objects that are already free
 *
 * @param flags If contains SLAB_RECLAIM_ALL, do aggressive freeing
 * @return Number of freed pages
 *
 * TODO: Add light reclaim
 */
static count_t _slab_reclaim(slab_cache_t *cache, int flags)
{
    int i;
    slab_magazine_t *mag;
    link_t *cur;
    count_t frames = 0;
   
    if (cache->flags & SLAB_CACHE_NOMAGAZINE)
        return 0; /* Nothing to do */
   
    /* First lock all cpu caches, then the complete cache lock */
    for (i=0; i < config.cpu_count; i++)
        spinlock_lock(&cache->mag_cache[i].lock);
    spinlock_lock(&cache->lock);
   
    if (flags & SLAB_RECLAIM_ALL) {
        /* Aggressive memfree */
 
        /* Destroy CPU magazines */
        for (i=0; i<config.cpu_count; i++) {
            mag = cache->mag_cache[i].current;
            if (mag)
                frames += magazine_destroy(cache, mag);
            cache->mag_cache[i].current = NULL;
           
            mag = cache->mag_cache[i].last;
            if (mag)
                frames += magazine_destroy(cache, mag);
            cache->mag_cache[i].last = NULL;
        }
    }
    /* Destroy full magazines */
    cur=cache->magazines.prev;
    while (cur!=&cache->magazines) {
        mag = list_get_instance(cur, slab_magazine_t, link);
       
        cur = cur->prev;
        list_remove(cur->next);
        frames += magazine_destroy(cache,mag);
        /* If we do not do full reclaim, break
         * as soon as something is freed */
        if (!(flags & SLAB_RECLAIM_ALL) && frames)
            break;
    }
   
    spinlock_unlock(&cache->lock);
    for (i=0; i < config.cpu_count; i++)
        spinlock_unlock(&cache->mag_cache[i].lock);
   
    return frames;
}
 
/** Check that there are no slabs and remove cache from system  */
void slab_cache_destroy(slab_cache_t *cache)
{
    /* Do not lock anything, we assume the software is correct and
     * does not touch the cache when it decides to destroy it */
   
    /* Destroy all magazines */
    _slab_reclaim(cache, SLAB_RECLAIM_ALL);
 
    /* All slabs must be empty */
    if (!list_empty(&cache->full_slabs) \
        || !list_empty(&cache->partial_slabs))
        panic("Destroying cache that is not empty.");
 
    spinlock_lock(&slab_cache_lock);
    list_remove(&cache->link);
    spinlock_unlock(&slab_cache_lock);
 
    free(cache);
}
 
/** Allocate new object from cache - if no flags given, always returns
    memory */
void * slab_alloc(slab_cache_t *cache, int flags)
{
    ipl_t ipl;
    void *result = NULL;
 
    /* Disable interrupts to avoid deadlocks with interrupt handlers */
    ipl = interrupts_disable();
   
    if (!cache->flags & SLAB_CACHE_NOMAGAZINE)
        result = magazine_obj_get(cache);
 
    if (!result) {
        spinlock_lock(&cache->lock);
        result = slab_obj_create(cache, flags);
        spinlock_unlock(&cache->lock);
    }
 
    if (result)
        atomic_inc(&cache->allocated_objs);
 
    interrupts_restore(ipl);
 
 
    return result;
}
 
/** Return object to cache  */
void slab_free(slab_cache_t *cache, void *obj)
{
    ipl_t ipl;
 
    ipl = interrupts_disable();
 
    if ((cache->flags & SLAB_CACHE_NOMAGAZINE) \
        || magazine_obj_put(cache, obj)) {
       
        spinlock_lock(&cache->lock);
        slab_obj_destroy(cache, obj, NULL);
        spinlock_unlock(&cache->lock);
    }
    atomic_dec(&cache->allocated_objs);
    interrupts_restore(ipl);
}
 
/* Go through all caches and reclaim what is possible */
count_t slab_reclaim(int flags)
{
    slab_cache_t *cache;
    link_t *cur;
    count_t frames = 0;
 
    spinlock_lock(&slab_cache_lock);
 
    for (cur = slab_cache_list.next;cur!=&slab_cache_list; cur=cur->next) {
        cache = list_get_instance(cur, slab_cache_t, link);
        frames += _slab_reclaim(cache, flags);
    }
 
    spinlock_unlock(&slab_cache_lock);
 
    return frames;
}
 
 
/* Print list of slabs */
void slab_print_list(void)
{
    slab_cache_t *cache;
    link_t *cur;
 
    spinlock_lock(&slab_cache_lock);
    printf("SLAB name\tOsize\tOrder\tOcnt\tSlabs\tAllocobjs\n");
    for (cur = slab_cache_list.next;cur!=&slab_cache_list; cur=cur->next) {
        cache = list_get_instance(cur, slab_cache_t, link);
        printf("%s\t%d\t%d\t%d\t%d\t%d\n", cache->name, cache->size,
               cache->order, cache->objects,
               atomic_get(&cache->allocated_slabs),
               atomic_get(&cache->allocated_objs));
    }
    spinlock_unlock(&slab_cache_lock);
}
 
void slab_cache_init(void)
{
    /* Initialize magazine cache */
    _slab_cache_create(&mag_cache,
               "slab_magazine",
               sizeof(slab_magazine_t)+SLAB_MAG_SIZE*sizeof(void*),
               sizeof(__address),
               NULL, NULL,
               SLAB_CACHE_NOMAGAZINE);
 
    /* Initialize structures for malloc */
}
 

Rev 763	Rev 764
1	/*	1	/*
2	* Copyright (C) 2006 Ondrej Palkovsky	2	* Copyright (C) 2006 Ondrej Palkovsky
3	* All rights reserved.	3	* All rights reserved.
4	*	4	*
5	* Redistribution and use in source and binary forms, with or without	5	* Redistribution and use in source and binary forms, with or without
6	* modification, are permitted provided that the following conditions	6	* modification, are permitted provided that the following conditions
7	* are met:	7	* are met:
8	*	8	*
9	* - Redistributions of source code must retain the above copyright	9	* - Redistributions of source code must retain the above copyright
10	* notice, this list of conditions and the following disclaimer.	10	* notice, this list of conditions and the following disclaimer.
11	* - Redistributions in binary form must reproduce the above copyright	11	* - Redistributions in binary form must reproduce the above copyright
12	* notice, this list of conditions and the following disclaimer in the	12	* notice, this list of conditions and the following disclaimer in the
13	* documentation and/or other materials provided with the distribution.	13	* documentation and/or other materials provided with the distribution.
14	* - The name of the author may not be used to endorse or promote products	14	* - The name of the author may not be used to endorse or promote products
15	* derived from this software without specific prior written permission.	15	* derived from this software without specific prior written permission.
16	*	16	*
17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR	17	* THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES	18	* IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.	19	* OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,	20	* IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT	21	* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,	22	* NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY	23	* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT	24	* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF	25	* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.	26	* THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27	*/	27	*/
28		28
29	#include <synch/spinlock.h>	29	#include <synch/spinlock.h>
30	#include <mm/slab.h>	30	#include <mm/slab.h>
31	#include <list.h>	31	#include <list.h>
32	#include <memstr.h>	32	#include <memstr.h>
33	#include <align.h>	33	#include <align.h>
34	#include <mm/heap.h>	34	#include <mm/heap.h>
35	#include <mm/frame.h>	35	#include <mm/frame.h>
36	#include <config.h>	36	#include <config.h>
37	#include <print.h>	37	#include <print.h>
38	#include <arch.h>	38	#include <arch.h>
39	#include <panic.h>	39	#include <panic.h>
40	#include <debug.h>	40	#include <debug.h>
41		41
42	SPINLOCK_INITIALIZE(slab_cache_lock);	42	SPINLOCK_INITIALIZE(slab_cache_lock);
43	LIST_INITIALIZE(slab_cache_list);	43	LIST_INITIALIZE(slab_cache_list);
44		44
45	slab_cache_t mag_cache;	45	slab_cache_t mag_cache;
46		46
47		47
48	typedef struct {	48	typedef struct {
49	slab_cache_t cache; /< Pointer to parent cache /	49	slab_cache_t cache; /< Pointer to parent cache /
50	link_t link; /* List of full/partial slabs */	50	link_t link; /* List of full/partial slabs */
51	void start; /< Start address of first available item /	51	void start; /< Start address of first available item /
52	count_t available; /*< Count of available items in this slab /	52	count_t available; /*< Count of available items in this slab /
53	index_t nextavail; /*< The index of next available item /	53	index_t nextavail; /*< The index of next available item /
54	}slab_t;	54	}slab_t;
55		55
56	/**************************************/	56	/**************************************/
57	/* SLAB allocation functions */	57	/* SLAB allocation functions */
58		58
59	/**	59	/**
60	* Allocate frames for slab space and initialize	60	* Allocate frames for slab space and initialize
61	*	61	*
62	* TODO: Change slab_t allocation to slab_alloc(????), malloc with flags!!	62	* TODO: Change slab_t allocation to slab_alloc(????), malloc with flags!!
63	*/	63	*/
64	static slab_t * slab_space_alloc(slab_cache_t *cache, int flags)	64	static slab_t * slab_space_alloc(slab_cache_t *cache, int flags)
65	{	65	{
66	void *data;	66	void *data;
67	slab_t *slab;	67	slab_t *slab;
68	size_t fsize;	68	size_t fsize;
69	int i;	69	int i;
70	zone_t *zone = NULL;	70	zone_t *zone = NULL;
71	int status;	71	int status;
-		72	frame_t *frame;
72		73
73	data = (void *)frame_alloc(FRAME_KA \| flags, cache->order, &status, &zone);	74	data = (void *)frame_alloc(FRAME_KA \| flags, cache->order, &status, &zone);
74	if (status != FRAME_OK)	75	if (status != FRAME_OK) {
75	return NULL;	76	return NULL;
76		77	}
77	if (! cache->flags & SLAB_CACHE_SLINSIDE) {	78	if (! cache->flags & SLAB_CACHE_SLINSIDE) {
78	slab = malloc(sizeof(*slab)); // , flags);	79	slab = malloc(sizeof(*slab)); // , flags);
79	if (!slab) {	80	if (!slab) {
80	frame_free((__address)data);	81	frame_free((__address)data);
81	return NULL;	82	return NULL;
82	}	83	}
83	} else {	84	} else {
84	fsize = (PAGE_SIZE << cache->order);	85	fsize = (PAGE_SIZE << cache->order);
85	slab = data + fsize - sizeof(*slab);	86	slab = data + fsize - sizeof(*slab);
86	}	87	}
87		88
88	/* Fill in slab structures */	89	/* Fill in slab structures */
89	/* TODO: some better way of accessing the frame */	90	/* TODO: some better way of accessing the frame */
90	for (i=0; i< (1<<cache->order); i++) {	91	for (i=0; i< (1<<cache->order); i++) {
91	ADDR2FRAME(zone, (__address)(data+i*PAGE_SIZE))->parent = slab;	92	frame = ADDR2FRAME(zone, KA2PA((__address)(data+i*PAGE_SIZE)));
-		93	frame->parent = slab;
92	}	94	}
93		95
94	slab->start = data;	96	slab->start = data;
95	slab->available = cache->objects;	97	slab->available = cache->objects;
96	slab->nextavail = 0;	98	slab->nextavail = 0;
97		99
98	for (i=0; i<cache->objects;i++)	100	for (i=0; i<cache->objects;i++)
99	((int ) (slab->start + i*cache->size)) = i+1;	101	((int ) (slab->start + i*cache->size)) = i+1;
-		102
-		103	atomic_inc(&cache->allocated_slabs);
-		104
100	return slab;	105	return slab;
101	}	106	}
102		107
103	/**	108	/**
104	* Free space associated with SLAB	109	* Free space associated with SLAB
105	*	110	*
106	* @return number of freed frames	111	* @return number of freed frames
107	*/	112	*/
108	static count_t slab_space_free(slab_cache_t cache, slab_t slab)	113	static count_t slab_space_free(slab_cache_t cache, slab_t slab)
109	{	114	{
110	frame_free((__address)slab->start);	115	frame_free((__address)slab->start);
111	if (! cache->flags & SLAB_CACHE_SLINSIDE)	116	if (! cache->flags & SLAB_CACHE_SLINSIDE)
112	free(slab);	117	free(slab);
-		118
-		119	atomic_dec(&cache->allocated_slabs);
-		120
113	return 1 << cache->order;	121	return 1 << cache->order;
114	}	122	}
115		123
116	/** Map object to slab structure */	124	/** Map object to slab structure */
117	static slab_t * obj2slab(void *obj)	125	static slab_t * obj2slab(void *obj)
118	{	126	{
119	frame_t *frame;	127	frame_t *frame;
120		128
121	frame = frame_addr2frame((__address)obj);	129	frame = frame_addr2frame((__address)obj);
122	return (slab_t *)frame->parent;	130	return (slab_t *)frame->parent;
123	}	131	}
124		132
125	/**************************************/	133	/**************************************/
126	/* SLAB functions */	134	/* SLAB functions */
127		135
128		136
129	/**	137	/**
130	* Return object to slab and call a destructor	138	* Return object to slab and call a destructor
131	*	139	*
132	* Assume the cache->lock is held;	140	* Assume the cache->lock is held;
133	*	141	*
134	* @param slab If the caller knows directly slab of the object, otherwise NULL	142	* @param slab If the caller knows directly slab of the object, otherwise NULL
135	*	143	*
136	* @return Number of freed pages	144	* @return Number of freed pages
137	*/	145	*/
138	static count_t slab_obj_destroy(slab_cache_t cache, void obj,	146	static count_t slab_obj_destroy(slab_cache_t cache, void obj,
139	slab_t *slab)	147	slab_t *slab)
140	{	148	{
141	count_t frames = 0;	149	count_t frames = 0;
142		150
143	if (!slab)	151	if (!slab)
144	slab = obj2slab(obj);	152	slab = obj2slab(obj);
145		153
146	spinlock_lock(&cache->lock);	154	spinlock_lock(&cache->lock);
147		155
148	((int )obj) = slab->nextavail;	156	((int )obj) = slab->nextavail;
149	slab->nextavail = (obj - slab->start)/cache->size;	157	slab->nextavail = (obj - slab->start)/cache->size;
150	slab->available++;	158	slab->available++;
151		159
152	/* Move it to correct list */	160	/* Move it to correct list */
153	if (slab->available == 1) {	161	if (slab->available == 1) {
154	/* It was in full, move to partial */	162	/* It was in full, move to partial */
155	list_remove(&slab->link);	163	list_remove(&slab->link);
156	list_prepend(&cache->partial_slabs, &slab->link);	164	list_prepend(&slab->link, &cache->partial_slabs);
157	}	165	}
158	if (slab->available == cache->objects) {	166	if (slab->available == cache->objects) {
159	/* Free associated memory */	167	/* Free associated memory */
160	list_remove(&slab->link);	168	list_remove(&slab->link);
161	/* Avoid deadlock */	169	/* Avoid deadlock */
162	spinlock_unlock(&cache->lock);	170	spinlock_unlock(&cache->lock);
163	frames = slab_space_free(cache, slab);	171	frames = slab_space_free(cache, slab);
164	spinlock_lock(&cache->lock);	172	spinlock_lock(&cache->lock);
165	}	173	}
166		174
167	spinlock_unlock(&cache->lock);	175	spinlock_unlock(&cache->lock);
168		176
169	return frames;	177	return frames;
170	}	178	}
171		179
172	/**	180	/**
173	* Take new object from slab or create new if needed	181	* Take new object from slab or create new if needed
174	*	182	*
175	* Assume cache->lock is held.	183	* Assume cache->lock is held.
176	*	184	*
177	* @return Object address or null	185	* @return Object address or null
178	*/	186	*/
179	static void * slab_obj_create(slab_cache_t *cache, int flags)	187	static void * slab_obj_create(slab_cache_t *cache, int flags)
180	{	188	{
181	slab_t *slab;	189	slab_t *slab;
182	void *obj;	190	void *obj;
183		191
184	if (list_empty(&cache->partial_slabs)) {	192	if (list_empty(&cache->partial_slabs)) {
185	/* Allow recursion and reclaiming	193	/* Allow recursion and reclaiming
186	* - this should work, as the SLAB control structures	194	* - this should work, as the SLAB control structures
187	* are small and do not need to allocte with anything	195	* are small and do not need to allocte with anything
188	* other ten frame_alloc when they are allocating,	196	* other ten frame_alloc when they are allocating,
189	* that's why we should get recursion at most 1-level deep	197	* that's why we should get recursion at most 1-level deep
190	*/	198	*/
191	spinlock_unlock(&cache->lock);	199	spinlock_unlock(&cache->lock);
192	slab = slab_space_alloc(cache, flags);	200	slab = slab_space_alloc(cache, flags);
193	spinlock_lock(&cache->lock);	201	spinlock_lock(&cache->lock);
194	if (!slab)	202	if (!slab) {
195	return NULL;	203	return NULL;
-		204	}
196	} else {	205	} else {
197	slab = list_get_instance(cache->partial_slabs.next,	206	slab = list_get_instance(cache->partial_slabs.next,
198	slab_t,	207	slab_t,
199	link);	208	link);
200	list_remove(&slab->link);	209	list_remove(&slab->link);
201	}	210	}
202	obj = slab->start + slab->nextavail * cache->size;	211	obj = slab->start + slab->nextavail * cache->size;
203	slab->nextavail = ((int )obj);	212	slab->nextavail = ((int )obj);
204	slab->available--;	213	slab->available--;
205	if (! slab->available)	214	if (! slab->available)
206	list_prepend(&cache->full_slabs, &slab->link);	215	list_prepend(&slab->link, &cache->full_slabs);
207	else	216	else
208	list_prepend(&cache->partial_slabs, &slab->link);	217	list_prepend(&slab->link, &cache->partial_slabs);
209	return obj;	218	return obj;
210	}	219	}
211		220
212	/**************************************/	221	/**************************************/
213	/* CPU-Cache slab functions */	222	/* CPU-Cache slab functions */
214		223
215	/**	224	/**
216	* Free all objects in magazine and free memory associated with magazine	225	* Free all objects in magazine and free memory associated with magazine
217	*	226	*
218	* Assume mag_cache[cpu].lock is locked	227	* Assume mag_cache[cpu].lock is locked
219	*	228	*
220	* @return Number of freed pages	229	* @return Number of freed pages
221	*/	230	*/
222	static count_t magazine_destroy(slab_cache_t *cache,	231	static count_t magazine_destroy(slab_cache_t *cache,
223	slab_magazine_t *mag)	232	slab_magazine_t *mag)
224	{	233	{
225	int i;	234	int i;
226	count_t frames = 0;	235	count_t frames = 0;
227		236
228	for (i=0;i < mag->busy; i++)	237	for (i=0;i < mag->busy; i++)
229	frames += slab_obj_destroy(cache, mag->objs[i], NULL);	238	frames += slab_obj_destroy(cache, mag->objs[i], NULL);
230		239
231	slab_free(&mag_cache, mag);	240	slab_free(&mag_cache, mag);
232		241
233	return frames;	242	return frames;
234	}	243	}
235		244
236	/**	245	/**
237	* Try to find object in CPU-cache magazines	246	* Try to find object in CPU-cache magazines
238	*	247	*
239	* @return Pointer to object or NULL if not available	248	* @return Pointer to object or NULL if not available
240	*/	249	*/
241	static void * magazine_obj_get(slab_cache_t *cache)	250	static void * magazine_obj_get(slab_cache_t *cache)
242	{	251	{
243	slab_magazine_t *mag;	252	slab_magazine_t *mag;
244		253
245	spinlock_lock(&cache->mag_cache[CPU->id].lock);	254	spinlock_lock(&cache->mag_cache[CPU->id].lock);
246		255
247	mag = cache->mag_cache[CPU->id].current;	256	mag = cache->mag_cache[CPU->id].current;
248	if (!mag)	257	if (!mag)
249	goto out;	258	goto out;
250		259
251	if (!mag->busy) {	260	if (!mag->busy) {
252	/* If current is empty && last exists && not empty, exchange */	261	/* If current is empty && last exists && not empty, exchange */
253	if (cache->mag_cache[CPU->id].last \	262	if (cache->mag_cache[CPU->id].last \
254	&& cache->mag_cache[CPU->id].last->busy) {	263	&& cache->mag_cache[CPU->id].last->busy) {
255	cache->mag_cache[CPU->id].current = cache->mag_cache[CPU->id].last;	264	cache->mag_cache[CPU->id].current = cache->mag_cache[CPU->id].last;
256	cache->mag_cache[CPU->id].last = mag;	265	cache->mag_cache[CPU->id].last = mag;
257	mag = cache->mag_cache[CPU->id].current;	266	mag = cache->mag_cache[CPU->id].current;
258	goto gotit;	267	goto gotit;
259	}	268	}
260	/* If still not busy, exchange current with some from	269	/* If still not busy, exchange current with some from
261	* other full magazines */	270	* other full magazines */
262	spinlock_lock(&cache->lock);	271	spinlock_lock(&cache->lock);
263	if (list_empty(&cache->magazines)) {	272	if (list_empty(&cache->magazines)) {
264	spinlock_unlock(&cache->lock);	273	spinlock_unlock(&cache->lock);
265	goto out;	274	goto out;
266	}	275	}
267	/* Free current magazine and take one from list */	276	/* Free current magazine and take one from list */
268	slab_free(&mag_cache, mag);	277	slab_free(&mag_cache, mag);
269	mag = list_get_instance(cache->magazines.next,	278	mag = list_get_instance(cache->magazines.next,
270	slab_magazine_t,	279	slab_magazine_t,
271	link);	280	link);
272	list_remove(&mag->link);	281	list_remove(&mag->link);
273		282
274	spinlock_unlock(&cache->lock);	283	spinlock_unlock(&cache->lock);
275	}	284	}
276	gotit:	285	gotit:
277	spinlock_unlock(&cache->mag_cache[CPU->id].lock);	286	spinlock_unlock(&cache->mag_cache[CPU->id].lock);
278	return mag->objs[--mag->busy];	287	return mag->objs[--mag->busy];
279	out:	288	out:
280	spinlock_unlock(&cache->mag_cache[CPU->id].lock);	289	spinlock_unlock(&cache->mag_cache[CPU->id].lock);
281	return NULL;	290	return NULL;
282	}	291	}
283		292
284	/**	293	/**
285	* Put object into CPU-cache magazine	294	* Put object into CPU-cache magazine
286	*	295	*
287	* We have 2 magazines bound to processor.	296	* We have 2 magazines bound to processor.
288	* First try the current.	297	* First try the current.
289	* If full, try the last.	298	* If full, try the last.
290	* If full, put to magazines list.	299	* If full, put to magazines list.
291	* allocate new, exchange last & current	300	* allocate new, exchange last & current
292	*	301	*
293	* @return 0 - success, -1 - could not get memory	302	* @return 0 - success, -1 - could not get memory
294	*/	303	*/
295	static int magazine_obj_put(slab_cache_t cache, void obj)	304	static int magazine_obj_put(slab_cache_t cache, void obj)
296	{	305	{
297	slab_magazine_t *mag;	306	slab_magazine_t *mag;
298		307
299	spinlock_lock(&cache->mag_cache[CPU->id].lock);	308	spinlock_lock(&cache->mag_cache[CPU->id].lock);
300		309
301	mag = cache->mag_cache[CPU->id].current;	310	mag = cache->mag_cache[CPU->id].current;
302	if (!mag) {	311	if (!mag) {
303	/* We do not want to sleep just because of caching */	312	/* We do not want to sleep just because of caching */
304	/* Especially we do not want reclaiming to start, as	313	/* Especially we do not want reclaiming to start, as
305	* this would deadlock */	314	* this would deadlock */
306	mag = slab_alloc(&mag_cache, FRAME_ATOMIC \| FRAME_NO_RECLAIM);	315	mag = slab_alloc(&mag_cache, FRAME_ATOMIC \| FRAME_NO_RECLAIM);
307	if (!mag) /* Allocation failed, give up on caching */	316	if (!mag) /* Allocation failed, give up on caching */
308	goto errout;	317	goto errout;
309		318
310	cache->mag_cache[CPU->id].current = mag;	319	cache->mag_cache[CPU->id].current = mag;
311	mag->size = SLAB_MAG_SIZE;	320	mag->size = SLAB_MAG_SIZE;
312	mag->busy = 0;	321	mag->busy = 0;
313	} else if (mag->busy == mag->size) {	322	} else if (mag->busy == mag->size) {
314	/* If the last is full \| empty, allocate new */	323	/* If the last is full \| empty, allocate new */
315	mag = cache->mag_cache[CPU->id].last;	324	mag = cache->mag_cache[CPU->id].last;
316	if (!mag \|\| mag->size == mag->busy) {	325	if (!mag \|\| mag->size == mag->busy) {
317	if (mag)	326	if (mag)
318	list_prepend(&cache->magazines, &mag->link);	327	list_prepend(&mag->link, &cache->magazines);
319		328
320	mag = slab_alloc(&mag_cache, FRAME_ATOMIC \| FRAME_NO_RECLAIM);	329	mag = slab_alloc(&mag_cache, FRAME_ATOMIC \| FRAME_NO_RECLAIM);
321	if (!mag)	330	if (!mag)
322	goto errout;	331	goto errout;
323		332
324	mag->size = SLAB_MAG_SIZE;	333	mag->size = SLAB_MAG_SIZE;
325	mag->busy = 0;	334	mag->busy = 0;
326	cache->mag_cache[CPU->id].last = mag;	335	cache->mag_cache[CPU->id].last = mag;
327	}	336	}
328	/* Exchange the 2 */	337	/* Exchange the 2 */
329	cache->mag_cache[CPU->id].last = cache->mag_cache[CPU->id].current;	338	cache->mag_cache[CPU->id].last = cache->mag_cache[CPU->id].current;
330	cache->mag_cache[CPU->id].current = mag;	339	cache->mag_cache[CPU->id].current = mag;
331	}	340	}
332	mag->objs[mag->busy++] = obj;	341	mag->objs[mag->busy++] = obj;
333		342
334	spinlock_unlock(&cache->mag_cache[CPU->id].lock);	343	spinlock_unlock(&cache->mag_cache[CPU->id].lock);
335	return 0;	344	return 0;
336	errout:	345	errout:
337	spinlock_unlock(&cache->mag_cache[CPU->id].lock);	346	spinlock_unlock(&cache->mag_cache[CPU->id].lock);
338	return -1;	347	return -1;
339	}	348	}
340		349
341		350
342	/**************************************/	351	/**************************************/
343	/* SLAB CACHE functions */	352	/* SLAB CACHE functions */
344		353
345	/** Return number of objects that fit in certain cache size */	354	/** Return number of objects that fit in certain cache size */
346	static int comp_objects(slab_cache_t *cache)	355	static int comp_objects(slab_cache_t *cache)
347	{	356	{
348	if (cache->flags & SLAB_CACHE_SLINSIDE)	357	if (cache->flags & SLAB_CACHE_SLINSIDE)
349	return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / cache->size;	358	return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / cache->size;
350	else	359	else
351	return (PAGE_SIZE << cache->order) / cache->size;	360	return (PAGE_SIZE << cache->order) / cache->size;
352	}	361	}
353		362
354	/** Return wasted space in slab */	363	/** Return wasted space in slab */
355	static int badness(slab_cache_t *cache)	364	static int badness(slab_cache_t *cache)
356	{	365	{
357	int objects;	366	int objects;
358	int ssize;	367	int ssize;
359		368
360	objects = comp_objects(cache);	369	objects = comp_objects(cache);
361	ssize = PAGE_SIZE << cache->order;	370	ssize = PAGE_SIZE << cache->order;
362	if (cache->flags & SLAB_CACHE_SLINSIDE)	371	if (cache->flags & SLAB_CACHE_SLINSIDE)
363	ssize -= sizeof(slab_t);	372	ssize -= sizeof(slab_t);
364	return ssize - objects*cache->size;	373	return ssize - objects*cache->size;
365	}	374	}
366		375
367	/** Initialize allocated memory as a slab cache */	376	/** Initialize allocated memory as a slab cache */
368	static void	377	static void
369	_slab_cache_create(slab_cache_t *cache,	378	_slab_cache_create(slab_cache_t *cache,
370	char *name,	379	char *name,
371	size_t size,	380	size_t size,
372	size_t align,	381	size_t align,
373	int (constructor)(void obj, int kmflag),	382	int (constructor)(void obj, int kmflag),
374	void (destructor)(void obj),	383	void (destructor)(void obj),
375	int flags)	384	int flags)
376	{	385	{
377	int i;	386	int i;
378		387
379	memsetb((__address)cache, sizeof(*cache), 0);	388	memsetb((__address)cache, sizeof(*cache), 0);
380	cache->name = name;	389	cache->name = name;
381		390
382	if (align)	391	if (align)
383	size = ALIGN_UP(size, align);	392	size = ALIGN_UP(size, align);
384	cache->size = size;	393	cache->size = size;
385		394
386	cache->constructor = constructor;	395	cache->constructor = constructor;
387	cache->destructor = destructor;	396	cache->destructor = destructor;
388	cache->flags = flags;	397	cache->flags = flags;
389		398
390	list_initialize(&cache->full_slabs);	399	list_initialize(&cache->full_slabs);
391	list_initialize(&cache->partial_slabs);	400	list_initialize(&cache->partial_slabs);
392	list_initialize(&cache->magazines);	401	list_initialize(&cache->magazines);
393	spinlock_initialize(&cache->lock, "cachelock");	402	spinlock_initialize(&cache->lock, "cachelock");
394	if (! cache->flags & SLAB_CACHE_NOMAGAZINE) {	403	if (! cache->flags & SLAB_CACHE_NOMAGAZINE) {
395	for (i=0; i< config.cpu_count; i++)	404	for (i=0; i< config.cpu_count; i++)
396	spinlock_initialize(&cache->mag_cache[i].lock,	405	spinlock_initialize(&cache->mag_cache[i].lock,
397	"cpucachelock");	406	"cpucachelock");
398	}	407	}
399		408
400	/* Compute slab sizes, object counts in slabs etc. */	409	/* Compute slab sizes, object counts in slabs etc. */
401	if (cache->size < SLAB_INSIDE_SIZE)	410	if (cache->size < SLAB_INSIDE_SIZE)
402	cache->flags \|= SLAB_CACHE_SLINSIDE;	411	cache->flags \|= SLAB_CACHE_SLINSIDE;
403		412
404	/* Minimum slab order */	413	/* Minimum slab order */
405	cache->order = (cache->size / PAGE_SIZE) + 1;	414	cache->order = (cache->size / PAGE_SIZE) + 1;
406		415
407	while (badness(cache) > SLAB_MAX_BADNESS(cache)) {	416	while (badness(cache) > SLAB_MAX_BADNESS(cache)) {
408	cache->order += 1;	417	cache->order += 1;
409	}	418	}
410		419
411	cache->objects = comp_objects(cache);	420	cache->objects = comp_objects(cache);
412		421
413	spinlock_lock(&slab_cache_lock);	422	spinlock_lock(&slab_cache_lock);
414		423
415	list_append(&cache->link, &slab_cache_list);	424	list_append(&cache->link, &slab_cache_list);
416		425
417	spinlock_unlock(&slab_cache_lock);	426	spinlock_unlock(&slab_cache_lock);
418	}	427	}
419		428
420	/** Create slab cache */	429	/** Create slab cache */
421	slab_cache_t * slab_cache_create(char *name,	430	slab_cache_t * slab_cache_create(char *name,
422	size_t size,	431	size_t size,
423	size_t align,	432	size_t align,
424	int (constructor)(void obj, int kmflag),	433	int (constructor)(void obj, int kmflag),
425	void (destructor)(void obj),	434	void (destructor)(void obj),
426	int flags)	435	int flags)
427	{	436	{
428	slab_cache_t *cache;	437	slab_cache_t *cache;
429		438
430	cache = malloc(sizeof(cache) + config.cpu_countsizeof(cache->mag_cache[0]));	439	cache = malloc(sizeof(cache) + config.cpu_countsizeof(cache->mag_cache[0]));
431	_slab_cache_create(cache, name, size, align, constructor, destructor,	440	_slab_cache_create(cache, name, size, align, constructor, destructor,
432	flags);	441	flags);
433	return cache;	442	return cache;
434	}	443	}
435		444
436	/**	445	/**
437	* Reclaim space occupied by objects that are already free	446	* Reclaim space occupied by objects that are already free
438	*	447	*
439	* @param flags If contains SLAB_RECLAIM_ALL, do aggressive freeing	448	* @param flags If contains SLAB_RECLAIM_ALL, do aggressive freeing
440	* @return Number of freed pages	449	* @return Number of freed pages
441	*	450	*
442	* TODO: Add light reclaim	451	* TODO: Add light reclaim
443	*/	452	*/
444	static count_t _slab_reclaim(slab_cache_t *cache, int flags)	453	static count_t _slab_reclaim(slab_cache_t *cache, int flags)
445	{	454	{
446	int i;	455	int i;
447	slab_magazine_t *mag;	456	slab_magazine_t *mag;
448	link_t *cur;	457	link_t *cur;
449	count_t frames = 0;	458	count_t frames = 0;
450		459
451	if (cache->flags & SLAB_CACHE_NOMAGAZINE)	460	if (cache->flags & SLAB_CACHE_NOMAGAZINE)
452	return 0; /* Nothing to do */	461	return 0; /* Nothing to do */
453		462
454	/* First lock all cpu caches, then the complete cache lock */	463	/* First lock all cpu caches, then the complete cache lock */
455	for (i=0; i < config.cpu_count; i++)	464	for (i=0; i < config.cpu_count; i++)
456	spinlock_lock(&cache->mag_cache[i].lock);	465	spinlock_lock(&cache->mag_cache[i].lock);
457	spinlock_lock(&cache->lock);	466	spinlock_lock(&cache->lock);
458		467
459	if (flags & SLAB_RECLAIM_ALL) {	468	if (flags & SLAB_RECLAIM_ALL) {
460	/* Aggressive memfree */	469	/* Aggressive memfree */
461		470
462	/* Destroy CPU magazines */	471	/* Destroy CPU magazines */
463	for (i=0; i<config.cpu_count; i++) {	472	for (i=0; i<config.cpu_count; i++) {
464	mag = cache->mag_cache[i].current;	473	mag = cache->mag_cache[i].current;
465	if (mag)	474	if (mag)
466	frames += magazine_destroy(cache, mag);	475	frames += magazine_destroy(cache, mag);
467	cache->mag_cache[i].current = NULL;	476	cache->mag_cache[i].current = NULL;
468		477
469	mag = cache->mag_cache[i].last;	478	mag = cache->mag_cache[i].last;
470	if (mag)	479	if (mag)
471	frames += magazine_destroy(cache, mag);	480	frames += magazine_destroy(cache, mag);
472	cache->mag_cache[i].last = NULL;	481	cache->mag_cache[i].last = NULL;
473	}	482	}
474	}	483	}
475	/* Destroy full magazines */	484	/* Destroy full magazines */
476	cur=cache->magazines.prev;	485	cur=cache->magazines.prev;
477	while (cur!=&cache->magazines) {	486	while (cur!=&cache->magazines) {
478	mag = list_get_instance(cur, slab_magazine_t, link);	487	mag = list_get_instance(cur, slab_magazine_t, link);
479		488
480	cur = cur->prev;	489	cur = cur->prev;
481	list_remove(cur->next);	490	list_remove(cur->next);
482	frames += magazine_destroy(cache,mag);	491	frames += magazine_destroy(cache,mag);
483	/* If we do not do full reclaim, break	492	/* If we do not do full reclaim, break
484	* as soon as something is freed */	493	* as soon as something is freed */
485	if (!(flags & SLAB_RECLAIM_ALL) && frames)	494	if (!(flags & SLAB_RECLAIM_ALL) && frames)
486	break;	495	break;
487	}	496	}
488		497
489	spinlock_unlock(&cache->lock);	498	spinlock_unlock(&cache->lock);
490	for (i=0; i < config.cpu_count; i++)	499	for (i=0; i < config.cpu_count; i++)
491	spinlock_unlock(&cache->mag_cache[i].lock);	500	spinlock_unlock(&cache->mag_cache[i].lock);

Subversion Repositories HelenOS-historic

(root)/kernel/trunk/generic/src/mm/slab.c – Rev 763 → 764