Rev 782 | Rev 785 | Go to most recent revision | Details | Compare with Previous | Last modification | View Log | RSS feed
Rev | Author | Line No. | Line |
---|---|---|---|
759 | palkovsky | 1 | /* |
2 | * Copyright (C) 2006 Ondrej Palkovsky |
||
3 | * All rights reserved. |
||
4 | * |
||
5 | * Redistribution and use in source and binary forms, with or without |
||
6 | * modification, are permitted provided that the following conditions |
||
7 | * are met: |
||
8 | * |
||
9 | * - Redistributions of source code must retain the above copyright |
||
10 | * notice, this list of conditions and the following disclaimer. |
||
11 | * - Redistributions in binary form must reproduce the above copyright |
||
12 | * notice, this list of conditions and the following disclaimer in the |
||
13 | * documentation and/or other materials provided with the distribution. |
||
14 | * - The name of the author may not be used to endorse or promote products |
||
15 | * derived from this software without specific prior written permission. |
||
16 | * |
||
17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
||
18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
||
19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
||
20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
||
21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
||
22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
||
23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
||
24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
||
25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
||
26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
||
27 | */ |
||
28 | |||
769 | palkovsky | 29 | /* |
30 | * The SLAB allocator is closely modelled after Opensolaris SLAB allocator |
||
31 | * http://www.usenix.org/events/usenix01/full_papers/bonwick/bonwick_html/ |
||
32 | * |
||
33 | * with the following exceptions: |
||
34 | * - empty SLABS are deallocated immediately |
||
35 | * (in Linux they are kept in linked list, in Solaris ???) |
||
36 | * - empty magazines are deallocated when not needed |
||
37 | * (in Solaris they are held in linked list in slab cache) |
||
38 | * |
||
39 | * Following features are not currently supported but would be easy to do: |
||
40 | * - cache coloring |
||
41 | * - dynamic magazine growing (different magazine sizes are already |
||
42 | * supported, but we would need to adjust allocating strategy) |
||
43 | * |
||
44 | * The SLAB allocator supports per-CPU caches ('magazines') to facilitate |
||
45 | * good SMP scaling. |
||
46 | * |
||
47 | * When a new object is being allocated, it is first checked, if it is |
||
48 | * available in CPU-bound magazine. If it is not found there, it is |
||
49 | * allocated from CPU-shared SLAB - if partial full is found, it is used, |
||
50 | * otherwise a new one is allocated. |
||
51 | * |
||
52 | * When an object is being deallocated, it is put to CPU-bound magazine. |
||
53 | * If there is no such magazine, new one is allocated (if it fails, |
||
54 | * the object is deallocated into SLAB). If the magazine is full, it is |
||
55 | * put into cpu-shared list of magazines and new one is allocated. |
||
56 | * |
||
57 | * The CPU-bound magazine is actually a pair of magazine to avoid |
||
58 | * thrashing when somebody is allocating/deallocating 1 item at the magazine |
||
59 | * size boundary. LIFO order is enforced, which should avoid fragmentation |
||
60 | * as much as possible. |
||
61 | * |
||
62 | * Every cache contains list of full slabs and list of partialy full slabs. |
||
63 | * Empty SLABS are immediately freed (thrashing will be avoided because |
||
64 | * of magazines). |
||
65 | * |
||
66 | * The SLAB information structure is kept inside the data area, if possible. |
||
67 | * The cache can be marked that it should not use magazines. This is used |
||
68 | * only for SLAB related caches to avoid deadlocks and infinite recursion |
||
69 | * (the SLAB allocator uses itself for allocating all it's control structures). |
||
70 | * |
||
71 | * The SLAB allocator allocates lot of space and does not free it. When |
||
72 | * frame allocator fails to allocate the frame, it calls slab_reclaim(). |
||
73 | * It tries 'light reclaim' first, then brutal reclaim. The light reclaim |
||
74 | * releases slabs from cpu-shared magazine-list, until at least 1 slab |
||
75 | * is deallocated in each cache (this algorithm should probably change). |
||
76 | * The brutal reclaim removes all cached objects, even from CPU-bound |
||
77 | * magazines. |
||
78 | * |
||
775 | palkovsky | 79 | * TODO: For better CPU-scaling the magazine allocation strategy should |
80 | * be extended. Currently, if the cache does not have magazine, it asks |
||
81 | * for non-cpu cached magazine cache to provide one. It might be feasible |
||
82 | * to add cpu-cached magazine cache (which would allocate it's magazines |
||
83 | * from non-cpu-cached mag. cache). This would provide a nice per-cpu |
||
84 | * buffer. The other possibility is to use the per-cache |
||
85 | * 'empty-magazine-list', which decreases competing for 1 per-system |
||
86 | * magazine cache. |
||
87 | * |
||
776 | palkovsky | 88 | * - it might be good to add granularity of locks even to slab level, |
89 | * we could then try_spinlock over all partial slabs and thus improve |
||
90 | * scalability even on slab level |
||
769 | palkovsky | 91 | */ |
92 | |||
93 | |||
759 | palkovsky | 94 | #include <synch/spinlock.h> |
95 | #include <mm/slab.h> |
||
96 | #include <list.h> |
||
97 | #include <memstr.h> |
||
98 | #include <align.h> |
||
99 | #include <mm/heap.h> |
||
762 | palkovsky | 100 | #include <mm/frame.h> |
759 | palkovsky | 101 | #include <config.h> |
102 | #include <print.h> |
||
103 | #include <arch.h> |
||
104 | #include <panic.h> |
||
762 | palkovsky | 105 | #include <debug.h> |
771 | palkovsky | 106 | #include <bitops.h> |
759 | palkovsky | 107 | |
108 | SPINLOCK_INITIALIZE(slab_cache_lock); |
||
769 | palkovsky | 109 | static LIST_INITIALIZE(slab_cache_list); |
759 | palkovsky | 110 | |
769 | palkovsky | 111 | /** Magazine cache */ |
112 | static slab_cache_t mag_cache; |
||
113 | /** Cache for cache descriptors */ |
||
114 | static slab_cache_t slab_cache_cache; |
||
759 | palkovsky | 115 | |
769 | palkovsky | 116 | /** Cache for external slab descriptors |
117 | * This time we want per-cpu cache, so do not make it static |
||
118 | * - using SLAB for internal SLAB structures will not deadlock, |
||
119 | * as all slab structures are 'small' - control structures of |
||
120 | * their caches do not require further allocation |
||
121 | */ |
||
122 | static slab_cache_t *slab_extern_cache; |
||
771 | palkovsky | 123 | /** Caches for malloc */ |
124 | static slab_cache_t *malloc_caches[SLAB_MAX_MALLOC_W-SLAB_MIN_MALLOC_W+1]; |
||
125 | char *malloc_names[] = { |
||
126 | "malloc-8","malloc-16","malloc-32","malloc-64","malloc-128", |
||
127 | "malloc-256","malloc-512","malloc-1K","malloc-2K", |
||
128 | "malloc-4K","malloc-8K","malloc-16K","malloc-32K", |
||
129 | "malloc-64K","malloc-128K" |
||
130 | }; |
||
762 | palkovsky | 131 | |
769 | palkovsky | 132 | /** Slab descriptor */ |
762 | palkovsky | 133 | typedef struct { |
134 | slab_cache_t *cache; /**< Pointer to parent cache */ |
||
135 | link_t link; /* List of full/partial slabs */ |
||
136 | void *start; /**< Start address of first available item */ |
||
137 | count_t available; /**< Count of available items in this slab */ |
||
138 | index_t nextavail; /**< The index of next available item */ |
||
139 | }slab_t; |
||
140 | |||
759 | palkovsky | 141 | /**************************************/ |
762 | palkovsky | 142 | /* SLAB allocation functions */ |
759 | palkovsky | 143 | |
762 | palkovsky | 144 | /** |
145 | * Allocate frames for slab space and initialize |
||
146 | * |
||
147 | */ |
||
148 | static slab_t * slab_space_alloc(slab_cache_t *cache, int flags) |
||
149 | { |
||
150 | void *data; |
||
151 | slab_t *slab; |
||
152 | size_t fsize; |
||
153 | int i; |
||
154 | zone_t *zone = NULL; |
||
155 | int status; |
||
764 | palkovsky | 156 | frame_t *frame; |
759 | palkovsky | 157 | |
762 | palkovsky | 158 | data = (void *)frame_alloc(FRAME_KA | flags, cache->order, &status, &zone); |
764 | palkovsky | 159 | if (status != FRAME_OK) { |
762 | palkovsky | 160 | return NULL; |
764 | palkovsky | 161 | } |
768 | palkovsky | 162 | if (! (cache->flags & SLAB_CACHE_SLINSIDE)) { |
769 | palkovsky | 163 | slab = slab_alloc(slab_extern_cache, flags); |
762 | palkovsky | 164 | if (!slab) { |
165 | frame_free((__address)data); |
||
166 | return NULL; |
||
167 | } |
||
168 | } else { |
||
169 | fsize = (PAGE_SIZE << cache->order); |
||
170 | slab = data + fsize - sizeof(*slab); |
||
171 | } |
||
764 | palkovsky | 172 | |
762 | palkovsky | 173 | /* Fill in slab structures */ |
763 | jermar | 174 | /* TODO: some better way of accessing the frame */ |
766 | palkovsky | 175 | for (i=0; i < (1 << cache->order); i++) { |
764 | palkovsky | 176 | frame = ADDR2FRAME(zone, KA2PA((__address)(data+i*PAGE_SIZE))); |
177 | frame->parent = slab; |
||
762 | palkovsky | 178 | } |
179 | |||
180 | slab->start = data; |
||
181 | slab->available = cache->objects; |
||
182 | slab->nextavail = 0; |
||
767 | palkovsky | 183 | slab->cache = cache; |
762 | palkovsky | 184 | |
185 | for (i=0; i<cache->objects;i++) |
||
186 | *((int *) (slab->start + i*cache->size)) = i+1; |
||
764 | palkovsky | 187 | |
188 | atomic_inc(&cache->allocated_slabs); |
||
762 | palkovsky | 189 | return slab; |
190 | } |
||
191 | |||
759 | palkovsky | 192 | /** |
766 | palkovsky | 193 | * Deallocate space associated with SLAB |
762 | palkovsky | 194 | * |
195 | * @return number of freed frames |
||
196 | */ |
||
197 | static count_t slab_space_free(slab_cache_t *cache, slab_t *slab) |
||
198 | { |
||
199 | frame_free((__address)slab->start); |
||
768 | palkovsky | 200 | if (! (cache->flags & SLAB_CACHE_SLINSIDE)) |
769 | palkovsky | 201 | slab_free(slab_extern_cache, slab); |
764 | palkovsky | 202 | |
203 | atomic_dec(&cache->allocated_slabs); |
||
204 | |||
762 | palkovsky | 205 | return 1 << cache->order; |
206 | } |
||
207 | |||
208 | /** Map object to slab structure */ |
||
209 | static slab_t * obj2slab(void *obj) |
||
210 | { |
||
211 | frame_t *frame; |
||
212 | |||
213 | frame = frame_addr2frame((__address)obj); |
||
214 | return (slab_t *)frame->parent; |
||
215 | } |
||
216 | |||
217 | /**************************************/ |
||
218 | /* SLAB functions */ |
||
219 | |||
220 | |||
221 | /** |
||
759 | palkovsky | 222 | * Return object to slab and call a destructor |
223 | * |
||
762 | palkovsky | 224 | * @param slab If the caller knows directly slab of the object, otherwise NULL |
225 | * |
||
759 | palkovsky | 226 | * @return Number of freed pages |
227 | */ |
||
762 | palkovsky | 228 | static count_t slab_obj_destroy(slab_cache_t *cache, void *obj, |
229 | slab_t *slab) |
||
759 | palkovsky | 230 | { |
762 | palkovsky | 231 | if (!slab) |
232 | slab = obj2slab(obj); |
||
233 | |||
767 | palkovsky | 234 | ASSERT(slab->cache == cache); |
780 | palkovsky | 235 | ASSERT(slab->available < cache->objects); |
767 | palkovsky | 236 | |
776 | palkovsky | 237 | spinlock_lock(&cache->slablock); |
238 | |||
762 | palkovsky | 239 | *((int *)obj) = slab->nextavail; |
240 | slab->nextavail = (obj - slab->start)/cache->size; |
||
241 | slab->available++; |
||
242 | |||
243 | /* Move it to correct list */ |
||
244 | if (slab->available == cache->objects) { |
||
245 | /* Free associated memory */ |
||
246 | list_remove(&slab->link); |
||
782 | palkovsky | 247 | spinlock_unlock(&cache->slablock); |
248 | |||
249 | return slab_space_free(cache, slab); |
||
250 | |||
780 | palkovsky | 251 | } else if (slab->available == 1) { |
252 | /* It was in full, move to partial */ |
||
253 | list_remove(&slab->link); |
||
254 | list_prepend(&slab->link, &cache->partial_slabs); |
||
762 | palkovsky | 255 | } |
783 | palkovsky | 256 | spinlock_unlock(&cache->slablock); |
782 | palkovsky | 257 | return 0; |
759 | palkovsky | 258 | } |
259 | |||
260 | /** |
||
261 | * Take new object from slab or create new if needed |
||
262 | * |
||
263 | * @return Object address or null |
||
264 | */ |
||
265 | static void * slab_obj_create(slab_cache_t *cache, int flags) |
||
266 | { |
||
762 | palkovsky | 267 | slab_t *slab; |
268 | void *obj; |
||
269 | |||
776 | palkovsky | 270 | spinlock_lock(&cache->slablock); |
271 | |||
762 | palkovsky | 272 | if (list_empty(&cache->partial_slabs)) { |
273 | /* Allow recursion and reclaiming |
||
274 | * - this should work, as the SLAB control structures |
||
275 | * are small and do not need to allocte with anything |
||
276 | * other ten frame_alloc when they are allocating, |
||
277 | * that's why we should get recursion at most 1-level deep |
||
278 | */ |
||
776 | palkovsky | 279 | spinlock_unlock(&cache->slablock); |
762 | palkovsky | 280 | slab = slab_space_alloc(cache, flags); |
780 | palkovsky | 281 | if (!slab) |
282 | return NULL; |
||
776 | palkovsky | 283 | spinlock_lock(&cache->slablock); |
762 | palkovsky | 284 | } else { |
285 | slab = list_get_instance(cache->partial_slabs.next, |
||
286 | slab_t, |
||
287 | link); |
||
288 | list_remove(&slab->link); |
||
289 | } |
||
290 | obj = slab->start + slab->nextavail * cache->size; |
||
291 | slab->nextavail = *((int *)obj); |
||
292 | slab->available--; |
||
293 | if (! slab->available) |
||
764 | palkovsky | 294 | list_prepend(&slab->link, &cache->full_slabs); |
762 | palkovsky | 295 | else |
764 | palkovsky | 296 | list_prepend(&slab->link, &cache->partial_slabs); |
776 | palkovsky | 297 | |
298 | spinlock_unlock(&cache->slablock); |
||
762 | palkovsky | 299 | return obj; |
759 | palkovsky | 300 | } |
301 | |||
302 | /**************************************/ |
||
303 | /* CPU-Cache slab functions */ |
||
304 | |||
305 | /** |
||
781 | palkovsky | 306 | * Finds a full magazine in cache, takes it from list |
307 | * and returns it |
||
308 | * |
||
309 | * @param first If true, return first, else last mag |
||
310 | */ |
||
311 | static slab_magazine_t * get_mag_from_cache(slab_cache_t *cache, |
||
312 | int first) |
||
313 | { |
||
314 | slab_magazine_t *mag = NULL; |
||
315 | link_t *cur; |
||
316 | |||
317 | spinlock_lock(&cache->maglock); |
||
318 | if (!list_empty(&cache->magazines)) { |
||
319 | if (first) |
||
320 | cur = cache->magazines.next; |
||
321 | else |
||
322 | cur = cache->magazines.prev; |
||
323 | mag = list_get_instance(cur, slab_magazine_t, link); |
||
324 | list_remove(&mag->link); |
||
325 | atomic_dec(&cache->magazine_counter); |
||
326 | } |
||
327 | spinlock_unlock(&cache->maglock); |
||
328 | return mag; |
||
329 | } |
||
330 | |||
331 | /** Prepend magazine to magazine list in cache */ |
||
332 | static void put_mag_to_cache(slab_cache_t *cache, slab_magazine_t *mag) |
||
333 | { |
||
334 | spinlock_lock(&cache->maglock); |
||
335 | |||
336 | list_prepend(&mag->link, &cache->magazines); |
||
337 | atomic_inc(&cache->magazine_counter); |
||
338 | |||
339 | spinlock_unlock(&cache->maglock); |
||
340 | } |
||
341 | |||
342 | /** |
||
759 | palkovsky | 343 | * Free all objects in magazine and free memory associated with magazine |
344 | * |
||
345 | * @return Number of freed pages |
||
346 | */ |
||
347 | static count_t magazine_destroy(slab_cache_t *cache, |
||
348 | slab_magazine_t *mag) |
||
349 | { |
||
350 | int i; |
||
351 | count_t frames = 0; |
||
352 | |||
767 | palkovsky | 353 | for (i=0;i < mag->busy; i++) { |
762 | palkovsky | 354 | frames += slab_obj_destroy(cache, mag->objs[i], NULL); |
767 | palkovsky | 355 | atomic_dec(&cache->cached_objs); |
356 | } |
||
759 | palkovsky | 357 | |
358 | slab_free(&mag_cache, mag); |
||
359 | |||
360 | return frames; |
||
361 | } |
||
362 | |||
363 | /** |
||
769 | palkovsky | 364 | * Find full magazine, set it as current and return it |
365 | * |
||
366 | * Assume cpu_magazine lock is held |
||
367 | */ |
||
368 | static slab_magazine_t * get_full_current_mag(slab_cache_t *cache) |
||
369 | { |
||
370 | slab_magazine_t *cmag, *lastmag, *newmag; |
||
371 | |||
372 | cmag = cache->mag_cache[CPU->id].current; |
||
373 | lastmag = cache->mag_cache[CPU->id].last; |
||
374 | if (cmag) { /* First try local CPU magazines */ |
||
375 | if (cmag->busy) |
||
376 | return cmag; |
||
377 | |||
378 | if (lastmag && lastmag->busy) { |
||
379 | cache->mag_cache[CPU->id].current = lastmag; |
||
380 | cache->mag_cache[CPU->id].last = cmag; |
||
381 | return lastmag; |
||
382 | } |
||
383 | } |
||
384 | /* Local magazines are empty, import one from magazine list */ |
||
781 | palkovsky | 385 | newmag = get_mag_from_cache(cache, 1); |
386 | if (!newmag) |
||
769 | palkovsky | 387 | return NULL; |
388 | |||
389 | if (lastmag) |
||
781 | palkovsky | 390 | magazine_destroy(cache, lastmag); |
391 | |||
769 | palkovsky | 392 | cache->mag_cache[CPU->id].last = cmag; |
393 | cache->mag_cache[CPU->id].current = newmag; |
||
394 | return newmag; |
||
395 | } |
||
396 | |||
397 | /** |
||
759 | palkovsky | 398 | * Try to find object in CPU-cache magazines |
399 | * |
||
400 | * @return Pointer to object or NULL if not available |
||
401 | */ |
||
402 | static void * magazine_obj_get(slab_cache_t *cache) |
||
403 | { |
||
404 | slab_magazine_t *mag; |
||
767 | palkovsky | 405 | void *obj; |
759 | palkovsky | 406 | |
772 | palkovsky | 407 | if (!CPU) |
408 | return NULL; |
||
409 | |||
759 | palkovsky | 410 | spinlock_lock(&cache->mag_cache[CPU->id].lock); |
411 | |||
769 | palkovsky | 412 | mag = get_full_current_mag(cache); |
413 | if (!mag) { |
||
414 | spinlock_unlock(&cache->mag_cache[CPU->id].lock); |
||
415 | return NULL; |
||
759 | palkovsky | 416 | } |
767 | palkovsky | 417 | obj = mag->objs[--mag->busy]; |
759 | palkovsky | 418 | spinlock_unlock(&cache->mag_cache[CPU->id].lock); |
767 | palkovsky | 419 | atomic_dec(&cache->cached_objs); |
420 | |||
421 | return obj; |
||
759 | palkovsky | 422 | } |
423 | |||
424 | /** |
||
768 | palkovsky | 425 | * Assure that the current magazine is empty, return pointer to it, or NULL if |
769 | palkovsky | 426 | * no empty magazine is available and cannot be allocated |
759 | palkovsky | 427 | * |
773 | palkovsky | 428 | * Assume mag_cache[CPU->id].lock is held |
429 | * |
||
759 | palkovsky | 430 | * We have 2 magazines bound to processor. |
431 | * First try the current. |
||
432 | * If full, try the last. |
||
433 | * If full, put to magazines list. |
||
434 | * allocate new, exchange last & current |
||
435 | * |
||
768 | palkovsky | 436 | */ |
437 | static slab_magazine_t * make_empty_current_mag(slab_cache_t *cache) |
||
438 | { |
||
439 | slab_magazine_t *cmag,*lastmag,*newmag; |
||
440 | |||
441 | cmag = cache->mag_cache[CPU->id].current; |
||
442 | lastmag = cache->mag_cache[CPU->id].last; |
||
443 | |||
444 | if (cmag) { |
||
445 | if (cmag->busy < cmag->size) |
||
446 | return cmag; |
||
447 | if (lastmag && lastmag->busy < lastmag->size) { |
||
448 | cache->mag_cache[CPU->id].last = cmag; |
||
449 | cache->mag_cache[CPU->id].current = lastmag; |
||
450 | return lastmag; |
||
451 | } |
||
452 | } |
||
453 | /* current | last are full | nonexistent, allocate new */ |
||
454 | /* We do not want to sleep just because of caching */ |
||
455 | /* Especially we do not want reclaiming to start, as |
||
456 | * this would deadlock */ |
||
457 | newmag = slab_alloc(&mag_cache, FRAME_ATOMIC | FRAME_NO_RECLAIM); |
||
458 | if (!newmag) |
||
459 | return NULL; |
||
460 | newmag->size = SLAB_MAG_SIZE; |
||
461 | newmag->busy = 0; |
||
462 | |||
463 | /* Flush last to magazine list */ |
||
781 | palkovsky | 464 | if (lastmag) |
465 | put_mag_to_cache(cache, lastmag); |
||
466 | |||
768 | palkovsky | 467 | /* Move current as last, save new as current */ |
468 | cache->mag_cache[CPU->id].last = cmag; |
||
469 | cache->mag_cache[CPU->id].current = newmag; |
||
470 | |||
471 | return newmag; |
||
472 | } |
||
473 | |||
474 | /** |
||
475 | * Put object into CPU-cache magazine |
||
476 | * |
||
759 | palkovsky | 477 | * @return 0 - success, -1 - could not get memory |
478 | */ |
||
479 | static int magazine_obj_put(slab_cache_t *cache, void *obj) |
||
480 | { |
||
481 | slab_magazine_t *mag; |
||
482 | |||
772 | palkovsky | 483 | if (!CPU) |
484 | return -1; |
||
485 | |||
759 | palkovsky | 486 | spinlock_lock(&cache->mag_cache[CPU->id].lock); |
768 | palkovsky | 487 | |
488 | mag = make_empty_current_mag(cache); |
||
769 | palkovsky | 489 | if (!mag) { |
490 | spinlock_unlock(&cache->mag_cache[CPU->id].lock); |
||
491 | return -1; |
||
492 | } |
||
759 | palkovsky | 493 | |
494 | mag->objs[mag->busy++] = obj; |
||
495 | |||
496 | spinlock_unlock(&cache->mag_cache[CPU->id].lock); |
||
767 | palkovsky | 497 | atomic_inc(&cache->cached_objs); |
759 | palkovsky | 498 | return 0; |
499 | } |
||
500 | |||
501 | |||
502 | /**************************************/ |
||
762 | palkovsky | 503 | /* SLAB CACHE functions */ |
759 | palkovsky | 504 | |
762 | palkovsky | 505 | /** Return number of objects that fit in certain cache size */ |
506 | static int comp_objects(slab_cache_t *cache) |
||
507 | { |
||
508 | if (cache->flags & SLAB_CACHE_SLINSIDE) |
||
509 | return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / cache->size; |
||
510 | else |
||
511 | return (PAGE_SIZE << cache->order) / cache->size; |
||
512 | } |
||
513 | |||
514 | /** Return wasted space in slab */ |
||
515 | static int badness(slab_cache_t *cache) |
||
516 | { |
||
517 | int objects; |
||
518 | int ssize; |
||
519 | |||
520 | objects = comp_objects(cache); |
||
521 | ssize = PAGE_SIZE << cache->order; |
||
522 | if (cache->flags & SLAB_CACHE_SLINSIDE) |
||
523 | ssize -= sizeof(slab_t); |
||
524 | return ssize - objects*cache->size; |
||
525 | } |
||
526 | |||
759 | palkovsky | 527 | /** Initialize allocated memory as a slab cache */ |
528 | static void |
||
529 | _slab_cache_create(slab_cache_t *cache, |
||
530 | char *name, |
||
531 | size_t size, |
||
532 | size_t align, |
||
533 | int (*constructor)(void *obj, int kmflag), |
||
534 | void (*destructor)(void *obj), |
||
535 | int flags) |
||
536 | { |
||
537 | int i; |
||
771 | palkovsky | 538 | int pages; |
783 | palkovsky | 539 | ipl_t ipl; |
759 | palkovsky | 540 | |
541 | memsetb((__address)cache, sizeof(*cache), 0); |
||
542 | cache->name = name; |
||
543 | |||
766 | palkovsky | 544 | if (align < sizeof(__native)) |
545 | align = sizeof(__native); |
||
546 | size = ALIGN_UP(size, align); |
||
547 | |||
762 | palkovsky | 548 | cache->size = size; |
759 | palkovsky | 549 | |
550 | cache->constructor = constructor; |
||
551 | cache->destructor = destructor; |
||
552 | cache->flags = flags; |
||
553 | |||
554 | list_initialize(&cache->full_slabs); |
||
555 | list_initialize(&cache->partial_slabs); |
||
556 | list_initialize(&cache->magazines); |
||
776 | palkovsky | 557 | spinlock_initialize(&cache->slablock, "slab_lock"); |
558 | spinlock_initialize(&cache->maglock, "slab_maglock"); |
||
768 | palkovsky | 559 | if (! (cache->flags & SLAB_CACHE_NOMAGAZINE)) { |
773 | palkovsky | 560 | for (i=0; i < config.cpu_count; i++) { |
772 | palkovsky | 561 | memsetb((__address)&cache->mag_cache[i], |
562 | sizeof(cache->mag_cache[i]), 0); |
||
759 | palkovsky | 563 | spinlock_initialize(&cache->mag_cache[i].lock, |
776 | palkovsky | 564 | "slab_maglock_cpu"); |
772 | palkovsky | 565 | } |
759 | palkovsky | 566 | } |
567 | |||
568 | /* Compute slab sizes, object counts in slabs etc. */ |
||
569 | if (cache->size < SLAB_INSIDE_SIZE) |
||
570 | cache->flags |= SLAB_CACHE_SLINSIDE; |
||
571 | |||
762 | palkovsky | 572 | /* Minimum slab order */ |
771 | palkovsky | 573 | pages = ((cache->size-1) >> PAGE_WIDTH) + 1; |
574 | cache->order = fnzb(pages); |
||
766 | palkovsky | 575 | |
762 | palkovsky | 576 | while (badness(cache) > SLAB_MAX_BADNESS(cache)) { |
577 | cache->order += 1; |
||
578 | } |
||
579 | cache->objects = comp_objects(cache); |
||
766 | palkovsky | 580 | /* If info fits in, put it inside */ |
581 | if (badness(cache) > sizeof(slab_t)) |
||
582 | cache->flags |= SLAB_CACHE_SLINSIDE; |
||
762 | palkovsky | 583 | |
783 | palkovsky | 584 | /* Add cache to cache list */ |
585 | ipl = interrupts_disable(); |
||
759 | palkovsky | 586 | spinlock_lock(&slab_cache_lock); |
587 | |||
588 | list_append(&cache->link, &slab_cache_list); |
||
589 | |||
590 | spinlock_unlock(&slab_cache_lock); |
||
783 | palkovsky | 591 | interrupts_restore(ipl); |
759 | palkovsky | 592 | } |
593 | |||
594 | /** Create slab cache */ |
||
595 | slab_cache_t * slab_cache_create(char *name, |
||
596 | size_t size, |
||
597 | size_t align, |
||
598 | int (*constructor)(void *obj, int kmflag), |
||
599 | void (*destructor)(void *obj), |
||
600 | int flags) |
||
601 | { |
||
602 | slab_cache_t *cache; |
||
603 | |||
769 | palkovsky | 604 | cache = slab_alloc(&slab_cache_cache, 0); |
759 | palkovsky | 605 | _slab_cache_create(cache, name, size, align, constructor, destructor, |
606 | flags); |
||
607 | return cache; |
||
608 | } |
||
609 | |||
610 | /** |
||
611 | * Reclaim space occupied by objects that are already free |
||
612 | * |
||
613 | * @param flags If contains SLAB_RECLAIM_ALL, do aggressive freeing |
||
614 | * @return Number of freed pages |
||
615 | */ |
||
616 | static count_t _slab_reclaim(slab_cache_t *cache, int flags) |
||
617 | { |
||
618 | int i; |
||
619 | slab_magazine_t *mag; |
||
620 | count_t frames = 0; |
||
781 | palkovsky | 621 | int magcount; |
759 | palkovsky | 622 | |
623 | if (cache->flags & SLAB_CACHE_NOMAGAZINE) |
||
624 | return 0; /* Nothing to do */ |
||
781 | palkovsky | 625 | |
626 | /* We count up to original magazine count to avoid |
||
627 | * endless loop |
||
628 | */ |
||
629 | magcount = atomic_get(&cache->magazine_counter); |
||
630 | while (magcount-- && (mag=get_mag_from_cache(cache,0))) { |
||
631 | frames += magazine_destroy(cache,mag); |
||
632 | if (!(flags & SLAB_RECLAIM_ALL) && frames) |
||
633 | break; |
||
769 | palkovsky | 634 | } |
759 | palkovsky | 635 | |
636 | if (flags & SLAB_RECLAIM_ALL) { |
||
781 | palkovsky | 637 | /* Free cpu-bound magazines */ |
759 | palkovsky | 638 | /* Destroy CPU magazines */ |
639 | for (i=0; i<config.cpu_count; i++) { |
||
781 | palkovsky | 640 | spinlock_lock(&cache->mag_cache[i].lock); |
641 | |||
759 | palkovsky | 642 | mag = cache->mag_cache[i].current; |
643 | if (mag) |
||
644 | frames += magazine_destroy(cache, mag); |
||
645 | cache->mag_cache[i].current = NULL; |
||
646 | |||
647 | mag = cache->mag_cache[i].last; |
||
648 | if (mag) |
||
649 | frames += magazine_destroy(cache, mag); |
||
650 | cache->mag_cache[i].last = NULL; |
||
781 | palkovsky | 651 | |
652 | spinlock_unlock(&cache->mag_cache[i].lock); |
||
759 | palkovsky | 653 | } |
654 | } |
||
767 | palkovsky | 655 | |
759 | palkovsky | 656 | return frames; |
657 | } |
||
658 | |||
659 | /** Check that there are no slabs and remove cache from system */ |
||
660 | void slab_cache_destroy(slab_cache_t *cache) |
||
661 | { |
||
781 | palkovsky | 662 | ipl_t ipl; |
663 | |||
664 | /* First remove cache from link, so that we don't need |
||
665 | * to disable interrupts later |
||
666 | */ |
||
667 | |||
668 | ipl = interrupts_disable(); |
||
669 | spinlock_lock(&slab_cache_lock); |
||
670 | |||
671 | list_remove(&cache->link); |
||
672 | |||
673 | spinlock_unlock(&slab_cache_lock); |
||
674 | interrupts_restore(ipl); |
||
675 | |||
759 | palkovsky | 676 | /* Do not lock anything, we assume the software is correct and |
677 | * does not touch the cache when it decides to destroy it */ |
||
678 | |||
679 | /* Destroy all magazines */ |
||
680 | _slab_reclaim(cache, SLAB_RECLAIM_ALL); |
||
681 | |||
682 | /* All slabs must be empty */ |
||
683 | if (!list_empty(&cache->full_slabs) \ |
||
684 | || !list_empty(&cache->partial_slabs)) |
||
685 | panic("Destroying cache that is not empty."); |
||
686 | |||
769 | palkovsky | 687 | slab_free(&slab_cache_cache, cache); |
759 | palkovsky | 688 | } |
689 | |||
690 | /** Allocate new object from cache - if no flags given, always returns |
||
691 | memory */ |
||
692 | void * slab_alloc(slab_cache_t *cache, int flags) |
||
693 | { |
||
694 | ipl_t ipl; |
||
695 | void *result = NULL; |
||
773 | palkovsky | 696 | |
759 | palkovsky | 697 | /* Disable interrupts to avoid deadlocks with interrupt handlers */ |
698 | ipl = interrupts_disable(); |
||
771 | palkovsky | 699 | |
772 | palkovsky | 700 | if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) |
759 | palkovsky | 701 | result = magazine_obj_get(cache); |
776 | palkovsky | 702 | if (!result) |
759 | palkovsky | 703 | result = slab_obj_create(cache, flags); |
704 | |||
769 | palkovsky | 705 | interrupts_restore(ipl); |
706 | |||
764 | palkovsky | 707 | if (result) |
708 | atomic_inc(&cache->allocated_objs); |
||
709 | |||
759 | palkovsky | 710 | return result; |
711 | } |
||
712 | |||
771 | palkovsky | 713 | /** Return object to cache, use slab if known */ |
714 | static void _slab_free(slab_cache_t *cache, void *obj, slab_t *slab) |
||
759 | palkovsky | 715 | { |
716 | ipl_t ipl; |
||
717 | |||
718 | ipl = interrupts_disable(); |
||
719 | |||
762 | palkovsky | 720 | if ((cache->flags & SLAB_CACHE_NOMAGAZINE) \ |
721 | || magazine_obj_put(cache, obj)) { |
||
776 | palkovsky | 722 | |
771 | palkovsky | 723 | slab_obj_destroy(cache, obj, slab); |
776 | palkovsky | 724 | |
759 | palkovsky | 725 | } |
769 | palkovsky | 726 | interrupts_restore(ipl); |
764 | palkovsky | 727 | atomic_dec(&cache->allocated_objs); |
759 | palkovsky | 728 | } |
729 | |||
771 | palkovsky | 730 | /** Return slab object to cache */ |
731 | void slab_free(slab_cache_t *cache, void *obj) |
||
732 | { |
||
733 | _slab_free(cache,obj,NULL); |
||
734 | } |
||
735 | |||
759 | palkovsky | 736 | /* Go through all caches and reclaim what is possible */ |
737 | count_t slab_reclaim(int flags) |
||
738 | { |
||
739 | slab_cache_t *cache; |
||
740 | link_t *cur; |
||
741 | count_t frames = 0; |
||
742 | |||
743 | spinlock_lock(&slab_cache_lock); |
||
744 | |||
776 | palkovsky | 745 | /* TODO: Add assert, that interrupts are disabled, otherwise |
746 | * memory allocation from interrupts can deadlock. |
||
747 | */ |
||
748 | |||
759 | palkovsky | 749 | for (cur = slab_cache_list.next;cur!=&slab_cache_list; cur=cur->next) { |
750 | cache = list_get_instance(cur, slab_cache_t, link); |
||
751 | frames += _slab_reclaim(cache, flags); |
||
752 | } |
||
753 | |||
754 | spinlock_unlock(&slab_cache_lock); |
||
755 | |||
756 | return frames; |
||
757 | } |
||
758 | |||
759 | |||
760 | /* Print list of slabs */ |
||
761 | void slab_print_list(void) |
||
762 | { |
||
763 | slab_cache_t *cache; |
||
764 | link_t *cur; |
||
783 | palkovsky | 765 | ipl_t ipl; |
766 | |||
767 | ipl = interrupts_disable(); |
||
759 | palkovsky | 768 | spinlock_lock(&slab_cache_lock); |
767 | palkovsky | 769 | printf("SLAB name\tOsize\tPages\tObj/pg\tSlabs\tCached\tAllocobjs\tCtl\n"); |
759 | palkovsky | 770 | for (cur = slab_cache_list.next;cur!=&slab_cache_list; cur=cur->next) { |
771 | cache = list_get_instance(cur, slab_cache_t, link); |
||
767 | palkovsky | 772 | printf("%s\t%d\t%d\t%d\t%d\t%d\t%d\t\t%s\n", cache->name, cache->size, |
766 | palkovsky | 773 | (1 << cache->order), cache->objects, |
767 | palkovsky | 774 | atomic_get(&cache->allocated_slabs), |
775 | atomic_get(&cache->cached_objs), |
||
766 | palkovsky | 776 | atomic_get(&cache->allocated_objs), |
777 | cache->flags & SLAB_CACHE_SLINSIDE ? "In" : "Out"); |
||
759 | palkovsky | 778 | } |
779 | spinlock_unlock(&slab_cache_lock); |
||
783 | palkovsky | 780 | interrupts_restore(ipl); |
759 | palkovsky | 781 | } |
782 | |||
778 | palkovsky | 783 | #ifdef CONFIG_DEBUG |
784 | static int _slab_initialized = 0; |
||
785 | #endif |
||
786 | |||
759 | palkovsky | 787 | void slab_cache_init(void) |
788 | { |
||
771 | palkovsky | 789 | int i, size; |
790 | |||
759 | palkovsky | 791 | /* Initialize magazine cache */ |
792 | _slab_cache_create(&mag_cache, |
||
793 | "slab_magazine", |
||
794 | sizeof(slab_magazine_t)+SLAB_MAG_SIZE*sizeof(void*), |
||
795 | sizeof(__address), |
||
796 | NULL, NULL, |
||
769 | palkovsky | 797 | SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE); |
798 | /* Initialize slab_cache cache */ |
||
799 | _slab_cache_create(&slab_cache_cache, |
||
800 | "slab_cache", |
||
801 | sizeof(slab_cache_cache) + config.cpu_count*sizeof(slab_cache_cache.mag_cache[0]), |
||
802 | sizeof(__address), |
||
803 | NULL, NULL, |
||
804 | SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE); |
||
805 | /* Initialize external slab cache */ |
||
806 | slab_extern_cache = slab_cache_create("slab_extern", |
||
807 | sizeof(slab_t), |
||
808 | 0, NULL, NULL, |
||
809 | SLAB_CACHE_SLINSIDE); |
||
759 | palkovsky | 810 | |
811 | /* Initialize structures for malloc */ |
||
771 | palkovsky | 812 | for (i=0, size=(1<<SLAB_MIN_MALLOC_W); |
813 | i < (SLAB_MAX_MALLOC_W-SLAB_MIN_MALLOC_W+1); |
||
814 | i++, size <<= 1) { |
||
815 | malloc_caches[i] = slab_cache_create(malloc_names[i], |
||
816 | size, 0, |
||
817 | NULL,NULL,0); |
||
818 | } |
||
778 | palkovsky | 819 | #ifdef CONFIG_DEBUG |
820 | _slab_initialized = 1; |
||
821 | #endif |
||
759 | palkovsky | 822 | } |
771 | palkovsky | 823 | |
824 | /**************************************/ |
||
825 | /* kalloc/kfree functions */ |
||
826 | void * kalloc(unsigned int size, int flags) |
||
827 | { |
||
828 | int idx; |
||
778 | palkovsky | 829 | |
830 | ASSERT(_slab_initialized); |
||
771 | palkovsky | 831 | ASSERT( size && size <= (1 << SLAB_MAX_MALLOC_W)); |
832 | |||
833 | if (size < (1 << SLAB_MIN_MALLOC_W)) |
||
834 | size = (1 << SLAB_MIN_MALLOC_W); |
||
835 | |||
836 | idx = fnzb(size-1) - SLAB_MIN_MALLOC_W + 1; |
||
837 | |||
838 | return slab_alloc(malloc_caches[idx], flags); |
||
839 | } |
||
840 | |||
841 | |||
842 | void kfree(void *obj) |
||
843 | { |
||
781 | palkovsky | 844 | slab_t *slab; |
845 | |||
846 | if (!obj) return; |
||
847 | |||
848 | slab = obj2slab(obj); |
||
771 | palkovsky | 849 | _slab_free(slab->cache, obj, slab); |
850 | } |