Rev 3343 | Only display areas with differences | Ignore whitespace | Details | Blame | Last modification | View Log | RSS feed
Rev 3343 | Rev 3862 | ||
---|---|---|---|
1 | /* |
1 | /* |
2 | * Copyright (c) 2006 Ondrej Palkovsky |
2 | * Copyright (c) 2006 Ondrej Palkovsky |
3 | * All rights reserved. |
3 | * All rights reserved. |
4 | * |
4 | * |
5 | * Redistribution and use in source and binary forms, with or without |
5 | * Redistribution and use in source and binary forms, with or without |
6 | * modification, are permitted provided that the following conditions |
6 | * modification, are permitted provided that the following conditions |
7 | * are met: |
7 | * are met: |
8 | * |
8 | * |
9 | * - Redistributions of source code must retain the above copyright |
9 | * - Redistributions of source code must retain the above copyright |
10 | * notice, this list of conditions and the following disclaimer. |
10 | * notice, this list of conditions and the following disclaimer. |
11 | * - Redistributions in binary form must reproduce the above copyright |
11 | * - Redistributions in binary form must reproduce the above copyright |
12 | * notice, this list of conditions and the following disclaimer in the |
12 | * notice, this list of conditions and the following disclaimer in the |
13 | * documentation and/or other materials provided with the distribution. |
13 | * documentation and/or other materials provided with the distribution. |
14 | * - The name of the author may not be used to endorse or promote products |
14 | * - The name of the author may not be used to endorse or promote products |
15 | * derived from this software without specific prior written permission. |
15 | * derived from this software without specific prior written permission. |
16 | * |
16 | * |
17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
17 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
18 | * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
19 | * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
20 | * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
21 | * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
22 | * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
23 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
24 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
25 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
26 | * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
27 | */ |
27 | */ |
28 | 28 | ||
29 | /** @addtogroup genericmm |
29 | /** @addtogroup genericmm |
30 | * @{ |
30 | * @{ |
31 | */ |
31 | */ |
32 | 32 | ||
33 | /** |
33 | /** |
34 | * @file |
34 | * @file |
35 | * @brief Slab allocator. |
35 | * @brief Slab allocator. |
36 | * |
36 | * |
37 | * The slab allocator is closely modelled after OpenSolaris slab allocator. |
37 | * The slab allocator is closely modelled after OpenSolaris slab allocator. |
38 | * @see http://www.usenix.org/events/usenix01/full_papers/bonwick/bonwick_html/ |
38 | * @see http://www.usenix.org/events/usenix01/full_papers/bonwick/bonwick_html/ |
39 | * |
39 | * |
40 | * with the following exceptions: |
40 | * with the following exceptions: |
41 | * @li empty slabs are deallocated immediately |
41 | * @li empty slabs are deallocated immediately |
42 | * (in Linux they are kept in linked list, in Solaris ???) |
42 | * (in Linux they are kept in linked list, in Solaris ???) |
43 | * @li empty magazines are deallocated when not needed |
43 | * @li empty magazines are deallocated when not needed |
44 | * (in Solaris they are held in linked list in slab cache) |
44 | * (in Solaris they are held in linked list in slab cache) |
45 | * |
45 | * |
46 | * Following features are not currently supported but would be easy to do: |
46 | * Following features are not currently supported but would be easy to do: |
47 | * @li cache coloring |
47 | * @li cache coloring |
48 | * @li dynamic magazine growing (different magazine sizes are already |
48 | * @li dynamic magazine growing (different magazine sizes are already |
49 | * supported, but we would need to adjust allocation strategy) |
49 | * supported, but we would need to adjust allocation strategy) |
50 | * |
50 | * |
51 | * The slab allocator supports per-CPU caches ('magazines') to facilitate |
51 | * The slab allocator supports per-CPU caches ('magazines') to facilitate |
52 | * good SMP scaling. |
52 | * good SMP scaling. |
53 | * |
53 | * |
54 | * When a new object is being allocated, it is first checked, if it is |
54 | * When a new object is being allocated, it is first checked, if it is |
55 | * available in a CPU-bound magazine. If it is not found there, it is |
55 | * available in a CPU-bound magazine. If it is not found there, it is |
56 | * allocated from a CPU-shared slab - if a partially full one is found, |
56 | * allocated from a CPU-shared slab - if a partially full one is found, |
57 | * it is used, otherwise a new one is allocated. |
57 | * it is used, otherwise a new one is allocated. |
58 | * |
58 | * |
59 | * When an object is being deallocated, it is put to a CPU-bound magazine. |
59 | * When an object is being deallocated, it is put to a CPU-bound magazine. |
60 | * If there is no such magazine, a new one is allocated (if this fails, |
60 | * If there is no such magazine, a new one is allocated (if this fails, |
61 | * the object is deallocated into slab). If the magazine is full, it is |
61 | * the object is deallocated into slab). If the magazine is full, it is |
62 | * put into cpu-shared list of magazines and a new one is allocated. |
62 | * put into cpu-shared list of magazines and a new one is allocated. |
63 | * |
63 | * |
64 | * The CPU-bound magazine is actually a pair of magazines in order to avoid |
64 | * The CPU-bound magazine is actually a pair of magazines in order to avoid |
65 | * thrashing when somebody is allocating/deallocating 1 item at the magazine |
65 | * thrashing when somebody is allocating/deallocating 1 item at the magazine |
66 | * size boundary. LIFO order is enforced, which should avoid fragmentation |
66 | * size boundary. LIFO order is enforced, which should avoid fragmentation |
67 | * as much as possible. |
67 | * as much as possible. |
68 | * |
68 | * |
69 | * Every cache contains list of full slabs and list of partially full slabs. |
69 | * Every cache contains list of full slabs and list of partially full slabs. |
70 | * Empty slabs are immediately freed (thrashing will be avoided because |
70 | * Empty slabs are immediately freed (thrashing will be avoided because |
71 | * of magazines). |
71 | * of magazines). |
72 | * |
72 | * |
73 | * The slab information structure is kept inside the data area, if possible. |
73 | * The slab information structure is kept inside the data area, if possible. |
74 | * The cache can be marked that it should not use magazines. This is used |
74 | * The cache can be marked that it should not use magazines. This is used |
75 | * only for slab related caches to avoid deadlocks and infinite recursion |
75 | * only for slab related caches to avoid deadlocks and infinite recursion |
76 | * (the slab allocator uses itself for allocating all it's control structures). |
76 | * (the slab allocator uses itself for allocating all it's control structures). |
77 | * |
77 | * |
78 | * The slab allocator allocates a lot of space and does not free it. When |
78 | * The slab allocator allocates a lot of space and does not free it. When |
79 | * the frame allocator fails to allocate a frame, it calls slab_reclaim(). |
79 | * the frame allocator fails to allocate a frame, it calls slab_reclaim(). |
80 | * It tries 'light reclaim' first, then brutal reclaim. The light reclaim |
80 | * It tries 'light reclaim' first, then brutal reclaim. The light reclaim |
81 | * releases slabs from cpu-shared magazine-list, until at least 1 slab |
81 | * releases slabs from cpu-shared magazine-list, until at least 1 slab |
82 | * is deallocated in each cache (this algorithm should probably change). |
82 | * is deallocated in each cache (this algorithm should probably change). |
83 | * The brutal reclaim removes all cached objects, even from CPU-bound |
83 | * The brutal reclaim removes all cached objects, even from CPU-bound |
84 | * magazines. |
84 | * magazines. |
85 | * |
85 | * |
86 | * @todo |
86 | * @todo |
87 | * For better CPU-scaling the magazine allocation strategy should |
87 | * For better CPU-scaling the magazine allocation strategy should |
88 | * be extended. Currently, if the cache does not have magazine, it asks |
88 | * be extended. Currently, if the cache does not have magazine, it asks |
89 | * for non-cpu cached magazine cache to provide one. It might be feasible |
89 | * for non-cpu cached magazine cache to provide one. It might be feasible |
90 | * to add cpu-cached magazine cache (which would allocate it's magazines |
90 | * to add cpu-cached magazine cache (which would allocate it's magazines |
91 | * from non-cpu-cached mag. cache). This would provide a nice per-cpu |
91 | * from non-cpu-cached mag. cache). This would provide a nice per-cpu |
92 | * buffer. The other possibility is to use the per-cache |
92 | * buffer. The other possibility is to use the per-cache |
93 | * 'empty-magazine-list', which decreases competing for 1 per-system |
93 | * 'empty-magazine-list', which decreases competing for 1 per-system |
94 | * magazine cache. |
94 | * magazine cache. |
95 | * |
95 | * |
96 | * @todo |
96 | * @todo |
97 | * it might be good to add granularity of locks even to slab level, |
97 | * it might be good to add granularity of locks even to slab level, |
98 | * we could then try_spinlock over all partial slabs and thus improve |
98 | * we could then try_spinlock over all partial slabs and thus improve |
99 | * scalability even on slab level |
99 | * scalability even on slab level |
100 | */ |
100 | */ |
101 | 101 | ||
102 | #include <synch/spinlock.h> |
102 | #include <synch/spinlock.h> |
103 | #include <mm/slab.h> |
103 | #include <mm/slab.h> |
104 | #include <adt/list.h> |
104 | #include <adt/list.h> |
105 | #include <memstr.h> |
105 | #include <memstr.h> |
106 | #include <align.h> |
106 | #include <align.h> |
107 | #include <mm/frame.h> |
107 | #include <mm/frame.h> |
108 | #include <config.h> |
108 | #include <config.h> |
109 | #include <print.h> |
109 | #include <print.h> |
110 | #include <arch.h> |
110 | #include <arch.h> |
111 | #include <panic.h> |
111 | #include <panic.h> |
112 | #include <debug.h> |
112 | #include <debug.h> |
113 | #include <bitops.h> |
113 | #include <bitops.h> |
114 | #include <macros.h> |
114 | #include <macros.h> |
- | 115 | #include <arch/asm.h> |
|
115 | 116 | ||
116 | SPINLOCK_INITIALIZE(slab_cache_lock); |
117 | SPINLOCK_INITIALIZE(slab_cache_lock); |
117 | static LIST_INITIALIZE(slab_cache_list); |
118 | static LIST_INITIALIZE(slab_cache_list); |
118 | 119 | ||
119 | /** Magazine cache */ |
120 | /** Magazine cache */ |
120 | static slab_cache_t mag_cache; |
121 | static slab_cache_t mag_cache; |
121 | /** Cache for cache descriptors */ |
122 | /** Cache for cache descriptors */ |
122 | static slab_cache_t slab_cache_cache; |
123 | static slab_cache_t slab_cache_cache; |
123 | /** Cache for external slab descriptors |
124 | /** Cache for external slab descriptors |
124 | * This time we want per-cpu cache, so do not make it static |
125 | * This time we want per-cpu cache, so do not make it static |
125 | * - using slab for internal slab structures will not deadlock, |
126 | * - using slab for internal slab structures will not deadlock, |
126 | * as all slab structures are 'small' - control structures of |
127 | * as all slab structures are 'small' - control structures of |
127 | * their caches do not require further allocation |
128 | * their caches do not require further allocation |
128 | */ |
129 | */ |
129 | static slab_cache_t *slab_extern_cache; |
130 | static slab_cache_t *slab_extern_cache; |
130 | /** Caches for malloc */ |
131 | /** Caches for malloc */ |
131 | static slab_cache_t *malloc_caches[SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1]; |
132 | static slab_cache_t *malloc_caches[SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1]; |
132 | char *malloc_names[] = { |
133 | char *malloc_names[] = { |
133 | "malloc-16", |
134 | "malloc-16", |
134 | "malloc-32", |
135 | "malloc-32", |
135 | "malloc-64", |
136 | "malloc-64", |
136 | "malloc-128", |
137 | "malloc-128", |
137 | "malloc-256", |
138 | "malloc-256", |
138 | "malloc-512", |
139 | "malloc-512", |
139 | "malloc-1K", |
140 | "malloc-1K", |
140 | "malloc-2K", |
141 | "malloc-2K", |
141 | "malloc-4K", |
142 | "malloc-4K", |
142 | "malloc-8K", |
143 | "malloc-8K", |
143 | "malloc-16K", |
144 | "malloc-16K", |
144 | "malloc-32K", |
145 | "malloc-32K", |
145 | "malloc-64K", |
146 | "malloc-64K", |
146 | "malloc-128K", |
147 | "malloc-128K", |
147 | "malloc-256K" |
148 | "malloc-256K" |
148 | }; |
149 | }; |
149 | 150 | ||
150 | /** Slab descriptor */ |
151 | /** Slab descriptor */ |
151 | typedef struct { |
152 | typedef struct { |
152 | slab_cache_t *cache; /**< Pointer to parent cache. */ |
153 | slab_cache_t *cache; /**< Pointer to parent cache. */ |
153 | link_t link; /**< List of full/partial slabs. */ |
154 | link_t link; /**< List of full/partial slabs. */ |
154 | void *start; /**< Start address of first available item. */ |
155 | void *start; /**< Start address of first available item. */ |
155 | count_t available; /**< Count of available items in this slab. */ |
156 | count_t available; /**< Count of available items in this slab. */ |
156 | index_t nextavail; /**< The index of next available item. */ |
157 | index_t nextavail; /**< The index of next available item. */ |
157 | } slab_t; |
158 | } slab_t; |
158 | 159 | ||
159 | #ifdef CONFIG_DEBUG |
160 | #ifdef CONFIG_DEBUG |
160 | static int _slab_initialized = 0; |
161 | static int _slab_initialized = 0; |
161 | #endif |
162 | #endif |
162 | 163 | ||
163 | /**************************************/ |
164 | /**************************************/ |
164 | /* Slab allocation functions */ |
165 | /* Slab allocation functions */ |
165 | 166 | ||
166 | /** |
167 | /** |
167 | * Allocate frames for slab space and initialize |
168 | * Allocate frames for slab space and initialize |
168 | * |
169 | * |
169 | */ |
170 | */ |
170 | static slab_t *slab_space_alloc(slab_cache_t *cache, int flags) |
171 | static slab_t *slab_space_alloc(slab_cache_t *cache, int flags) |
171 | { |
172 | { |
172 | void *data; |
173 | void *data; |
173 | slab_t *slab; |
174 | slab_t *slab; |
174 | size_t fsize; |
175 | size_t fsize; |
175 | unsigned int i; |
176 | unsigned int i; |
176 | unsigned int zone = 0; |
177 | unsigned int zone = 0; |
177 | 178 | ||
178 | data = frame_alloc_generic(cache->order, FRAME_KA | flags, &zone); |
179 | data = frame_alloc_generic(cache->order, FRAME_KA | flags, &zone); |
179 | if (!data) { |
180 | if (!data) { |
180 | return NULL; |
181 | return NULL; |
181 | } |
182 | } |
182 | if (!(cache->flags & SLAB_CACHE_SLINSIDE)) { |
183 | if (!(cache->flags & SLAB_CACHE_SLINSIDE)) { |
183 | slab = slab_alloc(slab_extern_cache, flags); |
184 | slab = slab_alloc(slab_extern_cache, flags); |
184 | if (!slab) { |
185 | if (!slab) { |
185 | frame_free(KA2PA(data)); |
186 | frame_free(KA2PA(data)); |
186 | return NULL; |
187 | return NULL; |
187 | } |
188 | } |
188 | } else { |
189 | } else { |
189 | fsize = (PAGE_SIZE << cache->order); |
190 | fsize = (PAGE_SIZE << cache->order); |
190 | slab = data + fsize - sizeof(*slab); |
191 | slab = data + fsize - sizeof(*slab); |
191 | } |
192 | } |
192 | 193 | ||
193 | /* Fill in slab structures */ |
194 | /* Fill in slab structures */ |
194 | for (i = 0; i < ((unsigned int) 1 << cache->order); i++) |
195 | for (i = 0; i < ((unsigned int) 1 << cache->order); i++) |
195 | frame_set_parent(ADDR2PFN(KA2PA(data)) + i, slab, zone); |
196 | frame_set_parent(ADDR2PFN(KA2PA(data)) + i, slab, zone); |
196 | 197 | ||
197 | slab->start = data; |
198 | slab->start = data; |
198 | slab->available = cache->objects; |
199 | slab->available = cache->objects; |
199 | slab->nextavail = 0; |
200 | slab->nextavail = 0; |
200 | slab->cache = cache; |
201 | slab->cache = cache; |
201 | 202 | ||
202 | for (i = 0; i < cache->objects; i++) |
203 | for (i = 0; i < cache->objects; i++) |
203 | *((int *) (slab->start + i*cache->size)) = i + 1; |
204 | *((int *) (slab->start + i*cache->size)) = i + 1; |
204 | 205 | ||
205 | atomic_inc(&cache->allocated_slabs); |
206 | atomic_inc(&cache->allocated_slabs); |
206 | return slab; |
207 | return slab; |
207 | } |
208 | } |
208 | 209 | ||
209 | /** |
210 | /** |
210 | * Deallocate space associated with slab |
211 | * Deallocate space associated with slab |
211 | * |
212 | * |
212 | * @return number of freed frames |
213 | * @return number of freed frames |
213 | */ |
214 | */ |
214 | static count_t slab_space_free(slab_cache_t *cache, slab_t *slab) |
215 | static count_t slab_space_free(slab_cache_t *cache, slab_t *slab) |
215 | { |
216 | { |
216 | frame_free(KA2PA(slab->start)); |
217 | frame_free(KA2PA(slab->start)); |
217 | if (! (cache->flags & SLAB_CACHE_SLINSIDE)) |
218 | if (! (cache->flags & SLAB_CACHE_SLINSIDE)) |
218 | slab_free(slab_extern_cache, slab); |
219 | slab_free(slab_extern_cache, slab); |
219 | 220 | ||
220 | atomic_dec(&cache->allocated_slabs); |
221 | atomic_dec(&cache->allocated_slabs); |
221 | 222 | ||
222 | return 1 << cache->order; |
223 | return 1 << cache->order; |
223 | } |
224 | } |
224 | 225 | ||
225 | /** Map object to slab structure */ |
226 | /** Map object to slab structure */ |
226 | static slab_t * obj2slab(void *obj) |
227 | static slab_t * obj2slab(void *obj) |
227 | { |
228 | { |
228 | return (slab_t *) frame_get_parent(ADDR2PFN(KA2PA(obj)), 0); |
229 | return (slab_t *) frame_get_parent(ADDR2PFN(KA2PA(obj)), 0); |
229 | } |
230 | } |
230 | 231 | ||
231 | /**************************************/ |
232 | /**************************************/ |
232 | /* Slab functions */ |
233 | /* Slab functions */ |
233 | 234 | ||
234 | 235 | ||
235 | /** |
236 | /** |
236 | * Return object to slab and call a destructor |
237 | * Return object to slab and call a destructor |
237 | * |
238 | * |
238 | * @param slab If the caller knows directly slab of the object, otherwise NULL |
239 | * @param slab If the caller knows directly slab of the object, otherwise NULL |
239 | * |
240 | * |
240 | * @return Number of freed pages |
241 | * @return Number of freed pages |
241 | */ |
242 | */ |
242 | static count_t slab_obj_destroy(slab_cache_t *cache, void *obj, slab_t *slab) |
243 | static count_t slab_obj_destroy(slab_cache_t *cache, void *obj, slab_t *slab) |
243 | { |
244 | { |
244 | int freed = 0; |
245 | int freed = 0; |
245 | 246 | ||
246 | if (!slab) |
247 | if (!slab) |
247 | slab = obj2slab(obj); |
248 | slab = obj2slab(obj); |
248 | 249 | ||
249 | ASSERT(slab->cache == cache); |
250 | ASSERT(slab->cache == cache); |
250 | 251 | ||
251 | if (cache->destructor) |
252 | if (cache->destructor) |
252 | freed = cache->destructor(obj); |
253 | freed = cache->destructor(obj); |
253 | 254 | ||
254 | spinlock_lock(&cache->slablock); |
255 | spinlock_lock(&cache->slablock); |
255 | ASSERT(slab->available < cache->objects); |
256 | ASSERT(slab->available < cache->objects); |
256 | 257 | ||
257 | *((int *)obj) = slab->nextavail; |
258 | *((int *)obj) = slab->nextavail; |
258 | slab->nextavail = (obj - slab->start) / cache->size; |
259 | slab->nextavail = (obj - slab->start) / cache->size; |
259 | slab->available++; |
260 | slab->available++; |
260 | 261 | ||
261 | /* Move it to correct list */ |
262 | /* Move it to correct list */ |
262 | if (slab->available == cache->objects) { |
263 | if (slab->available == cache->objects) { |
263 | /* Free associated memory */ |
264 | /* Free associated memory */ |
264 | list_remove(&slab->link); |
265 | list_remove(&slab->link); |
265 | spinlock_unlock(&cache->slablock); |
266 | spinlock_unlock(&cache->slablock); |
266 | 267 | ||
267 | return freed + slab_space_free(cache, slab); |
268 | return freed + slab_space_free(cache, slab); |
268 | 269 | ||
269 | } else if (slab->available == 1) { |
270 | } else if (slab->available == 1) { |
270 | /* It was in full, move to partial */ |
271 | /* It was in full, move to partial */ |
271 | list_remove(&slab->link); |
272 | list_remove(&slab->link); |
272 | list_prepend(&slab->link, &cache->partial_slabs); |
273 | list_prepend(&slab->link, &cache->partial_slabs); |
273 | } |
274 | } |
274 | spinlock_unlock(&cache->slablock); |
275 | spinlock_unlock(&cache->slablock); |
275 | return freed; |
276 | return freed; |
276 | } |
277 | } |
277 | 278 | ||
278 | /** |
279 | /** |
279 | * Take new object from slab or create new if needed |
280 | * Take new object from slab or create new if needed |
280 | * |
281 | * |
281 | * @return Object address or null |
282 | * @return Object address or null |
282 | */ |
283 | */ |
283 | static void *slab_obj_create(slab_cache_t *cache, int flags) |
284 | static void *slab_obj_create(slab_cache_t *cache, int flags) |
284 | { |
285 | { |
285 | slab_t *slab; |
286 | slab_t *slab; |
286 | void *obj; |
287 | void *obj; |
287 | 288 | ||
288 | spinlock_lock(&cache->slablock); |
289 | spinlock_lock(&cache->slablock); |
289 | 290 | ||
290 | if (list_empty(&cache->partial_slabs)) { |
291 | if (list_empty(&cache->partial_slabs)) { |
291 | /* Allow recursion and reclaiming |
292 | /* Allow recursion and reclaiming |
292 | * - this should work, as the slab control structures |
293 | * - this should work, as the slab control structures |
293 | * are small and do not need to allocate with anything |
294 | * are small and do not need to allocate with anything |
294 | * other than frame_alloc when they are allocating, |
295 | * other than frame_alloc when they are allocating, |
295 | * that's why we should get recursion at most 1-level deep |
296 | * that's why we should get recursion at most 1-level deep |
296 | */ |
297 | */ |
297 | spinlock_unlock(&cache->slablock); |
298 | spinlock_unlock(&cache->slablock); |
298 | slab = slab_space_alloc(cache, flags); |
299 | slab = slab_space_alloc(cache, flags); |
299 | if (!slab) |
300 | if (!slab) |
300 | return NULL; |
301 | return NULL; |
301 | spinlock_lock(&cache->slablock); |
302 | spinlock_lock(&cache->slablock); |
302 | } else { |
303 | } else { |
303 | slab = list_get_instance(cache->partial_slabs.next, slab_t, |
304 | slab = list_get_instance(cache->partial_slabs.next, slab_t, |
304 | link); |
305 | link); |
305 | list_remove(&slab->link); |
306 | list_remove(&slab->link); |
306 | } |
307 | } |
307 | obj = slab->start + slab->nextavail * cache->size; |
308 | obj = slab->start + slab->nextavail * cache->size; |
308 | slab->nextavail = *((int *)obj); |
309 | slab->nextavail = *((int *)obj); |
309 | slab->available--; |
310 | slab->available--; |
310 | 311 | ||
311 | if (!slab->available) |
312 | if (!slab->available) |
312 | list_prepend(&slab->link, &cache->full_slabs); |
313 | list_prepend(&slab->link, &cache->full_slabs); |
313 | else |
314 | else |
314 | list_prepend(&slab->link, &cache->partial_slabs); |
315 | list_prepend(&slab->link, &cache->partial_slabs); |
315 | 316 | ||
316 | spinlock_unlock(&cache->slablock); |
317 | spinlock_unlock(&cache->slablock); |
317 | 318 | ||
318 | if (cache->constructor && cache->constructor(obj, flags)) { |
319 | if (cache->constructor && cache->constructor(obj, flags)) { |
319 | /* Bad, bad, construction failed */ |
320 | /* Bad, bad, construction failed */ |
320 | slab_obj_destroy(cache, obj, slab); |
321 | slab_obj_destroy(cache, obj, slab); |
321 | return NULL; |
322 | return NULL; |
322 | } |
323 | } |
323 | return obj; |
324 | return obj; |
324 | } |
325 | } |
325 | 326 | ||
326 | /**************************************/ |
327 | /**************************************/ |
327 | /* CPU-Cache slab functions */ |
328 | /* CPU-Cache slab functions */ |
328 | 329 | ||
329 | /** |
330 | /** |
330 | * Finds a full magazine in cache, takes it from list |
331 | * Finds a full magazine in cache, takes it from list |
331 | * and returns it |
332 | * and returns it |
332 | * |
333 | * |
333 | * @param first If true, return first, else last mag |
334 | * @param first If true, return first, else last mag |
334 | */ |
335 | */ |
335 | static slab_magazine_t *get_mag_from_cache(slab_cache_t *cache, int first) |
336 | static slab_magazine_t *get_mag_from_cache(slab_cache_t *cache, int first) |
336 | { |
337 | { |
337 | slab_magazine_t *mag = NULL; |
338 | slab_magazine_t *mag = NULL; |
338 | link_t *cur; |
339 | link_t *cur; |
339 | 340 | ||
340 | spinlock_lock(&cache->maglock); |
341 | spinlock_lock(&cache->maglock); |
341 | if (!list_empty(&cache->magazines)) { |
342 | if (!list_empty(&cache->magazines)) { |
342 | if (first) |
343 | if (first) |
343 | cur = cache->magazines.next; |
344 | cur = cache->magazines.next; |
344 | else |
345 | else |
345 | cur = cache->magazines.prev; |
346 | cur = cache->magazines.prev; |
346 | mag = list_get_instance(cur, slab_magazine_t, link); |
347 | mag = list_get_instance(cur, slab_magazine_t, link); |
347 | list_remove(&mag->link); |
348 | list_remove(&mag->link); |
348 | atomic_dec(&cache->magazine_counter); |
349 | atomic_dec(&cache->magazine_counter); |
349 | } |
350 | } |
350 | spinlock_unlock(&cache->maglock); |
351 | spinlock_unlock(&cache->maglock); |
351 | return mag; |
352 | return mag; |
352 | } |
353 | } |
353 | 354 | ||
354 | /** Prepend magazine to magazine list in cache */ |
355 | /** Prepend magazine to magazine list in cache */ |
355 | static void put_mag_to_cache(slab_cache_t *cache, slab_magazine_t *mag) |
356 | static void put_mag_to_cache(slab_cache_t *cache, slab_magazine_t *mag) |
356 | { |
357 | { |
357 | spinlock_lock(&cache->maglock); |
358 | spinlock_lock(&cache->maglock); |
358 | 359 | ||
359 | list_prepend(&mag->link, &cache->magazines); |
360 | list_prepend(&mag->link, &cache->magazines); |
360 | atomic_inc(&cache->magazine_counter); |
361 | atomic_inc(&cache->magazine_counter); |
361 | 362 | ||
362 | spinlock_unlock(&cache->maglock); |
363 | spinlock_unlock(&cache->maglock); |
363 | } |
364 | } |
364 | 365 | ||
365 | /** |
366 | /** |
366 | * Free all objects in magazine and free memory associated with magazine |
367 | * Free all objects in magazine and free memory associated with magazine |
367 | * |
368 | * |
368 | * @return Number of freed pages |
369 | * @return Number of freed pages |
369 | */ |
370 | */ |
370 | static count_t magazine_destroy(slab_cache_t *cache, slab_magazine_t *mag) |
371 | static count_t magazine_destroy(slab_cache_t *cache, slab_magazine_t *mag) |
371 | { |
372 | { |
372 | unsigned int i; |
373 | unsigned int i; |
373 | count_t frames = 0; |
374 | count_t frames = 0; |
374 | 375 | ||
375 | for (i = 0; i < mag->busy; i++) { |
376 | for (i = 0; i < mag->busy; i++) { |
376 | frames += slab_obj_destroy(cache, mag->objs[i], NULL); |
377 | frames += slab_obj_destroy(cache, mag->objs[i], NULL); |
377 | atomic_dec(&cache->cached_objs); |
378 | atomic_dec(&cache->cached_objs); |
378 | } |
379 | } |
379 | 380 | ||
380 | slab_free(&mag_cache, mag); |
381 | slab_free(&mag_cache, mag); |
381 | 382 | ||
382 | return frames; |
383 | return frames; |
383 | } |
384 | } |
384 | 385 | ||
385 | /** |
386 | /** |
386 | * Find full magazine, set it as current and return it |
387 | * Find full magazine, set it as current and return it |
387 | * |
388 | * |
388 | * Assume cpu_magazine lock is held |
389 | * Assume cpu_magazine lock is held |
389 | */ |
390 | */ |
390 | static slab_magazine_t *get_full_current_mag(slab_cache_t *cache) |
391 | static slab_magazine_t *get_full_current_mag(slab_cache_t *cache) |
391 | { |
392 | { |
392 | slab_magazine_t *cmag, *lastmag, *newmag; |
393 | slab_magazine_t *cmag, *lastmag, *newmag; |
393 | 394 | ||
394 | cmag = cache->mag_cache[CPU->id].current; |
395 | cmag = cache->mag_cache[CPU->id].current; |
395 | lastmag = cache->mag_cache[CPU->id].last; |
396 | lastmag = cache->mag_cache[CPU->id].last; |
396 | if (cmag) { /* First try local CPU magazines */ |
397 | if (cmag) { /* First try local CPU magazines */ |
397 | if (cmag->busy) |
398 | if (cmag->busy) |
398 | return cmag; |
399 | return cmag; |
399 | 400 | ||
400 | if (lastmag && lastmag->busy) { |
401 | if (lastmag && lastmag->busy) { |
401 | cache->mag_cache[CPU->id].current = lastmag; |
402 | cache->mag_cache[CPU->id].current = lastmag; |
402 | cache->mag_cache[CPU->id].last = cmag; |
403 | cache->mag_cache[CPU->id].last = cmag; |
403 | return lastmag; |
404 | return lastmag; |
404 | } |
405 | } |
405 | } |
406 | } |
406 | /* Local magazines are empty, import one from magazine list */ |
407 | /* Local magazines are empty, import one from magazine list */ |
407 | newmag = get_mag_from_cache(cache, 1); |
408 | newmag = get_mag_from_cache(cache, 1); |
408 | if (!newmag) |
409 | if (!newmag) |
409 | return NULL; |
410 | return NULL; |
410 | 411 | ||
411 | if (lastmag) |
412 | if (lastmag) |
412 | magazine_destroy(cache, lastmag); |
413 | magazine_destroy(cache, lastmag); |
413 | 414 | ||
414 | cache->mag_cache[CPU->id].last = cmag; |
415 | cache->mag_cache[CPU->id].last = cmag; |
415 | cache->mag_cache[CPU->id].current = newmag; |
416 | cache->mag_cache[CPU->id].current = newmag; |
416 | return newmag; |
417 | return newmag; |
417 | } |
418 | } |
418 | 419 | ||
419 | /** |
420 | /** |
420 | * Try to find object in CPU-cache magazines |
421 | * Try to find object in CPU-cache magazines |
421 | * |
422 | * |
422 | * @return Pointer to object or NULL if not available |
423 | * @return Pointer to object or NULL if not available |
423 | */ |
424 | */ |
424 | static void *magazine_obj_get(slab_cache_t *cache) |
425 | static void *magazine_obj_get(slab_cache_t *cache) |
425 | { |
426 | { |
426 | slab_magazine_t *mag; |
427 | slab_magazine_t *mag; |
427 | void *obj; |
428 | void *obj; |
428 | 429 | ||
429 | if (!CPU) |
430 | if (!CPU) |
430 | return NULL; |
431 | return NULL; |
431 | 432 | ||
432 | spinlock_lock(&cache->mag_cache[CPU->id].lock); |
433 | spinlock_lock(&cache->mag_cache[CPU->id].lock); |
433 | 434 | ||
434 | mag = get_full_current_mag(cache); |
435 | mag = get_full_current_mag(cache); |
435 | if (!mag) { |
436 | if (!mag) { |
436 | spinlock_unlock(&cache->mag_cache[CPU->id].lock); |
437 | spinlock_unlock(&cache->mag_cache[CPU->id].lock); |
437 | return NULL; |
438 | return NULL; |
438 | } |
439 | } |
439 | obj = mag->objs[--mag->busy]; |
440 | obj = mag->objs[--mag->busy]; |
440 | spinlock_unlock(&cache->mag_cache[CPU->id].lock); |
441 | spinlock_unlock(&cache->mag_cache[CPU->id].lock); |
441 | atomic_dec(&cache->cached_objs); |
442 | atomic_dec(&cache->cached_objs); |
442 | 443 | ||
443 | return obj; |
444 | return obj; |
444 | } |
445 | } |
445 | 446 | ||
446 | /** |
447 | /** |
447 | * Assure that the current magazine is empty, return pointer to it, or NULL if |
448 | * Assure that the current magazine is empty, return pointer to it, or NULL if |
448 | * no empty magazine is available and cannot be allocated |
449 | * no empty magazine is available and cannot be allocated |
449 | * |
450 | * |
450 | * Assume mag_cache[CPU->id].lock is held |
451 | * Assume mag_cache[CPU->id].lock is held |
451 | * |
452 | * |
452 | * We have 2 magazines bound to processor. |
453 | * We have 2 magazines bound to processor. |
453 | * First try the current. |
454 | * First try the current. |
454 | * If full, try the last. |
455 | * If full, try the last. |
455 | * If full, put to magazines list. |
456 | * If full, put to magazines list. |
456 | * allocate new, exchange last & current |
457 | * allocate new, exchange last & current |
457 | * |
458 | * |
458 | */ |
459 | */ |
459 | static slab_magazine_t *make_empty_current_mag(slab_cache_t *cache) |
460 | static slab_magazine_t *make_empty_current_mag(slab_cache_t *cache) |
460 | { |
461 | { |
461 | slab_magazine_t *cmag,*lastmag,*newmag; |
462 | slab_magazine_t *cmag,*lastmag,*newmag; |
462 | 463 | ||
463 | cmag = cache->mag_cache[CPU->id].current; |
464 | cmag = cache->mag_cache[CPU->id].current; |
464 | lastmag = cache->mag_cache[CPU->id].last; |
465 | lastmag = cache->mag_cache[CPU->id].last; |
465 | 466 | ||
466 | if (cmag) { |
467 | if (cmag) { |
467 | if (cmag->busy < cmag->size) |
468 | if (cmag->busy < cmag->size) |
468 | return cmag; |
469 | return cmag; |
469 | if (lastmag && lastmag->busy < lastmag->size) { |
470 | if (lastmag && lastmag->busy < lastmag->size) { |
470 | cache->mag_cache[CPU->id].last = cmag; |
471 | cache->mag_cache[CPU->id].last = cmag; |
471 | cache->mag_cache[CPU->id].current = lastmag; |
472 | cache->mag_cache[CPU->id].current = lastmag; |
472 | return lastmag; |
473 | return lastmag; |
473 | } |
474 | } |
474 | } |
475 | } |
475 | /* current | last are full | nonexistent, allocate new */ |
476 | /* current | last are full | nonexistent, allocate new */ |
476 | /* We do not want to sleep just because of caching */ |
477 | /* We do not want to sleep just because of caching */ |
477 | /* Especially we do not want reclaiming to start, as |
478 | /* Especially we do not want reclaiming to start, as |
478 | * this would deadlock */ |
479 | * this would deadlock */ |
479 | newmag = slab_alloc(&mag_cache, FRAME_ATOMIC | FRAME_NO_RECLAIM); |
480 | newmag = slab_alloc(&mag_cache, FRAME_ATOMIC | FRAME_NO_RECLAIM); |
480 | if (!newmag) |
481 | if (!newmag) |
481 | return NULL; |
482 | return NULL; |
482 | newmag->size = SLAB_MAG_SIZE; |
483 | newmag->size = SLAB_MAG_SIZE; |
483 | newmag->busy = 0; |
484 | newmag->busy = 0; |
484 | 485 | ||
485 | /* Flush last to magazine list */ |
486 | /* Flush last to magazine list */ |
486 | if (lastmag) |
487 | if (lastmag) |
487 | put_mag_to_cache(cache, lastmag); |
488 | put_mag_to_cache(cache, lastmag); |
488 | 489 | ||
489 | /* Move current as last, save new as current */ |
490 | /* Move current as last, save new as current */ |
490 | cache->mag_cache[CPU->id].last = cmag; |
491 | cache->mag_cache[CPU->id].last = cmag; |
491 | cache->mag_cache[CPU->id].current = newmag; |
492 | cache->mag_cache[CPU->id].current = newmag; |
492 | 493 | ||
493 | return newmag; |
494 | return newmag; |
494 | } |
495 | } |
495 | 496 | ||
496 | /** |
497 | /** |
497 | * Put object into CPU-cache magazine |
498 | * Put object into CPU-cache magazine |
498 | * |
499 | * |
499 | * @return 0 - success, -1 - could not get memory |
500 | * @return 0 - success, -1 - could not get memory |
500 | */ |
501 | */ |
501 | static int magazine_obj_put(slab_cache_t *cache, void *obj) |
502 | static int magazine_obj_put(slab_cache_t *cache, void *obj) |
502 | { |
503 | { |
503 | slab_magazine_t *mag; |
504 | slab_magazine_t *mag; |
504 | 505 | ||
505 | if (!CPU) |
506 | if (!CPU) |
506 | return -1; |
507 | return -1; |
507 | 508 | ||
508 | spinlock_lock(&cache->mag_cache[CPU->id].lock); |
509 | spinlock_lock(&cache->mag_cache[CPU->id].lock); |
509 | 510 | ||
510 | mag = make_empty_current_mag(cache); |
511 | mag = make_empty_current_mag(cache); |
511 | if (!mag) { |
512 | if (!mag) { |
512 | spinlock_unlock(&cache->mag_cache[CPU->id].lock); |
513 | spinlock_unlock(&cache->mag_cache[CPU->id].lock); |
513 | return -1; |
514 | return -1; |
514 | } |
515 | } |
515 | 516 | ||
516 | mag->objs[mag->busy++] = obj; |
517 | mag->objs[mag->busy++] = obj; |
517 | 518 | ||
518 | spinlock_unlock(&cache->mag_cache[CPU->id].lock); |
519 | spinlock_unlock(&cache->mag_cache[CPU->id].lock); |
519 | atomic_inc(&cache->cached_objs); |
520 | atomic_inc(&cache->cached_objs); |
520 | return 0; |
521 | return 0; |
521 | } |
522 | } |
522 | 523 | ||
523 | 524 | ||
524 | /**************************************/ |
525 | /**************************************/ |
525 | /* Slab cache functions */ |
526 | /* Slab cache functions */ |
526 | 527 | ||
527 | /** Return number of objects that fit in certain cache size */ |
528 | /** Return number of objects that fit in certain cache size */ |
528 | static unsigned int comp_objects(slab_cache_t *cache) |
529 | static unsigned int comp_objects(slab_cache_t *cache) |
529 | { |
530 | { |
530 | if (cache->flags & SLAB_CACHE_SLINSIDE) |
531 | if (cache->flags & SLAB_CACHE_SLINSIDE) |
531 | return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / |
532 | return ((PAGE_SIZE << cache->order) - sizeof(slab_t)) / |
532 | cache->size; |
533 | cache->size; |
533 | else |
534 | else |
534 | return (PAGE_SIZE << cache->order) / cache->size; |
535 | return (PAGE_SIZE << cache->order) / cache->size; |
535 | } |
536 | } |
536 | 537 | ||
537 | /** Return wasted space in slab */ |
538 | /** Return wasted space in slab */ |
538 | static unsigned int badness(slab_cache_t *cache) |
539 | static unsigned int badness(slab_cache_t *cache) |
539 | { |
540 | { |
540 | unsigned int objects; |
541 | unsigned int objects; |
541 | unsigned int ssize; |
542 | unsigned int ssize; |
542 | 543 | ||
543 | objects = comp_objects(cache); |
544 | objects = comp_objects(cache); |
544 | ssize = PAGE_SIZE << cache->order; |
545 | ssize = PAGE_SIZE << cache->order; |
545 | if (cache->flags & SLAB_CACHE_SLINSIDE) |
546 | if (cache->flags & SLAB_CACHE_SLINSIDE) |
546 | ssize -= sizeof(slab_t); |
547 | ssize -= sizeof(slab_t); |
547 | return ssize - objects * cache->size; |
548 | return ssize - objects * cache->size; |
548 | } |
549 | } |
549 | 550 | ||
550 | /** |
551 | /** |
551 | * Initialize mag_cache structure in slab cache |
552 | * Initialize mag_cache structure in slab cache |
552 | */ |
553 | */ |
553 | static void make_magcache(slab_cache_t *cache) |
554 | static void make_magcache(slab_cache_t *cache) |
554 | { |
555 | { |
555 | unsigned int i; |
556 | unsigned int i; |
556 | 557 | ||
557 | ASSERT(_slab_initialized >= 2); |
558 | ASSERT(_slab_initialized >= 2); |
558 | 559 | ||
559 | cache->mag_cache = malloc(sizeof(slab_mag_cache_t) * config.cpu_count, |
560 | cache->mag_cache = malloc(sizeof(slab_mag_cache_t) * config.cpu_count, |
560 | 0); |
561 | 0); |
561 | for (i = 0; i < config.cpu_count; i++) { |
562 | for (i = 0; i < config.cpu_count; i++) { |
562 | memsetb(&cache->mag_cache[i], sizeof(cache->mag_cache[i]), 0); |
563 | memsetb(&cache->mag_cache[i], sizeof(cache->mag_cache[i]), 0); |
563 | spinlock_initialize(&cache->mag_cache[i].lock, |
564 | spinlock_initialize(&cache->mag_cache[i].lock, |
564 | "slab_maglock_cpu"); |
565 | "slab_maglock_cpu"); |
565 | } |
566 | } |
566 | } |
567 | } |
567 | 568 | ||
568 | /** Initialize allocated memory as a slab cache */ |
569 | /** Initialize allocated memory as a slab cache */ |
569 | static void |
570 | static void |
570 | _slab_cache_create(slab_cache_t *cache, char *name, size_t size, size_t align, |
571 | _slab_cache_create(slab_cache_t *cache, char *name, size_t size, size_t align, |
571 | int (*constructor)(void *obj, int kmflag), int (*destructor)(void *obj), |
572 | int (*constructor)(void *obj, int kmflag), int (*destructor)(void *obj), |
572 | int flags) |
573 | int flags) |
573 | { |
574 | { |
574 | int pages; |
575 | int pages; |
575 | ipl_t ipl; |
576 | ipl_t ipl; |
576 | 577 | ||
577 | memsetb(cache, sizeof(*cache), 0); |
578 | memsetb(cache, sizeof(*cache), 0); |
578 | cache->name = name; |
579 | cache->name = name; |
579 | 580 | ||
580 | if (align < sizeof(unative_t)) |
581 | if (align < sizeof(unative_t)) |
581 | align = sizeof(unative_t); |
582 | align = sizeof(unative_t); |
582 | size = ALIGN_UP(size, align); |
583 | size = ALIGN_UP(size, align); |
583 | 584 | ||
584 | cache->size = size; |
585 | cache->size = size; |
585 | 586 | ||
586 | cache->constructor = constructor; |
587 | cache->constructor = constructor; |
587 | cache->destructor = destructor; |
588 | cache->destructor = destructor; |
588 | cache->flags = flags; |
589 | cache->flags = flags; |
589 | 590 | ||
590 | list_initialize(&cache->full_slabs); |
591 | list_initialize(&cache->full_slabs); |
591 | list_initialize(&cache->partial_slabs); |
592 | list_initialize(&cache->partial_slabs); |
592 | list_initialize(&cache->magazines); |
593 | list_initialize(&cache->magazines); |
593 | spinlock_initialize(&cache->slablock, "slab_lock"); |
594 | spinlock_initialize(&cache->slablock, "slab_lock"); |
594 | spinlock_initialize(&cache->maglock, "slab_maglock"); |
595 | spinlock_initialize(&cache->maglock, "slab_maglock"); |
595 | if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) |
596 | if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) |
596 | make_magcache(cache); |
597 | make_magcache(cache); |
597 | 598 | ||
598 | /* Compute slab sizes, object counts in slabs etc. */ |
599 | /* Compute slab sizes, object counts in slabs etc. */ |
599 | if (cache->size < SLAB_INSIDE_SIZE) |
600 | if (cache->size < SLAB_INSIDE_SIZE) |
600 | cache->flags |= SLAB_CACHE_SLINSIDE; |
601 | cache->flags |= SLAB_CACHE_SLINSIDE; |
601 | 602 | ||
602 | /* Minimum slab order */ |
603 | /* Minimum slab order */ |
603 | pages = SIZE2FRAMES(cache->size); |
604 | pages = SIZE2FRAMES(cache->size); |
604 | /* We need the 2^order >= pages */ |
605 | /* We need the 2^order >= pages */ |
605 | if (pages == 1) |
606 | if (pages == 1) |
606 | cache->order = 0; |
607 | cache->order = 0; |
607 | else |
608 | else |
608 | cache->order = fnzb(pages - 1) + 1; |
609 | cache->order = fnzb(pages - 1) + 1; |
609 | 610 | ||
610 | while (badness(cache) > SLAB_MAX_BADNESS(cache)) { |
611 | while (badness(cache) > SLAB_MAX_BADNESS(cache)) { |
611 | cache->order += 1; |
612 | cache->order += 1; |
612 | } |
613 | } |
613 | cache->objects = comp_objects(cache); |
614 | cache->objects = comp_objects(cache); |
614 | /* If info fits in, put it inside */ |
615 | /* If info fits in, put it inside */ |
615 | if (badness(cache) > sizeof(slab_t)) |
616 | if (badness(cache) > sizeof(slab_t)) |
616 | cache->flags |= SLAB_CACHE_SLINSIDE; |
617 | cache->flags |= SLAB_CACHE_SLINSIDE; |
617 | 618 | ||
618 | /* Add cache to cache list */ |
619 | /* Add cache to cache list */ |
619 | ipl = interrupts_disable(); |
620 | ipl = interrupts_disable(); |
620 | spinlock_lock(&slab_cache_lock); |
621 | spinlock_lock(&slab_cache_lock); |
621 | 622 | ||
622 | list_append(&cache->link, &slab_cache_list); |
623 | list_append(&cache->link, &slab_cache_list); |
623 | 624 | ||
624 | spinlock_unlock(&slab_cache_lock); |
625 | spinlock_unlock(&slab_cache_lock); |
625 | interrupts_restore(ipl); |
626 | interrupts_restore(ipl); |
626 | } |
627 | } |
627 | 628 | ||
628 | /** Create slab cache */ |
629 | /** Create slab cache */ |
629 | slab_cache_t * |
630 | slab_cache_t * |
630 | slab_cache_create(char *name, size_t size, size_t align, |
631 | slab_cache_create(char *name, size_t size, size_t align, |
631 | int (*constructor)(void *obj, int kmflag), int (*destructor)(void *obj), |
632 | int (*constructor)(void *obj, int kmflag), int (*destructor)(void *obj), |
632 | int flags) |
633 | int flags) |
633 | { |
634 | { |
634 | slab_cache_t *cache; |
635 | slab_cache_t *cache; |
635 | 636 | ||
636 | cache = slab_alloc(&slab_cache_cache, 0); |
637 | cache = slab_alloc(&slab_cache_cache, 0); |
637 | _slab_cache_create(cache, name, size, align, constructor, destructor, |
638 | _slab_cache_create(cache, name, size, align, constructor, destructor, |
638 | flags); |
639 | flags); |
639 | return cache; |
640 | return cache; |
640 | } |
641 | } |
641 | 642 | ||
642 | /** |
643 | /** |
643 | * Reclaim space occupied by objects that are already free |
644 | * Reclaim space occupied by objects that are already free |
644 | * |
645 | * |
645 | * @param flags If contains SLAB_RECLAIM_ALL, do aggressive freeing |
646 | * @param flags If contains SLAB_RECLAIM_ALL, do aggressive freeing |
646 | * @return Number of freed pages |
647 | * @return Number of freed pages |
647 | */ |
648 | */ |
648 | static count_t _slab_reclaim(slab_cache_t *cache, int flags) |
649 | static count_t _slab_reclaim(slab_cache_t *cache, int flags) |
649 | { |
650 | { |
650 | unsigned int i; |
651 | unsigned int i; |
651 | slab_magazine_t *mag; |
652 | slab_magazine_t *mag; |
652 | count_t frames = 0; |
653 | count_t frames = 0; |
653 | int magcount; |
654 | int magcount; |
654 | 655 | ||
655 | if (cache->flags & SLAB_CACHE_NOMAGAZINE) |
656 | if (cache->flags & SLAB_CACHE_NOMAGAZINE) |
656 | return 0; /* Nothing to do */ |
657 | return 0; /* Nothing to do */ |
657 | 658 | ||
658 | /* We count up to original magazine count to avoid |
659 | /* We count up to original magazine count to avoid |
659 | * endless loop |
660 | * endless loop |
660 | */ |
661 | */ |
661 | magcount = atomic_get(&cache->magazine_counter); |
662 | magcount = atomic_get(&cache->magazine_counter); |
662 | while (magcount-- && (mag=get_mag_from_cache(cache, 0))) { |
663 | while (magcount-- && (mag=get_mag_from_cache(cache, 0))) { |
663 | frames += magazine_destroy(cache,mag); |
664 | frames += magazine_destroy(cache,mag); |
664 | if (!(flags & SLAB_RECLAIM_ALL) && frames) |
665 | if (!(flags & SLAB_RECLAIM_ALL) && frames) |
665 | break; |
666 | break; |
666 | } |
667 | } |
667 | 668 | ||
668 | if (flags & SLAB_RECLAIM_ALL) { |
669 | if (flags & SLAB_RECLAIM_ALL) { |
669 | /* Free cpu-bound magazines */ |
670 | /* Free cpu-bound magazines */ |
670 | /* Destroy CPU magazines */ |
671 | /* Destroy CPU magazines */ |
671 | for (i = 0; i < config.cpu_count; i++) { |
672 | for (i = 0; i < config.cpu_count; i++) { |
672 | spinlock_lock(&cache->mag_cache[i].lock); |
673 | spinlock_lock(&cache->mag_cache[i].lock); |
673 | 674 | ||
674 | mag = cache->mag_cache[i].current; |
675 | mag = cache->mag_cache[i].current; |
675 | if (mag) |
676 | if (mag) |
676 | frames += magazine_destroy(cache, mag); |
677 | frames += magazine_destroy(cache, mag); |
677 | cache->mag_cache[i].current = NULL; |
678 | cache->mag_cache[i].current = NULL; |
678 | 679 | ||
679 | mag = cache->mag_cache[i].last; |
680 | mag = cache->mag_cache[i].last; |
680 | if (mag) |
681 | if (mag) |
681 | frames += magazine_destroy(cache, mag); |
682 | frames += magazine_destroy(cache, mag); |
682 | cache->mag_cache[i].last = NULL; |
683 | cache->mag_cache[i].last = NULL; |
683 | 684 | ||
684 | spinlock_unlock(&cache->mag_cache[i].lock); |
685 | spinlock_unlock(&cache->mag_cache[i].lock); |
685 | } |
686 | } |
686 | } |
687 | } |
687 | 688 | ||
688 | return frames; |
689 | return frames; |
689 | } |
690 | } |
690 | 691 | ||
691 | /** Check that there are no slabs and remove cache from system */ |
692 | /** Check that there are no slabs and remove cache from system */ |
692 | void slab_cache_destroy(slab_cache_t *cache) |
693 | void slab_cache_destroy(slab_cache_t *cache) |
693 | { |
694 | { |
694 | ipl_t ipl; |
695 | ipl_t ipl; |
695 | 696 | ||
696 | /* First remove cache from link, so that we don't need |
697 | /* First remove cache from link, so that we don't need |
697 | * to disable interrupts later |
698 | * to disable interrupts later |
698 | */ |
699 | */ |
699 | 700 | ||
700 | ipl = interrupts_disable(); |
701 | ipl = interrupts_disable(); |
701 | spinlock_lock(&slab_cache_lock); |
702 | spinlock_lock(&slab_cache_lock); |
702 | 703 | ||
703 | list_remove(&cache->link); |
704 | list_remove(&cache->link); |
704 | 705 | ||
705 | spinlock_unlock(&slab_cache_lock); |
706 | spinlock_unlock(&slab_cache_lock); |
706 | interrupts_restore(ipl); |
707 | interrupts_restore(ipl); |
707 | 708 | ||
708 | /* Do not lock anything, we assume the software is correct and |
709 | /* Do not lock anything, we assume the software is correct and |
709 | * does not touch the cache when it decides to destroy it */ |
710 | * does not touch the cache when it decides to destroy it */ |
710 | 711 | ||
711 | /* Destroy all magazines */ |
712 | /* Destroy all magazines */ |
712 | _slab_reclaim(cache, SLAB_RECLAIM_ALL); |
713 | _slab_reclaim(cache, SLAB_RECLAIM_ALL); |
713 | 714 | ||
714 | /* All slabs must be empty */ |
715 | /* All slabs must be empty */ |
715 | if (!list_empty(&cache->full_slabs) || |
716 | if (!list_empty(&cache->full_slabs) || |
716 | !list_empty(&cache->partial_slabs)) |
717 | !list_empty(&cache->partial_slabs)) |
717 | panic("Destroying cache that is not empty."); |
718 | panic("Destroying cache that is not empty."); |
718 | 719 | ||
719 | if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) |
720 | if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) |
720 | free(cache->mag_cache); |
721 | free(cache->mag_cache); |
721 | slab_free(&slab_cache_cache, cache); |
722 | slab_free(&slab_cache_cache, cache); |
722 | } |
723 | } |
723 | 724 | ||
724 | /** Allocate new object from cache - if no flags given, always returns memory */ |
725 | /** Allocate new object from cache - if no flags given, always returns memory */ |
725 | void *slab_alloc(slab_cache_t *cache, int flags) |
726 | void *slab_alloc(slab_cache_t *cache, int flags) |
726 | { |
727 | { |
727 | ipl_t ipl; |
728 | ipl_t ipl; |
728 | void *result = NULL; |
729 | void *result = NULL; |
729 | 730 | ||
730 | /* Disable interrupts to avoid deadlocks with interrupt handlers */ |
731 | /* Disable interrupts to avoid deadlocks with interrupt handlers */ |
731 | ipl = interrupts_disable(); |
732 | ipl = interrupts_disable(); |
732 | 733 | ||
733 | if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) { |
734 | if (!(cache->flags & SLAB_CACHE_NOMAGAZINE)) { |
734 | result = magazine_obj_get(cache); |
735 | result = magazine_obj_get(cache); |
735 | } |
736 | } |
736 | if (!result) |
737 | if (!result) |
737 | result = slab_obj_create(cache, flags); |
738 | result = slab_obj_create(cache, flags); |
738 | 739 | ||
739 | interrupts_restore(ipl); |
740 | interrupts_restore(ipl); |
740 | 741 | ||
741 | if (result) |
742 | if (result) |
742 | atomic_inc(&cache->allocated_objs); |
743 | atomic_inc(&cache->allocated_objs); |
743 | 744 | ||
744 | return result; |
745 | return result; |
745 | } |
746 | } |
746 | 747 | ||
747 | /** Return object to cache, use slab if known */ |
748 | /** Return object to cache, use slab if known */ |
748 | static void _slab_free(slab_cache_t *cache, void *obj, slab_t *slab) |
749 | static void _slab_free(slab_cache_t *cache, void *obj, slab_t *slab) |
749 | { |
750 | { |
750 | ipl_t ipl; |
751 | ipl_t ipl; |
751 | 752 | ||
752 | ipl = interrupts_disable(); |
753 | ipl = interrupts_disable(); |
753 | 754 | ||
754 | if ((cache->flags & SLAB_CACHE_NOMAGAZINE) || |
755 | if ((cache->flags & SLAB_CACHE_NOMAGAZINE) || |
755 | magazine_obj_put(cache, obj)) { |
756 | magazine_obj_put(cache, obj)) { |
756 | slab_obj_destroy(cache, obj, slab); |
757 | slab_obj_destroy(cache, obj, slab); |
757 | 758 | ||
758 | } |
759 | } |
759 | interrupts_restore(ipl); |
760 | interrupts_restore(ipl); |
760 | atomic_dec(&cache->allocated_objs); |
761 | atomic_dec(&cache->allocated_objs); |
761 | } |
762 | } |
762 | 763 | ||
763 | /** Return slab object to cache */ |
764 | /** Return slab object to cache */ |
764 | void slab_free(slab_cache_t *cache, void *obj) |
765 | void slab_free(slab_cache_t *cache, void *obj) |
765 | { |
766 | { |
766 | _slab_free(cache, obj, NULL); |
767 | _slab_free(cache, obj, NULL); |
767 | } |
768 | } |
768 | 769 | ||
769 | /* Go through all caches and reclaim what is possible */ |
770 | /* Go through all caches and reclaim what is possible */ |
770 | count_t slab_reclaim(int flags) |
771 | count_t slab_reclaim(int flags) |
771 | { |
772 | { |
772 | slab_cache_t *cache; |
773 | slab_cache_t *cache; |
773 | link_t *cur; |
774 | link_t *cur; |
774 | count_t frames = 0; |
775 | count_t frames = 0; |
775 | 776 | ||
776 | spinlock_lock(&slab_cache_lock); |
777 | spinlock_lock(&slab_cache_lock); |
777 | 778 | ||
778 | /* TODO: Add assert, that interrupts are disabled, otherwise |
779 | /* TODO: Add assert, that interrupts are disabled, otherwise |
779 | * memory allocation from interrupts can deadlock. |
780 | * memory allocation from interrupts can deadlock. |
780 | */ |
781 | */ |
781 | 782 | ||
782 | for (cur = slab_cache_list.next; cur != &slab_cache_list; |
783 | for (cur = slab_cache_list.next; cur != &slab_cache_list; |
783 | cur = cur->next) { |
784 | cur = cur->next) { |
784 | cache = list_get_instance(cur, slab_cache_t, link); |
785 | cache = list_get_instance(cur, slab_cache_t, link); |
785 | frames += _slab_reclaim(cache, flags); |
786 | frames += _slab_reclaim(cache, flags); |
786 | } |
787 | } |
787 | 788 | ||
788 | spinlock_unlock(&slab_cache_lock); |
789 | spinlock_unlock(&slab_cache_lock); |
789 | 790 | ||
790 | return frames; |
791 | return frames; |
791 | } |
792 | } |
792 | 793 | ||
793 | 794 | ||
794 | /* Print list of slabs */ |
795 | /* Print list of slabs */ |
795 | void slab_print_list(void) |
796 | void slab_print_list(void) |
796 | { |
797 | { |
797 | int skip = 0; |
798 | int skip = 0; |
798 | 799 | ||
799 | printf("slab name size pages obj/pg slabs cached allocated" |
800 | printf("slab name size pages obj/pg slabs cached allocated" |
800 | " ctl\n"); |
801 | " ctl\n"); |
801 | printf("---------------- -------- ------ ------ ------ ------ ---------" |
802 | printf("---------------- -------- ------ ------ ------ ------ ---------" |
802 | " ---\n"); |
803 | " ---\n"); |
803 | 804 | ||
804 | while (true) { |
805 | while (true) { |
805 | slab_cache_t *cache; |
806 | slab_cache_t *cache; |
806 | link_t *cur; |
807 | link_t *cur; |
807 | ipl_t ipl; |
808 | ipl_t ipl; |
808 | int i; |
809 | int i; |
809 | 810 | ||
810 | /* |
811 | /* |
811 | * We must not hold the slab_cache_lock spinlock when printing |
812 | * We must not hold the slab_cache_lock spinlock when printing |
812 | * the statistics. Otherwise we can easily deadlock if the print |
813 | * the statistics. Otherwise we can easily deadlock if the print |
813 | * needs to allocate memory. |
814 | * needs to allocate memory. |
814 | * |
815 | * |
815 | * Therefore, we walk through the slab cache list, skipping some |
816 | * Therefore, we walk through the slab cache list, skipping some |
816 | * amount of already processed caches during each iteration and |
817 | * amount of already processed caches during each iteration and |
817 | * gathering statistics about the first unprocessed cache. For |
818 | * gathering statistics about the first unprocessed cache. For |
818 | * the sake of printing the statistics, we realese the |
819 | * the sake of printing the statistics, we realese the |
819 | * slab_cache_lock and reacquire it afterwards. Then the walk |
820 | * slab_cache_lock and reacquire it afterwards. Then the walk |
820 | * starts again. |
821 | * starts again. |
821 | * |
822 | * |
822 | * This limits both the efficiency and also accuracy of the |
823 | * This limits both the efficiency and also accuracy of the |
823 | * obtained statistics. The efficiency is decreased because the |
824 | * obtained statistics. The efficiency is decreased because the |
824 | * time complexity of the algorithm is quadratic instead of |
825 | * time complexity of the algorithm is quadratic instead of |
825 | * linear. The accuracy is impacted because we drop the lock |
826 | * linear. The accuracy is impacted because we drop the lock |
826 | * after processing one cache. If there is someone else |
827 | * after processing one cache. If there is someone else |
827 | * manipulating the cache list, we might omit an arbitrary |
828 | * manipulating the cache list, we might omit an arbitrary |
828 | * number of caches or process one cache multiple times. |
829 | * number of caches or process one cache multiple times. |
829 | * However, we don't bleed for this algorithm for it is only |
830 | * However, we don't bleed for this algorithm for it is only |
830 | * statistics. |
831 | * statistics. |
831 | */ |
832 | */ |
832 | 833 | ||
833 | ipl = interrupts_disable(); |
834 | ipl = interrupts_disable(); |
834 | spinlock_lock(&slab_cache_lock); |
835 | spinlock_lock(&slab_cache_lock); |
835 | 836 | ||
836 | for (i = 0, cur = slab_cache_list.next; |
837 | for (i = 0, cur = slab_cache_list.next; |
837 | i < skip && cur != &slab_cache_list; |
838 | i < skip && cur != &slab_cache_list; |
838 | i++, cur = cur->next) |
839 | i++, cur = cur->next) |
839 | ; |
840 | ; |
840 | 841 | ||
841 | if (cur == &slab_cache_list) { |
842 | if (cur == &slab_cache_list) { |
842 | spinlock_unlock(&slab_cache_lock); |
843 | spinlock_unlock(&slab_cache_lock); |
843 | interrupts_restore(ipl); |
844 | interrupts_restore(ipl); |
844 | break; |
845 | break; |
845 | } |
846 | } |
846 | 847 | ||
847 | skip++; |
848 | skip++; |
848 | 849 | ||
849 | cache = list_get_instance(cur, slab_cache_t, link); |
850 | cache = list_get_instance(cur, slab_cache_t, link); |
850 | 851 | ||
851 | char *name = cache->name; |
852 | char *name = cache->name; |
852 | uint8_t order = cache->order; |
853 | uint8_t order = cache->order; |
853 | size_t size = cache->size; |
854 | size_t size = cache->size; |
854 | unsigned int objects = cache->objects; |
855 | unsigned int objects = cache->objects; |
855 | long allocated_slabs = atomic_get(&cache->allocated_slabs); |
856 | long allocated_slabs = atomic_get(&cache->allocated_slabs); |
856 | long cached_objs = atomic_get(&cache->cached_objs); |
857 | long cached_objs = atomic_get(&cache->cached_objs); |
857 | long allocated_objs = atomic_get(&cache->allocated_objs); |
858 | long allocated_objs = atomic_get(&cache->allocated_objs); |
858 | int flags = cache->flags; |
859 | int flags = cache->flags; |
859 | 860 | ||
860 | spinlock_unlock(&slab_cache_lock); |
861 | spinlock_unlock(&slab_cache_lock); |
861 | interrupts_restore(ipl); |
862 | interrupts_restore(ipl); |
862 | 863 | ||
863 | printf("%-16s %8" PRIs " %6d %6u %6ld %6ld %9ld %-3s\n", |
864 | printf("%-16s %8" PRIs " %6d %6u %6ld %6ld %9ld %-3s\n", |
864 | name, size, (1 << order), objects, allocated_slabs, |
865 | name, size, (1 << order), objects, allocated_slabs, |
865 | cached_objs, allocated_objs, |
866 | cached_objs, allocated_objs, |
866 | flags & SLAB_CACHE_SLINSIDE ? "in" : "out"); |
867 | flags & SLAB_CACHE_SLINSIDE ? "in" : "out"); |
867 | } |
868 | } |
868 | } |
869 | } |
869 | 870 | ||
870 | void slab_cache_init(void) |
871 | void slab_cache_init(void) |
871 | { |
872 | { |
872 | int i, size; |
873 | int i, size; |
873 | 874 | ||
874 | /* Initialize magazine cache */ |
875 | /* Initialize magazine cache */ |
875 | _slab_cache_create(&mag_cache, "slab_magazine", |
876 | _slab_cache_create(&mag_cache, "slab_magazine", |
876 | sizeof(slab_magazine_t) + SLAB_MAG_SIZE * sizeof(void*), |
877 | sizeof(slab_magazine_t) + SLAB_MAG_SIZE * sizeof(void*), |
877 | sizeof(uintptr_t), NULL, NULL, SLAB_CACHE_NOMAGAZINE | |
878 | sizeof(uintptr_t), NULL, NULL, SLAB_CACHE_NOMAGAZINE | |
878 | SLAB_CACHE_SLINSIDE); |
879 | SLAB_CACHE_SLINSIDE); |
879 | /* Initialize slab_cache cache */ |
880 | /* Initialize slab_cache cache */ |
880 | _slab_cache_create(&slab_cache_cache, "slab_cache", |
881 | _slab_cache_create(&slab_cache_cache, "slab_cache", |
881 | sizeof(slab_cache_cache), sizeof(uintptr_t), NULL, NULL, |
882 | sizeof(slab_cache_cache), sizeof(uintptr_t), NULL, NULL, |
882 | SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE); |
883 | SLAB_CACHE_NOMAGAZINE | SLAB_CACHE_SLINSIDE); |
883 | /* Initialize external slab cache */ |
884 | /* Initialize external slab cache */ |
884 | slab_extern_cache = slab_cache_create("slab_extern", sizeof(slab_t), 0, |
885 | slab_extern_cache = slab_cache_create("slab_extern", sizeof(slab_t), 0, |
885 | NULL, NULL, SLAB_CACHE_SLINSIDE | SLAB_CACHE_MAGDEFERRED); |
886 | NULL, NULL, SLAB_CACHE_SLINSIDE | SLAB_CACHE_MAGDEFERRED); |
886 | 887 | ||
887 | /* Initialize structures for malloc */ |
888 | /* Initialize structures for malloc */ |
888 | for (i = 0, size = (1 << SLAB_MIN_MALLOC_W); |
889 | for (i = 0, size = (1 << SLAB_MIN_MALLOC_W); |
889 | i < (SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1); |
890 | i < (SLAB_MAX_MALLOC_W - SLAB_MIN_MALLOC_W + 1); |
890 | i++, size <<= 1) { |
891 | i++, size <<= 1) { |
891 | malloc_caches[i] = slab_cache_create(malloc_names[i], size, 0, |
892 | malloc_caches[i] = slab_cache_create(malloc_names[i], size, 0, |
892 | NULL, NULL, SLAB_CACHE_MAGDEFERRED); |
893 | NULL, NULL, SLAB_CACHE_MAGDEFERRED); |
893 | } |
894 | } |
894 | #ifdef CONFIG_DEBUG |
895 | #ifdef CONFIG_DEBUG |
895 | _slab_initialized = 1; |
896 | _slab_initialized = 1; |
896 | #endif |
897 | #endif |
897 | } |
898 | } |
898 | 899 | ||
899 | /** Enable cpu_cache |
900 | /** Enable cpu_cache |
900 | * |
901 | * |
901 | * Kernel calls this function, when it knows the real number of |
902 | * Kernel calls this function, when it knows the real number of |
902 | * processors. |
903 | * processors. |
903 | * Allocate slab for cpucache and enable it on all existing |
904 | * Allocate slab for cpucache and enable it on all existing |
904 | * slabs that are SLAB_CACHE_MAGDEFERRED |
905 | * slabs that are SLAB_CACHE_MAGDEFERRED |
905 | */ |
906 | */ |
906 | void slab_enable_cpucache(void) |
907 | void slab_enable_cpucache(void) |
907 | { |
908 | { |
908 | link_t *cur; |
909 | link_t *cur; |
909 | slab_cache_t *s; |
910 | slab_cache_t *s; |
910 | 911 | ||
911 | #ifdef CONFIG_DEBUG |
912 | #ifdef CONFIG_DEBUG |
912 | _slab_initialized = 2; |
913 | _slab_initialized = 2; |
913 | #endif |
914 | #endif |
914 | 915 | ||
915 | spinlock_lock(&slab_cache_lock); |
916 | spinlock_lock(&slab_cache_lock); |
916 | 917 | ||
917 | for (cur = slab_cache_list.next; cur != &slab_cache_list; |
918 | for (cur = slab_cache_list.next; cur != &slab_cache_list; |
918 | cur = cur->next){ |
919 | cur = cur->next){ |
919 | s = list_get_instance(cur, slab_cache_t, link); |
920 | s = list_get_instance(cur, slab_cache_t, link); |
920 | if ((s->flags & SLAB_CACHE_MAGDEFERRED) != |
921 | if ((s->flags & SLAB_CACHE_MAGDEFERRED) != |
921 | SLAB_CACHE_MAGDEFERRED) |
922 | SLAB_CACHE_MAGDEFERRED) |
922 | continue; |
923 | continue; |
923 | make_magcache(s); |
924 | make_magcache(s); |
924 | s->flags &= ~SLAB_CACHE_MAGDEFERRED; |
925 | s->flags &= ~SLAB_CACHE_MAGDEFERRED; |
925 | } |
926 | } |
926 | 927 | ||
927 | spinlock_unlock(&slab_cache_lock); |
928 | spinlock_unlock(&slab_cache_lock); |
928 | } |
929 | } |
929 | 930 | ||
930 | /**************************************/ |
931 | /**************************************/ |
931 | /* kalloc/kfree functions */ |
932 | /* kalloc/kfree functions */ |
932 | void *malloc(unsigned int size, int flags) |
933 | void *malloc(unsigned int size, int flags) |
933 | { |
934 | { |
934 | ASSERT(_slab_initialized); |
935 | ASSERT(_slab_initialized); |
935 | ASSERT(size && size <= (1 << SLAB_MAX_MALLOC_W)); |
936 | ASSERT(size && size <= (1 << SLAB_MAX_MALLOC_W)); |
936 | 937 | ||
937 | if (size < (1 << SLAB_MIN_MALLOC_W)) |
938 | if (size < (1 << SLAB_MIN_MALLOC_W)) |
938 | size = (1 << SLAB_MIN_MALLOC_W); |
939 | size = (1 << SLAB_MIN_MALLOC_W); |
939 | 940 | ||
940 | int idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; |
941 | int idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; |
941 | 942 | ||
942 | return slab_alloc(malloc_caches[idx], flags); |
943 | return slab_alloc(malloc_caches[idx], flags); |
943 | } |
944 | } |
944 | 945 | ||
945 | void *realloc(void *ptr, unsigned int size, int flags) |
946 | void *realloc(void *ptr, unsigned int size, int flags) |
946 | { |
947 | { |
947 | ASSERT(_slab_initialized); |
948 | ASSERT(_slab_initialized); |
948 | ASSERT(size <= (1 << SLAB_MAX_MALLOC_W)); |
949 | ASSERT(size <= (1 << SLAB_MAX_MALLOC_W)); |
949 | 950 | ||
950 | void *new_ptr; |
951 | void *new_ptr; |
951 | 952 | ||
952 | if (size > 0) { |
953 | if (size > 0) { |
953 | if (size < (1 << SLAB_MIN_MALLOC_W)) |
954 | if (size < (1 << SLAB_MIN_MALLOC_W)) |
954 | size = (1 << SLAB_MIN_MALLOC_W); |
955 | size = (1 << SLAB_MIN_MALLOC_W); |
955 | int idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; |
956 | int idx = fnzb(size - 1) - SLAB_MIN_MALLOC_W + 1; |
956 | 957 | ||
957 | new_ptr = slab_alloc(malloc_caches[idx], flags); |
958 | new_ptr = slab_alloc(malloc_caches[idx], flags); |
958 | } else |
959 | } else |
959 | new_ptr = NULL; |
960 | new_ptr = NULL; |
960 | 961 | ||
961 | if ((new_ptr != NULL) && (ptr != NULL)) { |
962 | if ((new_ptr != NULL) && (ptr != NULL)) { |
962 | slab_t *slab = obj2slab(ptr); |
963 | slab_t *slab = obj2slab(ptr); |
963 | memcpy(new_ptr, ptr, min(size, slab->cache->size)); |
964 | memcpy(new_ptr, ptr, min(size, slab->cache->size)); |
964 | } |
965 | } |
965 | 966 | ||
966 | if (ptr != NULL) |
967 | if (ptr != NULL) |
967 | free(ptr); |
968 | free(ptr); |
968 | 969 | ||
969 | return new_ptr; |
970 | return new_ptr; |
970 | } |
971 | } |
971 | 972 | ||
972 | void free(void *ptr) |
973 | void free(void *ptr) |
973 | { |
974 | { |
974 | if (!ptr) |
975 | if (!ptr) |
975 | return; |
976 | return; |
976 | 977 | ||
977 | slab_t *slab = obj2slab(ptr); |
978 | slab_t *slab = obj2slab(ptr); |
978 | _slab_free(slab->cache, ptr, slab); |
979 | _slab_free(slab->cache, ptr, slab); |
979 | } |
980 | } |
980 | 981 | ||
981 | /** @} |
982 | /** @} |
982 | */ |
983 | */ |
983 | 984 |