3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /*
26 * Kernel memory allocator, as described in the following two papers and a
27 * statement about the consolidator:
28 *
29 * Jeff Bonwick,
30 * The Slab Allocator: An Object-Caching Kernel Memory Allocator.
31 * Proceedings of the Summer 1994 Usenix Conference.
32 * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf.
33 *
34 * Jeff Bonwick and Jonathan Adams,
35 * Magazines and vmem: Extending the Slab Allocator to Many CPUs and
36 * Arbitrary Resources.
37 * Proceedings of the 2001 Usenix Conference.
38 * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf.
39 *
40 * kmem Slab Consolidator Big Theory Statement:
41 *
42 * 1. Motivation
2152 mlp->ml_list = mp;
2153 mlp->ml_total++;
2154 mutex_exit(&cp->cache_depot_lock);
2155 }
2156
2157 /*
2158 * Update the working set statistics for cp's depot.
2159 */
2160 static void
2161 kmem_depot_ws_update(kmem_cache_t *cp)
2162 {
2163 mutex_enter(&cp->cache_depot_lock);
2164 cp->cache_full.ml_reaplimit = cp->cache_full.ml_min;
2165 cp->cache_full.ml_min = cp->cache_full.ml_total;
2166 cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min;
2167 cp->cache_empty.ml_min = cp->cache_empty.ml_total;
2168 mutex_exit(&cp->cache_depot_lock);
2169 }
2170
2171 /*
2172 * Reap all magazines that have fallen out of the depot's working set.
2173 */
2174 static void
2175 kmem_depot_ws_reap(kmem_cache_t *cp)
2176 {
2177 long reap;
2178 kmem_magazine_t *mp;
2179
2180 ASSERT(!list_link_active(&cp->cache_link) ||
2181 taskq_member(kmem_taskq, curthread));
2182
2183 reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min);
2184 while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_full)) != NULL)
2185 kmem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize);
2186
2187 reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min);
2188 while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_empty)) != NULL)
2189 kmem_magazine_destroy(cp, mp, 0);
2190 }
2191
3221 ccp = &cp->cache_cpu[cpu_seqid];
3222
3223 mutex_enter(&ccp->cc_lock);
3224 mp = ccp->cc_loaded;
3225 pmp = ccp->cc_ploaded;
3226 rounds = ccp->cc_rounds;
3227 prounds = ccp->cc_prounds;
3228 ccp->cc_loaded = NULL;
3229 ccp->cc_ploaded = NULL;
3230 ccp->cc_rounds = -1;
3231 ccp->cc_prounds = -1;
3232 ccp->cc_magsize = 0;
3233 mutex_exit(&ccp->cc_lock);
3234
3235 if (mp)
3236 kmem_magazine_destroy(cp, mp, rounds);
3237 if (pmp)
3238 kmem_magazine_destroy(cp, pmp, prounds);
3239 }
3240
3241 /*
3242 * Updating the working set statistics twice in a row has the
3243 * effect of setting the working set size to zero, so everything
3244 * is eligible for reaping.
3245 */
3246 kmem_depot_ws_update(cp);
3247 kmem_depot_ws_update(cp);
3248
3249 kmem_depot_ws_reap(cp);
3250 }
3251
3252 /*
3253 * Enable per-cpu magazines on a cache.
3254 */
3255 static void
3256 kmem_cache_magazine_enable(kmem_cache_t *cp)
3257 {
3258 int cpu_seqid;
3259
3260 if (cp->cache_flags & KMF_NOMAGAZINE)
3261 return;
3262
3263 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
3264 kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
3265 mutex_enter(&ccp->cc_lock);
3266 ccp->cc_magsize = cp->cache_magtype->mt_magsize;
3267 mutex_exit(&ccp->cc_lock);
3268 }
3269
3270 }
3271
3272 /*
3273 * Reap (almost) everything right now. See kmem_cache_magazine_purge()
3274 * for explanation of the back-to-back kmem_depot_ws_update() calls.
3275 */
3276 void
3277 kmem_cache_reap_now(kmem_cache_t *cp)
3278 {
3279 ASSERT(list_link_active(&cp->cache_link));
3280
3281 kmem_depot_ws_update(cp);
3282 kmem_depot_ws_update(cp);
3283
3284 (void) taskq_dispatch(kmem_taskq,
3285 (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP);
3286 taskq_wait(kmem_taskq);
3287 }
3288
3289 /*
3290 * Recompute a cache's magazine size. The trade-off is that larger magazines
3291 * provide a higher transfer rate with the depot, while smaller magazines
3292 * reduce memory consumption. Magazine resizing is an expensive operation;
3293 * it should not be done frequently.
3294 *
3295 * Changes to the magazine size are serialized by the kmem_taskq lock.
3296 *
3297 * Note: at present this only grows the magazine size. It might be useful
3298 * to allow shrinkage too.
3299 */
3300 static void
3301 kmem_cache_magazine_resize(kmem_cache_t *cp)
3302 {
|
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
24 */
25
26 /*
27 * Kernel memory allocator, as described in the following two papers and a
28 * statement about the consolidator:
29 *
30 * Jeff Bonwick,
31 * The Slab Allocator: An Object-Caching Kernel Memory Allocator.
32 * Proceedings of the Summer 1994 Usenix Conference.
33 * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf.
34 *
35 * Jeff Bonwick and Jonathan Adams,
36 * Magazines and vmem: Extending the Slab Allocator to Many CPUs and
37 * Arbitrary Resources.
38 * Proceedings of the 2001 Usenix Conference.
39 * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf.
40 *
41 * kmem Slab Consolidator Big Theory Statement:
42 *
43 * 1. Motivation
2153 mlp->ml_list = mp;
2154 mlp->ml_total++;
2155 mutex_exit(&cp->cache_depot_lock);
2156 }
2157
2158 /*
2159 * Update the working set statistics for cp's depot.
2160 */
2161 static void
2162 kmem_depot_ws_update(kmem_cache_t *cp)
2163 {
2164 mutex_enter(&cp->cache_depot_lock);
2165 cp->cache_full.ml_reaplimit = cp->cache_full.ml_min;
2166 cp->cache_full.ml_min = cp->cache_full.ml_total;
2167 cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min;
2168 cp->cache_empty.ml_min = cp->cache_empty.ml_total;
2169 mutex_exit(&cp->cache_depot_lock);
2170 }
2171
2172 /*
2173 * Set the working set statistics for cp's depot to zero. (Everything is
2174 * eligible for reaping.)
2175 */
2176 static void
2177 kmem_depot_ws_zero(kmem_cache_t *cp)
2178 {
2179 mutex_enter(&cp->cache_depot_lock);
2180 cp->cache_full.ml_reaplimit = cp->cache_full.ml_total;
2181 cp->cache_full.ml_min = cp->cache_full.ml_total;
2182 cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_total;
2183 cp->cache_empty.ml_min = cp->cache_empty.ml_total;
2184 mutex_exit(&cp->cache_depot_lock);
2185 }
2186
2187 /*
2188 * Reap all magazines that have fallen out of the depot's working set.
2189 */
2190 static void
2191 kmem_depot_ws_reap(kmem_cache_t *cp)
2192 {
2193 long reap;
2194 kmem_magazine_t *mp;
2195
2196 ASSERT(!list_link_active(&cp->cache_link) ||
2197 taskq_member(kmem_taskq, curthread));
2198
2199 reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min);
2200 while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_full)) != NULL)
2201 kmem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize);
2202
2203 reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min);
2204 while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_empty)) != NULL)
2205 kmem_magazine_destroy(cp, mp, 0);
2206 }
2207
3237 ccp = &cp->cache_cpu[cpu_seqid];
3238
3239 mutex_enter(&ccp->cc_lock);
3240 mp = ccp->cc_loaded;
3241 pmp = ccp->cc_ploaded;
3242 rounds = ccp->cc_rounds;
3243 prounds = ccp->cc_prounds;
3244 ccp->cc_loaded = NULL;
3245 ccp->cc_ploaded = NULL;
3246 ccp->cc_rounds = -1;
3247 ccp->cc_prounds = -1;
3248 ccp->cc_magsize = 0;
3249 mutex_exit(&ccp->cc_lock);
3250
3251 if (mp)
3252 kmem_magazine_destroy(cp, mp, rounds);
3253 if (pmp)
3254 kmem_magazine_destroy(cp, pmp, prounds);
3255 }
3256
3257 kmem_depot_ws_zero(cp);
3258 kmem_depot_ws_reap(cp);
3259 }
3260
3261 /*
3262 * Enable per-cpu magazines on a cache.
3263 */
3264 static void
3265 kmem_cache_magazine_enable(kmem_cache_t *cp)
3266 {
3267 int cpu_seqid;
3268
3269 if (cp->cache_flags & KMF_NOMAGAZINE)
3270 return;
3271
3272 for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
3273 kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
3274 mutex_enter(&ccp->cc_lock);
3275 ccp->cc_magsize = cp->cache_magtype->mt_magsize;
3276 mutex_exit(&ccp->cc_lock);
3277 }
3278
3279 }
3280
3281 /*
3282 * Reap (almost) everything right now.
3283 */
3284 void
3285 kmem_cache_reap_now(kmem_cache_t *cp)
3286 {
3287 ASSERT(list_link_active(&cp->cache_link));
3288
3289 kmem_depot_ws_zero(cp);
3290
3291 (void) taskq_dispatch(kmem_taskq,
3292 (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP);
3293 taskq_wait(kmem_taskq);
3294 }
3295
3296 /*
3297 * Recompute a cache's magazine size. The trade-off is that larger magazines
3298 * provide a higher transfer rate with the depot, while smaller magazines
3299 * reduce memory consumption. Magazine resizing is an expensive operation;
3300 * it should not be done frequently.
3301 *
3302 * Changes to the magazine size are serialized by the kmem_taskq lock.
3303 *
3304 * Note: at present this only grows the magazine size. It might be useful
3305 * to allow shrinkage too.
3306 */
3307 static void
3308 kmem_cache_magazine_resize(kmem_cache_t *cp)
3309 {
|