Print this page
XXXX kmem: double-calling kmem_depot_ws_update isn't obvious
While the double-call is documented in a comment, it's not obvious what
exactly it is trying to accomplish.  The easiest way to address this is to
introduce a new function that "zeroes-out" the working set statistics to
force everything to be eligible for reaping.


   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.

  23  */
  24 
  25 /*
  26  * Kernel memory allocator, as described in the following two papers and a
  27  * statement about the consolidator:
  28  *
  29  * Jeff Bonwick,
  30  * The Slab Allocator: An Object-Caching Kernel Memory Allocator.
  31  * Proceedings of the Summer 1994 Usenix Conference.
  32  * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf.
  33  *
  34  * Jeff Bonwick and Jonathan Adams,
  35  * Magazines and vmem: Extending the Slab Allocator to Many CPUs and
  36  * Arbitrary Resources.
  37  * Proceedings of the 2001 Usenix Conference.
  38  * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf.
  39  *
  40  * kmem Slab Consolidator Big Theory Statement:
  41  *
  42  * 1. Motivation


2152         mlp->ml_list = mp;
2153         mlp->ml_total++;
2154         mutex_exit(&cp->cache_depot_lock);
2155 }
2156 
2157 /*
2158  * Update the working set statistics for cp's depot.
2159  */
2160 static void
2161 kmem_depot_ws_update(kmem_cache_t *cp)
2162 {
2163         mutex_enter(&cp->cache_depot_lock);
2164         cp->cache_full.ml_reaplimit = cp->cache_full.ml_min;
2165         cp->cache_full.ml_min = cp->cache_full.ml_total;
2166         cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min;
2167         cp->cache_empty.ml_min = cp->cache_empty.ml_total;
2168         mutex_exit(&cp->cache_depot_lock);
2169 }
2170 
2171 /*















2172  * Reap all magazines that have fallen out of the depot's working set.
2173  */
2174 static void
2175 kmem_depot_ws_reap(kmem_cache_t *cp)
2176 {
2177         long reap;
2178         kmem_magazine_t *mp;
2179 
2180         ASSERT(!list_link_active(&cp->cache_link) ||
2181             taskq_member(kmem_taskq, curthread));
2182 
2183         reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min);
2184         while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_full)) != NULL)
2185                 kmem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize);
2186 
2187         reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min);
2188         while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_empty)) != NULL)
2189                 kmem_magazine_destroy(cp, mp, 0);
2190 }
2191 


3221                 ccp = &cp->cache_cpu[cpu_seqid];
3222 
3223                 mutex_enter(&ccp->cc_lock);
3224                 mp = ccp->cc_loaded;
3225                 pmp = ccp->cc_ploaded;
3226                 rounds = ccp->cc_rounds;
3227                 prounds = ccp->cc_prounds;
3228                 ccp->cc_loaded = NULL;
3229                 ccp->cc_ploaded = NULL;
3230                 ccp->cc_rounds = -1;
3231                 ccp->cc_prounds = -1;
3232                 ccp->cc_magsize = 0;
3233                 mutex_exit(&ccp->cc_lock);
3234 
3235                 if (mp)
3236                         kmem_magazine_destroy(cp, mp, rounds);
3237                 if (pmp)
3238                         kmem_magazine_destroy(cp, pmp, prounds);
3239         }
3240 
3241         /*
3242          * Updating the working set statistics twice in a row has the
3243          * effect of setting the working set size to zero, so everything
3244          * is eligible for reaping.
3245          */
3246         kmem_depot_ws_update(cp);
3247         kmem_depot_ws_update(cp);
3248 
3249         kmem_depot_ws_reap(cp);
3250 }
3251 
3252 /*
3253  * Enable per-cpu magazines on a cache.
3254  */
3255 static void
3256 kmem_cache_magazine_enable(kmem_cache_t *cp)
3257 {
3258         int cpu_seqid;
3259 
3260         if (cp->cache_flags & KMF_NOMAGAZINE)
3261                 return;
3262 
3263         for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
3264                 kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
3265                 mutex_enter(&ccp->cc_lock);
3266                 ccp->cc_magsize = cp->cache_magtype->mt_magsize;
3267                 mutex_exit(&ccp->cc_lock);
3268         }
3269 
3270 }
3271 
3272 /*
3273  * Reap (almost) everything right now.  See kmem_cache_magazine_purge()
3274  * for explanation of the back-to-back kmem_depot_ws_update() calls.
3275  */
3276 void
3277 kmem_cache_reap_now(kmem_cache_t *cp)
3278 {
3279         ASSERT(list_link_active(&cp->cache_link));
3280 
3281         kmem_depot_ws_update(cp);
3282         kmem_depot_ws_update(cp);
3283 
3284         (void) taskq_dispatch(kmem_taskq,
3285             (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP);
3286         taskq_wait(kmem_taskq);
3287 }
3288 
3289 /*
3290  * Recompute a cache's magazine size.  The trade-off is that larger magazines
3291  * provide a higher transfer rate with the depot, while smaller magazines
3292  * reduce memory consumption.  Magazine resizing is an expensive operation;
3293  * it should not be done frequently.
3294  *
3295  * Changes to the magazine size are serialized by the kmem_taskq lock.
3296  *
3297  * Note: at present this only grows the magazine size.  It might be useful
3298  * to allow shrinkage too.
3299  */
3300 static void
3301 kmem_cache_magazine_resize(kmem_cache_t *cp)
3302 {




   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  24  */
  25 
  26 /*
  27  * Kernel memory allocator, as described in the following two papers and a
  28  * statement about the consolidator:
  29  *
  30  * Jeff Bonwick,
  31  * The Slab Allocator: An Object-Caching Kernel Memory Allocator.
  32  * Proceedings of the Summer 1994 Usenix Conference.
  33  * Available as /shared/sac/PSARC/1994/028/materials/kmem.pdf.
  34  *
  35  * Jeff Bonwick and Jonathan Adams,
  36  * Magazines and vmem: Extending the Slab Allocator to Many CPUs and
  37  * Arbitrary Resources.
  38  * Proceedings of the 2001 Usenix Conference.
  39  * Available as /shared/sac/PSARC/2000/550/materials/vmem.pdf.
  40  *
  41  * kmem Slab Consolidator Big Theory Statement:
  42  *
  43  * 1. Motivation


2153         mlp->ml_list = mp;
2154         mlp->ml_total++;
2155         mutex_exit(&cp->cache_depot_lock);
2156 }
2157 
2158 /*
2159  * Update the working set statistics for cp's depot.
2160  */
2161 static void
2162 kmem_depot_ws_update(kmem_cache_t *cp)
2163 {
2164         mutex_enter(&cp->cache_depot_lock);
2165         cp->cache_full.ml_reaplimit = cp->cache_full.ml_min;
2166         cp->cache_full.ml_min = cp->cache_full.ml_total;
2167         cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_min;
2168         cp->cache_empty.ml_min = cp->cache_empty.ml_total;
2169         mutex_exit(&cp->cache_depot_lock);
2170 }
2171 
2172 /*
2173  * Set the working set statistics for cp's depot to zero.  (Everything is
2174  * eligible for reaping.)
2175  */
2176 static void
2177 kmem_depot_ws_zero(kmem_cache_t *cp)
2178 {
2179         mutex_enter(&cp->cache_depot_lock);
2180         cp->cache_full.ml_reaplimit = cp->cache_full.ml_total;
2181         cp->cache_full.ml_min = cp->cache_full.ml_total;
2182         cp->cache_empty.ml_reaplimit = cp->cache_empty.ml_total;
2183         cp->cache_empty.ml_min = cp->cache_empty.ml_total;
2184         mutex_exit(&cp->cache_depot_lock);
2185 }
2186 
2187 /*
2188  * Reap all magazines that have fallen out of the depot's working set.
2189  */
2190 static void
2191 kmem_depot_ws_reap(kmem_cache_t *cp)
2192 {
2193         long reap;
2194         kmem_magazine_t *mp;
2195 
2196         ASSERT(!list_link_active(&cp->cache_link) ||
2197             taskq_member(kmem_taskq, curthread));
2198 
2199         reap = MIN(cp->cache_full.ml_reaplimit, cp->cache_full.ml_min);
2200         while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_full)) != NULL)
2201                 kmem_magazine_destroy(cp, mp, cp->cache_magtype->mt_magsize);
2202 
2203         reap = MIN(cp->cache_empty.ml_reaplimit, cp->cache_empty.ml_min);
2204         while (reap-- && (mp = kmem_depot_alloc(cp, &cp->cache_empty)) != NULL)
2205                 kmem_magazine_destroy(cp, mp, 0);
2206 }
2207 


3237                 ccp = &cp->cache_cpu[cpu_seqid];
3238 
3239                 mutex_enter(&ccp->cc_lock);
3240                 mp = ccp->cc_loaded;
3241                 pmp = ccp->cc_ploaded;
3242                 rounds = ccp->cc_rounds;
3243                 prounds = ccp->cc_prounds;
3244                 ccp->cc_loaded = NULL;
3245                 ccp->cc_ploaded = NULL;
3246                 ccp->cc_rounds = -1;
3247                 ccp->cc_prounds = -1;
3248                 ccp->cc_magsize = 0;
3249                 mutex_exit(&ccp->cc_lock);
3250 
3251                 if (mp)
3252                         kmem_magazine_destroy(cp, mp, rounds);
3253                 if (pmp)
3254                         kmem_magazine_destroy(cp, pmp, prounds);
3255         }
3256 
3257         kmem_depot_ws_zero(cp);







3258         kmem_depot_ws_reap(cp);
3259 }
3260 
3261 /*
3262  * Enable per-cpu magazines on a cache.
3263  */
3264 static void
3265 kmem_cache_magazine_enable(kmem_cache_t *cp)
3266 {
3267         int cpu_seqid;
3268 
3269         if (cp->cache_flags & KMF_NOMAGAZINE)
3270                 return;
3271 
3272         for (cpu_seqid = 0; cpu_seqid < max_ncpus; cpu_seqid++) {
3273                 kmem_cpu_cache_t *ccp = &cp->cache_cpu[cpu_seqid];
3274                 mutex_enter(&ccp->cc_lock);
3275                 ccp->cc_magsize = cp->cache_magtype->mt_magsize;
3276                 mutex_exit(&ccp->cc_lock);
3277         }
3278 
3279 }
3280 
3281 /*
3282  * Reap (almost) everything right now.

3283  */
3284 void
3285 kmem_cache_reap_now(kmem_cache_t *cp)
3286 {
3287         ASSERT(list_link_active(&cp->cache_link));
3288 
3289         kmem_depot_ws_zero(cp);

3290 
3291         (void) taskq_dispatch(kmem_taskq,
3292             (task_func_t *)kmem_depot_ws_reap, cp, TQ_SLEEP);
3293         taskq_wait(kmem_taskq);
3294 }
3295 
3296 /*
3297  * Recompute a cache's magazine size.  The trade-off is that larger magazines
3298  * provide a higher transfer rate with the depot, while smaller magazines
3299  * reduce memory consumption.  Magazine resizing is an expensive operation;
3300  * it should not be done frequently.
3301  *
3302  * Changes to the magazine size are serialized by the kmem_taskq lock.
3303  *
3304  * Note: at present this only grows the magazine size.  It might be useful
3305  * to allow shrinkage too.
3306  */
3307 static void
3308 kmem_cache_magazine_resize(kmem_cache_t *cp)
3309 {