1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License, Version 1.0 only
   6  * (the "License").  You may not use this file except in compliance
   7  * with the License.
   8  *
   9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
  10  * or http://www.opensolaris.org/os/licensing.
  11  * See the License for the specific language governing permissions
  12  * and limitations under the License.
  13  *
  14  * When distributing Covered Code, include this CDDL HEADER in each
  15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  16  * If applicable, add the following below this CDDL HEADER, with the
  17  * fields enclosed by brackets "[]" replaced with your own identifying
  18  * information: Portions Copyright [yyyy] [name of copyright owner]
  19  *
  20  * CDDL HEADER END
  21  */
  22 /*
  23  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 #pragma ident   "%Z%%M% %I%     %E% SMI"
  28 
  29 /*
  30  * Kernel Physical Mapping (kpm) segment driver (segkpm).
  31  *
  32  * This driver delivers along with the hat_kpm* interfaces an alternative
  33  * mechanism for kernel mappings within the 64-bit Solaris operating system,
  34  * which allows the mapping of all physical memory into the kernel address
  35  * space at once. This is feasible in 64 bit kernels, e.g. for Ultrasparc II
  36  * and beyond processors, since the available VA range is much larger than
  37  * possible physical memory. Momentarily all physical memory is supported,
  38  * that is represented by the list of memory segments (memsegs).
  39  *
  40  * Segkpm mappings have also very low overhead and large pages are used
  41  * (when possible) to minimize the TLB and TSB footprint. It is also
  42  * extentable for other than Sparc architectures (e.g. AMD64). Main
  43  * advantage is the avoidance of the TLB-shootdown X-calls, which are
  44  * normally needed when a kernel (global) mapping has to be removed.
  45  *
  46  * First example of a kernel facility that uses the segkpm mapping scheme
  47  * is seg_map, where it is used as an alternative to hat_memload().
  48  * See also hat layer for more information about the hat_kpm* routines.
  49  * The kpm facilty can be turned off at boot time (e.g. /etc/system).
  50  */
  51 
  52 #include <sys/types.h>
  53 #include <sys/param.h>
  54 #include <sys/sysmacros.h>
  55 #include <sys/systm.h>
  56 #include <sys/vnode.h>
  57 #include <sys/cmn_err.h>
  58 #include <sys/debug.h>
  59 #include <sys/thread.h>
  60 #include <sys/cpuvar.h>
  61 #include <sys/bitmap.h>
  62 #include <sys/atomic.h>
  63 #include <sys/lgrp.h>
  64 
  65 #include <vm/seg_kmem.h>
  66 #include <vm/seg_kpm.h>
  67 #include <vm/hat.h>
  68 #include <vm/as.h>
  69 #include <vm/seg.h>
  70 #include <vm/page.h>
  71 
  72 /*
  73  * Global kpm controls.
  74  * See also platform and mmu specific controls.
  75  *
  76  * kpm_enable -- global on/off switch for segkpm.
  77  * . Set by default on 64bit platforms that have kpm support.
  78  * . Will be disabled from platform layer if not supported.
  79  * . Can be disabled via /etc/system.
  80  *
  81  * kpm_smallpages -- use only regular/system pagesize for kpm mappings.
  82  * . Can be useful for critical debugging of kpm clients.
  83  * . Set to zero by default for platforms that support kpm large pages.
  84  *   The use of kpm large pages reduces the footprint of kpm meta data
  85  *   and has all the other advantages of using large pages (e.g TLB
  86  *   miss reduction).
  87  * . Set by default for platforms that don't support kpm large pages or
  88  *   where large pages cannot be used for other reasons (e.g. there are
  89  *   only few full associative TLB entries available for large pages).
  90  *
  91  * segmap_kpm -- separate on/off switch for segmap using segkpm:
  92  * . Set by default.
  93  * . Will be disabled when kpm_enable is zero.
  94  * . Will be disabled when MAXBSIZE != PAGESIZE.
  95  * . Can be disabled via /etc/system.
  96  *
  97  */
  98 int kpm_enable = 1;
  99 int kpm_smallpages = 0;
 100 int segmap_kpm = 1;
 101 
 102 /*
 103  * Private seg op routines.
 104  */
 105 faultcode_t segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr,
 106                         size_t len, enum fault_type type, enum seg_rw rw);
 107 static void     segkpm_dump(struct seg *);
 108 static void     segkpm_badop(void);
 109 static int      segkpm_notsup(void);
 110 static int      segkpm_capable(struct seg *, segcapability_t);
 111 
 112 #define SEGKPM_BADOP(t) (t(*)())segkpm_badop
 113 #define SEGKPM_NOTSUP   (int(*)())segkpm_notsup
 114 
 115 static struct seg_ops segkpm_ops = {
 116         SEGKPM_BADOP(int),      /* dup */
 117         SEGKPM_BADOP(int),      /* unmap */
 118         SEGKPM_BADOP(void),     /* free */
 119         segkpm_fault,
 120         SEGKPM_BADOP(int),      /* faulta */
 121         SEGKPM_BADOP(int),      /* setprot */
 122         SEGKPM_BADOP(int),      /* checkprot */
 123         SEGKPM_BADOP(int),      /* kluster */
 124         SEGKPM_BADOP(size_t),   /* swapout */
 125         SEGKPM_BADOP(int),      /* sync */
 126         SEGKPM_BADOP(size_t),   /* incore */
 127         SEGKPM_BADOP(int),      /* lockop */
 128         SEGKPM_BADOP(int),      /* getprot */
 129         SEGKPM_BADOP(u_offset_t), /* getoffset */
 130         SEGKPM_BADOP(int),      /* gettype */
 131         SEGKPM_BADOP(int),      /* getvp */
 132         SEGKPM_BADOP(int),      /* advise */
 133         segkpm_dump,            /* dump */
 134         SEGKPM_NOTSUP,          /* pagelock */
 135         SEGKPM_BADOP(int),      /* setpgsz */
 136         SEGKPM_BADOP(int),      /* getmemid */
 137         SEGKPM_BADOP(lgrp_mem_policy_info_t *), /* getpolicy */
 138         segkpm_capable,         /* capable */
 139 };
 140 
 141 /*
 142  * kpm_pgsz and kpm_pgshft are set by platform layer.
 143  */
 144 size_t          kpm_pgsz;       /* kpm page size */
 145 uint_t          kpm_pgshft;     /* kpm page shift */
 146 u_offset_t      kpm_pgoff;      /* kpm page offset mask */
 147 uint_t          kpmp2pshft;     /* kpm page to page shift */
 148 pgcnt_t         kpmpnpgs;       /* how many pages per kpm page */
 149 
 150 
 151 #ifdef  SEGKPM_SUPPORT
 152 
 153 int
 154 segkpm_create(struct seg *seg, void *argsp)
 155 {
 156         struct segkpm_data *skd;
 157         struct segkpm_crargs *b = (struct segkpm_crargs *)argsp;
 158         ushort_t *p;
 159         int i, j;
 160 
 161         ASSERT(seg->s_as && RW_WRITE_HELD(&seg->s_as->a_lock));
 162         ASSERT(btokpmp(seg->s_size) >= 1 &&
 163                 kpmpageoff((uintptr_t)seg->s_base) == 0 &&
 164                 kpmpageoff((uintptr_t)seg->s_base + seg->s_size) == 0);
 165 
 166         skd = kmem_zalloc(sizeof (struct segkpm_data), KM_SLEEP);
 167 
 168         seg->s_data = (void *)skd;
 169         seg->s_ops = &segkpm_ops;
 170         skd->skd_prot = b->prot;
 171 
 172         /*
 173          * (1) Segkpm virtual addresses are based on physical adresses.
 174          * From this and in opposite to other segment drivers it is
 175          * often required to allocate a page first to be able to
 176          * calculate the final segkpm virtual address.
 177          * (2) Page  allocation is done by calling page_create_va(),
 178          * one important input argument is a virtual address (also
 179          * expressed by the "va" in the function name). This function
 180          * is highly optimized to select the right page for an optimal
 181          * processor and platform support (e.g. virtual addressed
 182          * caches (VAC), physical addressed caches, NUMA).
 183          *
 184          * Because of (1) the approach is to generate a faked virtual
 185          * address for calling page_create_va(). In order to exploit
 186          * the abilities of (2), especially to utilize the cache
 187          * hierarchy (3) and to avoid VAC alias conflicts (4) the
 188          * selection has to be done carefully. For each virtual color
 189          * a separate counter is provided (4). The count values are
 190          * used for the utilization of all cache lines (3) and are
 191          * corresponding to the cache bins.
 192          */
 193         skd->skd_nvcolors = b->nvcolors;
 194 
 195         p = skd->skd_va_select =
 196                 kmem_zalloc(NCPU * b->nvcolors * sizeof (ushort_t), KM_SLEEP);
 197 
 198         for (i = 0; i < NCPU; i++)
 199                 for (j = 0; j < b->nvcolors; j++, p++)
 200                         *p = j;
 201 
 202         return (0);
 203 }
 204 
 205 /*
 206  * This routine is called via a machine specific fault handling
 207  * routine.
 208  */
 209 /* ARGSUSED */
 210 faultcode_t
 211 segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len,
 212         enum fault_type type, enum seg_rw rw)
 213 {
 214         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
 215 
 216         switch (type) {
 217         case F_INVAL:
 218                 return (hat_kpm_fault(hat, addr));
 219         case F_SOFTLOCK:
 220         case F_SOFTUNLOCK:
 221                 return (0);
 222         default:
 223                 return (FC_NOSUPPORT);
 224         }
 225         /*NOTREACHED*/
 226 }
 227 
 228 #define addr_to_vcolor(addr, vcolors) \
 229         ((int)(((uintptr_t)(addr) & ((vcolors << PAGESHIFT) - 1)) >> PAGESHIFT))
 230 
 231 /*
 232  * Create a virtual address that can be used for invocations of
 233  * page_create_va. Goal is to utilize the cache hierarchy (round
 234  * robin bins) and to select the right color for virtual indexed
 235  * caches. It isn't exact since we also increment the bin counter
 236  * when the caller uses VOP_GETPAGE and gets a hit in the page
 237  * cache, but we keep the bins turning for cache distribution
 238  * (see also segkpm_create block comment).
 239  */
 240 caddr_t
 241 segkpm_create_va(u_offset_t off)
 242 {
 243         int vcolor;
 244         ushort_t *p;
 245         struct segkpm_data *skd = (struct segkpm_data *)segkpm->s_data;
 246         int nvcolors = skd->skd_nvcolors;
 247         caddr_t va;
 248 
 249         vcolor = (nvcolors > 1) ? addr_to_vcolor(off, nvcolors) : 0;
 250         p = &skd->skd_va_select[(CPU->cpu_id * nvcolors) + vcolor];
 251         va = (caddr_t)ptob(*p);
 252 
 253         atomic_add_16(p, nvcolors);
 254 
 255         return (va);
 256 }
 257 
 258 /*
 259  * Unload mapping if the instance has an active kpm mapping.
 260  */
 261 void
 262 segkpm_mapout_validkpme(struct kpme *kpme)
 263 {
 264         caddr_t vaddr;
 265         page_t *pp;
 266 
 267 retry:
 268         if ((pp = kpme->kpe_page) == NULL) {
 269                 return;
 270         }
 271 
 272         if (page_lock(pp, SE_SHARED, (kmutex_t *)NULL, P_RECLAIM) == 0)
 273                 goto retry;
 274 
 275         /*
 276          * Check if segkpm mapping is not unloaded in the meantime
 277          */
 278         if (kpme->kpe_page == NULL) {
 279                 page_unlock(pp);
 280                 return;
 281         }
 282 
 283         vaddr = hat_kpm_page2va(pp, 1);
 284         hat_kpm_mapout(pp, kpme, vaddr);
 285         page_unlock(pp);
 286 }
 287 
 288 static void
 289 segkpm_badop()
 290 {
 291         panic("segkpm_badop");
 292 }
 293 
 294 #else   /* SEGKPM_SUPPORT */
 295 
 296 /* segkpm stubs */
 297 
 298 /*ARGSUSED*/
 299 int segkpm_create(struct seg *seg, void *argsp) { return (0); }
 300 
 301 /* ARGSUSED */
 302 faultcode_t
 303 segkpm_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len,
 304         enum fault_type type, enum seg_rw rw)
 305 {
 306         return ((faultcode_t)0);
 307 }
 308 
 309 /* ARGSUSED */
 310 caddr_t segkpm_create_va(u_offset_t off) { return (NULL); }
 311 
 312 /* ARGSUSED */
 313 void segkpm_mapout_validkpme(struct kpme *kpme) {}
 314 
 315 static void
 316 segkpm_badop() {}
 317 
 318 #endif  /* SEGKPM_SUPPORT */
 319 
 320 static int
 321 segkpm_notsup()
 322 {
 323         return (ENOTSUP);
 324 }
 325 
 326 /*
 327  * segkpm pages are not dumped, so we just return
 328  */
 329 /*ARGSUSED*/
 330 static void
 331 segkpm_dump(struct seg *seg)
 332 {}
 333 
 334 /*
 335  * We claim to have no special capabilities.
 336  */
 337 /*ARGSUSED*/
 338 static int
 339 segkpm_capable(struct seg *seg, segcapability_t capability)
 340 {
 341         return (0);
 342 }