5045 use atomic_{inc,dec}_* instead of atomic_add_*

   1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  */
  25 
  26 #ifndef _VM_HAT_I86_H
  27 #define _VM_HAT_I86_H
  28 
  29 
  30 #ifdef  __cplusplus
  31 extern "C" {
  32 #endif
  33 
  34 /*
  35  * VM - Hardware Address Translation management.
  36  *
  37  * This file describes the contents of the x86_64 HAT data structures.
  38  */
  39 #include <sys/types.h>
  40 #include <sys/t_lock.h>
  41 #include <sys/cpuvar.h>
  42 #include <sys/x_call.h>
  43 #include <vm/seg.h>
  44 #include <vm/page.h>
  45 #include <sys/vmparam.h>
  46 #include <sys/vm_machparam.h>
  47 #include <sys/promif.h>
  48 #include <vm/hat_pte.h>
  49 #include <vm/htable.h>
  50 #include <vm/hment.h>
  51 
  52 /*
  53  * The essential data types involved:
  54  *
  55  * htable_t     - There is one of these for each page table and it is used
  56  *              by the HAT to manage the page table.
  57  *
  58  * hment_t      - Links together multiple PTEs to a single page.
  59  */
  60 
  61 /*
  62  * VLP processes have a 32 bit address range, so their top level is 2 and
  63  * with only 4 PTEs in that table.
  64  */
  65 #define VLP_LEVEL       (2)
  66 #define VLP_NUM_PTES    (4)
  67 #define VLP_SIZE        (VLP_NUM_PTES * sizeof (x86pte_t))
  68 #define TOP_LEVEL(h)    (((h)->hat_flags & HAT_VLP) ? VLP_LEVEL : mmu.max_level)
  69 #define VLP_COPY(fromptep, toptep) { \
  70         toptep[0] = fromptep[0]; \
  71         toptep[1] = fromptep[1]; \
  72         toptep[2] = fromptep[2]; \
  73         toptep[3] = fromptep[3]; \
  74 }
  75 
  76 /*
  77  * The hat struct exists for each address space.
  78  */
  79 struct hat {
  80         kmutex_t        hat_mutex;
  81         struct as       *hat_as;
  82         uint_t          hat_stats;
  83         pgcnt_t         hat_pages_mapped[MAX_PAGE_LEVEL + 1];
  84         pgcnt_t         hat_ism_pgcnt;
  85         cpuset_t        hat_cpus;
  86         uint16_t        hat_flags;
  87         htable_t        *hat_htable;    /* top level htable */
  88         struct hat      *hat_next;
  89         struct hat      *hat_prev;
  90         uint_t          hat_num_hash;   /* number of htable hash buckets */
  91         htable_t        **hat_ht_hash;  /* htable hash buckets */
  92         htable_t        *hat_ht_cached; /* cached free htables */
  93         x86pte_t        hat_vlp_ptes[VLP_NUM_PTES];
  94 #if defined(__amd64) && defined(__xpv)
  95         pfn_t           hat_user_ptable; /* alt top ptable for user mode */
  96 #endif
  97 };
  98 typedef struct hat hat_t;
  99 
 100 #define PGCNT_INC(hat, level)   \
 101         atomic_add_long(&(hat)->hat_pages_mapped[level], 1);
 102 #define PGCNT_DEC(hat, level)   \
 103         atomic_add_long(&(hat)->hat_pages_mapped[level], -1);
 104 
 105 /*
 106  * Flags for the hat_flags field
 107  *
 108  * HAT_FREEING - set when HAT is being destroyed - mostly used to detect that
 109  *      demap()s can be avoided.
 110  *
 111  * HAT_VLP - indicates a 32 bit process has a virtual address range less than
 112  *      the hardware's physical address range. (VLP->Virtual Less-than Physical)
 113  *      Note - never used on the hypervisor.
 114  *
 115  * HAT_VICTIM - This is set while a hat is being examined for page table
 116  *      stealing and prevents it from being freed.
 117  *
 118  * HAT_SHARED - The hat has exported it's page tables via hat_share()
 119  *
 120  * HAT_PINNED - On the hypervisor, indicates the top page table has been pinned.
 121  */
 122 #define HAT_FREEING     (0x0001)
 123 #define HAT_VLP         (0x0002)
 124 #define HAT_VICTIM      (0x0004)
 125 #define HAT_SHARED      (0x0008)
 126 #define HAT_PINNED      (0x0010)
 127 
 128 /*
 129  * Additional platform attribute for hat_devload() to force no caching.
 130  */
 131 #define HAT_PLAT_NOCACHE        (0x100000)
 132 
 133 /*
 134  * Simple statistics for the HAT. These are just counters that are
 135  * atomically incremented. They can be reset directly from the kernel
 136  * debugger.
 137  */
 138 struct hatstats {
 139         ulong_t hs_reap_attempts;
 140         ulong_t hs_reaped;
 141         ulong_t hs_steals;
 142         ulong_t hs_ptable_allocs;
 143         ulong_t hs_ptable_frees;
 144         ulong_t hs_htable_rgets;        /* allocs from reserve */
 145         ulong_t hs_htable_rputs;        /* putbacks to reserve */
 146         ulong_t hs_htable_shared;       /* number of htables shared */
 147         ulong_t hs_htable_unshared;     /* number of htables unshared */
 148         ulong_t hs_hm_alloc;
 149         ulong_t hs_hm_free;
 150         ulong_t hs_hm_put_reserve;
 151         ulong_t hs_hm_get_reserve;
 152         ulong_t hs_hm_steals;
 153         ulong_t hs_hm_steal_exam;
 154         ulong_t hs_tlb_inval_delayed;
 155 };
 156 extern struct hatstats hatstat;
 157 #ifdef DEBUG
 158 #define HATSTAT_INC(x)  (++hatstat.x)
 159 #else
 160 #define HATSTAT_INC(x)  (0)
 161 #endif
 162 
 163 #if defined(_KERNEL)
 164 
 165 /*
 166  * Useful macro to align hat_XXX() address arguments to a page boundary
 167  */
 168 #define ALIGN2PAGE(a)           ((uintptr_t)(a) & MMU_PAGEMASK)
 169 #define IS_PAGEALIGNED(a)       (((uintptr_t)(a) & MMU_PAGEOFFSET) == 0)
 170 
 171 extern uint_t   khat_running;   /* set at end of hat_kern_setup() */
 172 extern cpuset_t khat_cpuset;    /* cpuset for kernal address demap Xcalls */
 173 extern kmutex_t hat_list_lock;
 174 extern kcondvar_t hat_list_cv;
 175 
 176 
 177 
 178 /*
 179  * Interfaces to setup a cpu private mapping (ie. preemption disabled).
 180  * The attr and flags arguments are the same as for hat_devload().
 181  * setup() must be called once, then any number of calls to remap(),
 182  * followed by a final call to release()
 183  *
 184  * Used by ppcopy(), page_zero(), the memscrubber, and the kernel debugger.
 185  */
 186 typedef paddr_t hat_mempte_t;                           /* phys addr of PTE */
 187 extern hat_mempte_t hat_mempte_setup(caddr_t addr);
 188 extern void hat_mempte_remap(pfn_t, caddr_t, hat_mempte_t,
 189         uint_t attr, uint_t flags);
 190 extern void hat_mempte_release(caddr_t addr, hat_mempte_t);
 191 
 192 /*
 193  * Interfaces to manage which thread has access to htable and hment reserves.
 194  * The USE_HAT_RESERVES macro should always be recomputed in full. Its value
 195  * (due to curthread) can change after any call into kmem/vmem.
 196  */
 197 extern uint_t can_steal_post_boot;
 198 extern uint_t use_boot_reserve;
 199 #define USE_HAT_RESERVES()                                      \
 200         (use_boot_reserve || curthread->t_hatdepth > 1 || \
 201         panicstr != NULL || vmem_is_populator())
 202 
 203 /*
 204  * initialization stuff needed by by startup, mp_startup...
 205  */
 206 extern void hat_cpu_online(struct cpu *);
 207 extern void hat_cpu_offline(struct cpu *);
 208 extern void setup_vaddr_for_ppcopy(struct cpu *);
 209 extern void teardown_vaddr_for_ppcopy(struct cpu *);
 210 extern void clear_boot_mappings(uintptr_t, uintptr_t);
 211 
 212 /*
 213  * magic value to indicate that all TLB entries should be demapped.
 214  */
 215 #define DEMAP_ALL_ADDR  (~(uintptr_t)0)
 216 
 217 /*
 218  * not in any include file???
 219  */
 220 extern void halt(char *fmt);
 221 
 222 /*
 223  * x86 specific routines for use online in setup or i86pc/vm files
 224  */
 225 extern void hat_kern_alloc(caddr_t segmap_base, size_t segmap_size,
 226         caddr_t ekernelheap);
 227 extern void hat_kern_setup(void);
 228 extern void hat_tlb_inval(struct hat *hat, uintptr_t va);
 229 extern void hat_pte_unmap(htable_t *ht, uint_t entry, uint_t flags,
 230         x86pte_t old_pte, void *pte_ptr);
 231 extern void hat_init_finish(void);
 232 extern caddr_t hat_kpm_pfn2va(pfn_t pfn);
 233 extern pfn_t hat_kpm_va2pfn(caddr_t);
 234 extern page_t *hat_kpm_vaddr2page(caddr_t);
 235 extern uintptr_t hat_kernelbase(uintptr_t);
 236 extern void hat_kmap_init(uintptr_t base, size_t len);
 237 
 238 extern hment_t *hati_page_unmap(page_t *pp, htable_t *ht, uint_t entry);
 239 
 240 #if !defined(__xpv)
 241 /*
 242  * routines to deal with delayed TLB invalidations for idle CPUs
 243  */
 244 extern void tlb_going_idle(void);
 245 extern void tlb_service(void);
 246 #endif
 247 
 248 /*
 249  * Hat switch function invoked to load a new context into %cr3
 250  */
 251 extern void hat_switch(struct hat *hat);
 252 
 253 #ifdef __xpv
 254 /*
 255  * Interfaces to use around code that maps/unmaps grant table references.
 256  */
 257 extern void hat_prepare_mapping(hat_t *, caddr_t, uint64_t *);
 258 extern void hat_release_mapping(hat_t *, caddr_t);
 259 
 260 #define XPV_DISALLOW_MIGRATE()  xen_block_migrate()
 261 #define XPV_ALLOW_MIGRATE()     xen_allow_migrate()
 262 
 263 #else
 264 
 265 #define XPV_DISALLOW_MIGRATE()  /* nothing */
 266 #define XPV_ALLOW_MIGRATE()     /* nothing */
 267 
 268 #define pfn_is_foreign(pfn)     __lintzero
 269 
 270 #endif
 271 
 272 
 273 #endif  /* _KERNEL */
 274 
 275 #ifdef  __cplusplus
 276 }
 277 #endif
 278 
 279 #endif  /* _VM_HAT_I86_H */
--- EOF ---