1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 #include <sys/systm.h>
27 #include <sys/sysmacros.h>
28 #include <sys/bootconf.h>
29 #include <sys/atomic.h>
30 #include <sys/lgrp.h>
31 #include <sys/memlist.h>
32 #include <sys/memnode.h>
33 #include <sys/platform_module.h>
34 #include <vm/vm_dep.h>
35
36 int max_mem_nodes = 1;
37
38 struct mem_node_conf mem_node_config[MAX_MEM_NODES];
39 int mem_node_pfn_shift;
40 /*
41 * num_memnodes should be updated atomically and always >=
42 * the number of bits in memnodes_mask or the algorithm may fail.
43 */
44 uint16_t num_memnodes;
45 mnodeset_t memnodes_mask; /* assumes 8*(sizeof(mnodeset_t)) >= MAX_MEM_NODES */
46
47 /*
48 * If set, mem_node_physalign should be a power of two, and
49 * should reflect the minimum address alignment of each node.
50 */
51 uint64_t mem_node_physalign;
52
53 /*
54 * Platform hooks we will need.
55 */
56
57 #pragma weak plat_build_mem_nodes
58 #pragma weak plat_slice_add
59 #pragma weak plat_slice_del
60
61 /*
62 * Adjust the memnode config after a DR operation.
63 *
64 * It is rather tricky to do these updates since we can't
65 * protect the memnode structures with locks, so we must
66 * be mindful of the order in which updates and reads to
67 * these values can occur.
68 */
69
70 void
71 mem_node_add_slice(pfn_t start, pfn_t end)
72 {
73 int mnode;
74 mnodeset_t newmask, oldmask;
75
76 /*
77 * DR will pass us the first pfn that is allocatable.
78 * We need to round down to get the real start of
79 * the slice.
80 */
81 if (mem_node_physalign) {
82 start &= ~(btop(mem_node_physalign) - 1);
83 end = roundup(end, btop(mem_node_physalign)) - 1;
84 }
85
86 mnode = PFN_2_MEM_NODE(start);
87 ASSERT(mnode >= 0 && mnode < max_mem_nodes);
88
89 if (cas32((uint32_t *)&mem_node_config[mnode].exists, 0, 1)) {
90 /*
91 * Add slice to existing node.
92 */
93 if (start < mem_node_config[mnode].physbase)
94 mem_node_config[mnode].physbase = start;
95 if (end > mem_node_config[mnode].physmax)
96 mem_node_config[mnode].physmax = end;
97 } else {
98 mem_node_config[mnode].physbase = start;
99 mem_node_config[mnode].physmax = end;
100 atomic_add_16(&num_memnodes, 1);
101 do {
102 oldmask = memnodes_mask;
103 newmask = memnodes_mask | (1ull << mnode);
104 } while (cas64(&memnodes_mask, oldmask, newmask) != oldmask);
105 }
106
107 /*
108 * Inform the common lgrp framework about the new memory
109 */
110 lgrp_config(LGRP_CONFIG_MEM_ADD, mnode, MEM_NODE_2_LGRPHAND(mnode));
111 }
112
113 /*
114 * Remove a PFN range from a memnode. On some platforms,
115 * the memnode will be created with physbase at the first
116 * allocatable PFN, but later deleted with the MC slice
117 * base address converted to a PFN, in which case we need
118 * to assume physbase and up.
119 */
120 void
121 mem_node_del_slice(pfn_t start, pfn_t end)
122 {
123 int mnode;
124 pgcnt_t delta_pgcnt, node_size;
125 mnodeset_t omask, nmask;
126
127 if (mem_node_physalign) {
128 start &= ~(btop(mem_node_physalign) - 1);
129 end = roundup(end, btop(mem_node_physalign)) - 1;
130 }
131 mnode = PFN_2_MEM_NODE(start);
132
133 ASSERT(mnode >= 0 && mnode < max_mem_nodes);
134 ASSERT(mem_node_config[mnode].exists == 1);
135
136 delta_pgcnt = end - start;
137 node_size = mem_node_config[mnode].physmax -
138 mem_node_config[mnode].physbase;
139
140 if (node_size > delta_pgcnt) {
141 /*
142 * Subtract the slice from the memnode.
143 */
144 if (start <= mem_node_config[mnode].physbase)
145 mem_node_config[mnode].physbase = end + 1;
146 ASSERT(end <= mem_node_config[mnode].physmax);
147 if (end == mem_node_config[mnode].physmax)
148 mem_node_config[mnode].physmax = start - 1;
149 } else {
150 /*
151 * Let the common lgrp framework know this mnode is
152 * leaving
153 */
154 lgrp_config(LGRP_CONFIG_MEM_DEL,
155 mnode, MEM_NODE_2_LGRPHAND(mnode));
156
157 /*
158 * Delete the whole node.
159 */
160 ASSERT(MNODE_PGCNT(mnode) == 0);
161 do {
162 omask = memnodes_mask;
163 nmask = omask & ~(1ull << mnode);
164 } while (cas64(&memnodes_mask, omask, nmask) != omask);
165 atomic_add_16(&num_memnodes, -1);
166 mem_node_config[mnode].exists = 0;
167 }
168 }
169
170 void
171 mem_node_add_range(pfn_t start, pfn_t end)
172 {
173 if (&plat_slice_add)
174 plat_slice_add(start, end);
175 else
176 mem_node_add_slice(start, end);
177 }
178
179 void
180 mem_node_del_range(pfn_t start, pfn_t end)
181 {
182 if (&plat_slice_del)
183 plat_slice_del(start, end);
184 else
185 mem_node_del_slice(start, end);
186 }
187
188 void
189 startup_build_mem_nodes(struct memlist *list)
190 {
191 pfn_t start, end;
192
193 /* LINTED: ASSERT will always true or false */
194 ASSERT(NBBY * sizeof (mnodeset_t) >= max_mem_nodes);
195
196 if (&plat_build_mem_nodes) {
197 plat_build_mem_nodes(list);
198 } else {
199 /*
200 * Boot install lists are arranged <addr, len>, ...
201 */
202 while (list) {
203 start = list->ml_address >> PAGESHIFT;
204 if (start > physmax)
205 continue;
206 end =
207 (list->ml_address + list->ml_size - 1) >> PAGESHIFT;
208 if (end > physmax)
209 end = physmax;
210 mem_node_add_range(start, end);
211 list = list->ml_next;
212 }
213 mem_node_physalign = 0;
214 mem_node_pfn_shift = 0;
215 }
216 }
217
218 /*
219 * Allocate an unassigned memnode.
220 */
221 int
222 mem_node_alloc()
223 {
224 int mnode;
225 mnodeset_t newmask, oldmask;
226
227 /*
228 * Find an unused memnode. Update it atomically to prevent
229 * a first time memnode creation race.
230 */
231 for (mnode = 0; mnode < max_mem_nodes; mnode++)
232 if (cas32((uint32_t *)&mem_node_config[mnode].exists,
233 0, 1) == 0)
234 break;
235
236 if (mnode >= max_mem_nodes)
237 panic("Out of free memnodes\n");
238
239 mem_node_config[mnode].physbase = (pfn_t)-1l;
240 mem_node_config[mnode].physmax = 0;
241 atomic_add_16(&num_memnodes, 1);
242 do {
243 oldmask = memnodes_mask;
244 newmask = memnodes_mask | (1ull << mnode);
245 } while (cas64(&memnodes_mask, oldmask, newmask) != oldmask);
246
247 return (mnode);
248 }
249
250 /*
251 * Find the intersection between a memnode and a memlist
252 * and returns the number of pages that overlap.
253 *
254 * Assumes the list is protected from DR operations by
255 * the memlist lock.
256 */
257 pgcnt_t
258 mem_node_memlist_pages(int mnode, struct memlist *mlist)
259 {
260 pfn_t base, end;
261 pfn_t cur_base, cur_end;
262 pgcnt_t npgs;
263 struct memlist *pmem;
264
265 base = mem_node_config[mnode].physbase;
266 end = mem_node_config[mnode].physmax;
267 npgs = 0;
268
269 memlist_read_lock();
270
271 for (pmem = mlist; pmem; pmem = pmem->ml_next) {
272 cur_base = btop(pmem->ml_address);
273 cur_end = cur_base + btop(pmem->ml_size) - 1;
274 if (end < cur_base || base > cur_end)
275 continue;
276 npgs = npgs + (MIN(cur_end, end) -
277 MAX(cur_base, base)) + 1;
278 }
279
280 memlist_read_unlock();
281
282 return (npgs);
283 }