1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2013, Joyent, Inc. All rights reserved.
25 */
26
27 #include <sys/types.h>
28 #include <sys/param.h>
29 #include <sys/sysmacros.h>
30 #include <sys/cred.h>
31 #include <sys/proc.h>
32 #include <sys/strsubr.h>
33 #include <sys/priocntl.h>
34 #include <sys/class.h>
35 #include <sys/disp.h>
36 #include <sys/procset.h>
37 #include <sys/debug.h>
38 #include <sys/kmem.h>
39 #include <sys/errno.h>
40 #include <sys/systm.h>
41 #include <sys/schedctl.h>
42 #include <sys/vmsystm.h>
43 #include <sys/atomic.h>
44 #include <sys/project.h>
45 #include <sys/modctl.h>
46 #include <sys/fss.h>
47 #include <sys/fsspriocntl.h>
48 #include <sys/cpupart.h>
49 #include <sys/zone.h>
50 #include <vm/rm.h>
51 #include <vm/seg_kmem.h>
52 #include <sys/tnf_probe.h>
53 #include <sys/policy.h>
54 #include <sys/sdt.h>
55 #include <sys/cpucaps.h>
56
57 /*
58 * FSS Data Structures:
59 *
60 * fsszone
61 * ----- -----
62 * ----- | | | |
63 * | |-------->| |<------->| |<---->...
64 * | | ----- -----
65 * | | ^ ^ ^
66 * | |--- | \ \
67 * ----- | | \ \
68 * fsspset | | \ \
69 * | | \ \
70 * | ----- ----- -----
71 * -->| |<--->| |<--->| |
72 * | | | | | |
73 * ----- ----- -----
74 * fssproj
75 *
76 *
77 * That is, fsspsets contain a list of fsszone's that are currently active in
78 * the pset, and a list of fssproj's, corresponding to projects with runnable
79 * threads on the pset. fssproj's in turn point to the fsszone which they
80 * are a member of.
81 *
82 * An fssproj_t is removed when there are no threads in it.
83 *
84 * An fsszone_t is removed when there are no projects with threads in it.
85 *
86 * Projects in a zone compete with each other for cpu time, receiving cpu
87 * allocation within a zone proportional to fssproj->fssp_shares
88 * (project.cpu-shares); at a higher level zones compete with each other,
89 * receiving allocation in a pset proportional to fsszone->fssz_shares
90 * (zone.cpu-shares). See fss_decay_usage() for the precise formula.
91 */
92
93 static pri_t fss_init(id_t, int, classfuncs_t **);
94
95 static struct sclass fss = {
96 "FSS",
97 fss_init,
98 0
99 };
100
101 extern struct mod_ops mod_schedops;
102
103 /*
104 * Module linkage information for the kernel.
105 */
106 static struct modlsched modlsched = {
107 &mod_schedops, "fair share scheduling class", &fss
108 };
109
110 static struct modlinkage modlinkage = {
111 MODREV_1, (void *)&modlsched, NULL
112 };
113
114 #define FSS_MAXUPRI 60
115
116 /*
117 * The fssproc_t structures are kept in an array of circular doubly linked
118 * lists. A hash on the thread pointer is used to determine which list each
119 * thread should be placed in. Each list has a dummy "head" which is never
120 * removed, so the list is never empty. fss_update traverses these lists to
121 * update the priorities of threads that have been waiting on the run queue.
122 */
123 #define FSS_LISTS 16 /* number of lists, must be power of 2 */
124 #define FSS_LIST_HASH(t) (((uintptr_t)(t) >> 9) & (FSS_LISTS - 1))
125 #define FSS_LIST_NEXT(i) (((i) + 1) & (FSS_LISTS - 1))
126
127 #define FSS_LIST_INSERT(fssproc) \
128 { \
129 int index = FSS_LIST_HASH(fssproc->fss_tp); \
130 kmutex_t *lockp = &fss_listlock[index]; \
131 fssproc_t *headp = &fss_listhead[index]; \
132 mutex_enter(lockp); \
133 fssproc->fss_next = headp->fss_next; \
134 fssproc->fss_prev = headp; \
135 headp->fss_next->fss_prev = fssproc; \
136 headp->fss_next = fssproc; \
137 mutex_exit(lockp); \
138 }
139
140 #define FSS_LIST_DELETE(fssproc) \
141 { \
142 int index = FSS_LIST_HASH(fssproc->fss_tp); \
143 kmutex_t *lockp = &fss_listlock[index]; \
144 mutex_enter(lockp); \
145 fssproc->fss_prev->fss_next = fssproc->fss_next; \
146 fssproc->fss_next->fss_prev = fssproc->fss_prev; \
147 mutex_exit(lockp); \
148 }
149
150 #define FSS_TICK_COST 1000 /* tick cost for threads with nice level = 0 */
151
152 /*
153 * Decay rate percentages are based on n/128 rather than n/100 so that
154 * calculations can avoid having to do an integer divide by 100 (divide
155 * by FSS_DECAY_BASE == 128 optimizes to an arithmetic shift).
156 *
157 * FSS_DECAY_MIN = 83/128 ~= 65%
158 * FSS_DECAY_MAX = 108/128 ~= 85%
159 * FSS_DECAY_USG = 96/128 ~= 75%
160 */
161 #define FSS_DECAY_MIN 83 /* fsspri decay pct for threads w/ nice -20 */
162 #define FSS_DECAY_MAX 108 /* fsspri decay pct for threads w/ nice +19 */
163 #define FSS_DECAY_USG 96 /* fssusage decay pct for projects */
164 #define FSS_DECAY_BASE 128 /* base for decay percentages above */
165
166 #define FSS_NICE_MIN 0
167 #define FSS_NICE_MAX (2 * NZERO - 1)
168 #define FSS_NICE_RANGE (FSS_NICE_MAX - FSS_NICE_MIN + 1)
169
170 static int fss_nice_tick[FSS_NICE_RANGE];
171 static int fss_nice_decay[FSS_NICE_RANGE];
172
173 static pri_t fss_maxupri = FSS_MAXUPRI; /* maximum FSS user priority */
174 static pri_t fss_maxumdpri; /* maximum user mode fss priority */
175 static pri_t fss_maxglobpri; /* maximum global priority used by fss class */
176 static pri_t fss_minglobpri; /* minimum global priority */
177
178 static fssproc_t fss_listhead[FSS_LISTS];
179 static kmutex_t fss_listlock[FSS_LISTS];
180
181 static fsspset_t *fsspsets;
182 static kmutex_t fsspsets_lock; /* protects fsspsets */
183
184 static id_t fss_cid;
185
186 static time_t fss_minrun = 2; /* t_pri becomes 59 within 2 secs */
187 static time_t fss_minslp = 2; /* min time on sleep queue for hardswap */
188 static int fss_quantum = 11;
189
190 static void fss_newpri(fssproc_t *);
191 static void fss_update(void *);
192 static int fss_update_list(int);
193 static void fss_change_priority(kthread_t *, fssproc_t *);
194
195 static int fss_admin(caddr_t, cred_t *);
196 static int fss_getclinfo(void *);
197 static int fss_parmsin(void *);
198 static int fss_parmsout(void *, pc_vaparms_t *);
199 static int fss_vaparmsin(void *, pc_vaparms_t *);
200 static int fss_vaparmsout(void *, pc_vaparms_t *);
201 static int fss_getclpri(pcpri_t *);
202 static int fss_alloc(void **, int);
203 static void fss_free(void *);
204
205 static int fss_enterclass(kthread_t *, id_t, void *, cred_t *, void *);
206 static void fss_exitclass(void *);
207 static int fss_canexit(kthread_t *, cred_t *);
208 static int fss_fork(kthread_t *, kthread_t *, void *);
209 static void fss_forkret(kthread_t *, kthread_t *);
210 static void fss_parmsget(kthread_t *, void *);
211 static int fss_parmsset(kthread_t *, void *, id_t, cred_t *);
212 static void fss_stop(kthread_t *, int, int);
213 static void fss_exit(kthread_t *);
214 static void fss_active(kthread_t *);
215 static void fss_inactive(kthread_t *);
216 static void fss_trapret(kthread_t *);
217 static void fss_preempt(kthread_t *);
218 static void fss_setrun(kthread_t *);
219 static void fss_sleep(kthread_t *);
220 static void fss_tick(kthread_t *);
221 static void fss_wakeup(kthread_t *);
222 static int fss_donice(kthread_t *, cred_t *, int, int *);
223 static int fss_doprio(kthread_t *, cred_t *, int, int *);
224 static pri_t fss_globpri(kthread_t *);
225 static void fss_yield(kthread_t *);
226 static void fss_nullsys();
227
228 static struct classfuncs fss_classfuncs = {
229 /* class functions */
230 fss_admin,
231 fss_getclinfo,
232 fss_parmsin,
233 fss_parmsout,
234 fss_vaparmsin,
235 fss_vaparmsout,
236 fss_getclpri,
237 fss_alloc,
238 fss_free,
239
240 /* thread functions */
241 fss_enterclass,
242 fss_exitclass,
243 fss_canexit,
244 fss_fork,
245 fss_forkret,
246 fss_parmsget,
247 fss_parmsset,
248 fss_stop,
249 fss_exit,
250 fss_active,
251 fss_inactive,
252 fss_trapret,
253 fss_preempt,
254 fss_setrun,
255 fss_sleep,
256 fss_tick,
257 fss_wakeup,
258 fss_donice,
259 fss_globpri,
260 fss_nullsys, /* set_process_group */
261 fss_yield,
262 fss_doprio,
263 };
264
265 int
266 _init()
267 {
268 return (mod_install(&modlinkage));
269 }
270
271 int
272 _fini()
273 {
274 return (EBUSY);
275 }
276
277 int
278 _info(struct modinfo *modinfop)
279 {
280 return (mod_info(&modlinkage, modinfop));
281 }
282
283 /*ARGSUSED*/
284 static int
285 fss_project_walker(kproject_t *kpj, void *buf)
286 {
287 return (0);
288 }
289
290 void *
291 fss_allocbuf(int op, int type)
292 {
293 fssbuf_t *fssbuf;
294 void **fsslist;
295 int cnt;
296 int i;
297 size_t size;
298
299 ASSERT(op == FSS_NPSET_BUF || op == FSS_NPROJ_BUF || op == FSS_ONE_BUF);
300 ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
301 ASSERT(MUTEX_HELD(&cpu_lock));
302
303 fssbuf = kmem_zalloc(sizeof (fssbuf_t), KM_SLEEP);
304 switch (op) {
305 case FSS_NPSET_BUF:
306 cnt = cpupart_list(NULL, 0, CP_NONEMPTY);
307 break;
308 case FSS_NPROJ_BUF:
309 cnt = project_walk_all(ALL_ZONES, fss_project_walker, NULL);
310 break;
311 case FSS_ONE_BUF:
312 cnt = 1;
313 break;
314 }
315
316 switch (type) {
317 case FSS_ALLOC_PROJ:
318 size = sizeof (fssproj_t);
319 break;
320 case FSS_ALLOC_ZONE:
321 size = sizeof (fsszone_t);
322 break;
323 }
324 fsslist = kmem_zalloc(cnt * sizeof (void *), KM_SLEEP);
325 fssbuf->fssb_size = cnt;
326 fssbuf->fssb_list = fsslist;
327 for (i = 0; i < cnt; i++)
328 fsslist[i] = kmem_zalloc(size, KM_SLEEP);
329 return (fssbuf);
330 }
331
332 void
333 fss_freebuf(fssbuf_t *fssbuf, int type)
334 {
335 void **fsslist;
336 int i;
337 size_t size;
338
339 ASSERT(fssbuf != NULL);
340 ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
341 fsslist = fssbuf->fssb_list;
342
343 switch (type) {
344 case FSS_ALLOC_PROJ:
345 size = sizeof (fssproj_t);
346 break;
347 case FSS_ALLOC_ZONE:
348 size = sizeof (fsszone_t);
349 break;
350 }
351
352 for (i = 0; i < fssbuf->fssb_size; i++) {
353 if (fsslist[i] != NULL)
354 kmem_free(fsslist[i], size);
355 }
356 kmem_free(fsslist, sizeof (void *) * fssbuf->fssb_size);
357 kmem_free(fssbuf, sizeof (fssbuf_t));
358 }
359
360 static fsspset_t *
361 fss_find_fsspset(cpupart_t *cpupart)
362 {
363 int i;
364 fsspset_t *fsspset = NULL;
365 int found = 0;
366
367 ASSERT(cpupart != NULL);
368 ASSERT(MUTEX_HELD(&fsspsets_lock));
369
370 /*
371 * Search for the cpupart pointer in the array of fsspsets.
372 */
373 for (i = 0; i < max_ncpus; i++) {
374 fsspset = &fsspsets[i];
375 if (fsspset->fssps_cpupart == cpupart) {
376 ASSERT(fsspset->fssps_nproj > 0);
377 found = 1;
378 break;
379 }
380 }
381 if (found == 0) {
382 /*
383 * If we didn't find anything, then use the first
384 * available slot in the fsspsets array.
385 */
386 for (i = 0; i < max_ncpus; i++) {
387 fsspset = &fsspsets[i];
388 if (fsspset->fssps_cpupart == NULL) {
389 ASSERT(fsspset->fssps_nproj == 0);
390 found = 1;
391 break;
392 }
393 }
394 fsspset->fssps_cpupart = cpupart;
395 }
396 ASSERT(found == 1);
397 return (fsspset);
398 }
399
400 static void
401 fss_del_fsspset(fsspset_t *fsspset)
402 {
403 ASSERT(MUTEX_HELD(&fsspsets_lock));
404 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
405 ASSERT(fsspset->fssps_nproj == 0);
406 ASSERT(fsspset->fssps_list == NULL);
407 ASSERT(fsspset->fssps_zones == NULL);
408 fsspset->fssps_cpupart = NULL;
409 fsspset->fssps_maxfsspri = 0;
410 fsspset->fssps_shares = 0;
411 }
412
413 /*
414 * The following routine returns a pointer to the fsszone structure which
415 * belongs to zone "zone" and cpu partition fsspset, if such structure exists.
416 */
417 static fsszone_t *
418 fss_find_fsszone(fsspset_t *fsspset, zone_t *zone)
419 {
420 fsszone_t *fsszone;
421
422 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
423
424 if (fsspset->fssps_list != NULL) {
425 /*
426 * There are projects/zones active on this cpu partition
427 * already. Try to find our zone among them.
428 */
429 fsszone = fsspset->fssps_zones;
430 do {
431 if (fsszone->fssz_zone == zone) {
432 return (fsszone);
433 }
434 fsszone = fsszone->fssz_next;
435 } while (fsszone != fsspset->fssps_zones);
436 }
437 return (NULL);
438 }
439
440 /*
441 * The following routine links new fsszone structure into doubly linked list of
442 * zones active on the specified cpu partition.
443 */
444 static void
445 fss_insert_fsszone(fsspset_t *fsspset, zone_t *zone, fsszone_t *fsszone)
446 {
447 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
448
449 fsszone->fssz_zone = zone;
450 fsszone->fssz_rshares = zone->zone_shares;
451
452 if (fsspset->fssps_zones == NULL) {
453 /*
454 * This will be the first fsszone for this fsspset
455 */
456 fsszone->fssz_next = fsszone->fssz_prev = fsszone;
457 fsspset->fssps_zones = fsszone;
458 } else {
459 /*
460 * Insert this fsszone to the doubly linked list.
461 */
462 fsszone_t *fssz_head = fsspset->fssps_zones;
463
464 fsszone->fssz_next = fssz_head;
465 fsszone->fssz_prev = fssz_head->fssz_prev;
466 fssz_head->fssz_prev->fssz_next = fsszone;
467 fssz_head->fssz_prev = fsszone;
468 fsspset->fssps_zones = fsszone;
469 }
470 }
471
472 /*
473 * The following routine removes a single fsszone structure from the doubly
474 * linked list of zones active on the specified cpu partition. Note that
475 * global fsspsets_lock must be held in case this fsszone structure is the last
476 * on the above mentioned list. Also note that the fsszone structure is not
477 * freed here, it is the responsibility of the caller to call kmem_free for it.
478 */
479 static void
480 fss_remove_fsszone(fsspset_t *fsspset, fsszone_t *fsszone)
481 {
482 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
483 ASSERT(fsszone->fssz_nproj == 0);
484 ASSERT(fsszone->fssz_shares == 0);
485 ASSERT(fsszone->fssz_runnable == 0);
486
487 if (fsszone->fssz_next != fsszone) {
488 /*
489 * This is not the last zone in the list.
490 */
491 fsszone->fssz_prev->fssz_next = fsszone->fssz_next;
492 fsszone->fssz_next->fssz_prev = fsszone->fssz_prev;
493 if (fsspset->fssps_zones == fsszone)
494 fsspset->fssps_zones = fsszone->fssz_next;
495 } else {
496 /*
497 * This was the last zone active in this cpu partition.
498 */
499 fsspset->fssps_zones = NULL;
500 }
501 }
502
503 /*
504 * The following routine returns a pointer to the fssproj structure
505 * which belongs to project kpj and cpu partition fsspset, if such structure
506 * exists.
507 */
508 static fssproj_t *
509 fss_find_fssproj(fsspset_t *fsspset, kproject_t *kpj)
510 {
511 fssproj_t *fssproj;
512
513 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
514
515 if (fsspset->fssps_list != NULL) {
516 /*
517 * There are projects running on this cpu partition already.
518 * Try to find our project among them.
519 */
520 fssproj = fsspset->fssps_list;
521 do {
522 if (fssproj->fssp_proj == kpj) {
523 ASSERT(fssproj->fssp_pset == fsspset);
524 return (fssproj);
525 }
526 fssproj = fssproj->fssp_next;
527 } while (fssproj != fsspset->fssps_list);
528 }
529 return (NULL);
530 }
531
532 /*
533 * The following routine links new fssproj structure into doubly linked list
534 * of projects running on the specified cpu partition.
535 */
536 static void
537 fss_insert_fssproj(fsspset_t *fsspset, kproject_t *kpj, fsszone_t *fsszone,
538 fssproj_t *fssproj)
539 {
540 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
541
542 fssproj->fssp_pset = fsspset;
543 fssproj->fssp_proj = kpj;
544 fssproj->fssp_shares = kpj->kpj_shares;
545
546 fsspset->fssps_nproj++;
547
548 if (fsspset->fssps_list == NULL) {
549 /*
550 * This will be the first fssproj for this fsspset
551 */
552 fssproj->fssp_next = fssproj->fssp_prev = fssproj;
553 fsspset->fssps_list = fssproj;
554 } else {
555 /*
556 * Insert this fssproj to the doubly linked list.
557 */
558 fssproj_t *fssp_head = fsspset->fssps_list;
559
560 fssproj->fssp_next = fssp_head;
561 fssproj->fssp_prev = fssp_head->fssp_prev;
562 fssp_head->fssp_prev->fssp_next = fssproj;
563 fssp_head->fssp_prev = fssproj;
564 fsspset->fssps_list = fssproj;
565 }
566 fssproj->fssp_fsszone = fsszone;
567 fsszone->fssz_nproj++;
568 ASSERT(fsszone->fssz_nproj != 0);
569 }
570
571 /*
572 * The following routine removes a single fssproj structure from the doubly
573 * linked list of projects running on the specified cpu partition. Note that
574 * global fsspsets_lock must be held in case if this fssproj structure is the
575 * last on the above mentioned list. Also note that the fssproj structure is
576 * not freed here, it is the responsibility of the caller to call kmem_free
577 * for it.
578 */
579 static void
580 fss_remove_fssproj(fsspset_t *fsspset, fssproj_t *fssproj)
581 {
582 fsszone_t *fsszone;
583
584 ASSERT(MUTEX_HELD(&fsspsets_lock));
585 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
586 ASSERT(fssproj->fssp_runnable == 0);
587
588 fsspset->fssps_nproj--;
589
590 fsszone = fssproj->fssp_fsszone;
591 fsszone->fssz_nproj--;
592
593 if (fssproj->fssp_next != fssproj) {
594 /*
595 * This is not the last part in the list.
596 */
597 fssproj->fssp_prev->fssp_next = fssproj->fssp_next;
598 fssproj->fssp_next->fssp_prev = fssproj->fssp_prev;
599 if (fsspset->fssps_list == fssproj)
600 fsspset->fssps_list = fssproj->fssp_next;
601 if (fsszone->fssz_nproj == 0)
602 fss_remove_fsszone(fsspset, fsszone);
603 } else {
604 /*
605 * This was the last project part running
606 * at this cpu partition.
607 */
608 fsspset->fssps_list = NULL;
609 ASSERT(fsspset->fssps_nproj == 0);
610 ASSERT(fsszone->fssz_nproj == 0);
611 fss_remove_fsszone(fsspset, fsszone);
612 fss_del_fsspset(fsspset);
613 }
614 }
615
616 static void
617 fss_inactive(kthread_t *t)
618 {
619 fssproc_t *fssproc;
620 fssproj_t *fssproj;
621 fsspset_t *fsspset;
622 fsszone_t *fsszone;
623
624 ASSERT(THREAD_LOCK_HELD(t));
625 fssproc = FSSPROC(t);
626 fssproj = FSSPROC2FSSPROJ(fssproc);
627 if (fssproj == NULL) /* if this thread already exited */
628 return;
629 fsspset = FSSPROJ2FSSPSET(fssproj);
630 fsszone = fssproj->fssp_fsszone;
631 disp_lock_enter_high(&fsspset->fssps_displock);
632 ASSERT(fssproj->fssp_runnable > 0);
633 if (--fssproj->fssp_runnable == 0) {
634 fsszone->fssz_shares -= fssproj->fssp_shares;
635 if (--fsszone->fssz_runnable == 0)
636 fsspset->fssps_shares -= fsszone->fssz_rshares;
637 }
638 ASSERT(fssproc->fss_runnable == 1);
639 fssproc->fss_runnable = 0;
640 disp_lock_exit_high(&fsspset->fssps_displock);
641 }
642
643 static void
644 fss_active(kthread_t *t)
645 {
646 fssproc_t *fssproc;
647 fssproj_t *fssproj;
648 fsspset_t *fsspset;
649 fsszone_t *fsszone;
650
651 ASSERT(THREAD_LOCK_HELD(t));
652 fssproc = FSSPROC(t);
653 fssproj = FSSPROC2FSSPROJ(fssproc);
654 if (fssproj == NULL) /* if this thread already exited */
655 return;
656 fsspset = FSSPROJ2FSSPSET(fssproj);
657 fsszone = fssproj->fssp_fsszone;
658 disp_lock_enter_high(&fsspset->fssps_displock);
659 if (++fssproj->fssp_runnable == 1) {
660 fsszone->fssz_shares += fssproj->fssp_shares;
661 if (++fsszone->fssz_runnable == 1)
662 fsspset->fssps_shares += fsszone->fssz_rshares;
663 }
664 ASSERT(fssproc->fss_runnable == 0);
665 fssproc->fss_runnable = 1;
666 disp_lock_exit_high(&fsspset->fssps_displock);
667 }
668
669 /*
670 * Fair share scheduler initialization. Called by dispinit() at boot time.
671 * We can ignore clparmsz argument since we know that the smallest possible
672 * parameter buffer is big enough for us.
673 */
674 /*ARGSUSED*/
675 static pri_t
676 fss_init(id_t cid, int clparmsz, classfuncs_t **clfuncspp)
677 {
678 int i;
679
680 ASSERT(MUTEX_HELD(&cpu_lock));
681
682 fss_cid = cid;
683 fss_maxumdpri = minclsyspri - 1;
684 fss_maxglobpri = minclsyspri;
685 fss_minglobpri = 0;
686 fsspsets = kmem_zalloc(sizeof (fsspset_t) * max_ncpus, KM_SLEEP);
687
688 /*
689 * Initialize the fssproc hash table.
690 */
691 for (i = 0; i < FSS_LISTS; i++)
692 fss_listhead[i].fss_next = fss_listhead[i].fss_prev =
693 &fss_listhead[i];
694
695 *clfuncspp = &fss_classfuncs;
696
697 /*
698 * Fill in fss_nice_tick and fss_nice_decay arrays:
699 * The cost of a tick is lower at positive nice values (so that it
700 * will not increase its project's usage as much as normal) with 50%
701 * drop at the maximum level and 50% increase at the minimum level.
702 * The fsspri decay is slower at positive nice values. fsspri values
703 * of processes with negative nice levels must decay faster to receive
704 * time slices more frequently than normal.
705 */
706 for (i = 0; i < FSS_NICE_RANGE; i++) {
707 fss_nice_tick[i] = (FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2)
708 - i)) / FSS_NICE_RANGE;
709 fss_nice_decay[i] = FSS_DECAY_MIN +
710 ((FSS_DECAY_MAX - FSS_DECAY_MIN) * i) /
711 (FSS_NICE_RANGE - 1);
712 }
713
714 return (fss_maxglobpri);
715 }
716
717 /*
718 * Calculate the new cpupri based on the usage, the number of shares and
719 * the number of active threads. Reset the tick counter for this thread.
720 */
721 static void
722 fss_newpri(fssproc_t *fssproc)
723 {
724 kthread_t *tp;
725 fssproj_t *fssproj;
726 fsspset_t *fsspset;
727 fsszone_t *fsszone;
728 fsspri_t fsspri, maxfsspri;
729 pri_t invpri;
730 uint32_t ticks;
731
732 tp = fssproc->fss_tp;
733 ASSERT(tp != NULL);
734
735 if (tp->t_cid != fss_cid)
736 return;
737
738 ASSERT(THREAD_LOCK_HELD(tp));
739
740 fssproj = FSSPROC2FSSPROJ(fssproc);
741 fsszone = FSSPROJ2FSSZONE(fssproj);
742 if (fssproj == NULL)
743 /*
744 * No need to change priority of exited threads.
745 */
746 return;
747
748 fsspset = FSSPROJ2FSSPSET(fssproj);
749 disp_lock_enter_high(&fsspset->fssps_displock);
750
751 if (fssproj->fssp_shares == 0 || fsszone->fssz_rshares == 0) {
752 /*
753 * Special case: threads with no shares.
754 */
755 fssproc->fss_umdpri = fss_minglobpri;
756 fssproc->fss_ticks = 0;
757 disp_lock_exit_high(&fsspset->fssps_displock);
758 return;
759 }
760
761 /*
762 * fsspri += shusage * nrunnable * ticks
763 */
764 ticks = fssproc->fss_ticks;
765 fssproc->fss_ticks = 0;
766 fsspri = fssproc->fss_fsspri;
767 fsspri += fssproj->fssp_shusage * fssproj->fssp_runnable * ticks;
768 fssproc->fss_fsspri = fsspri;
769
770 if (fsspri < fss_maxumdpri)
771 fsspri = fss_maxumdpri; /* so that maxfsspri is != 0 */
772
773 /*
774 * The general priority formula:
775 *
776 * (fsspri * umdprirange)
777 * pri = maxumdpri - ------------------------
778 * maxfsspri
779 *
780 * If this thread's fsspri is greater than the previous largest
781 * fsspri, then record it as the new high and priority for this
782 * thread will be one (the lowest priority assigned to a thread
783 * that has non-zero shares).
784 * Note that this formula cannot produce out of bounds priority
785 * values; if it is changed, additional checks may need to be
786 * added.
787 */
788 maxfsspri = fsspset->fssps_maxfsspri;
789 if (fsspri >= maxfsspri) {
790 fsspset->fssps_maxfsspri = fsspri;
791 disp_lock_exit_high(&fsspset->fssps_displock);
792 fssproc->fss_umdpri = 1;
793 } else {
794 disp_lock_exit_high(&fsspset->fssps_displock);
795 invpri = (fsspri * (fss_maxumdpri - 1)) / maxfsspri;
796 fssproc->fss_umdpri = fss_maxumdpri - invpri;
797 }
798 }
799
800 /*
801 * Decays usages of all running projects and resets their tick counters.
802 * Called once per second from fss_update() after updating priorities.
803 */
804 static void
805 fss_decay_usage()
806 {
807 uint32_t zone_ext_shares, zone_int_shares;
808 uint32_t kpj_shares, pset_shares;
809 fsspset_t *fsspset;
810 fssproj_t *fssproj;
811 fsszone_t *fsszone;
812 fsspri_t maxfsspri;
813 int psetid;
814
815 mutex_enter(&fsspsets_lock);
816 /*
817 * Go through all active processor sets and decay usages of projects
818 * running on them.
819 */
820 for (psetid = 0; psetid < max_ncpus; psetid++) {
821 fsspset = &fsspsets[psetid];
822 mutex_enter(&fsspset->fssps_lock);
823
824 if (fsspset->fssps_cpupart == NULL ||
825 (fssproj = fsspset->fssps_list) == NULL) {
826 mutex_exit(&fsspset->fssps_lock);
827 continue;
828 }
829
830 /*
831 * Decay maxfsspri for this cpu partition with the
832 * fastest possible decay rate.
833 */
834 disp_lock_enter(&fsspset->fssps_displock);
835
836 maxfsspri = (fsspset->fssps_maxfsspri *
837 fss_nice_decay[NZERO]) / FSS_DECAY_BASE;
838 if (maxfsspri < fss_maxumdpri)
839 maxfsspri = fss_maxumdpri;
840 fsspset->fssps_maxfsspri = maxfsspri;
841
842 do {
843 /*
844 * Decay usage for each project running on
845 * this cpu partition.
846 */
847 fssproj->fssp_usage =
848 (fssproj->fssp_usage * FSS_DECAY_USG) /
849 FSS_DECAY_BASE + fssproj->fssp_ticks;
850 fssproj->fssp_ticks = 0;
851
852 fsszone = fssproj->fssp_fsszone;
853 /*
854 * Readjust the project's number of shares if it has
855 * changed since we checked it last time.
856 */
857 kpj_shares = fssproj->fssp_proj->kpj_shares;
858 if (fssproj->fssp_shares != kpj_shares) {
859 if (fssproj->fssp_runnable != 0) {
860 fsszone->fssz_shares -=
861 fssproj->fssp_shares;
862 fsszone->fssz_shares += kpj_shares;
863 }
864 fssproj->fssp_shares = kpj_shares;
865 }
866
867 /*
868 * Readjust the zone's number of shares if it
869 * has changed since we checked it last time.
870 */
871 zone_ext_shares = fsszone->fssz_zone->zone_shares;
872 if (fsszone->fssz_rshares != zone_ext_shares) {
873 if (fsszone->fssz_runnable != 0) {
874 fsspset->fssps_shares -=
875 fsszone->fssz_rshares;
876 fsspset->fssps_shares +=
877 zone_ext_shares;
878 }
879 fsszone->fssz_rshares = zone_ext_shares;
880 }
881 zone_int_shares = fsszone->fssz_shares;
882 pset_shares = fsspset->fssps_shares;
883 /*
884 * Calculate fssp_shusage value to be used
885 * for fsspri increments for the next second.
886 */
887 if (kpj_shares == 0 || zone_ext_shares == 0) {
888 fssproj->fssp_shusage = 0;
889 } else if (FSSPROJ2KPROJ(fssproj) == proj0p) {
890 /*
891 * Project 0 in the global zone has 50%
892 * of its zone.
893 */
894 fssproj->fssp_shusage = (fssproj->fssp_usage *
895 zone_int_shares * zone_int_shares) /
896 (zone_ext_shares * zone_ext_shares);
897 } else {
898 /*
899 * Thread's priority is based on its project's
900 * normalized usage (shusage) value which gets
901 * calculated this way:
902 *
903 * pset_shares^2 zone_int_shares^2
904 * usage * ------------- * ------------------
905 * kpj_shares^2 zone_ext_shares^2
906 *
907 * Where zone_int_shares is the sum of shares
908 * of all active projects within the zone (and
909 * the pset), and zone_ext_shares is the number
910 * of zone shares (ie, zone.cpu-shares).
911 *
912 * If there is only one zone active on the pset
913 * the above reduces to:
914 *
915 * zone_int_shares^2
916 * shusage = usage * ---------------------
917 * kpj_shares^2
918 *
919 * If there's only one project active in the
920 * zone this formula reduces to:
921 *
922 * pset_shares^2
923 * shusage = usage * ----------------------
924 * zone_ext_shares^2
925 */
926 fssproj->fssp_shusage = fssproj->fssp_usage *
927 pset_shares * zone_int_shares;
928 fssproj->fssp_shusage /=
929 kpj_shares * zone_ext_shares;
930 fssproj->fssp_shusage *=
931 pset_shares * zone_int_shares;
932 fssproj->fssp_shusage /=
933 kpj_shares * zone_ext_shares;
934 }
935 fssproj = fssproj->fssp_next;
936 } while (fssproj != fsspset->fssps_list);
937
938 disp_lock_exit(&fsspset->fssps_displock);
939 mutex_exit(&fsspset->fssps_lock);
940 }
941 mutex_exit(&fsspsets_lock);
942 }
943
944 static void
945 fss_change_priority(kthread_t *t, fssproc_t *fssproc)
946 {
947 pri_t new_pri;
948
949 ASSERT(THREAD_LOCK_HELD(t));
950 new_pri = fssproc->fss_umdpri;
951 ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
952
953 t->t_cpri = fssproc->fss_upri;
954 fssproc->fss_flags &= ~FSSRESTORE;
955 if (t == curthread || t->t_state == TS_ONPROC) {
956 /*
957 * curthread is always onproc
958 */
959 cpu_t *cp = t->t_disp_queue->disp_cpu;
960 THREAD_CHANGE_PRI(t, new_pri);
961 if (t == cp->cpu_dispthread)
962 cp->cpu_dispatch_pri = DISP_PRIO(t);
963 if (DISP_MUST_SURRENDER(t)) {
964 fssproc->fss_flags |= FSSBACKQ;
965 cpu_surrender(t);
966 } else {
967 fssproc->fss_timeleft = fss_quantum;
968 }
969 } else {
970 /*
971 * When the priority of a thread is changed, it may be
972 * necessary to adjust its position on a sleep queue or
973 * dispatch queue. The function thread_change_pri accomplishes
974 * this.
975 */
976 if (thread_change_pri(t, new_pri, 0)) {
977 /*
978 * The thread was on a run queue.
979 */
980 fssproc->fss_timeleft = fss_quantum;
981 } else {
982 fssproc->fss_flags |= FSSBACKQ;
983 }
984 }
985 }
986
987 /*
988 * Update priorities of all fair-sharing threads that are currently runnable
989 * at a user mode priority based on the number of shares and current usage.
990 * Called once per second via timeout which we reset here.
991 *
992 * There are several lists of fair-sharing threads broken up by a hash on the
993 * thread pointer. Each list has its own lock. This avoids blocking all
994 * fss_enterclass, fss_fork, and fss_exitclass operations while fss_update runs.
995 * fss_update traverses each list in turn.
996 */
997 static void
998 fss_update(void *arg)
999 {
1000 int i;
1001 int new_marker = -1;
1002 static int fss_update_marker;
1003
1004 /*
1005 * Decay and update usages for all projects.
1006 */
1007 fss_decay_usage();
1008
1009 /*
1010 * Start with the fss_update_marker list, then do the rest.
1011 */
1012 i = fss_update_marker;
1013
1014 /*
1015 * Go around all threads, set new priorities and decay
1016 * per-thread CPU usages.
1017 */
1018 do {
1019 /*
1020 * If this is the first list after the current marker to have
1021 * threads with priorities updates, advance the marker to this
1022 * list for the next time fss_update runs.
1023 */
1024 if (fss_update_list(i) &&
1025 new_marker == -1 && i != fss_update_marker)
1026 new_marker = i;
1027 } while ((i = FSS_LIST_NEXT(i)) != fss_update_marker);
1028
1029 /*
1030 * Advance marker for the next fss_update call
1031 */
1032 if (new_marker != -1)
1033 fss_update_marker = new_marker;
1034
1035 (void) timeout(fss_update, arg, hz);
1036 }
1037
1038 /*
1039 * Updates priority for a list of threads. Returns 1 if the priority of one
1040 * of the threads was actually updated, 0 if none were for various reasons
1041 * (thread is no longer in the FSS class, is not runnable, has the preemption
1042 * control no-preempt bit set, etc.)
1043 */
1044 static int
1045 fss_update_list(int i)
1046 {
1047 fssproc_t *fssproc;
1048 fssproj_t *fssproj;
1049 fsspri_t fsspri;
1050 kthread_t *t;
1051 int updated = 0;
1052
1053 mutex_enter(&fss_listlock[i]);
1054 for (fssproc = fss_listhead[i].fss_next; fssproc != &fss_listhead[i];
1055 fssproc = fssproc->fss_next) {
1056 t = fssproc->fss_tp;
1057 /*
1058 * Lock the thread and verify the state.
1059 */
1060 thread_lock(t);
1061 /*
1062 * Skip the thread if it is no longer in the FSS class or
1063 * is running with kernel mode priority.
1064 */
1065 if (t->t_cid != fss_cid)
1066 goto next;
1067 if ((fssproc->fss_flags & FSSKPRI) != 0)
1068 goto next;
1069
1070 fssproj = FSSPROC2FSSPROJ(fssproc);
1071 if (fssproj == NULL)
1072 goto next;
1073 if (fssproj->fssp_shares != 0) {
1074 /*
1075 * Decay fsspri value.
1076 */
1077 fsspri = fssproc->fss_fsspri;
1078 fsspri = (fsspri * fss_nice_decay[fssproc->fss_nice]) /
1079 FSS_DECAY_BASE;
1080 fssproc->fss_fsspri = fsspri;
1081 }
1082
1083 if (t->t_schedctl && schedctl_get_nopreempt(t))
1084 goto next;
1085 if (t->t_state != TS_RUN && t->t_state != TS_WAIT) {
1086 /*
1087 * Make next syscall/trap call fss_trapret
1088 */
1089 t->t_trapret = 1;
1090 aston(t);
1091 goto next;
1092 }
1093 fss_newpri(fssproc);
1094 updated = 1;
1095
1096 /*
1097 * Only dequeue the thread if it needs to be moved; otherwise
1098 * it should just round-robin here.
1099 */
1100 if (t->t_pri != fssproc->fss_umdpri)
1101 fss_change_priority(t, fssproc);
1102 next:
1103 thread_unlock(t);
1104 }
1105 mutex_exit(&fss_listlock[i]);
1106 return (updated);
1107 }
1108
1109 /*ARGSUSED*/
1110 static int
1111 fss_admin(caddr_t uaddr, cred_t *reqpcredp)
1112 {
1113 fssadmin_t fssadmin;
1114
1115 if (copyin(uaddr, &fssadmin, sizeof (fssadmin_t)))
1116 return (EFAULT);
1117
1118 switch (fssadmin.fss_cmd) {
1119 case FSS_SETADMIN:
1120 if (secpolicy_dispadm(reqpcredp) != 0)
1121 return (EPERM);
1122 if (fssadmin.fss_quantum <= 0 || fssadmin.fss_quantum >= hz)
1123 return (EINVAL);
1124 fss_quantum = fssadmin.fss_quantum;
1125 break;
1126 case FSS_GETADMIN:
1127 fssadmin.fss_quantum = fss_quantum;
1128 if (copyout(&fssadmin, uaddr, sizeof (fssadmin_t)))
1129 return (EFAULT);
1130 break;
1131 default:
1132 return (EINVAL);
1133 }
1134 return (0);
1135 }
1136
1137 static int
1138 fss_getclinfo(void *infop)
1139 {
1140 fssinfo_t *fssinfo = (fssinfo_t *)infop;
1141 fssinfo->fss_maxupri = fss_maxupri;
1142 return (0);
1143 }
1144
1145 static int
1146 fss_parmsin(void *parmsp)
1147 {
1148 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1149
1150 /*
1151 * Check validity of parameters.
1152 */
1153 if ((fssparmsp->fss_uprilim > fss_maxupri ||
1154 fssparmsp->fss_uprilim < -fss_maxupri) &&
1155 fssparmsp->fss_uprilim != FSS_NOCHANGE)
1156 return (EINVAL);
1157
1158 if ((fssparmsp->fss_upri > fss_maxupri ||
1159 fssparmsp->fss_upri < -fss_maxupri) &&
1160 fssparmsp->fss_upri != FSS_NOCHANGE)
1161 return (EINVAL);
1162
1163 return (0);
1164 }
1165
1166 /*ARGSUSED*/
1167 static int
1168 fss_parmsout(void *parmsp, pc_vaparms_t *vaparmsp)
1169 {
1170 return (0);
1171 }
1172
1173 static int
1174 fss_vaparmsin(void *parmsp, pc_vaparms_t *vaparmsp)
1175 {
1176 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1177 int priflag = 0;
1178 int limflag = 0;
1179 uint_t cnt;
1180 pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
1181
1182 /*
1183 * FSS_NOCHANGE (-32768) is outside of the range of values for
1184 * fss_uprilim and fss_upri. If the structure fssparms_t is changed,
1185 * FSS_NOCHANGE should be replaced by a flag word.
1186 */
1187 fssparmsp->fss_uprilim = FSS_NOCHANGE;
1188 fssparmsp->fss_upri = FSS_NOCHANGE;
1189
1190 /*
1191 * Get the varargs parameter and check validity of parameters.
1192 */
1193 if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
1194 return (EINVAL);
1195
1196 for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
1197 switch (vpp->pc_key) {
1198 case FSS_KY_UPRILIM:
1199 if (limflag++)
1200 return (EINVAL);
1201 fssparmsp->fss_uprilim = (pri_t)vpp->pc_parm;
1202 if (fssparmsp->fss_uprilim > fss_maxupri ||
1203 fssparmsp->fss_uprilim < -fss_maxupri)
1204 return (EINVAL);
1205 break;
1206 case FSS_KY_UPRI:
1207 if (priflag++)
1208 return (EINVAL);
1209 fssparmsp->fss_upri = (pri_t)vpp->pc_parm;
1210 if (fssparmsp->fss_upri > fss_maxupri ||
1211 fssparmsp->fss_upri < -fss_maxupri)
1212 return (EINVAL);
1213 break;
1214 default:
1215 return (EINVAL);
1216 }
1217 }
1218
1219 if (vaparmsp->pc_vaparmscnt == 0) {
1220 /*
1221 * Use default parameters.
1222 */
1223 fssparmsp->fss_upri = fssparmsp->fss_uprilim = 0;
1224 }
1225
1226 return (0);
1227 }
1228
1229 /*
1230 * Copy all selected fair-sharing class parameters to the user. The parameters
1231 * are specified by a key.
1232 */
1233 static int
1234 fss_vaparmsout(void *parmsp, pc_vaparms_t *vaparmsp)
1235 {
1236 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1237 int priflag = 0;
1238 int limflag = 0;
1239 uint_t cnt;
1240 pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
1241
1242 ASSERT(MUTEX_NOT_HELD(&curproc->p_lock));
1243
1244 if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
1245 return (EINVAL);
1246
1247 for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
1248 switch (vpp->pc_key) {
1249 case FSS_KY_UPRILIM:
1250 if (limflag++)
1251 return (EINVAL);
1252 if (copyout(&fssparmsp->fss_uprilim,
1253 (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
1254 return (EFAULT);
1255 break;
1256 case FSS_KY_UPRI:
1257 if (priflag++)
1258 return (EINVAL);
1259 if (copyout(&fssparmsp->fss_upri,
1260 (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
1261 return (EFAULT);
1262 break;
1263 default:
1264 return (EINVAL);
1265 }
1266 }
1267
1268 return (0);
1269 }
1270
1271 /*
1272 * Return the user mode scheduling priority range.
1273 */
1274 static int
1275 fss_getclpri(pcpri_t *pcprip)
1276 {
1277 pcprip->pc_clpmax = fss_maxupri;
1278 pcprip->pc_clpmin = -fss_maxupri;
1279 return (0);
1280 }
1281
1282 static int
1283 fss_alloc(void **p, int flag)
1284 {
1285 void *bufp;
1286
1287 if ((bufp = kmem_zalloc(sizeof (fssproc_t), flag)) == NULL) {
1288 return (ENOMEM);
1289 } else {
1290 *p = bufp;
1291 return (0);
1292 }
1293 }
1294
1295 static void
1296 fss_free(void *bufp)
1297 {
1298 if (bufp)
1299 kmem_free(bufp, sizeof (fssproc_t));
1300 }
1301
1302 /*
1303 * Thread functions
1304 */
1305 static int
1306 fss_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp,
1307 void *bufp)
1308 {
1309 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1310 fssproc_t *fssproc;
1311 pri_t reqfssuprilim;
1312 pri_t reqfssupri;
1313 static uint32_t fssexists = 0;
1314 fsspset_t *fsspset;
1315 fssproj_t *fssproj;
1316 fsszone_t *fsszone;
1317 kproject_t *kpj;
1318 zone_t *zone;
1319 int fsszone_allocated = 0;
1320
1321 fssproc = (fssproc_t *)bufp;
1322 ASSERT(fssproc != NULL);
1323
1324 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1325
1326 /*
1327 * Only root can move threads to FSS class.
1328 */
1329 if (reqpcredp != NULL && secpolicy_setpriority(reqpcredp) != 0)
1330 return (EPERM);
1331 /*
1332 * Initialize the fssproc structure.
1333 */
1334 fssproc->fss_umdpri = fss_maxumdpri / 2;
1335
1336 if (fssparmsp == NULL) {
1337 /*
1338 * Use default values.
1339 */
1340 fssproc->fss_nice = NZERO;
1341 fssproc->fss_uprilim = fssproc->fss_upri = 0;
1342 } else {
1343 /*
1344 * Use supplied values.
1345 */
1346 if (fssparmsp->fss_uprilim == FSS_NOCHANGE) {
1347 reqfssuprilim = 0;
1348 } else {
1349 if (fssparmsp->fss_uprilim > 0 &&
1350 secpolicy_setpriority(reqpcredp) != 0)
1351 return (EPERM);
1352 reqfssuprilim = fssparmsp->fss_uprilim;
1353 }
1354 if (fssparmsp->fss_upri == FSS_NOCHANGE) {
1355 reqfssupri = reqfssuprilim;
1356 } else {
1357 if (fssparmsp->fss_upri > 0 &&
1358 secpolicy_setpriority(reqpcredp) != 0)
1359 return (EPERM);
1360 /*
1361 * Set the user priority to the requested value or
1362 * the upri limit, whichever is lower.
1363 */
1364 reqfssupri = fssparmsp->fss_upri;
1365 if (reqfssupri > reqfssuprilim)
1366 reqfssupri = reqfssuprilim;
1367 }
1368 fssproc->fss_uprilim = reqfssuprilim;
1369 fssproc->fss_upri = reqfssupri;
1370 fssproc->fss_nice = NZERO - (NZERO * reqfssupri) / fss_maxupri;
1371 if (fssproc->fss_nice > FSS_NICE_MAX)
1372 fssproc->fss_nice = FSS_NICE_MAX;
1373 }
1374
1375 fssproc->fss_timeleft = fss_quantum;
1376 fssproc->fss_tp = t;
1377 cpucaps_sc_init(&fssproc->fss_caps);
1378
1379 /*
1380 * Put a lock on our fsspset structure.
1381 */
1382 mutex_enter(&fsspsets_lock);
1383 fsspset = fss_find_fsspset(t->t_cpupart);
1384 mutex_enter(&fsspset->fssps_lock);
1385 mutex_exit(&fsspsets_lock);
1386
1387 zone = ttoproc(t)->p_zone;
1388 if ((fsszone = fss_find_fsszone(fsspset, zone)) == NULL) {
1389 if ((fsszone = kmem_zalloc(sizeof (fsszone_t), KM_NOSLEEP))
1390 == NULL) {
1391 mutex_exit(&fsspset->fssps_lock);
1392 return (ENOMEM);
1393 } else {
1394 fsszone_allocated = 1;
1395 fss_insert_fsszone(fsspset, zone, fsszone);
1396 }
1397 }
1398 kpj = ttoproj(t);
1399 if ((fssproj = fss_find_fssproj(fsspset, kpj)) == NULL) {
1400 if ((fssproj = kmem_zalloc(sizeof (fssproj_t), KM_NOSLEEP))
1401 == NULL) {
1402 if (fsszone_allocated) {
1403 fss_remove_fsszone(fsspset, fsszone);
1404 kmem_free(fsszone, sizeof (fsszone_t));
1405 }
1406 mutex_exit(&fsspset->fssps_lock);
1407 return (ENOMEM);
1408 } else {
1409 fss_insert_fssproj(fsspset, kpj, fsszone, fssproj);
1410 }
1411 }
1412 fssproj->fssp_threads++;
1413 fssproc->fss_proj = fssproj;
1414
1415 /*
1416 * Reset priority. Process goes to a "user mode" priority here
1417 * regardless of whether or not it has slept since entering the kernel.
1418 */
1419 thread_lock(t);
1420 t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
1421 t->t_cid = cid;
1422 t->t_cldata = (void *)fssproc;
1423 t->t_schedflag |= TS_RUNQMATCH;
1424 fss_change_priority(t, fssproc);
1425 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
1426 t->t_state == TS_WAIT)
1427 fss_active(t);
1428 thread_unlock(t);
1429
1430 mutex_exit(&fsspset->fssps_lock);
1431
1432 /*
1433 * Link new structure into fssproc list.
1434 */
1435 FSS_LIST_INSERT(fssproc);
1436
1437 /*
1438 * If this is the first fair-sharing thread to occur since boot,
1439 * we set up the initial call to fss_update() here. Use an atomic
1440 * compare-and-swap since that's easier and faster than a mutex
1441 * (but check with an ordinary load first since most of the time
1442 * this will already be done).
1443 */
1444 if (fssexists == 0 && cas32(&fssexists, 0, 1) == 0)
1445 (void) timeout(fss_update, NULL, hz);
1446
1447 return (0);
1448 }
1449
1450 /*
1451 * Remove fssproc_t from the list.
1452 */
1453 static void
1454 fss_exitclass(void *procp)
1455 {
1456 fssproc_t *fssproc = (fssproc_t *)procp;
1457 fssproj_t *fssproj;
1458 fsspset_t *fsspset;
1459 fsszone_t *fsszone;
1460 kthread_t *t = fssproc->fss_tp;
1461
1462 /*
1463 * We should be either getting this thread off the deathrow or
1464 * this thread has already moved to another scheduling class and
1465 * we're being called with its old cldata buffer pointer. In both
1466 * cases, the content of this buffer can not be changed while we're
1467 * here.
1468 */
1469 mutex_enter(&fsspsets_lock);
1470 thread_lock(t);
1471 if (t->t_cid != fss_cid) {
1472 /*
1473 * We're being called as a result of the priocntl() system
1474 * call -- someone is trying to move our thread to another
1475 * scheduling class. We can't call fss_inactive() here
1476 * because our thread's t_cldata pointer already points
1477 * to another scheduling class specific data.
1478 */
1479 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1480
1481 fssproj = FSSPROC2FSSPROJ(fssproc);
1482 fsspset = FSSPROJ2FSSPSET(fssproj);
1483 fsszone = fssproj->fssp_fsszone;
1484
1485 if (fssproc->fss_runnable) {
1486 disp_lock_enter_high(&fsspset->fssps_displock);
1487 if (--fssproj->fssp_runnable == 0) {
1488 fsszone->fssz_shares -= fssproj->fssp_shares;
1489 if (--fsszone->fssz_runnable == 0)
1490 fsspset->fssps_shares -=
1491 fsszone->fssz_rshares;
1492 }
1493 disp_lock_exit_high(&fsspset->fssps_displock);
1494 }
1495 thread_unlock(t);
1496
1497 mutex_enter(&fsspset->fssps_lock);
1498 if (--fssproj->fssp_threads == 0) {
1499 fss_remove_fssproj(fsspset, fssproj);
1500 if (fsszone->fssz_nproj == 0)
1501 kmem_free(fsszone, sizeof (fsszone_t));
1502 kmem_free(fssproj, sizeof (fssproj_t));
1503 }
1504 mutex_exit(&fsspset->fssps_lock);
1505
1506 } else {
1507 ASSERT(t->t_state == TS_FREE);
1508 /*
1509 * We're being called from thread_free() when our thread
1510 * is removed from the deathrow. There is nothing we need
1511 * do here since everything should've been done earlier
1512 * in fss_exit().
1513 */
1514 thread_unlock(t);
1515 }
1516 mutex_exit(&fsspsets_lock);
1517
1518 FSS_LIST_DELETE(fssproc);
1519 fss_free(fssproc);
1520 }
1521
1522 /*ARGSUSED*/
1523 static int
1524 fss_canexit(kthread_t *t, cred_t *credp)
1525 {
1526 /*
1527 * A thread is allowed to exit FSS only if we have sufficient
1528 * privileges.
1529 */
1530 if (credp != NULL && secpolicy_setpriority(credp) != 0)
1531 return (EPERM);
1532 else
1533 return (0);
1534 }
1535
1536 /*
1537 * Initialize fair-share class specific proc structure for a child.
1538 */
1539 static int
1540 fss_fork(kthread_t *pt, kthread_t *ct, void *bufp)
1541 {
1542 fssproc_t *pfssproc; /* ptr to parent's fssproc structure */
1543 fssproc_t *cfssproc; /* ptr to child's fssproc structure */
1544 fssproj_t *fssproj;
1545 fsspset_t *fsspset;
1546
1547 ASSERT(MUTEX_HELD(&ttoproc(pt)->p_lock));
1548 ASSERT(ct->t_state == TS_STOPPED);
1549
1550 cfssproc = (fssproc_t *)bufp;
1551 ASSERT(cfssproc != NULL);
1552 bzero(cfssproc, sizeof (fssproc_t));
1553
1554 thread_lock(pt);
1555 pfssproc = FSSPROC(pt);
1556 fssproj = FSSPROC2FSSPROJ(pfssproc);
1557 fsspset = FSSPROJ2FSSPSET(fssproj);
1558 thread_unlock(pt);
1559
1560 mutex_enter(&fsspset->fssps_lock);
1561 /*
1562 * Initialize child's fssproc structure.
1563 */
1564 thread_lock(pt);
1565 ASSERT(FSSPROJ(pt) == fssproj);
1566 cfssproc->fss_proj = fssproj;
1567 cfssproc->fss_timeleft = fss_quantum;
1568 cfssproc->fss_umdpri = pfssproc->fss_umdpri;
1569 cfssproc->fss_fsspri = 0;
1570 cfssproc->fss_uprilim = pfssproc->fss_uprilim;
1571 cfssproc->fss_upri = pfssproc->fss_upri;
1572 cfssproc->fss_tp = ct;
1573 cfssproc->fss_nice = pfssproc->fss_nice;
1574 cpucaps_sc_init(&cfssproc->fss_caps);
1575
1576 cfssproc->fss_flags =
1577 pfssproc->fss_flags & ~(FSSKPRI | FSSBACKQ | FSSRESTORE);
1578 ct->t_cldata = (void *)cfssproc;
1579 ct->t_schedflag |= TS_RUNQMATCH;
1580 thread_unlock(pt);
1581
1582 fssproj->fssp_threads++;
1583 mutex_exit(&fsspset->fssps_lock);
1584
1585 /*
1586 * Link new structure into fssproc hash table.
1587 */
1588 FSS_LIST_INSERT(cfssproc);
1589 return (0);
1590 }
1591
1592 /*
1593 * Child is placed at back of dispatcher queue and parent gives up processor
1594 * so that the child runs first after the fork. This allows the child
1595 * immediately execing to break the multiple use of copy on write pages with no
1596 * disk home. The parent will get to steal them back rather than uselessly
1597 * copying them.
1598 */
1599 static void
1600 fss_forkret(kthread_t *t, kthread_t *ct)
1601 {
1602 proc_t *pp = ttoproc(t);
1603 proc_t *cp = ttoproc(ct);
1604 fssproc_t *fssproc;
1605
1606 ASSERT(t == curthread);
1607 ASSERT(MUTEX_HELD(&pidlock));
1608
1609 /*
1610 * Grab the child's p_lock before dropping pidlock to ensure the
1611 * process does not disappear before we set it running.
1612 */
1613 mutex_enter(&cp->p_lock);
1614 continuelwps(cp);
1615 mutex_exit(&cp->p_lock);
1616
1617 mutex_enter(&pp->p_lock);
1618 mutex_exit(&pidlock);
1619 continuelwps(pp);
1620
1621 thread_lock(t);
1622
1623 fssproc = FSSPROC(t);
1624 fss_newpri(fssproc);
1625 fssproc->fss_timeleft = fss_quantum;
1626 t->t_pri = fssproc->fss_umdpri;
1627 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
1628 fssproc->fss_flags &= ~FSSKPRI;
1629 THREAD_TRANSITION(t);
1630
1631 /*
1632 * We don't want to call fss_setrun(t) here because it may call
1633 * fss_active, which we don't need.
1634 */
1635 fssproc->fss_flags &= ~FSSBACKQ;
1636
1637 if (t->t_disp_time != ddi_get_lbolt())
1638 setbackdq(t);
1639 else
1640 setfrontdq(t);
1641
1642 thread_unlock(t);
1643 /*
1644 * Safe to drop p_lock now since it is safe to change
1645 * the scheduling class after this point.
1646 */
1647 mutex_exit(&pp->p_lock);
1648
1649 swtch();
1650 }
1651
1652 /*
1653 * Get the fair-sharing parameters of the thread pointed to by fssprocp into
1654 * the buffer pointed by fssparmsp.
1655 */
1656 static void
1657 fss_parmsget(kthread_t *t, void *parmsp)
1658 {
1659 fssproc_t *fssproc = FSSPROC(t);
1660 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1661
1662 fssparmsp->fss_uprilim = fssproc->fss_uprilim;
1663 fssparmsp->fss_upri = fssproc->fss_upri;
1664 }
1665
1666 /*ARGSUSED*/
1667 static int
1668 fss_parmsset(kthread_t *t, void *parmsp, id_t reqpcid, cred_t *reqpcredp)
1669 {
1670 char nice;
1671 pri_t reqfssuprilim;
1672 pri_t reqfssupri;
1673 fssproc_t *fssproc = FSSPROC(t);
1674 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1675
1676 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
1677
1678 if (fssparmsp->fss_uprilim == FSS_NOCHANGE)
1679 reqfssuprilim = fssproc->fss_uprilim;
1680 else
1681 reqfssuprilim = fssparmsp->fss_uprilim;
1682
1683 if (fssparmsp->fss_upri == FSS_NOCHANGE)
1684 reqfssupri = fssproc->fss_upri;
1685 else
1686 reqfssupri = fssparmsp->fss_upri;
1687
1688 /*
1689 * Make sure the user priority doesn't exceed the upri limit.
1690 */
1691 if (reqfssupri > reqfssuprilim)
1692 reqfssupri = reqfssuprilim;
1693
1694 /*
1695 * Basic permissions enforced by generic kernel code for all classes
1696 * require that a thread attempting to change the scheduling parameters
1697 * of a target thread be privileged or have a real or effective UID
1698 * matching that of the target thread. We are not called unless these
1699 * basic permission checks have already passed. The fair-sharing class
1700 * requires in addition that the calling thread be privileged if it
1701 * is attempting to raise the upri limit above its current value.
1702 * This may have been checked previously but if our caller passed us
1703 * a non-NULL credential pointer we assume it hasn't and we check it
1704 * here.
1705 */
1706 if ((reqpcredp != NULL) &&
1707 (reqfssuprilim > fssproc->fss_uprilim) &&
1708 secpolicy_raisepriority(reqpcredp) != 0)
1709 return (EPERM);
1710
1711 /*
1712 * Set fss_nice to the nice value corresponding to the user priority we
1713 * are setting. Note that setting the nice field of the parameter
1714 * struct won't affect upri or nice.
1715 */
1716 nice = NZERO - (reqfssupri * NZERO) / fss_maxupri;
1717 if (nice > FSS_NICE_MAX)
1718 nice = FSS_NICE_MAX;
1719
1720 thread_lock(t);
1721
1722 fssproc->fss_uprilim = reqfssuprilim;
1723 fssproc->fss_upri = reqfssupri;
1724 fssproc->fss_nice = nice;
1725 fss_newpri(fssproc);
1726
1727 if ((fssproc->fss_flags & FSSKPRI) != 0) {
1728 thread_unlock(t);
1729 return (0);
1730 }
1731
1732 fss_change_priority(t, fssproc);
1733 thread_unlock(t);
1734 return (0);
1735
1736 }
1737
1738 /*
1739 * The thread is being stopped.
1740 */
1741 /*ARGSUSED*/
1742 static void
1743 fss_stop(kthread_t *t, int why, int what)
1744 {
1745 ASSERT(THREAD_LOCK_HELD(t));
1746 ASSERT(t == curthread);
1747
1748 fss_inactive(t);
1749 }
1750
1751 /*
1752 * The current thread is exiting, do necessary adjustments to its project
1753 */
1754 static void
1755 fss_exit(kthread_t *t)
1756 {
1757 fsspset_t *fsspset;
1758 fssproj_t *fssproj;
1759 fssproc_t *fssproc;
1760 fsszone_t *fsszone;
1761 int free = 0;
1762
1763 /*
1764 * Thread t here is either a current thread (in which case we hold
1765 * its process' p_lock), or a thread being destroyed by forklwp_fail(),
1766 * in which case we hold pidlock and thread is no longer on the
1767 * thread list.
1768 */
1769 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock) || MUTEX_HELD(&pidlock));
1770
1771 fssproc = FSSPROC(t);
1772 fssproj = FSSPROC2FSSPROJ(fssproc);
1773 fsspset = FSSPROJ2FSSPSET(fssproj);
1774 fsszone = fssproj->fssp_fsszone;
1775
1776 mutex_enter(&fsspsets_lock);
1777 mutex_enter(&fsspset->fssps_lock);
1778
1779 thread_lock(t);
1780 disp_lock_enter_high(&fsspset->fssps_displock);
1781 if (t->t_state == TS_ONPROC || t->t_state == TS_RUN) {
1782 if (--fssproj->fssp_runnable == 0) {
1783 fsszone->fssz_shares -= fssproj->fssp_shares;
1784 if (--fsszone->fssz_runnable == 0)
1785 fsspset->fssps_shares -= fsszone->fssz_rshares;
1786 }
1787 ASSERT(fssproc->fss_runnable == 1);
1788 fssproc->fss_runnable = 0;
1789 }
1790 if (--fssproj->fssp_threads == 0) {
1791 fss_remove_fssproj(fsspset, fssproj);
1792 free = 1;
1793 }
1794 disp_lock_exit_high(&fsspset->fssps_displock);
1795 fssproc->fss_proj = NULL; /* mark this thread as already exited */
1796 thread_unlock(t);
1797
1798 if (free) {
1799 if (fsszone->fssz_nproj == 0)
1800 kmem_free(fsszone, sizeof (fsszone_t));
1801 kmem_free(fssproj, sizeof (fssproj_t));
1802 }
1803 mutex_exit(&fsspset->fssps_lock);
1804 mutex_exit(&fsspsets_lock);
1805
1806 /*
1807 * A thread could be exiting in between clock ticks, so we need to
1808 * calculate how much CPU time it used since it was charged last time.
1809 *
1810 * CPU caps are not enforced on exiting processes - it is usually
1811 * desirable to exit as soon as possible to free resources.
1812 */
1813 if (CPUCAPS_ON()) {
1814 thread_lock(t);
1815 fssproc = FSSPROC(t);
1816 (void) cpucaps_charge(t, &fssproc->fss_caps,
1817 CPUCAPS_CHARGE_ONLY);
1818 thread_unlock(t);
1819 }
1820 }
1821
1822 static void
1823 fss_nullsys()
1824 {
1825 }
1826
1827 /*
1828 * If thread is currently at a kernel mode priority (has slept) and is
1829 * returning to the userland we assign it the appropriate user mode priority
1830 * and time quantum here. If we're lowering the thread's priority below that
1831 * of other runnable threads then we will set runrun via cpu_surrender() to
1832 * cause preemption.
1833 */
1834 static void
1835 fss_trapret(kthread_t *t)
1836 {
1837 fssproc_t *fssproc = FSSPROC(t);
1838 cpu_t *cp = CPU;
1839
1840 ASSERT(THREAD_LOCK_HELD(t));
1841 ASSERT(t == curthread);
1842 ASSERT(cp->cpu_dispthread == t);
1843 ASSERT(t->t_state == TS_ONPROC);
1844
1845 t->t_kpri_req = 0;
1846 if (fssproc->fss_flags & FSSKPRI) {
1847 /*
1848 * If thread has blocked in the kernel
1849 */
1850 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
1851 cp->cpu_dispatch_pri = DISP_PRIO(t);
1852 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
1853 fssproc->fss_flags &= ~FSSKPRI;
1854
1855 if (DISP_MUST_SURRENDER(t))
1856 cpu_surrender(t);
1857 }
1858 }
1859
1860 /*
1861 * Arrange for thread to be placed in appropriate location on dispatcher queue.
1862 * This is called with the current thread in TS_ONPROC and locked.
1863 */
1864 static void
1865 fss_preempt(kthread_t *t)
1866 {
1867 fssproc_t *fssproc = FSSPROC(t);
1868 klwp_t *lwp;
1869 uint_t flags;
1870
1871 ASSERT(t == curthread);
1872 ASSERT(THREAD_LOCK_HELD(curthread));
1873 ASSERT(t->t_state == TS_ONPROC);
1874
1875 /*
1876 * If preempted in the kernel, make sure the thread has a kernel
1877 * priority if needed.
1878 */
1879 lwp = curthread->t_lwp;
1880 if (!(fssproc->fss_flags & FSSKPRI) && lwp != NULL && t->t_kpri_req) {
1881 fssproc->fss_flags |= FSSKPRI;
1882 THREAD_CHANGE_PRI(t, minclsyspri);
1883 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
1884 t->t_trapret = 1; /* so that fss_trapret will run */
1885 aston(t);
1886 }
1887
1888 /*
1889 * This thread may be placed on wait queue by CPU Caps. In this case we
1890 * do not need to do anything until it is removed from the wait queue.
1891 * Do not enforce CPU caps on threads running at a kernel priority
1892 */
1893 if (CPUCAPS_ON()) {
1894 (void) cpucaps_charge(t, &fssproc->fss_caps,
1895 CPUCAPS_CHARGE_ENFORCE);
1896
1897 if (!(fssproc->fss_flags & FSSKPRI) && CPUCAPS_ENFORCE(t))
1898 return;
1899 }
1900
1901 /*
1902 * Check to see if we're doing "preemption control" here. If
1903 * we are, and if the user has requested that this thread not
1904 * be preempted, and if preemptions haven't been put off for
1905 * too long, let the preemption happen here but try to make
1906 * sure the thread is rescheduled as soon as possible. We do
1907 * this by putting it on the front of the highest priority run
1908 * queue in the FSS class. If the preemption has been put off
1909 * for too long, clear the "nopreempt" bit and let the thread
1910 * be preempted.
1911 */
1912 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
1913 if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
1914 DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
1915 if (!(fssproc->fss_flags & FSSKPRI)) {
1916 /*
1917 * If not already remembered, remember current
1918 * priority for restoration in fss_yield().
1919 */
1920 if (!(fssproc->fss_flags & FSSRESTORE)) {
1921 fssproc->fss_scpri = t->t_pri;
1922 fssproc->fss_flags |= FSSRESTORE;
1923 }
1924 THREAD_CHANGE_PRI(t, fss_maxumdpri);
1925 }
1926 schedctl_set_yield(t, 1);
1927 setfrontdq(t);
1928 return;
1929 } else {
1930 if (fssproc->fss_flags & FSSRESTORE) {
1931 THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
1932 fssproc->fss_flags &= ~FSSRESTORE;
1933 }
1934 schedctl_set_nopreempt(t, 0);
1935 DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
1936 /*
1937 * Fall through and be preempted below.
1938 */
1939 }
1940 }
1941
1942 flags = fssproc->fss_flags & (FSSBACKQ | FSSKPRI);
1943
1944 if (flags == FSSBACKQ) {
1945 fssproc->fss_timeleft = fss_quantum;
1946 fssproc->fss_flags &= ~FSSBACKQ;
1947 setbackdq(t);
1948 } else if (flags == (FSSBACKQ | FSSKPRI)) {
1949 fssproc->fss_flags &= ~FSSBACKQ;
1950 setbackdq(t);
1951 } else {
1952 setfrontdq(t);
1953 }
1954 }
1955
1956 /*
1957 * Called when a thread is waking up and is to be placed on the run queue.
1958 */
1959 static void
1960 fss_setrun(kthread_t *t)
1961 {
1962 fssproc_t *fssproc = FSSPROC(t);
1963
1964 ASSERT(THREAD_LOCK_HELD(t)); /* t should be in transition */
1965
1966 if (t->t_state == TS_SLEEP || t->t_state == TS_STOPPED)
1967 fss_active(t);
1968
1969 fssproc->fss_timeleft = fss_quantum;
1970
1971 fssproc->fss_flags &= ~FSSBACKQ;
1972 /*
1973 * If previously were running at the kernel priority then keep that
1974 * priority and the fss_timeleft doesn't matter.
1975 */
1976 if ((fssproc->fss_flags & FSSKPRI) == 0)
1977 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
1978
1979 if (t->t_disp_time != ddi_get_lbolt())
1980 setbackdq(t);
1981 else
1982 setfrontdq(t);
1983 }
1984
1985 /*
1986 * Prepare thread for sleep. We reset the thread priority so it will run at the
1987 * kernel priority level when it wakes up.
1988 */
1989 static void
1990 fss_sleep(kthread_t *t)
1991 {
1992 fssproc_t *fssproc = FSSPROC(t);
1993
1994 ASSERT(t == curthread);
1995 ASSERT(THREAD_LOCK_HELD(t));
1996
1997 ASSERT(t->t_state == TS_ONPROC);
1998
1999 /*
2000 * Account for time spent on CPU before going to sleep.
2001 */
2002 (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2003
2004 fss_inactive(t);
2005
2006 /*
2007 * Assign a system priority to the thread and arrange for it to be
2008 * retained when the thread is next placed on the run queue (i.e.,
2009 * when it wakes up) instead of being given a new pri. Also arrange
2010 * for trapret processing as the thread leaves the system call so it
2011 * will drop back to normal priority range.
2012 */
2013 if (t->t_kpri_req) {
2014 THREAD_CHANGE_PRI(t, minclsyspri);
2015 fssproc->fss_flags |= FSSKPRI;
2016 t->t_trapret = 1; /* so that fss_trapret will run */
2017 aston(t);
2018 } else if (fssproc->fss_flags & FSSKPRI) {
2019 /*
2020 * The thread has done a THREAD_KPRI_REQUEST(), slept, then
2021 * done THREAD_KPRI_RELEASE() (so no t_kpri_req is 0 again),
2022 * then slept again all without finishing the current system
2023 * call so trapret won't have cleared FSSKPRI
2024 */
2025 fssproc->fss_flags &= ~FSSKPRI;
2026 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2027 if (DISP_MUST_SURRENDER(curthread))
2028 cpu_surrender(t);
2029 }
2030 }
2031
2032 /*
2033 * A tick interrupt has ocurrend on a running thread. Check to see if our
2034 * time slice has expired.
2035 */
2036 static void
2037 fss_tick(kthread_t *t)
2038 {
2039 fssproc_t *fssproc;
2040 fssproj_t *fssproj;
2041 klwp_t *lwp;
2042 boolean_t call_cpu_surrender = B_FALSE;
2043 boolean_t cpucaps_enforce = B_FALSE;
2044
2045 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
2046
2047 /*
2048 * It's safe to access fsspset and fssproj structures because we're
2049 * holding our p_lock here.
2050 */
2051 thread_lock(t);
2052 fssproc = FSSPROC(t);
2053 fssproj = FSSPROC2FSSPROJ(fssproc);
2054 if (fssproj != NULL) {
2055 fsspset_t *fsspset = FSSPROJ2FSSPSET(fssproj);
2056 disp_lock_enter_high(&fsspset->fssps_displock);
2057 fssproj->fssp_ticks += fss_nice_tick[fssproc->fss_nice];
2058 fssproc->fss_ticks++;
2059 disp_lock_exit_high(&fsspset->fssps_displock);
2060 }
2061
2062 /*
2063 * Keep track of thread's project CPU usage. Note that projects
2064 * get charged even when threads are running in the kernel.
2065 * Do not surrender CPU if running in the SYS class.
2066 */
2067 if (CPUCAPS_ON()) {
2068 cpucaps_enforce = cpucaps_charge(t,
2069 &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE) &&
2070 !(fssproc->fss_flags & FSSKPRI);
2071 }
2072
2073 /*
2074 * A thread's execution time for threads running in the SYS class
2075 * is not tracked.
2076 */
2077 if ((fssproc->fss_flags & FSSKPRI) == 0) {
2078 /*
2079 * If thread is not in kernel mode, decrement its fss_timeleft
2080 */
2081 if (--fssproc->fss_timeleft <= 0) {
2082 pri_t new_pri;
2083
2084 /*
2085 * If we're doing preemption control and trying to
2086 * avoid preempting this thread, just note that the
2087 * thread should yield soon and let it keep running
2088 * (unless it's been a while).
2089 */
2090 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2091 if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2092 DTRACE_SCHED1(schedctl__nopreempt,
2093 kthread_t *, t);
2094 schedctl_set_yield(t, 1);
2095 thread_unlock_nopreempt(t);
2096 return;
2097 }
2098 }
2099 fssproc->fss_flags &= ~FSSRESTORE;
2100
2101 fss_newpri(fssproc);
2102 new_pri = fssproc->fss_umdpri;
2103 ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
2104
2105 /*
2106 * When the priority of a thread is changed, it may
2107 * be necessary to adjust its position on a sleep queue
2108 * or dispatch queue. The function thread_change_pri
2109 * accomplishes this.
2110 */
2111 if (thread_change_pri(t, new_pri, 0)) {
2112 fssproc->fss_timeleft = fss_quantum;
2113 } else {
2114 call_cpu_surrender = B_TRUE;
2115 }
2116 } else if (t->t_state == TS_ONPROC &&
2117 t->t_pri < t->t_disp_queue->disp_maxrunpri) {
2118 /*
2119 * If there is a higher-priority thread which is
2120 * waiting for a processor, then thread surrenders
2121 * the processor.
2122 */
2123 call_cpu_surrender = B_TRUE;
2124 }
2125 }
2126
2127 if (cpucaps_enforce && 2 * fssproc->fss_timeleft > fss_quantum) {
2128 /*
2129 * The thread used more than half of its quantum, so assume that
2130 * it used the whole quantum.
2131 *
2132 * Update thread's priority just before putting it on the wait
2133 * queue so that it gets charged for the CPU time from its
2134 * quantum even before that quantum expires.
2135 */
2136 fss_newpri(fssproc);
2137 if (t->t_pri != fssproc->fss_umdpri)
2138 fss_change_priority(t, fssproc);
2139
2140 /*
2141 * We need to call cpu_surrender for this thread due to cpucaps
2142 * enforcement, but fss_change_priority may have already done
2143 * so. In this case FSSBACKQ is set and there is no need to call
2144 * cpu-surrender again.
2145 */
2146 if (!(fssproc->fss_flags & FSSBACKQ))
2147 call_cpu_surrender = B_TRUE;
2148 }
2149
2150 if (call_cpu_surrender) {
2151 fssproc->fss_flags |= FSSBACKQ;
2152 cpu_surrender(t);
2153 }
2154
2155 thread_unlock_nopreempt(t); /* clock thread can't be preempted */
2156 }
2157
2158 /*
2159 * Processes waking up go to the back of their queue. We don't need to assign
2160 * a time quantum here because thread is still at a kernel mode priority and
2161 * the time slicing is not done for threads running in the kernel after
2162 * sleeping. The proper time quantum will be assigned by fss_trapret before the
2163 * thread returns to user mode.
2164 */
2165 static void
2166 fss_wakeup(kthread_t *t)
2167 {
2168 fssproc_t *fssproc;
2169
2170 ASSERT(THREAD_LOCK_HELD(t));
2171 ASSERT(t->t_state == TS_SLEEP);
2172
2173 fss_active(t);
2174
2175 fssproc = FSSPROC(t);
2176 fssproc->fss_flags &= ~FSSBACKQ;
2177
2178 if (fssproc->fss_flags & FSSKPRI) {
2179 /*
2180 * If we already have a kernel priority assigned, then we
2181 * just use it.
2182 */
2183 setbackdq(t);
2184 } else if (t->t_kpri_req) {
2185 /*
2186 * Give thread a priority boost if we were asked.
2187 */
2188 fssproc->fss_flags |= FSSKPRI;
2189 THREAD_CHANGE_PRI(t, minclsyspri);
2190 setbackdq(t);
2191 t->t_trapret = 1; /* so that fss_trapret will run */
2192 aston(t);
2193 } else {
2194 /*
2195 * Otherwise, we recalculate the priority.
2196 */
2197 if (t->t_disp_time == ddi_get_lbolt()) {
2198 setfrontdq(t);
2199 } else {
2200 fssproc->fss_timeleft = fss_quantum;
2201 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2202 setbackdq(t);
2203 }
2204 }
2205 }
2206
2207 /*
2208 * fss_donice() is called when a nice(1) command is issued on the thread to
2209 * alter the priority. The nice(1) command exists in Solaris for compatibility.
2210 * Thread priority adjustments should be done via priocntl(1).
2211 */
2212 static int
2213 fss_donice(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2214 {
2215 int newnice;
2216 fssproc_t *fssproc = FSSPROC(t);
2217 fssparms_t fssparms;
2218
2219 /*
2220 * If there is no change to priority, just return current setting.
2221 */
2222 if (incr == 0) {
2223 if (retvalp)
2224 *retvalp = fssproc->fss_nice - NZERO;
2225 return (0);
2226 }
2227
2228 if ((incr < 0 || incr > 2 * NZERO) && secpolicy_raisepriority(cr) != 0)
2229 return (EPERM);
2230
2231 /*
2232 * Specifying a nice increment greater than the upper limit of
2233 * FSS_NICE_MAX (== 2 * NZERO - 1) will result in the thread's nice
2234 * value being set to the upper limit. We check for this before
2235 * computing the new value because otherwise we could get overflow
2236 * if a privileged user specified some ridiculous increment.
2237 */
2238 if (incr > FSS_NICE_MAX)
2239 incr = FSS_NICE_MAX;
2240
2241 newnice = fssproc->fss_nice + incr;
2242 if (newnice > FSS_NICE_MAX)
2243 newnice = FSS_NICE_MAX;
2244 else if (newnice < FSS_NICE_MIN)
2245 newnice = FSS_NICE_MIN;
2246
2247 fssparms.fss_uprilim = fssparms.fss_upri =
2248 -((newnice - NZERO) * fss_maxupri) / NZERO;
2249
2250 /*
2251 * Reset the uprilim and upri values of the thread.
2252 */
2253 (void) fss_parmsset(t, (void *)&fssparms, (id_t)0, (cred_t *)NULL);
2254
2255 /*
2256 * Although fss_parmsset already reset fss_nice it may not have been
2257 * set to precisely the value calculated above because fss_parmsset
2258 * determines the nice value from the user priority and we may have
2259 * truncated during the integer conversion from nice value to user
2260 * priority and back. We reset fss_nice to the value we calculated
2261 * above.
2262 */
2263 fssproc->fss_nice = (char)newnice;
2264
2265 if (retvalp)
2266 *retvalp = newnice - NZERO;
2267 return (0);
2268 }
2269
2270 /*
2271 * Increment the priority of the specified thread by incr and
2272 * return the new value in *retvalp.
2273 */
2274 static int
2275 fss_doprio(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2276 {
2277 int newpri;
2278 fssproc_t *fssproc = FSSPROC(t);
2279 fssparms_t fssparms;
2280
2281 /*
2282 * If there is no change to priority, just return current setting.
2283 */
2284 if (incr == 0) {
2285 *retvalp = fssproc->fss_upri;
2286 return (0);
2287 }
2288
2289 newpri = fssproc->fss_upri + incr;
2290 if (newpri > fss_maxupri || newpri < -fss_maxupri)
2291 return (EINVAL);
2292
2293 *retvalp = newpri;
2294 fssparms.fss_uprilim = fssparms.fss_upri = newpri;
2295
2296 /*
2297 * Reset the uprilim and upri values of the thread.
2298 */
2299 return (fss_parmsset(t, &fssparms, (id_t)0, cr));
2300 }
2301
2302 /*
2303 * Return the global scheduling priority that would be assigned to a thread
2304 * entering the fair-sharing class with the fss_upri.
2305 */
2306 /*ARGSUSED*/
2307 static pri_t
2308 fss_globpri(kthread_t *t)
2309 {
2310 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2311
2312 return (fss_maxumdpri / 2);
2313 }
2314
2315 /*
2316 * Called from the yield(2) system call when a thread is yielding (surrendering)
2317 * the processor. The kernel thread is placed at the back of a dispatch queue.
2318 */
2319 static void
2320 fss_yield(kthread_t *t)
2321 {
2322 fssproc_t *fssproc = FSSPROC(t);
2323
2324 ASSERT(t == curthread);
2325 ASSERT(THREAD_LOCK_HELD(t));
2326
2327 /*
2328 * Collect CPU usage spent before yielding
2329 */
2330 (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2331
2332 /*
2333 * Clear the preemption control "yield" bit since the user is
2334 * doing a yield.
2335 */
2336 if (t->t_schedctl)
2337 schedctl_set_yield(t, 0);
2338 /*
2339 * If fss_preempt() artifically increased the thread's priority
2340 * to avoid preemption, restore the original priority now.
2341 */
2342 if (fssproc->fss_flags & FSSRESTORE) {
2343 THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2344 fssproc->fss_flags &= ~FSSRESTORE;
2345 }
2346 if (fssproc->fss_timeleft < 0) {
2347 /*
2348 * Time slice was artificially extended to avoid preemption,
2349 * so pretend we're preempting it now.
2350 */
2351 DTRACE_SCHED1(schedctl__yield, int, -fssproc->fss_timeleft);
2352 fssproc->fss_timeleft = fss_quantum;
2353 }
2354 fssproc->fss_flags &= ~FSSBACKQ;
2355 setbackdq(t);
2356 }
2357
2358 void
2359 fss_changeproj(kthread_t *t, void *kp, void *zp, fssbuf_t *projbuf,
2360 fssbuf_t *zonebuf)
2361 {
2362 kproject_t *kpj_new = kp;
2363 zone_t *zone = zp;
2364 fssproj_t *fssproj_old, *fssproj_new;
2365 fsspset_t *fsspset;
2366 kproject_t *kpj_old;
2367 fssproc_t *fssproc;
2368 fsszone_t *fsszone_old, *fsszone_new;
2369 int free = 0;
2370 int id;
2371
2372 ASSERT(MUTEX_HELD(&cpu_lock));
2373 ASSERT(MUTEX_HELD(&pidlock));
2374 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2375
2376 if (t->t_cid != fss_cid)
2377 return;
2378
2379 fssproc = FSSPROC(t);
2380 mutex_enter(&fsspsets_lock);
2381 fssproj_old = FSSPROC2FSSPROJ(fssproc);
2382 if (fssproj_old == NULL) {
2383 mutex_exit(&fsspsets_lock);
2384 return;
2385 }
2386
2387 fsspset = FSSPROJ2FSSPSET(fssproj_old);
2388 mutex_enter(&fsspset->fssps_lock);
2389 kpj_old = FSSPROJ2KPROJ(fssproj_old);
2390 fsszone_old = fssproj_old->fssp_fsszone;
2391
2392 ASSERT(t->t_cpupart == fsspset->fssps_cpupart);
2393
2394 if (kpj_old == kpj_new) {
2395 mutex_exit(&fsspset->fssps_lock);
2396 mutex_exit(&fsspsets_lock);
2397 return;
2398 }
2399
2400 if ((fsszone_new = fss_find_fsszone(fsspset, zone)) == NULL) {
2401 /*
2402 * If the zone for the new project is not currently active on
2403 * the cpu partition we're on, get one of the pre-allocated
2404 * buffers and link it in our per-pset zone list. Such buffers
2405 * should already exist.
2406 */
2407 for (id = 0; id < zonebuf->fssb_size; id++) {
2408 if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
2409 fss_insert_fsszone(fsspset, zone, fsszone_new);
2410 zonebuf->fssb_list[id] = NULL;
2411 break;
2412 }
2413 }
2414 }
2415 ASSERT(fsszone_new != NULL);
2416 if ((fssproj_new = fss_find_fssproj(fsspset, kpj_new)) == NULL) {
2417 /*
2418 * If our new project is not currently running
2419 * on the cpu partition we're on, get one of the
2420 * pre-allocated buffers and link it in our new cpu
2421 * partition doubly linked list. Such buffers should already
2422 * exist.
2423 */
2424 for (id = 0; id < projbuf->fssb_size; id++) {
2425 if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
2426 fss_insert_fssproj(fsspset, kpj_new,
2427 fsszone_new, fssproj_new);
2428 projbuf->fssb_list[id] = NULL;
2429 break;
2430 }
2431 }
2432 }
2433 ASSERT(fssproj_new != NULL);
2434
2435 thread_lock(t);
2436 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2437 t->t_state == TS_WAIT)
2438 fss_inactive(t);
2439 ASSERT(fssproj_old->fssp_threads > 0);
2440 if (--fssproj_old->fssp_threads == 0) {
2441 fss_remove_fssproj(fsspset, fssproj_old);
2442 free = 1;
2443 }
2444 fssproc->fss_proj = fssproj_new;
2445 fssproc->fss_fsspri = 0;
2446 fssproj_new->fssp_threads++;
2447 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2448 t->t_state == TS_WAIT)
2449 fss_active(t);
2450 thread_unlock(t);
2451 if (free) {
2452 if (fsszone_old->fssz_nproj == 0)
2453 kmem_free(fsszone_old, sizeof (fsszone_t));
2454 kmem_free(fssproj_old, sizeof (fssproj_t));
2455 }
2456
2457 mutex_exit(&fsspset->fssps_lock);
2458 mutex_exit(&fsspsets_lock);
2459 }
2460
2461 void
2462 fss_changepset(kthread_t *t, void *newcp, fssbuf_t *projbuf,
2463 fssbuf_t *zonebuf)
2464 {
2465 fsspset_t *fsspset_old, *fsspset_new;
2466 fssproj_t *fssproj_old, *fssproj_new;
2467 fsszone_t *fsszone_old, *fsszone_new;
2468 fssproc_t *fssproc;
2469 kproject_t *kpj;
2470 zone_t *zone;
2471 int id;
2472
2473 ASSERT(MUTEX_HELD(&cpu_lock));
2474 ASSERT(MUTEX_HELD(&pidlock));
2475 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2476
2477 if (t->t_cid != fss_cid)
2478 return;
2479
2480 fssproc = FSSPROC(t);
2481 zone = ttoproc(t)->p_zone;
2482 mutex_enter(&fsspsets_lock);
2483 fssproj_old = FSSPROC2FSSPROJ(fssproc);
2484 if (fssproj_old == NULL) {
2485 mutex_exit(&fsspsets_lock);
2486 return;
2487 }
2488 fsszone_old = fssproj_old->fssp_fsszone;
2489 fsspset_old = FSSPROJ2FSSPSET(fssproj_old);
2490 kpj = FSSPROJ2KPROJ(fssproj_old);
2491
2492 if (fsspset_old->fssps_cpupart == newcp) {
2493 mutex_exit(&fsspsets_lock);
2494 return;
2495 }
2496
2497 ASSERT(ttoproj(t) == kpj);
2498
2499 fsspset_new = fss_find_fsspset(newcp);
2500
2501 mutex_enter(&fsspset_new->fssps_lock);
2502 if ((fsszone_new = fss_find_fsszone(fsspset_new, zone)) == NULL) {
2503 for (id = 0; id < zonebuf->fssb_size; id++) {
2504 if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
2505 fss_insert_fsszone(fsspset_new, zone,
2506 fsszone_new);
2507 zonebuf->fssb_list[id] = NULL;
2508 break;
2509 }
2510 }
2511 }
2512 ASSERT(fsszone_new != NULL);
2513 if ((fssproj_new = fss_find_fssproj(fsspset_new, kpj)) == NULL) {
2514 for (id = 0; id < projbuf->fssb_size; id++) {
2515 if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
2516 fss_insert_fssproj(fsspset_new, kpj,
2517 fsszone_new, fssproj_new);
2518 projbuf->fssb_list[id] = NULL;
2519 break;
2520 }
2521 }
2522 }
2523 ASSERT(fssproj_new != NULL);
2524
2525 fssproj_new->fssp_threads++;
2526 thread_lock(t);
2527 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2528 t->t_state == TS_WAIT)
2529 fss_inactive(t);
2530 fssproc->fss_proj = fssproj_new;
2531 fssproc->fss_fsspri = 0;
2532 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2533 t->t_state == TS_WAIT)
2534 fss_active(t);
2535 thread_unlock(t);
2536 mutex_exit(&fsspset_new->fssps_lock);
2537
2538 mutex_enter(&fsspset_old->fssps_lock);
2539 if (--fssproj_old->fssp_threads == 0) {
2540 fss_remove_fssproj(fsspset_old, fssproj_old);
2541 if (fsszone_old->fssz_nproj == 0)
2542 kmem_free(fsszone_old, sizeof (fsszone_t));
2543 kmem_free(fssproj_old, sizeof (fssproj_t));
2544 }
2545 mutex_exit(&fsspset_old->fssps_lock);
2546
2547 mutex_exit(&fsspsets_lock);
2548 }