Print this page
5253 kmem_alloc/kmem_zalloc won't fail with KM_SLEEP
5254 getrbuf won't fail with KM_SLEEP
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/intel/io/drm/i915_gem.c
+++ new/usr/src/uts/intel/io/drm/i915_gem.c
1 1 /* BEGIN CSTYLED */
2 2
3 3 /*
4 4 * Copyright (c) 2009, Intel Corporation.
5 5 * All Rights Reserved.
6 6 *
7 7 * Permission is hereby granted, free of charge, to any person obtaining a
8 8 * copy of this software and associated documentation files (the "Software"),
9 9 * to deal in the Software without restriction, including without limitation
10 10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 11 * and/or sell copies of the Software, and to permit persons to whom the
12 12 * Software is furnished to do so, subject to the following conditions:
13 13 *
14 14 * The above copyright notice and this permission notice (including the next
15 15 * paragraph) shall be included in all copies or substantial portions of the
16 16 * Software.
17 17 *
18 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 21 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22 22 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
23 23 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24 24 * IN THE SOFTWARE.
25 25 *
26 26 * Authors:
27 27 * Eric Anholt <eric@anholt.net>
28 28 *
29 29 */
30 30
31 31 /*
32 32 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
33 33 * Use is subject to license terms.
34 34 */
35 35
36 36 #include <sys/x86_archext.h>
37 37 #include <sys/vfs_opreg.h>
38 38 #include "drmP.h"
39 39 #include "drm.h"
40 40 #include "i915_drm.h"
41 41 #include "i915_drv.h"
42 42
43 43 #ifndef roundup
44 44 #define roundup(x, y) ((((x)+((y)-1))/(y))*(y))
45 45 #endif /* !roundup */
46 46
47 47 #define I915_GEM_GPU_DOMAINS (~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
48 48
49 49 static timeout_id_t worktimer_id = NULL;
50 50
51 51 extern int drm_mm_init(struct drm_mm *mm,
52 52 unsigned long start, unsigned long size);
53 53 extern void drm_mm_put_block(struct drm_mm_node *cur);
54 54 extern int choose_addr(struct as *as, caddr_t *addrp, size_t len, offset_t off,
55 55 int vacalign, uint_t flags);
56 56
57 57 static void
58 58 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
59 59 uint32_t read_domains,
60 60 uint32_t write_domain);
61 61 static void i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj);
62 62 static void i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj);
63 63 static void i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj);
64 64 static int i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj,
65 65 int write);
66 66 static int i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj,
67 67 int write);
68 68 static int i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
69 69 uint64_t offset,
70 70 uint64_t size);
71 71 static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj);
72 72 static void i915_gem_object_free_page_list(struct drm_gem_object *obj);
73 73 static int i915_gem_object_wait_rendering(struct drm_gem_object *obj);
74 74 static int i915_gem_object_get_page_list(struct drm_gem_object *obj);
75 75
76 76 static void
77 77 i915_gem_cleanup_ringbuffer(struct drm_device *dev);
78 78
79 79 /*ARGSUSED*/
80 80 int
81 81 i915_gem_init_ioctl(DRM_IOCTL_ARGS)
82 82 {
83 83 DRM_DEVICE;
84 84 drm_i915_private_t *dev_priv = dev->dev_private;
85 85 struct drm_i915_gem_init args;
86 86
87 87 if (dev->driver->use_gem != 1)
88 88 return ENODEV;
89 89
90 90 DRM_COPYFROM_WITH_RETURN(&args,
91 91 (struct drm_i915_gem_init *) data, sizeof(args));
92 92
93 93 spin_lock(&dev->struct_mutex);
94 94
95 95 if ((args.gtt_start >= args.gtt_end) ||
96 96 ((args.gtt_start & (PAGE_SIZE - 1)) != 0) ||
97 97 ((args.gtt_end & (PAGE_SIZE - 1)) != 0)) {
98 98 spin_unlock(&dev->struct_mutex);
99 99 DRM_ERROR("i915_gem_init_ioctel invalid arg 0x%lx args.start 0x%lx end 0x%lx", &args, args.gtt_start, args.gtt_end);
100 100 return EINVAL;
101 101 }
102 102
103 103 dev->gtt_total = (uint32_t) (args.gtt_end - args.gtt_start);
104 104
105 105 (void) drm_mm_init(&dev_priv->mm.gtt_space,
106 106 (unsigned long) args.gtt_start, dev->gtt_total);
107 107 DRM_DEBUG("i915_gem_init_ioctl dev->gtt_total %x, dev_priv->mm.gtt_space 0x%x gtt_start 0x%lx", dev->gtt_total, dev_priv->mm.gtt_space, args.gtt_start);
108 108 ASSERT(dev->gtt_total != 0);
109 109
110 110 spin_unlock(&dev->struct_mutex);
111 111
112 112
113 113 return 0;
114 114 }
115 115
116 116 /*ARGSUSED*/
117 117 int
118 118 i915_gem_get_aperture_ioctl(DRM_IOCTL_ARGS)
119 119 {
120 120 DRM_DEVICE;
121 121 struct drm_i915_gem_get_aperture args;
122 122 int ret;
123 123
124 124 if (dev->driver->use_gem != 1)
125 125 return ENODEV;
126 126
127 127 args.aper_size = (uint64_t)dev->gtt_total;
128 128 args.aper_available_size = (args.aper_size -
129 129 atomic_read(&dev->pin_memory));
130 130
131 131 ret = DRM_COPY_TO_USER((struct drm_i915_gem_get_aperture __user *) data, &args, sizeof(args));
132 132
133 133 if ( ret != 0)
134 134 DRM_ERROR(" i915_gem_get_aperture_ioctl error! %d", ret);
135 135
136 136 DRM_DEBUG("i915_gem_get_aaperture_ioctl called sizeof %d, aper_size 0x%x, aper_available_size 0x%x\n", sizeof(args), dev->gtt_total, args.aper_available_size);
137 137
138 138 return 0;
139 139 }
140 140
141 141 /**
142 142 * Creates a new mm object and returns a handle to it.
143 143 */
144 144 /*ARGSUSED*/
145 145 int
146 146 i915_gem_create_ioctl(DRM_IOCTL_ARGS)
147 147 {
148 148 DRM_DEVICE;
149 149 struct drm_i915_gem_create args;
150 150 struct drm_gem_object *obj;
151 151 int handlep;
152 152 int ret;
153 153
154 154 if (dev->driver->use_gem != 1)
155 155 return ENODEV;
156 156
157 157 DRM_COPYFROM_WITH_RETURN(&args,
158 158 (struct drm_i915_gem_create *) data, sizeof(args));
159 159
160 160
161 161 args.size = (uint64_t) roundup(args.size, PAGE_SIZE);
162 162
163 163 if (args.size == 0) {
164 164 DRM_ERROR("Invalid obj size %d", args.size);
165 165 return EINVAL;
166 166 }
167 167 /* Allocate the new object */
168 168 obj = drm_gem_object_alloc(dev, args.size);
169 169 if (obj == NULL) {
170 170 DRM_ERROR("Failed to alloc obj");
171 171 return ENOMEM;
172 172 }
173 173
174 174 ret = drm_gem_handle_create(fpriv, obj, &handlep);
175 175 spin_lock(&dev->struct_mutex);
176 176 drm_gem_object_handle_unreference(obj);
177 177 spin_unlock(&dev->struct_mutex);
178 178 if (ret)
179 179 return ret;
180 180
181 181 args.handle = handlep;
182 182
183 183 ret = DRM_COPY_TO_USER((struct drm_i915_gem_create *) data, &args, sizeof(args));
184 184
185 185 if ( ret != 0)
186 186 DRM_ERROR(" gem create error! %d", ret);
187 187
188 188 DRM_DEBUG("i915_gem_create_ioctl object name %d, size 0x%lx, list 0x%lx, obj 0x%lx",handlep, args.size, &fpriv->object_idr, obj);
189 189
190 190 return 0;
191 191 }
192 192
193 193 /**
194 194 * Reads data from the object referenced by handle.
195 195 *
196 196 * On error, the contents of *data are undefined.
197 197 */
198 198 /*ARGSUSED*/
199 199 int
200 200 i915_gem_pread_ioctl(DRM_IOCTL_ARGS)
201 201 {
202 202 DRM_DEVICE;
203 203 struct drm_i915_gem_pread args;
204 204 struct drm_gem_object *obj;
205 205 int ret;
206 206
207 207 if (dev->driver->use_gem != 1)
208 208 return ENODEV;
209 209
210 210 DRM_COPYFROM_WITH_RETURN(&args,
211 211 (struct drm_i915_gem_pread __user *) data, sizeof(args));
212 212
213 213 obj = drm_gem_object_lookup(fpriv, args.handle);
214 214 if (obj == NULL)
215 215 return EBADF;
216 216
217 217 /* Bounds check source.
218 218 *
219 219 * XXX: This could use review for overflow issues...
220 220 */
221 221 if (args.offset > obj->size || args.size > obj->size ||
222 222 args.offset + args.size > obj->size) {
223 223 drm_gem_object_unreference(obj);
224 224 DRM_ERROR("i915_gem_pread_ioctl invalid args");
225 225 return EINVAL;
226 226 }
227 227
228 228 spin_lock(&dev->struct_mutex);
229 229
230 230 ret = i915_gem_object_set_cpu_read_domain_range(obj, args.offset, args.size);
231 231 if (ret != 0) {
232 232 drm_gem_object_unreference(obj);
233 233 spin_unlock(&dev->struct_mutex);
234 234 DRM_ERROR("pread failed to read domain range ret %d!!!", ret);
235 235 return EFAULT;
236 236 }
237 237
238 238 unsigned long unwritten = 0;
239 239 uint32_t *user_data;
240 240 user_data = (uint32_t *) (uintptr_t) args.data_ptr;
241 241
242 242 unwritten = DRM_COPY_TO_USER(user_data, obj->kaddr + args.offset, args.size);
243 243 if (unwritten) {
244 244 ret = EFAULT;
245 245 DRM_ERROR("i915_gem_pread error!!! unwritten %d", unwritten);
246 246 }
247 247
248 248 drm_gem_object_unreference(obj);
249 249 spin_unlock(&dev->struct_mutex);
250 250
251 251 return ret;
252 252 }
253 253
254 254 /*ARGSUSED*/
255 255 static int
256 256 i915_gem_gtt_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
257 257 struct drm_i915_gem_pwrite *args,
258 258 struct drm_file *file_priv)
259 259 {
260 260 uint32_t *user_data;
261 261 int ret = 0;
262 262 unsigned long unwritten = 0;
263 263
264 264 user_data = (uint32_t *) (uintptr_t) args->data_ptr;
265 265 spin_lock(&dev->struct_mutex);
266 266 ret = i915_gem_object_pin(obj, 0);
267 267 if (ret) {
268 268 spin_unlock(&dev->struct_mutex);
269 269 DRM_ERROR("i915_gem_gtt_pwrite failed to pin ret %d", ret);
270 270 return ret;
271 271 }
272 272
273 273 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
274 274 if (ret)
275 275 goto err;
276 276
277 277 DRM_DEBUG("obj %d write domain 0x%x read domain 0x%x", obj->name, obj->write_domain, obj->read_domains);
278 278
279 279 unwritten = DRM_COPY_FROM_USER(obj->kaddr + args->offset, user_data, args->size);
280 280 if (unwritten) {
281 281 ret = EFAULT;
282 282 DRM_ERROR("i915_gem_gtt_pwrite error!!! unwritten %d", unwritten);
283 283 goto err;
284 284 }
285 285
286 286 err:
287 287 i915_gem_object_unpin(obj);
288 288 spin_unlock(&dev->struct_mutex);
289 289 if (ret)
290 290 DRM_ERROR("i915_gem_gtt_pwrite error %d", ret);
291 291 return ret;
292 292 }
293 293
294 294 /*ARGSUSED*/
295 295 int
296 296 i915_gem_shmem_pwrite(struct drm_device *dev, struct drm_gem_object *obj,
297 297 struct drm_i915_gem_pwrite *args,
298 298 struct drm_file *file_priv)
299 299 {
300 300 DRM_ERROR(" i915_gem_shmem_pwrite Not support");
301 301 return -1;
302 302 }
303 303
304 304 /**
305 305 * Writes data to the object referenced by handle.
306 306 *
307 307 * On error, the contents of the buffer that were to be modified are undefined.
308 308 */
309 309 /*ARGSUSED*/
310 310 int
311 311 i915_gem_pwrite_ioctl(DRM_IOCTL_ARGS)
312 312 {
313 313 DRM_DEVICE;
314 314 struct drm_i915_gem_pwrite args;
315 315 struct drm_gem_object *obj;
316 316 struct drm_i915_gem_object *obj_priv;
317 317 int ret = 0;
318 318
319 319 if (dev->driver->use_gem != 1)
320 320 return ENODEV;
321 321
322 322 ret = DRM_COPY_FROM_USER(&args,
323 323 (struct drm_i915_gem_pwrite __user *) data, sizeof(args));
324 324 if (ret)
325 325 DRM_ERROR("i915_gem_pwrite_ioctl failed to copy from user");
326 326 obj = drm_gem_object_lookup(fpriv, args.handle);
327 327 if (obj == NULL)
328 328 return EBADF;
329 329 obj_priv = obj->driver_private;
330 330 DRM_DEBUG("i915_gem_pwrite_ioctl, obj->name %d",obj->name);
331 331
332 332 /* Bounds check destination.
333 333 *
334 334 * XXX: This could use review for overflow issues...
335 335 */
336 336 if (args.offset > obj->size || args.size > obj->size ||
337 337 args.offset + args.size > obj->size) {
338 338 drm_gem_object_unreference(obj);
339 339 DRM_ERROR("i915_gem_pwrite_ioctl invalid arg");
340 340 return EINVAL;
341 341 }
342 342
343 343 /* We can only do the GTT pwrite on untiled buffers, as otherwise
344 344 * it would end up going through the fenced access, and we'll get
345 345 * different detiling behavior between reading and writing.
346 346 * pread/pwrite currently are reading and writing from the CPU
347 347 * perspective, requiring manual detiling by the client.
348 348 */
349 349 if (obj_priv->tiling_mode == I915_TILING_NONE &&
350 350 dev->gtt_total != 0)
351 351 ret = i915_gem_gtt_pwrite(dev, obj, &args, fpriv);
352 352 else
353 353 ret = i915_gem_shmem_pwrite(dev, obj, &args, fpriv);
354 354
355 355 if (ret)
356 356 DRM_ERROR("pwrite failed %d\n", ret);
357 357
358 358 drm_gem_object_unreference(obj);
359 359
360 360 return ret;
361 361 }
362 362
363 363 /**
364 364 * Called when user space prepares to use an object with the CPU, either
365 365 * through the mmap ioctl's mapping or a GTT mapping.
366 366 */
367 367 /*ARGSUSED*/
368 368 int
369 369 i915_gem_set_domain_ioctl(DRM_IOCTL_ARGS)
370 370 {
371 371 DRM_DEVICE;
372 372 struct drm_i915_gem_set_domain args;
373 373 struct drm_gem_object *obj;
374 374 int ret = 0;
375 375
376 376 if (dev->driver->use_gem != 1)
377 377 return ENODEV;
378 378
379 379 DRM_COPYFROM_WITH_RETURN(&args,
380 380 (struct drm_i915_gem_set_domain __user *) data, sizeof(args));
381 381
382 382 uint32_t read_domains = args.read_domains;
383 383 uint32_t write_domain = args.write_domain;
384 384
385 385 /* Only handle setting domains to types used by the CPU. */
386 386 if (write_domain & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
387 387 ret = EINVAL;
388 388
389 389 if (read_domains & ~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
390 390 ret = EINVAL;
391 391
392 392 /* Having something in the write domain implies it's in the read
393 393 * domain, and only that read domain. Enforce that in the request.
394 394 */
395 395 if (write_domain != 0 && read_domains != write_domain)
396 396 ret = EINVAL;
397 397 if (ret) {
398 398 DRM_ERROR("set_domain invalid read or write");
399 399 return EINVAL;
400 400 }
401 401
402 402 obj = drm_gem_object_lookup(fpriv, args.handle);
403 403 if (obj == NULL)
404 404 return EBADF;
405 405
406 406 spin_lock(&dev->struct_mutex);
407 407 DRM_DEBUG("set_domain_ioctl %p(name %d size 0x%x), %08x %08x\n",
408 408 obj, obj->name, obj->size, args.read_domains, args.write_domain);
409 409
410 410 if (read_domains & I915_GEM_DOMAIN_GTT) {
411 411 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
412 412
413 413 /* Silently promote "you're not bound, there was nothing to do"
414 414 * to success, since the client was just asking us to
415 415 * make sure everything was done.
416 416 */
417 417 if (ret == EINVAL)
418 418 ret = 0;
419 419 } else {
420 420 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
421 421 }
422 422
423 423 drm_gem_object_unreference(obj);
424 424 spin_unlock(&dev->struct_mutex);
425 425 if (ret)
426 426 DRM_ERROR("i915_set_domain_ioctl ret %d", ret);
427 427 return ret;
428 428 }
429 429
430 430 /**
431 431 * Called when user space has done writes to this buffer
432 432 */
433 433 /*ARGSUSED*/
434 434 int
435 435 i915_gem_sw_finish_ioctl(DRM_IOCTL_ARGS)
436 436 {
437 437 DRM_DEVICE;
438 438 struct drm_i915_gem_sw_finish args;
439 439 struct drm_gem_object *obj;
440 440 struct drm_i915_gem_object *obj_priv;
441 441 int ret = 0;
442 442
443 443 if (dev->driver->use_gem != 1)
444 444 return ENODEV;
445 445
446 446 DRM_COPYFROM_WITH_RETURN(&args,
447 447 (struct drm_i915_gem_sw_finish __user *) data, sizeof(args));
448 448
449 449 spin_lock(&dev->struct_mutex);
450 450 obj = drm_gem_object_lookup(fpriv, args.handle);
451 451 if (obj == NULL) {
452 452 spin_unlock(&dev->struct_mutex);
453 453 return EBADF;
454 454 }
455 455
456 456 DRM_DEBUG("%s: sw_finish %d (%p name %d size 0x%x)\n",
457 457 __func__, args.handle, obj, obj->name, obj->size);
458 458
459 459 obj_priv = obj->driver_private;
460 460 /* Pinned buffers may be scanout, so flush the cache */
461 461 if (obj_priv->pin_count)
462 462 {
463 463 i915_gem_object_flush_cpu_write_domain(obj);
464 464 }
465 465
466 466 drm_gem_object_unreference(obj);
467 467 spin_unlock(&dev->struct_mutex);
468 468 return ret;
469 469 }
470 470
471 471 /**
472 472 * Maps the contents of an object, returning the address it is mapped
473 473 * into.
474 474 *
475 475 * While the mapping holds a reference on the contents of the object, it doesn't
476 476 * imply a ref on the object itself.
477 477 */
478 478 /*ARGSUSED*/
479 479 int
480 480 i915_gem_mmap_ioctl(DRM_IOCTL_ARGS)
481 481 {
482 482 DRM_DEVICE;
483 483 struct drm_i915_gem_mmap args;
484 484 struct drm_gem_object *obj;
485 485 caddr_t vvaddr = NULL;
486 486 int ret;
487 487
488 488 if (dev->driver->use_gem != 1)
489 489 return ENODEV;
490 490
491 491 DRM_COPYFROM_WITH_RETURN(
492 492 &args, (struct drm_i915_gem_mmap __user *)data,
493 493 sizeof (struct drm_i915_gem_mmap));
494 494
495 495 obj = drm_gem_object_lookup(fpriv, args.handle);
496 496 if (obj == NULL)
497 497 return EBADF;
498 498
499 499 ret = ddi_devmap_segmap(fpriv->dev, (off_t)obj->map->handle,
500 500 ttoproc(curthread)->p_as, &vvaddr, obj->map->size,
501 501 PROT_ALL, PROT_ALL, MAP_SHARED, fpriv->credp);
502 502 if (ret)
503 503 return ret;
504 504
505 505 spin_lock(&dev->struct_mutex);
506 506 drm_gem_object_unreference(obj);
507 507 spin_unlock(&dev->struct_mutex);
508 508
509 509 args.addr_ptr = (uint64_t)(uintptr_t)vvaddr;
510 510
511 511 DRM_COPYTO_WITH_RETURN(
512 512 (struct drm_i915_gem_mmap __user *)data,
513 513 &args, sizeof (struct drm_i915_gem_mmap));
514 514
515 515 return 0;
516 516 }
517 517
518 518 static void
519 519 i915_gem_object_free_page_list(struct drm_gem_object *obj)
520 520 {
521 521 struct drm_i915_gem_object *obj_priv = obj->driver_private;
522 522 if (obj_priv->page_list == NULL)
523 523 return;
524 524
525 525 kmem_free(obj_priv->page_list,
526 526 btop(obj->size) * sizeof(caddr_t));
527 527
528 528 obj_priv->page_list = NULL;
529 529 }
530 530
531 531 static void
532 532 i915_gem_object_move_to_active(struct drm_gem_object *obj, uint32_t seqno)
533 533 {
534 534 struct drm_device *dev = obj->dev;
535 535 drm_i915_private_t *dev_priv = dev->dev_private;
536 536 struct drm_i915_gem_object *obj_priv = obj->driver_private;
537 537
538 538 /* Add a reference if we're newly entering the active list. */
539 539 if (!obj_priv->active) {
540 540 drm_gem_object_reference(obj);
541 541 obj_priv->active = 1;
542 542 }
543 543 /* Move from whatever list we were on to the tail of execution. */
544 544 list_move_tail(&obj_priv->list,
545 545 &dev_priv->mm.active_list, (caddr_t)obj_priv);
546 546 obj_priv->last_rendering_seqno = seqno;
547 547 }
548 548
549 549 static void
550 550 i915_gem_object_move_to_flushing(struct drm_gem_object *obj)
551 551 {
552 552 struct drm_device *dev = obj->dev;
553 553 drm_i915_private_t *dev_priv = dev->dev_private;
554 554 struct drm_i915_gem_object *obj_priv = obj->driver_private;
555 555
556 556 list_move_tail(&obj_priv->list, &dev_priv->mm.flushing_list, (caddr_t)obj_priv);
557 557 obj_priv->last_rendering_seqno = 0;
558 558 }
559 559
560 560 static void
561 561 i915_gem_object_move_to_inactive(struct drm_gem_object *obj)
562 562 {
563 563 struct drm_device *dev = obj->dev;
564 564 drm_i915_private_t *dev_priv = dev->dev_private;
565 565 struct drm_i915_gem_object *obj_priv = obj->driver_private;
566 566
567 567 if (obj_priv->pin_count != 0)
568 568 {
569 569 list_del_init(&obj_priv->list);
570 570 } else {
571 571 list_move_tail(&obj_priv->list, &dev_priv->mm.inactive_list, (caddr_t)obj_priv);
572 572 }
573 573 obj_priv->last_rendering_seqno = 0;
574 574 if (obj_priv->active) {
575 575 obj_priv->active = 0;
576 576 drm_gem_object_unreference(obj);
577 577 }
578 578 }
579 579
580 580 /**
581 581 * Creates a new sequence number, emitting a write of it to the status page
582 582 * plus an interrupt, which will trigger i915_user_interrupt_handler.
583 583 *
584 584 * Must be called with struct_lock held.
585 585 *
586 586 * Returned sequence numbers are nonzero on success.
587 587 */
588 588 static uint32_t
589 589 i915_add_request(struct drm_device *dev, uint32_t flush_domains)
590 590 {
591 591 drm_i915_private_t *dev_priv = dev->dev_private;
592 592 struct drm_i915_gem_request *request;
593 593 uint32_t seqno;
594 594 int was_empty;
595 595 RING_LOCALS;
596 596
597 597 request = drm_calloc(1, sizeof(*request), DRM_MEM_DRIVER);
598 598 if (request == NULL) {
599 599 DRM_ERROR("Failed to alloc request");
600 600 return 0;
601 601 }
602 602 /* Grab the seqno we're going to make this request be, and bump the
603 603 * next (skipping 0 so it can be the reserved no-seqno value).
604 604 */
605 605 seqno = dev_priv->mm.next_gem_seqno;
606 606 dev_priv->mm.next_gem_seqno++;
607 607 if (dev_priv->mm.next_gem_seqno == 0)
608 608 dev_priv->mm.next_gem_seqno++;
609 609
610 610 DRM_DEBUG("add_request seqno = %d dev 0x%lx", seqno, dev);
611 611
612 612 BEGIN_LP_RING(4);
613 613 OUT_RING(MI_STORE_DWORD_INDEX);
614 614 OUT_RING(I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
615 615 OUT_RING(seqno);
616 616 OUT_RING(0);
617 617 ADVANCE_LP_RING();
618 618
619 619 BEGIN_LP_RING(2);
620 620 OUT_RING(0);
621 621 OUT_RING(MI_USER_INTERRUPT);
622 622 ADVANCE_LP_RING();
623 623
624 624 request->seqno = seqno;
625 625 request->emitted_jiffies = jiffies;
626 626 was_empty = list_empty(&dev_priv->mm.request_list);
627 627 list_add_tail(&request->list, &dev_priv->mm.request_list, (caddr_t)request);
628 628
629 629 /* Associate any objects on the flushing list matching the write
630 630 * domain we're flushing with our flush.
631 631 */
632 632 if (flush_domains != 0) {
633 633 struct drm_i915_gem_object *obj_priv, *next;
634 634
635 635 obj_priv = list_entry(dev_priv->mm.flushing_list.next, struct drm_i915_gem_object, list),
636 636 next = list_entry(obj_priv->list.next, struct drm_i915_gem_object, list);
637 637 for(; &obj_priv->list != &dev_priv->mm.flushing_list;
638 638 obj_priv = next,
639 639 next = list_entry(next->list.next, struct drm_i915_gem_object, list)) {
640 640 struct drm_gem_object *obj = obj_priv->obj;
641 641
642 642 if ((obj->write_domain & flush_domains) ==
643 643 obj->write_domain) {
644 644 obj->write_domain = 0;
645 645 i915_gem_object_move_to_active(obj, seqno);
646 646 }
647 647 }
648 648
649 649 }
650 650
651 651 if (was_empty && !dev_priv->mm.suspended)
652 652 {
653 653 /* change to delay HZ and then run work (not insert to workqueue of Linux) */
654 654 worktimer_id = timeout(i915_gem_retire_work_handler, (void *) dev, DRM_HZ);
655 655 DRM_DEBUG("i915_gem: schedule_delayed_work");
656 656 }
657 657 return seqno;
658 658 }
659 659
660 660 /**
661 661 * Command execution barrier
662 662 *
663 663 * Ensures that all commands in the ring are finished
664 664 * before signalling the CPU
665 665 */
666 666 uint32_t
667 667 i915_retire_commands(struct drm_device *dev)
668 668 {
669 669 drm_i915_private_t *dev_priv = dev->dev_private;
670 670 uint32_t cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
671 671 uint32_t flush_domains = 0;
672 672 RING_LOCALS;
673 673
674 674 /* The sampler always gets flushed on i965 (sigh) */
675 675 if (IS_I965G(dev))
676 676 flush_domains |= I915_GEM_DOMAIN_SAMPLER;
677 677 BEGIN_LP_RING(2);
678 678 OUT_RING(cmd);
679 679 OUT_RING(0); /* noop */
680 680 ADVANCE_LP_RING();
681 681
682 682 return flush_domains;
683 683 }
684 684
685 685 /**
686 686 * Moves buffers associated only with the given active seqno from the active
687 687 * to inactive list, potentially freeing them.
688 688 */
689 689 static void
690 690 i915_gem_retire_request(struct drm_device *dev,
691 691 struct drm_i915_gem_request *request)
692 692 {
693 693 drm_i915_private_t *dev_priv = dev->dev_private;
694 694 /* Move any buffers on the active list that are no longer referenced
695 695 * by the ringbuffer to the flushing/inactive lists as appropriate.
696 696 */
697 697 while (!list_empty(&dev_priv->mm.active_list)) {
698 698 struct drm_gem_object *obj;
699 699 struct drm_i915_gem_object *obj_priv;
700 700
701 701 obj_priv = list_entry(dev_priv->mm.active_list.next,
702 702 struct drm_i915_gem_object,
703 703 list);
704 704 obj = obj_priv->obj;
705 705
706 706 /* If the seqno being retired doesn't match the oldest in the
707 707 * list, then the oldest in the list must still be newer than
708 708 * this seqno.
709 709 */
710 710 if (obj_priv->last_rendering_seqno != request->seqno)
711 711 return;
712 712
713 713 DRM_DEBUG("%s: retire %d moves to inactive list %p\n",
714 714 __func__, request->seqno, obj);
715 715
716 716 if (obj->write_domain != 0) {
717 717 i915_gem_object_move_to_flushing(obj);
718 718 } else {
719 719 i915_gem_object_move_to_inactive(obj);
720 720 }
721 721 }
722 722 }
723 723
724 724 /**
725 725 * Returns true if seq1 is later than seq2.
726 726 */
727 727 static int
728 728 i915_seqno_passed(uint32_t seq1, uint32_t seq2)
729 729 {
730 730 return (int32_t)(seq1 - seq2) >= 0;
731 731 }
732 732
733 733 uint32_t
734 734 i915_get_gem_seqno(struct drm_device *dev)
735 735 {
736 736 drm_i915_private_t *dev_priv = dev->dev_private;
737 737
738 738 return READ_HWSP(dev_priv, I915_GEM_HWS_INDEX);
739 739 }
740 740
741 741 /**
742 742 * This function clears the request list as sequence numbers are passed.
743 743 */
744 744 void
745 745 i915_gem_retire_requests(struct drm_device *dev)
746 746 {
747 747 drm_i915_private_t *dev_priv = dev->dev_private;
748 748 uint32_t seqno;
749 749
750 750 seqno = i915_get_gem_seqno(dev);
751 751
752 752 while (!list_empty(&dev_priv->mm.request_list)) {
753 753 struct drm_i915_gem_request *request;
754 754 uint32_t retiring_seqno;
755 755 request = (struct drm_i915_gem_request *)(uintptr_t)(dev_priv->mm.request_list.next->contain_ptr);
756 756 retiring_seqno = request->seqno;
757 757
758 758 if (i915_seqno_passed(seqno, retiring_seqno) ||
759 759 dev_priv->mm.wedged) {
760 760 i915_gem_retire_request(dev, request);
761 761
762 762 list_del(&request->list);
763 763 drm_free(request, sizeof(*request), DRM_MEM_DRIVER);
764 764 } else
765 765 break;
766 766 }
767 767 }
768 768
769 769 void
770 770 i915_gem_retire_work_handler(void *device)
771 771 {
772 772 struct drm_device *dev = (struct drm_device *)device;
773 773 drm_i915_private_t *dev_priv = dev->dev_private;
774 774
775 775 spin_lock(&dev->struct_mutex);
776 776
777 777 /* Return if gem idle */
778 778 if (worktimer_id == NULL) {
779 779 spin_unlock(&dev->struct_mutex);
780 780 return;
781 781 }
782 782
783 783 i915_gem_retire_requests(dev);
784 784 if (!dev_priv->mm.suspended && !list_empty(&dev_priv->mm.request_list))
785 785 {
786 786 DRM_DEBUG("i915_gem: schedule_delayed_work");
787 787 worktimer_id = timeout(i915_gem_retire_work_handler, (void *) dev, DRM_HZ);
788 788 }
789 789 spin_unlock(&dev->struct_mutex);
790 790 }
791 791
792 792 /**
793 793 * i965_reset - reset chip after a hang
794 794 * @dev: drm device to reset
795 795 * @flags: reset domains
796 796 *
797 797 * Reset the chip. Useful if a hang is detected.
798 798 *
799 799 * Procedure is fairly simple:
800 800 * - reset the chip using the reset reg
801 801 * - re-init context state
802 802 * - re-init hardware status page
803 803 * - re-init ring buffer
804 804 * - re-init interrupt state
805 805 * - re-init display
806 806 */
807 807 void i965_reset(struct drm_device *dev, u8 flags)
808 808 {
809 809 ddi_acc_handle_t conf_hdl;
810 810 drm_i915_private_t *dev_priv = dev->dev_private;
811 811 int timeout = 0;
812 812 uint8_t gdrst;
813 813
814 814 if (flags & GDRST_FULL)
815 815 i915_save_display(dev);
816 816
817 817 if (pci_config_setup(dev->dip, &conf_hdl) != DDI_SUCCESS) {
818 818 DRM_ERROR(("i915_reset: pci_config_setup fail"));
819 819 return;
820 820 }
821 821
822 822 /*
823 823 * Set the reset bit, wait for reset, then clear it. Hardware
824 824 * will clear the status bit (bit 1) when it's actually ready
825 825 * for action again.
826 826 */
827 827 gdrst = pci_config_get8(conf_hdl, GDRST);
828 828 pci_config_put8(conf_hdl, GDRST, gdrst | flags);
829 829 drv_usecwait(50);
830 830 pci_config_put8(conf_hdl, GDRST, gdrst | 0xfe);
831 831
832 832 /* ...we don't want to loop forever though, 500ms should be plenty */
833 833 do {
834 834 drv_usecwait(100);
835 835 gdrst = pci_config_get8(conf_hdl, GDRST);
836 836 } while ((gdrst & 2) && (timeout++ < 5));
837 837
838 838 /* Ok now get things going again... */
839 839
840 840 /*
841 841 * Everything depends on having the GTT running, so we need to start
842 842 * there. Fortunately we don't need to do this unless we reset the
843 843 * chip at a PCI level.
844 844 *
845 845 * Next we need to restore the context, but we don't use those
846 846 * yet either...
847 847 *
848 848 * Ring buffer needs to be re-initialized in the KMS case, or if X
849 849 * was running at the time of the reset (i.e. we weren't VT
850 850 * switched away).
851 851 */
852 852 if (!dev_priv->mm.suspended) {
853 853 drm_i915_ring_buffer_t *ring = &dev_priv->ring;
854 854 struct drm_gem_object *obj = ring->ring_obj;
855 855 struct drm_i915_gem_object *obj_priv = obj->driver_private;
856 856 dev_priv->mm.suspended = 0;
857 857
858 858 /* Stop the ring if it's running. */
859 859 I915_WRITE(PRB0_CTL, 0);
860 860 I915_WRITE(PRB0_TAIL, 0);
861 861 I915_WRITE(PRB0_HEAD, 0);
862 862
863 863 /* Initialize the ring. */
864 864 I915_WRITE(PRB0_START, obj_priv->gtt_offset);
865 865 I915_WRITE(PRB0_CTL,
866 866 ((obj->size - 4096) & RING_NR_PAGES) |
867 867 RING_NO_REPORT |
868 868 RING_VALID);
869 869 i915_kernel_lost_context(dev);
870 870
871 871 (void) drm_irq_install(dev);
872 872 }
873 873
874 874 /*
875 875 * Display needs restore too...
876 876 */
877 877 if (flags & GDRST_FULL)
878 878 i915_restore_display(dev);
879 879 }
880 880
881 881 /**
882 882 * Waits for a sequence number to be signaled, and cleans up the
883 883 * request and object lists appropriately for that event.
884 884 */
885 885 int
886 886 i915_wait_request(struct drm_device *dev, uint32_t seqno)
887 887 {
888 888 drm_i915_private_t *dev_priv = dev->dev_private;
889 889 u32 ier;
890 890 int ret = 0;
891 891
892 892 ASSERT(seqno != 0);
893 893
894 894 if (!i915_seqno_passed(i915_get_gem_seqno(dev), seqno)) {
895 895 if (IS_IGDNG(dev))
896 896 ier = I915_READ(DEIER) | I915_READ(GTIER);
897 897 else
898 898 ier = I915_READ(IER);
899 899 if (!ier) {
900 900 DRM_ERROR("something (likely vbetool) disabled "
901 901 "interrupts, re-enabling\n");
902 902 (void) i915_driver_irq_preinstall(dev);
903 903 i915_driver_irq_postinstall(dev);
904 904 }
905 905
906 906 dev_priv->mm.waiting_gem_seqno = seqno;
907 907 i915_user_irq_on(dev);
908 908 DRM_WAIT(ret, &dev_priv->irq_queue,
909 909 (i915_seqno_passed(i915_get_gem_seqno(dev), seqno) ||
910 910 dev_priv->mm.wedged));
911 911 i915_user_irq_off(dev);
912 912 dev_priv->mm.waiting_gem_seqno = 0;
913 913 }
914 914 if (dev_priv->mm.wedged) {
915 915 ret = EIO;
916 916 }
917 917
918 918 /* GPU maybe hang, reset needed*/
919 919 if (ret == -2 && (seqno > i915_get_gem_seqno(dev))) {
920 920 if (IS_I965G(dev)) {
921 921 DRM_ERROR("GPU hang detected try to reset ... wait for irq_queue seqno %d, now seqno %d", seqno, i915_get_gem_seqno(dev));
922 922 dev_priv->mm.wedged = 1;
923 923 i965_reset(dev, GDRST_RENDER);
924 924 i915_gem_retire_requests(dev);
925 925 dev_priv->mm.wedged = 0;
926 926 }
927 927 else
928 928 DRM_ERROR("GPU hang detected.... reboot required");
929 929 return 0;
930 930 }
931 931 /* Directly dispatch request retiring. While we have the work queue
932 932 * to handle this, the waiter on a request often wants an associated
933 933 * buffer to have made it to the inactive list, and we would need
934 934 * a separate wait queue to handle that.
935 935 */
936 936 if (ret == 0)
937 937 i915_gem_retire_requests(dev);
938 938
939 939 return ret;
940 940 }
941 941
942 942 static void
943 943 i915_gem_flush(struct drm_device *dev,
944 944 uint32_t invalidate_domains,
945 945 uint32_t flush_domains)
946 946 {
947 947 drm_i915_private_t *dev_priv = dev->dev_private;
948 948 uint32_t cmd;
949 949 RING_LOCALS;
950 950
951 951 DRM_DEBUG("%s: invalidate %08x flush %08x\n", __func__,
952 952 invalidate_domains, flush_domains);
953 953
954 954 if (flush_domains & I915_GEM_DOMAIN_CPU)
955 955 drm_agp_chipset_flush(dev);
956 956
957 957 if ((invalidate_domains | flush_domains) & ~(I915_GEM_DOMAIN_CPU |
958 958 I915_GEM_DOMAIN_GTT)) {
959 959 /*
960 960 * read/write caches:
961 961 *
962 962 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
963 963 * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
964 964 * also flushed at 2d versus 3d pipeline switches.
965 965 *
966 966 * read-only caches:
967 967 *
968 968 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
969 969 * MI_READ_FLUSH is set, and is always flushed on 965.
970 970 *
971 971 * I915_GEM_DOMAIN_COMMAND may not exist?
972 972 *
973 973 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
974 974 * invalidated when MI_EXE_FLUSH is set.
975 975 *
976 976 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
977 977 * invalidated with every MI_FLUSH.
978 978 *
979 979 * TLBs:
980 980 *
981 981 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
982 982 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
983 983 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
984 984 * are flushed at any MI_FLUSH.
985 985 */
986 986
987 987 cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
988 988 if ((invalidate_domains|flush_domains) &
989 989 I915_GEM_DOMAIN_RENDER)
990 990 cmd &= ~MI_NO_WRITE_FLUSH;
991 991 if (!IS_I965G(dev)) {
992 992 /*
993 993 * On the 965, the sampler cache always gets flushed
994 994 * and this bit is reserved.
995 995 */
996 996 if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
997 997 cmd |= MI_READ_FLUSH;
998 998 }
999 999 if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
1000 1000 cmd |= MI_EXE_FLUSH;
1001 1001
1002 1002 DRM_DEBUG("%s: queue flush %08x to ring\n", __func__, cmd);
1003 1003
1004 1004 BEGIN_LP_RING(2);
1005 1005 OUT_RING(cmd);
1006 1006 OUT_RING(0); /* noop */
1007 1007 ADVANCE_LP_RING();
1008 1008 }
1009 1009 }
1010 1010
1011 1011 /**
1012 1012 * Ensures that all rendering to the object has completed and the object is
1013 1013 * safe to unbind from the GTT or access from the CPU.
1014 1014 */
1015 1015 static int
1016 1016 i915_gem_object_wait_rendering(struct drm_gem_object *obj)
1017 1017 {
1018 1018 struct drm_device *dev = obj->dev;
1019 1019 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1020 1020 int ret, seqno;
1021 1021
1022 1022 /* This function only exists to support waiting for existing rendering,
1023 1023 * not for emitting required flushes.
1024 1024 */
1025 1025
1026 1026 if((obj->write_domain & I915_GEM_GPU_DOMAINS) != 0) {
1027 1027 DRM_ERROR("write domain should not be GPU DOMAIN %d", obj_priv->active);
1028 1028 return 0;
1029 1029 }
1030 1030
1031 1031 /* If there is rendering queued on the buffer being evicted, wait for
1032 1032 * it.
1033 1033 */
1034 1034 if (obj_priv->active) {
1035 1035 DRM_DEBUG("%s: object %d %p wait for seqno %08x\n",
1036 1036 __func__, obj->name, obj, obj_priv->last_rendering_seqno);
1037 1037
1038 1038 seqno = obj_priv->last_rendering_seqno;
1039 1039 if (seqno == 0) {
1040 1040 DRM_DEBUG("last rendering maybe finished");
1041 1041 return 0;
1042 1042 }
1043 1043 ret = i915_wait_request(dev, seqno);
1044 1044 if (ret != 0) {
1045 1045 DRM_ERROR("%s: i915_wait_request request->seqno %d now %d\n", __func__, seqno, i915_get_gem_seqno(dev));
1046 1046 return ret;
1047 1047 }
1048 1048 }
1049 1049
1050 1050 return 0;
1051 1051 }
1052 1052
1053 1053 /**
1054 1054 * Unbinds an object from the GTT aperture.
1055 1055 */
1056 1056 int
1057 1057 i915_gem_object_unbind(struct drm_gem_object *obj, uint32_t type)
1058 1058 {
1059 1059 struct drm_device *dev = obj->dev;
1060 1060 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1061 1061 int ret = 0;
1062 1062
1063 1063 if (obj_priv->gtt_space == NULL)
1064 1064 return 0;
1065 1065
1066 1066 if (obj_priv->pin_count != 0) {
1067 1067 DRM_ERROR("Attempting to unbind pinned buffer\n");
1068 1068 return EINVAL;
1069 1069 }
1070 1070
1071 1071 /* Wait for any rendering to complete
1072 1072 */
1073 1073 ret = i915_gem_object_wait_rendering(obj);
1074 1074 if (ret) {
1075 1075 DRM_ERROR("wait_rendering failed: %d\n", ret);
1076 1076 return ret;
1077 1077 }
1078 1078
1079 1079 /* Move the object to the CPU domain to ensure that
1080 1080 * any possible CPU writes while it's not in the GTT
1081 1081 * are flushed when we go to remap it. This will
1082 1082 * also ensure that all pending GPU writes are finished
1083 1083 * before we unbind.
1084 1084 */
1085 1085 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1086 1086 if (ret) {
1087 1087 DRM_ERROR("set_domain failed: %d\n", ret);
1088 1088 return ret;
1089 1089 }
1090 1090
1091 1091 if (!obj_priv->agp_mem) {
1092 1092 (void) drm_agp_unbind_pages(dev, obj->size / PAGE_SIZE,
1093 1093 obj_priv->gtt_offset, type);
1094 1094 obj_priv->agp_mem = -1;
1095 1095 }
1096 1096
1097 1097 ASSERT(!obj_priv->active);
1098 1098
1099 1099 i915_gem_object_free_page_list(obj);
1100 1100
1101 1101 if (obj_priv->gtt_space) {
1102 1102 atomic_dec(&dev->gtt_count);
1103 1103 atomic_sub(obj->size, &dev->gtt_memory);
1104 1104 drm_mm_put_block(obj_priv->gtt_space);
1105 1105 obj_priv->gtt_space = NULL;
1106 1106 }
1107 1107
1108 1108 /* Remove ourselves from the LRU list if present. */
1109 1109 if (!list_empty(&obj_priv->list))
1110 1110 list_del_init(&obj_priv->list);
1111 1111
1112 1112 return 0;
1113 1113 }
1114 1114
1115 1115 static int
1116 1116 i915_gem_evict_something(struct drm_device *dev)
1117 1117 {
1118 1118 drm_i915_private_t *dev_priv = dev->dev_private;
1119 1119 struct drm_gem_object *obj;
1120 1120 struct drm_i915_gem_object *obj_priv;
1121 1121 int ret = 0;
1122 1122
1123 1123 for (;;) {
1124 1124 /* If there's an inactive buffer available now, grab it
1125 1125 * and be done.
1126 1126 */
1127 1127 if (!list_empty(&dev_priv->mm.inactive_list)) {
1128 1128 obj_priv = list_entry(dev_priv->mm.inactive_list.next,
1129 1129 struct drm_i915_gem_object,
1130 1130 list);
1131 1131 obj = obj_priv->obj;
1132 1132 ASSERT(!(obj_priv->pin_count != 0));
1133 1133 DRM_DEBUG("%s: evicting %d\n", __func__, obj->name);
1134 1134 ASSERT(!(obj_priv->active));
1135 1135 /* Wait on the rendering and unbind the buffer. */
1136 1136 ret = i915_gem_object_unbind(obj, 1);
1137 1137 break;
1138 1138 }
1139 1139 /* If we didn't get anything, but the ring is still processing
1140 1140 * things, wait for one of those things to finish and hopefully
1141 1141 * leave us a buffer to evict.
1142 1142 */
1143 1143 if (!list_empty(&dev_priv->mm.request_list)) {
1144 1144 struct drm_i915_gem_request *request;
1145 1145
1146 1146 request = list_entry(dev_priv->mm.request_list.next,
1147 1147 struct drm_i915_gem_request,
1148 1148 list);
1149 1149
1150 1150 ret = i915_wait_request(dev, request->seqno);
1151 1151 if (ret) {
1152 1152 break;
1153 1153 }
1154 1154 /* if waiting caused an object to become inactive,
1155 1155 * then loop around and wait for it. Otherwise, we
1156 1156 * assume that waiting freed and unbound something,
1157 1157 * so there should now be some space in the GTT
1158 1158 */
1159 1159 if (!list_empty(&dev_priv->mm.inactive_list))
1160 1160 continue;
1161 1161 break;
1162 1162 }
1163 1163
1164 1164 /* If we didn't have anything on the request list but there
1165 1165 * are buffers awaiting a flush, emit one and try again.
1166 1166 * When we wait on it, those buffers waiting for that flush
1167 1167 * will get moved to inactive.
1168 1168 */
1169 1169 if (!list_empty(&dev_priv->mm.flushing_list)) {
1170 1170 obj_priv = list_entry(dev_priv->mm.flushing_list.next,
1171 1171 struct drm_i915_gem_object,
1172 1172 list);
1173 1173 obj = obj_priv->obj;
1174 1174
1175 1175 i915_gem_flush(dev,
1176 1176 obj->write_domain,
1177 1177 obj->write_domain);
1178 1178 (void) i915_add_request(dev, obj->write_domain);
1179 1179
1180 1180 obj = NULL;
1181 1181 continue;
1182 1182 }
1183 1183
1184 1184 DRM_ERROR("inactive empty %d request empty %d "
1185 1185 "flushing empty %d\n",
1186 1186 list_empty(&dev_priv->mm.inactive_list),
1187 1187 list_empty(&dev_priv->mm.request_list),
1188 1188 list_empty(&dev_priv->mm.flushing_list));
1189 1189 /* If we didn't do any of the above, there's nothing to be done
1190 1190 * and we just can't fit it in.
1191 1191 */
1192 1192 return ENOMEM;
1193 1193 }
1194 1194 return ret;
1195 1195 }
1196 1196
1197 1197 static int
1198 1198 i915_gem_evict_everything(struct drm_device *dev)
1199 1199 {
1200 1200 int ret;
1201 1201
1202 1202 for (;;) {
1203 1203 ret = i915_gem_evict_something(dev);
1204 1204 if (ret != 0)
1205 1205 break;
1206 1206 }
1207 1207 if (ret == ENOMEM)
1208 1208 return 0;
1209 1209 else
1210 1210 DRM_ERROR("evict_everything ret %d", ret);
1211 1211 return ret;
1212 1212 }
1213 1213
1214 1214 /**
1215 1215 * Finds free space in the GTT aperture and binds the object there.
1216 1216 */
1217 1217 static int
1218 1218 i915_gem_object_bind_to_gtt(struct drm_gem_object *obj, uint32_t alignment)
1219 1219 {
1220 1220 struct drm_device *dev = obj->dev;
1221 1221 drm_i915_private_t *dev_priv = dev->dev_private;
1222 1222 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1223 1223 struct drm_mm_node *free_space;
1224 1224 int page_count, ret;
1225 1225
1226 1226 if (dev_priv->mm.suspended)
1227 1227 return EBUSY;
1228 1228 if (alignment == 0)
1229 1229 alignment = PAGE_SIZE;
1230 1230 if (alignment & (PAGE_SIZE - 1)) {
1231 1231 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
1232 1232 return EINVAL;
1233 1233 }
1234 1234
1235 1235 if (obj_priv->gtt_space) {
1236 1236 DRM_ERROR("Already bind!!");
1237 1237 return 0;
1238 1238 }
1239 1239 search_free:
1240 1240 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
1241 1241 (unsigned long) obj->size, alignment, 0);
1242 1242 if (free_space != NULL) {
1243 1243 obj_priv->gtt_space = drm_mm_get_block(free_space, (unsigned long) obj->size,
1244 1244 alignment);
1245 1245 if (obj_priv->gtt_space != NULL) {
1246 1246 obj_priv->gtt_space->private = obj;
1247 1247 obj_priv->gtt_offset = obj_priv->gtt_space->start;
1248 1248 }
1249 1249 }
1250 1250 if (obj_priv->gtt_space == NULL) {
1251 1251 /* If the gtt is empty and we're still having trouble
1252 1252 * fitting our object in, we're out of memory.
1253 1253 */
1254 1254 if (list_empty(&dev_priv->mm.inactive_list) &&
1255 1255 list_empty(&dev_priv->mm.flushing_list) &&
1256 1256 list_empty(&dev_priv->mm.active_list)) {
1257 1257 DRM_ERROR("GTT full, but LRU list empty\n");
1258 1258 return ENOMEM;
1259 1259 }
1260 1260
1261 1261 ret = i915_gem_evict_something(dev);
1262 1262 if (ret != 0) {
1263 1263 DRM_ERROR("Failed to evict a buffer %d\n", ret);
1264 1264 return ret;
1265 1265 }
1266 1266 goto search_free;
1267 1267 }
1268 1268
1269 1269 ret = i915_gem_object_get_page_list(obj);
1270 1270 if (ret) {
1271 1271 drm_mm_put_block(obj_priv->gtt_space);
1272 1272 obj_priv->gtt_space = NULL;
1273 1273 DRM_ERROR("bind to gtt failed to get page list");
1274 1274 return ret;
1275 1275 }
1276 1276
1277 1277 page_count = obj->size / PAGE_SIZE;
1278 1278 /* Create an AGP memory structure pointing at our pages, and bind it
1279 1279 * into the GTT.
1280 1280 */
1281 1281 DRM_DEBUG("Binding object %d of page_count %d at gtt_offset 0x%x obj->pfnarray = 0x%lx",
1282 1282 obj->name, page_count, obj_priv->gtt_offset, obj->pfnarray);
1283 1283
1284 1284 obj_priv->agp_mem = drm_agp_bind_pages(dev,
1285 1285 obj->pfnarray,
1286 1286 page_count,
1287 1287 obj_priv->gtt_offset);
1288 1288 if (obj_priv->agp_mem) {
1289 1289 i915_gem_object_free_page_list(obj);
1290 1290 drm_mm_put_block(obj_priv->gtt_space);
1291 1291 obj_priv->gtt_space = NULL;
1292 1292 DRM_ERROR("Failed to bind pages obj %d, obj 0x%lx", obj->name, obj);
1293 1293 return ENOMEM;
1294 1294 }
1295 1295 atomic_inc(&dev->gtt_count);
1296 1296 atomic_add(obj->size, &dev->gtt_memory);
1297 1297
1298 1298 /* Assert that the object is not currently in any GPU domain. As it
1299 1299 * wasn't in the GTT, there shouldn't be any way it could have been in
1300 1300 * a GPU cache
1301 1301 */
1302 1302 ASSERT(!(obj->read_domains & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)));
1303 1303 ASSERT(!(obj->write_domain & ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT)));
1304 1304
1305 1305 return 0;
1306 1306 }
1307 1307
1308 1308 void
1309 1309 i915_gem_clflush_object(struct drm_gem_object *obj)
1310 1310 {
1311 1311 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1312 1312
1313 1313 /* If we don't have a page list set up, then we're not pinned
1314 1314 * to GPU, and we can ignore the cache flush because it'll happen
1315 1315 * again at bind time.
1316 1316 */
1317 1317
1318 1318 if (obj_priv->page_list == NULL)
1319 1319 return;
1320 1320 drm_clflush_pages(obj_priv->page_list, obj->size / PAGE_SIZE);
1321 1321 }
1322 1322
1323 1323 /** Flushes any GPU write domain for the object if it's dirty. */
1324 1324 static void
1325 1325 i915_gem_object_flush_gpu_write_domain(struct drm_gem_object *obj)
1326 1326 {
1327 1327 struct drm_device *dev = obj->dev;
1328 1328 uint32_t seqno;
1329 1329
1330 1330 if ((obj->write_domain & I915_GEM_GPU_DOMAINS) == 0)
1331 1331 return;
1332 1332
1333 1333 /* Queue the GPU write cache flushing we need. */
1334 1334 i915_gem_flush(dev, 0, obj->write_domain);
1335 1335 seqno = i915_add_request(dev, obj->write_domain);
1336 1336 DRM_DEBUG("flush_gpu_write_domain seqno = %d", seqno);
1337 1337 obj->write_domain = 0;
1338 1338 i915_gem_object_move_to_active(obj, seqno);
1339 1339 }
1340 1340
1341 1341 /** Flushes the GTT write domain for the object if it's dirty. */
1342 1342 static void
1343 1343 i915_gem_object_flush_gtt_write_domain(struct drm_gem_object *obj)
1344 1344 {
1345 1345 if (obj->write_domain != I915_GEM_DOMAIN_GTT)
1346 1346 return;
1347 1347
1348 1348 /* No actual flushing is required for the GTT write domain. Writes
1349 1349 * to it immediately go to main memory as far as we know, so there's
1350 1350 * no chipset flush. It also doesn't land in render cache.
1351 1351 */
1352 1352 obj->write_domain = 0;
1353 1353 }
1354 1354
1355 1355 /** Flushes the CPU write domain for the object if it's dirty. */
1356 1356 static void
1357 1357 i915_gem_object_flush_cpu_write_domain(struct drm_gem_object *obj)
1358 1358 {
1359 1359 struct drm_device *dev = obj->dev;
1360 1360
1361 1361 if (obj->write_domain != I915_GEM_DOMAIN_CPU)
1362 1362 return;
1363 1363
1364 1364 i915_gem_clflush_object(obj);
1365 1365 drm_agp_chipset_flush(dev);
1366 1366 obj->write_domain = 0;
1367 1367 }
1368 1368
1369 1369 /**
1370 1370 * Moves a single object to the GTT read, and possibly write domain.
1371 1371 *
1372 1372 * This function returns when the move is complete, including waiting on
1373 1373 * flushes to occur.
1374 1374 */
1375 1375 static int
1376 1376 i915_gem_object_set_to_gtt_domain(struct drm_gem_object *obj, int write)
1377 1377 {
1378 1378 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1379 1379 int ret;
1380 1380
1381 1381 /* Not valid to be called on unbound objects. */
1382 1382 if (obj_priv->gtt_space == NULL)
1383 1383 return EINVAL;
1384 1384
1385 1385 i915_gem_object_flush_gpu_write_domain(obj);
1386 1386 /* Wait on any GPU rendering and flushing to occur. */
1387 1387 ret = i915_gem_object_wait_rendering(obj);
1388 1388 if (ret != 0) {
1389 1389 DRM_ERROR("set_to_gtt_domain wait_rendering ret %d", ret);
1390 1390 return ret;
1391 1391 }
1392 1392 /* If we're writing through the GTT domain, then CPU and GPU caches
1393 1393 * will need to be invalidated at next use.
1394 1394 */
1395 1395 if (write)
1396 1396 obj->read_domains &= I915_GEM_DOMAIN_GTT;
1397 1397 i915_gem_object_flush_cpu_write_domain(obj);
1398 1398
1399 1399 DRM_DEBUG("i915_gem_object_set_to_gtt_domain obj->read_domains %x ", obj->read_domains);
1400 1400 /* It should now be out of any other write domains, and we can update
1401 1401 * the domain values for our changes.
1402 1402 */
1403 1403 ASSERT(!((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0));
1404 1404 obj->read_domains |= I915_GEM_DOMAIN_GTT;
1405 1405 if (write) {
1406 1406 obj->write_domain = I915_GEM_DOMAIN_GTT;
1407 1407 obj_priv->dirty = 1;
1408 1408 }
1409 1409
1410 1410 return 0;
1411 1411 }
1412 1412
1413 1413 /**
1414 1414 * Moves a single object to the CPU read, and possibly write domain.
1415 1415 *
1416 1416 * This function returns when the move is complete, including waiting on
1417 1417 * flushes to occur.
1418 1418 */
1419 1419 static int
1420 1420 i915_gem_object_set_to_cpu_domain(struct drm_gem_object *obj, int write)
1421 1421 {
1422 1422 struct drm_device *dev = obj->dev;
1423 1423 int ret;
1424 1424
1425 1425
1426 1426 i915_gem_object_flush_gpu_write_domain(obj);
1427 1427 /* Wait on any GPU rendering and flushing to occur. */
1428 1428
1429 1429 ret = i915_gem_object_wait_rendering(obj);
1430 1430 if (ret != 0)
1431 1431 return ret;
1432 1432
1433 1433 i915_gem_object_flush_gtt_write_domain(obj);
1434 1434
1435 1435 /* If we have a partially-valid cache of the object in the CPU,
1436 1436 * finish invalidating it and free the per-page flags.
1437 1437 */
1438 1438 i915_gem_object_set_to_full_cpu_read_domain(obj);
1439 1439
1440 1440 /* Flush the CPU cache if it's still invalid. */
1441 1441 if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
1442 1442 i915_gem_clflush_object(obj);
1443 1443 drm_agp_chipset_flush(dev);
1444 1444 obj->read_domains |= I915_GEM_DOMAIN_CPU;
1445 1445 }
1446 1446
1447 1447 /* It should now be out of any other write domains, and we can update
1448 1448 * the domain values for our changes.
1449 1449 */
1450 1450 ASSERT(!((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0));
1451 1451
1452 1452 /* If we're writing through the CPU, then the GPU read domains will
1453 1453 * need to be invalidated at next use.
1454 1454 */
1455 1455 if (write) {
1456 1456 obj->read_domains &= I915_GEM_DOMAIN_CPU;
1457 1457 obj->write_domain = I915_GEM_DOMAIN_CPU;
1458 1458 }
1459 1459
1460 1460 return 0;
1461 1461 }
1462 1462
1463 1463 /*
1464 1464 * Set the next domain for the specified object. This
1465 1465 * may not actually perform the necessary flushing/invaliding though,
1466 1466 * as that may want to be batched with other set_domain operations
1467 1467 *
1468 1468 * This is (we hope) the only really tricky part of gem. The goal
1469 1469 * is fairly simple -- track which caches hold bits of the object
1470 1470 * and make sure they remain coherent. A few concrete examples may
1471 1471 * help to explain how it works. For shorthand, we use the notation
1472 1472 * (read_domains, write_domain), e.g. (CPU, CPU) to indicate the
1473 1473 * a pair of read and write domain masks.
1474 1474 *
1475 1475 * Case 1: the batch buffer
1476 1476 *
1477 1477 * 1. Allocated
1478 1478 * 2. Written by CPU
1479 1479 * 3. Mapped to GTT
1480 1480 * 4. Read by GPU
1481 1481 * 5. Unmapped from GTT
1482 1482 * 6. Freed
1483 1483 *
1484 1484 * Let's take these a step at a time
1485 1485 *
1486 1486 * 1. Allocated
1487 1487 * Pages allocated from the kernel may still have
1488 1488 * cache contents, so we set them to (CPU, CPU) always.
1489 1489 * 2. Written by CPU (using pwrite)
1490 1490 * The pwrite function calls set_domain (CPU, CPU) and
1491 1491 * this function does nothing (as nothing changes)
1492 1492 * 3. Mapped by GTT
1493 1493 * This function asserts that the object is not
1494 1494 * currently in any GPU-based read or write domains
1495 1495 * 4. Read by GPU
1496 1496 * i915_gem_execbuffer calls set_domain (COMMAND, 0).
1497 1497 * As write_domain is zero, this function adds in the
1498 1498 * current read domains (CPU+COMMAND, 0).
1499 1499 * flush_domains is set to CPU.
1500 1500 * invalidate_domains is set to COMMAND
1501 1501 * clflush is run to get data out of the CPU caches
1502 1502 * then i915_dev_set_domain calls i915_gem_flush to
1503 1503 * emit an MI_FLUSH and drm_agp_chipset_flush
1504 1504 * 5. Unmapped from GTT
1505 1505 * i915_gem_object_unbind calls set_domain (CPU, CPU)
1506 1506 * flush_domains and invalidate_domains end up both zero
1507 1507 * so no flushing/invalidating happens
1508 1508 * 6. Freed
1509 1509 * yay, done
1510 1510 *
1511 1511 * Case 2: The shared render buffer
1512 1512 *
1513 1513 * 1. Allocated
1514 1514 * 2. Mapped to GTT
1515 1515 * 3. Read/written by GPU
1516 1516 * 4. set_domain to (CPU,CPU)
1517 1517 * 5. Read/written by CPU
1518 1518 * 6. Read/written by GPU
1519 1519 *
1520 1520 * 1. Allocated
1521 1521 * Same as last example, (CPU, CPU)
1522 1522 * 2. Mapped to GTT
1523 1523 * Nothing changes (assertions find that it is not in the GPU)
1524 1524 * 3. Read/written by GPU
1525 1525 * execbuffer calls set_domain (RENDER, RENDER)
1526 1526 * flush_domains gets CPU
1527 1527 * invalidate_domains gets GPU
1528 1528 * clflush (obj)
1529 1529 * MI_FLUSH and drm_agp_chipset_flush
1530 1530 * 4. set_domain (CPU, CPU)
1531 1531 * flush_domains gets GPU
1532 1532 * invalidate_domains gets CPU
1533 1533 * wait_rendering (obj) to make sure all drawing is complete.
1534 1534 * This will include an MI_FLUSH to get the data from GPU
1535 1535 * to memory
1536 1536 * clflush (obj) to invalidate the CPU cache
1537 1537 * Another MI_FLUSH in i915_gem_flush (eliminate this somehow?)
1538 1538 * 5. Read/written by CPU
1539 1539 * cache lines are loaded and dirtied
1540 1540 * 6. Read written by GPU
1541 1541 * Same as last GPU access
1542 1542 *
1543 1543 * Case 3: The constant buffer
1544 1544 *
1545 1545 * 1. Allocated
1546 1546 * 2. Written by CPU
1547 1547 * 3. Read by GPU
1548 1548 * 4. Updated (written) by CPU again
1549 1549 * 5. Read by GPU
1550 1550 *
1551 1551 * 1. Allocated
1552 1552 * (CPU, CPU)
1553 1553 * 2. Written by CPU
1554 1554 * (CPU, CPU)
1555 1555 * 3. Read by GPU
1556 1556 * (CPU+RENDER, 0)
1557 1557 * flush_domains = CPU
1558 1558 * invalidate_domains = RENDER
1559 1559 * clflush (obj)
1560 1560 * MI_FLUSH
1561 1561 * drm_agp_chipset_flush
1562 1562 * 4. Updated (written) by CPU again
1563 1563 * (CPU, CPU)
1564 1564 * flush_domains = 0 (no previous write domain)
1565 1565 * invalidate_domains = 0 (no new read domains)
1566 1566 * 5. Read by GPU
1567 1567 * (CPU+RENDER, 0)
1568 1568 * flush_domains = CPU
1569 1569 * invalidate_domains = RENDER
1570 1570 * clflush (obj)
1571 1571 * MI_FLUSH
1572 1572 * drm_agp_chipset_flush
1573 1573 */
1574 1574 static void
1575 1575 i915_gem_object_set_to_gpu_domain(struct drm_gem_object *obj,
1576 1576 uint32_t read_domains,
1577 1577 uint32_t write_domain)
1578 1578 {
1579 1579 struct drm_device *dev = obj->dev;
1580 1580 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1581 1581 uint32_t invalidate_domains = 0;
1582 1582 uint32_t flush_domains = 0;
1583 1583
1584 1584 DRM_DEBUG("%s: object %p read %08x -> %08x write %08x -> %08x\n",
1585 1585 __func__, obj,
1586 1586 obj->read_domains, read_domains,
1587 1587 obj->write_domain, write_domain);
1588 1588 /*
1589 1589 * If the object isn't moving to a new write domain,
1590 1590 * let the object stay in multiple read domains
1591 1591 */
1592 1592 if (write_domain == 0)
1593 1593 read_domains |= obj->read_domains;
1594 1594 else
1595 1595 obj_priv->dirty = 1;
1596 1596
1597 1597 /*
1598 1598 * Flush the current write domain if
1599 1599 * the new read domains don't match. Invalidate
1600 1600 * any read domains which differ from the old
1601 1601 * write domain
1602 1602 */
1603 1603 if (obj->write_domain && obj->write_domain != read_domains) {
1604 1604 flush_domains |= obj->write_domain;
1605 1605 invalidate_domains |= read_domains & ~obj->write_domain;
1606 1606 }
1607 1607 /*
1608 1608 * Invalidate any read caches which may have
1609 1609 * stale data. That is, any new read domains.
1610 1610 */
1611 1611 invalidate_domains |= read_domains & ~obj->read_domains;
1612 1612 if ((flush_domains | invalidate_domains) & I915_GEM_DOMAIN_CPU) {
1613 1613 DRM_DEBUG("%s: CPU domain flush %08x invalidate %08x\n",
1614 1614 __func__, flush_domains, invalidate_domains);
1615 1615 i915_gem_clflush_object(obj);
1616 1616 }
1617 1617
1618 1618 if ((write_domain | flush_domains) != 0)
1619 1619 obj->write_domain = write_domain;
1620 1620 obj->read_domains = read_domains;
1621 1621
1622 1622 dev->invalidate_domains |= invalidate_domains;
1623 1623 dev->flush_domains |= flush_domains;
1624 1624
1625 1625 DRM_DEBUG("%s: read %08x write %08x invalidate %08x flush %08x\n",
1626 1626 __func__,
1627 1627 obj->read_domains, obj->write_domain,
1628 1628 dev->invalidate_domains, dev->flush_domains);
1629 1629
1630 1630 }
1631 1631
1632 1632 /**
1633 1633 * Moves the object from a partially CPU read to a full one.
1634 1634 *
1635 1635 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
1636 1636 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
1637 1637 */
1638 1638 static void
1639 1639 i915_gem_object_set_to_full_cpu_read_domain(struct drm_gem_object *obj)
1640 1640 {
1641 1641 struct drm_device *dev = obj->dev;
1642 1642 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1643 1643
1644 1644 if (!obj_priv->page_cpu_valid)
1645 1645 return;
1646 1646
1647 1647 /* If we're partially in the CPU read domain, finish moving it in.
1648 1648 */
1649 1649 if (obj->read_domains & I915_GEM_DOMAIN_CPU) {
1650 1650 int i;
1651 1651
1652 1652 for (i = 0; i <= (obj->size - 1) / PAGE_SIZE; i++) {
1653 1653 if (obj_priv->page_cpu_valid[i])
1654 1654 continue;
1655 1655 drm_clflush_pages(obj_priv->page_list + i, 1);
1656 1656 }
1657 1657 drm_agp_chipset_flush(dev);
1658 1658 }
1659 1659
1660 1660 /* Free the page_cpu_valid mappings which are now stale, whether
1661 1661 * or not we've got I915_GEM_DOMAIN_CPU.
1662 1662 */
1663 1663 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE,
1664 1664 DRM_MEM_DRIVER);
1665 1665 obj_priv->page_cpu_valid = NULL;
1666 1666 }
1667 1667
1668 1668 /**
1669 1669 * Set the CPU read domain on a range of the object.
1670 1670 *
1671 1671 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
1672 1672 * not entirely valid. The page_cpu_valid member of the object flags which
1673 1673 * pages have been flushed, and will be respected by
1674 1674 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
1675 1675 * of the whole object.
1676 1676 *
1677 1677 * This function returns when the move is complete, including waiting on
1678 1678 * flushes to occur.
1679 1679 */
1680 1680 static int
1681 1681 i915_gem_object_set_cpu_read_domain_range(struct drm_gem_object *obj,
1682 1682 uint64_t offset, uint64_t size)
1683 1683 {
1684 1684 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1685 1685 int i, ret;
1686 1686
1687 1687 if (offset == 0 && size == obj->size)
1688 1688 return i915_gem_object_set_to_cpu_domain(obj, 0);
1689 1689
1690 1690 i915_gem_object_flush_gpu_write_domain(obj);
1691 1691 /* Wait on any GPU rendering and flushing to occur. */
1692 1692 ret = i915_gem_object_wait_rendering(obj);
1693 1693 if (ret != 0)
1694 1694 return ret;
1695 1695 i915_gem_object_flush_gtt_write_domain(obj);
1696 1696
1697 1697 /* If we're already fully in the CPU read domain, we're done. */
1698 1698 if (obj_priv->page_cpu_valid == NULL &&
1699 1699 (obj->read_domains & I915_GEM_DOMAIN_CPU) != 0)
1700 1700 return 0;
1701 1701
1702 1702 /* Otherwise, create/clear the per-page CPU read domain flag if we're
1703 1703 * newly adding I915_GEM_DOMAIN_CPU
1704 1704 */
1705 1705 if (obj_priv->page_cpu_valid == NULL) {
1706 1706 obj_priv->page_cpu_valid = drm_calloc(1, obj->size / PAGE_SIZE,
1707 1707 DRM_MEM_DRIVER);
1708 1708 if (obj_priv->page_cpu_valid == NULL)
1709 1709 return ENOMEM;
1710 1710 } else if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0)
1711 1711 (void) memset(obj_priv->page_cpu_valid, 0, obj->size / PAGE_SIZE);
1712 1712
1713 1713 /* Flush the cache on any pages that are still invalid from the CPU's
1714 1714 * perspective.
1715 1715 */
1716 1716 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
1717 1717 i++) {
1718 1718 if (obj_priv->page_cpu_valid[i])
1719 1719 continue;
1720 1720
1721 1721 drm_clflush_pages(obj_priv->page_list + i, 1);
1722 1722 obj_priv->page_cpu_valid[i] = 1;
1723 1723 }
1724 1724
1725 1725 /* It should now be out of any other write domains, and we can update
1726 1726 * the domain values for our changes.
1727 1727 */
1728 1728 ASSERT(!((obj->write_domain & ~I915_GEM_DOMAIN_CPU) != 0));
1729 1729
1730 1730 obj->read_domains |= I915_GEM_DOMAIN_CPU;
1731 1731
1732 1732 return 0;
1733 1733 }
1734 1734
1735 1735 /**
1736 1736 * Pin an object to the GTT and evaluate the relocations landing in it.
1737 1737 */
1738 1738 static int
1739 1739 i915_gem_object_pin_and_relocate(struct drm_gem_object *obj,
1740 1740 struct drm_file *file_priv,
1741 1741 struct drm_i915_gem_exec_object *entry)
1742 1742 {
1743 1743 struct drm_i915_gem_relocation_entry reloc;
1744 1744 struct drm_i915_gem_relocation_entry __user *relocs;
1745 1745 struct drm_i915_gem_object *obj_priv = obj->driver_private;
1746 1746 int i, ret;
1747 1747
1748 1748 /* Choose the GTT offset for our buffer and put it there. */
1749 1749 ret = i915_gem_object_pin(obj, (uint32_t) entry->alignment);
1750 1750 if (ret) {
1751 1751 DRM_ERROR("failed to pin");
1752 1752 return ret;
1753 1753 }
1754 1754 entry->offset = obj_priv->gtt_offset;
1755 1755
1756 1756 relocs = (struct drm_i915_gem_relocation_entry __user *)
1757 1757 (uintptr_t) entry->relocs_ptr;
1758 1758 /* Apply the relocations, using the GTT aperture to avoid cache
1759 1759 * flushing requirements.
1760 1760 */
1761 1761 for (i = 0; i < entry->relocation_count; i++) {
1762 1762 struct drm_gem_object *target_obj;
1763 1763 struct drm_i915_gem_object *target_obj_priv;
1764 1764 uint32_t reloc_val, reloc_offset, *reloc_entry;
1765 1765
1766 1766 ret = DRM_COPY_FROM_USER(&reloc, relocs + i, sizeof(reloc));
1767 1767 if (ret != 0) {
1768 1768 i915_gem_object_unpin(obj);
1769 1769 DRM_ERROR("failed to copy from user");
1770 1770 return ret;
1771 1771 }
1772 1772
1773 1773 target_obj = drm_gem_object_lookup(file_priv,
1774 1774 reloc.target_handle);
1775 1775 if (target_obj == NULL) {
1776 1776 i915_gem_object_unpin(obj);
1777 1777 return EBADF;
1778 1778 }
1779 1779 target_obj_priv = target_obj->driver_private;
1780 1780
1781 1781 /* The target buffer should have appeared before us in the
1782 1782 * exec_object list, so it should have a GTT space bound by now.
1783 1783 */
1784 1784 if (target_obj_priv->gtt_space == NULL) {
1785 1785 DRM_ERROR("No GTT space found for object %d\n",
1786 1786 reloc.target_handle);
1787 1787 drm_gem_object_unreference(target_obj);
1788 1788 i915_gem_object_unpin(obj);
1789 1789 return EINVAL;
1790 1790 }
1791 1791
1792 1792 if (reloc.offset > obj->size - 4) {
1793 1793 DRM_ERROR("Relocation beyond object bounds: "
1794 1794 "obj %p target %d offset %d size %d.\n",
1795 1795 obj, reloc.target_handle,
1796 1796 (int) reloc.offset, (int) obj->size);
1797 1797 drm_gem_object_unreference(target_obj);
1798 1798 i915_gem_object_unpin(obj);
1799 1799 return EINVAL;
1800 1800 }
1801 1801 if (reloc.offset & 3) {
1802 1802 DRM_ERROR("Relocation not 4-byte aligned: "
1803 1803 "obj %p target %d offset %d.\n",
1804 1804 obj, reloc.target_handle,
1805 1805 (int) reloc.offset);
1806 1806 drm_gem_object_unreference(target_obj);
1807 1807 i915_gem_object_unpin(obj);
1808 1808 return EINVAL;
1809 1809 }
1810 1810
1811 1811 if (reloc.write_domain & I915_GEM_DOMAIN_CPU ||
1812 1812 reloc.read_domains & I915_GEM_DOMAIN_CPU) {
1813 1813 DRM_ERROR("reloc with read/write CPU domains: "
1814 1814 "obj %p target %d offset %d "
1815 1815 "read %08x write %08x",
1816 1816 obj, reloc.target_handle,
1817 1817 (int) reloc.offset,
1818 1818 reloc.read_domains,
1819 1819 reloc.write_domain);
1820 1820 drm_gem_object_unreference(target_obj);
1821 1821 i915_gem_object_unpin(obj);
1822 1822 return EINVAL;
1823 1823 }
1824 1824
1825 1825 if (reloc.write_domain && target_obj->pending_write_domain &&
1826 1826 reloc.write_domain != target_obj->pending_write_domain) {
1827 1827 DRM_ERROR("Write domain conflict: "
1828 1828 "obj %p target %d offset %d "
1829 1829 "new %08x old %08x\n",
1830 1830 obj, reloc.target_handle,
1831 1831 (int) reloc.offset,
1832 1832 reloc.write_domain,
1833 1833 target_obj->pending_write_domain);
1834 1834 drm_gem_object_unreference(target_obj);
1835 1835 i915_gem_object_unpin(obj);
1836 1836 return EINVAL;
1837 1837 }
1838 1838 DRM_DEBUG("%s: obj %p offset %08x target %d "
1839 1839 "read %08x write %08x gtt %08x "
1840 1840 "presumed %08x delta %08x\n",
1841 1841 __func__,
1842 1842 obj,
1843 1843 (int) reloc.offset,
1844 1844 (int) reloc.target_handle,
1845 1845 (int) reloc.read_domains,
1846 1846 (int) reloc.write_domain,
1847 1847 (int) target_obj_priv->gtt_offset,
1848 1848 (int) reloc.presumed_offset,
1849 1849 reloc.delta);
1850 1850
1851 1851 target_obj->pending_read_domains |= reloc.read_domains;
1852 1852 target_obj->pending_write_domain |= reloc.write_domain;
1853 1853
1854 1854 /* If the relocation already has the right value in it, no
1855 1855 * more work needs to be done.
1856 1856 */
1857 1857 if (target_obj_priv->gtt_offset == reloc.presumed_offset) {
1858 1858 drm_gem_object_unreference(target_obj);
1859 1859 continue;
1860 1860 }
1861 1861
1862 1862 ret = i915_gem_object_set_to_gtt_domain(obj, 1);
1863 1863 if (ret != 0) {
1864 1864 drm_gem_object_unreference(target_obj);
1865 1865 i915_gem_object_unpin(obj);
1866 1866 return EINVAL;
1867 1867 }
1868 1868
1869 1869 /* Map the page containing the relocation we're going to
1870 1870 * perform.
1871 1871 */
1872 1872
1873 1873 int reloc_base = (reloc.offset & ~(PAGE_SIZE-1));
1874 1874 reloc_offset = reloc.offset & (PAGE_SIZE-1);
1875 1875 reloc_entry = (uint32_t *)(uintptr_t)(obj_priv->page_list[reloc_base/PAGE_SIZE] + reloc_offset);
1876 1876 reloc_val = target_obj_priv->gtt_offset + reloc.delta;
1877 1877 *reloc_entry = reloc_val;
1878 1878
1879 1879 /* Write the updated presumed offset for this entry back out
1880 1880 * to the user.
1881 1881 */
1882 1882 reloc.presumed_offset = target_obj_priv->gtt_offset;
1883 1883 ret = DRM_COPY_TO_USER(relocs + i, &reloc, sizeof(reloc));
1884 1884 if (ret != 0) {
1885 1885 drm_gem_object_unreference(target_obj);
1886 1886 i915_gem_object_unpin(obj);
1887 1887 DRM_ERROR("%s: Failed to copy to user ret %d", __func__, ret);
1888 1888 return ret;
1889 1889 }
1890 1890
1891 1891 drm_gem_object_unreference(target_obj);
1892 1892 }
1893 1893
1894 1894 return 0;
1895 1895 }
1896 1896
1897 1897 /** Dispatch a batchbuffer to the ring
1898 1898 */
1899 1899 static int
1900 1900 i915_dispatch_gem_execbuffer(struct drm_device *dev,
1901 1901 struct drm_i915_gem_execbuffer *exec,
1902 1902 uint64_t exec_offset)
1903 1903 {
1904 1904 drm_i915_private_t *dev_priv = dev->dev_private;
1905 1905 struct drm_clip_rect __user *boxes = (struct drm_clip_rect __user *)
1906 1906 (uintptr_t) exec->cliprects_ptr;
1907 1907 int nbox = exec->num_cliprects;
1908 1908 int i = 0, count;
1909 1909 uint64_t exec_start, exec_len;
1910 1910 RING_LOCALS;
1911 1911
1912 1912 exec_start = exec_offset + exec->batch_start_offset;
1913 1913 exec_len = exec->batch_len;
1914 1914
1915 1915 if ((exec_start | exec_len) & 0x7) {
1916 1916 DRM_ERROR("alignment\n");
1917 1917 return EINVAL;
1918 1918 }
1919 1919
1920 1920 if (!exec_start) {
1921 1921 DRM_ERROR("wrong arg");
1922 1922 return EINVAL;
1923 1923 }
1924 1924
1925 1925 count = nbox ? nbox : 1;
1926 1926
1927 1927 for (i = 0; i < count; i++) {
1928 1928 if (i < nbox) {
1929 1929 int ret = i915_emit_box(dev, boxes, i,
1930 1930 exec->DR1, exec->DR4);
1931 1931 if (ret) {
1932 1932 DRM_ERROR("i915_emit_box %d DR1 0x%lx DRI2 0x%lx", ret, exec->DR1, exec->DR4);
1933 1933 return ret;
1934 1934 }
1935 1935 }
1936 1936 if (IS_I830(dev) || IS_845G(dev)) {
1937 1937 BEGIN_LP_RING(4);
1938 1938 OUT_RING(MI_BATCH_BUFFER);
1939 1939 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
1940 1940 OUT_RING(exec_start + exec_len - 4);
1941 1941 OUT_RING(0);
1942 1942 ADVANCE_LP_RING();
1943 1943 } else {
1944 1944 BEGIN_LP_RING(2);
1945 1945 if (IS_I965G(dev)) {
1946 1946 OUT_RING(MI_BATCH_BUFFER_START |
1947 1947 (2 << 6) |
1948 1948 (3 << 9) |
1949 1949 MI_BATCH_NON_SECURE_I965);
1950 1950 OUT_RING(exec_start);
1951 1951
1952 1952 } else {
1953 1953 OUT_RING(MI_BATCH_BUFFER_START |
1954 1954 (2 << 6));
1955 1955 OUT_RING(exec_start | MI_BATCH_NON_SECURE);
1956 1956 }
1957 1957 ADVANCE_LP_RING();
1958 1958 }
1959 1959 }
1960 1960 /* XXX breadcrumb */
1961 1961 return 0;
1962 1962 }
1963 1963
1964 1964 /* Throttle our rendering by waiting until the ring has completed our requests
1965 1965 * emitted over 20 msec ago.
1966 1966 *
1967 1967 * This should get us reasonable parallelism between CPU and GPU but also
1968 1968 * relatively low latency when blocking on a particular request to finish.
1969 1969 */
1970 1970 static int
1971 1971 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file_priv)
1972 1972 {
1973 1973 struct drm_i915_file_private *i915_file_priv = file_priv->driver_priv;
1974 1974 int ret = 0;
1975 1975 uint32_t seqno;
1976 1976
1977 1977 spin_lock(&dev->struct_mutex);
1978 1978 seqno = i915_file_priv->mm.last_gem_throttle_seqno;
1979 1979 i915_file_priv->mm.last_gem_throttle_seqno =
1980 1980 i915_file_priv->mm.last_gem_seqno;
1981 1981 if (seqno) {
1982 1982 ret = i915_wait_request(dev, seqno);
1983 1983 if (ret != 0)
1984 1984 DRM_ERROR("%s: i915_wait_request request->seqno %d now %d\n", __func__, seqno, i915_get_gem_seqno(dev));
1985 1985 }
1986 1986 spin_unlock(&dev->struct_mutex);
1987 1987 return ret;
1988 1988 }
1989 1989
1990 1990 /*ARGSUSED*/
1991 1991 int
1992 1992 i915_gem_execbuffer(DRM_IOCTL_ARGS)
1993 1993 {
1994 1994 DRM_DEVICE;
1995 1995 drm_i915_private_t *dev_priv = dev->dev_private;
1996 1996 struct drm_i915_file_private *i915_file_priv = fpriv->driver_priv;
1997 1997 struct drm_i915_gem_execbuffer args;
1998 1998 struct drm_i915_gem_exec_object *exec_list = NULL;
1999 1999 struct drm_gem_object **object_list = NULL;
2000 2000 struct drm_gem_object *batch_obj;
2001 2001 struct drm_i915_gem_object *obj_priv;
2002 2002 int ret = 0, i, pinned = 0;
2003 2003 uint64_t exec_offset;
2004 2004 uint32_t seqno, flush_domains;
2005 2005 int pin_tries;
2006 2006
2007 2007 if (dev->driver->use_gem != 1)
2008 2008 return ENODEV;
2009 2009
2010 2010 DRM_COPYFROM_WITH_RETURN(&args,
2011 2011 (struct drm_i915_gem_execbuffer __user *) data, sizeof(args));
2012 2012
2013 2013 DRM_DEBUG("buffer_count %d len %x\n", args.buffer_count, args.batch_len);
2014 2014
2015 2015 if (args.buffer_count < 1) {
2016 2016 DRM_ERROR("execbuf with %d buffers\n", args.buffer_count);
2017 2017 return EINVAL;
2018 2018 }
2019 2019 /* Copy in the exec list from userland */
2020 2020 exec_list = drm_calloc(sizeof(*exec_list), args.buffer_count,
2021 2021 DRM_MEM_DRIVER);
2022 2022 object_list = drm_calloc(sizeof(*object_list), args.buffer_count,
2023 2023 DRM_MEM_DRIVER);
2024 2024 if (exec_list == NULL || object_list == NULL) {
2025 2025 DRM_ERROR("Failed to allocate exec or object list "
2026 2026 "for %d buffers\n",
2027 2027 args.buffer_count);
2028 2028 ret = ENOMEM;
2029 2029 goto pre_mutex_err;
2030 2030 }
2031 2031
2032 2032 ret = DRM_COPY_FROM_USER(exec_list,
2033 2033 (struct drm_i915_gem_exec_object __user *)
2034 2034 (uintptr_t) args.buffers_ptr,
2035 2035 sizeof(*exec_list) * args.buffer_count);
2036 2036 if (ret != 0) {
2037 2037 DRM_ERROR("copy %d exec entries failed %d\n",
2038 2038 args.buffer_count, ret);
2039 2039 goto pre_mutex_err;
2040 2040 }
2041 2041 spin_lock(&dev->struct_mutex);
2042 2042
2043 2043 if (dev_priv->mm.wedged) {
2044 2044 DRM_ERROR("Execbuf while wedged\n");
2045 2045 spin_unlock(&dev->struct_mutex);
2046 2046 return EIO;
2047 2047 }
2048 2048
2049 2049 if (dev_priv->mm.suspended) {
2050 2050 DRM_ERROR("Execbuf while VT-switched.\n");
2051 2051 spin_unlock(&dev->struct_mutex);
2052 2052 return EBUSY;
2053 2053 }
2054 2054
2055 2055 /* Look up object handles */
2056 2056 for (i = 0; i < args.buffer_count; i++) {
2057 2057 object_list[i] = drm_gem_object_lookup(fpriv,
2058 2058 exec_list[i].handle);
2059 2059 if (object_list[i] == NULL) {
2060 2060 DRM_ERROR("Invalid object handle %d at index %d\n",
2061 2061 exec_list[i].handle, i);
2062 2062 ret = EBADF;
2063 2063 goto err;
2064 2064 }
2065 2065 obj_priv = object_list[i]->driver_private;
2066 2066 if (obj_priv->in_execbuffer) {
2067 2067 DRM_ERROR("Object[%d] (%d) %p appears more than once in object list in args.buffer_count %d \n",
2068 2068 i, object_list[i]->name, object_list[i], args.buffer_count);
2069 2069
2070 2070 ret = EBADF;
2071 2071 goto err;
2072 2072 }
2073 2073
2074 2074 obj_priv->in_execbuffer = 1;
2075 2075 }
2076 2076
2077 2077 /* Pin and relocate */
2078 2078 for (pin_tries = 0; ; pin_tries++) {
2079 2079 ret = 0;
2080 2080 for (i = 0; i < args.buffer_count; i++) {
2081 2081 object_list[i]->pending_read_domains = 0;
2082 2082 object_list[i]->pending_write_domain = 0;
2083 2083 ret = i915_gem_object_pin_and_relocate(object_list[i],
2084 2084 fpriv,
2085 2085 &exec_list[i]);
2086 2086 if (ret) {
2087 2087 DRM_ERROR("Not all object pinned");
2088 2088 break;
2089 2089 }
2090 2090 pinned = i + 1;
2091 2091 }
2092 2092 /* success */
2093 2093 if (ret == 0)
2094 2094 {
2095 2095 DRM_DEBUG("gem_execbuffer pin_relocate success");
2096 2096 break;
2097 2097 }
2098 2098 /* error other than GTT full, or we've already tried again */
2099 2099 if (ret != ENOMEM || pin_tries >= 1) {
2100 2100 if (ret != ERESTART)
2101 2101 DRM_ERROR("Failed to pin buffers %d\n", ret);
2102 2102 goto err;
2103 2103 }
2104 2104
2105 2105 /* unpin all of our buffers */
2106 2106 for (i = 0; i < pinned; i++)
2107 2107 i915_gem_object_unpin(object_list[i]);
2108 2108 pinned = 0;
2109 2109
2110 2110 /* evict everyone we can from the aperture */
2111 2111 ret = i915_gem_evict_everything(dev);
2112 2112 if (ret)
2113 2113 goto err;
2114 2114 }
2115 2115
2116 2116 /* Set the pending read domains for the batch buffer to COMMAND */
2117 2117 batch_obj = object_list[args.buffer_count-1];
2118 2118 batch_obj->pending_read_domains = I915_GEM_DOMAIN_COMMAND;
2119 2119 batch_obj->pending_write_domain = 0;
2120 2120
2121 2121 /* Zero the gloabl flush/invalidate flags. These
2122 2122 * will be modified as each object is bound to the
2123 2123 * gtt
2124 2124 */
2125 2125 dev->invalidate_domains = 0;
2126 2126 dev->flush_domains = 0;
2127 2127
2128 2128 for (i = 0; i < args.buffer_count; i++) {
2129 2129 struct drm_gem_object *obj = object_list[i];
2130 2130
2131 2131 /* Compute new gpu domains and update invalidate/flush */
2132 2132 i915_gem_object_set_to_gpu_domain(obj,
2133 2133 obj->pending_read_domains,
2134 2134 obj->pending_write_domain);
2135 2135 }
2136 2136
2137 2137 if (dev->invalidate_domains | dev->flush_domains) {
2138 2138
2139 2139 DRM_DEBUG("%s: invalidate_domains %08x flush_domains %08x Then flush\n",
2140 2140 __func__,
2141 2141 dev->invalidate_domains,
2142 2142 dev->flush_domains);
2143 2143 i915_gem_flush(dev,
2144 2144 dev->invalidate_domains,
2145 2145 dev->flush_domains);
2146 2146 if (dev->flush_domains) {
2147 2147 (void) i915_add_request(dev, dev->flush_domains);
2148 2148
2149 2149 }
2150 2150 }
2151 2151
2152 2152 for (i = 0; i < args.buffer_count; i++) {
2153 2153 struct drm_gem_object *obj = object_list[i];
2154 2154
2155 2155 obj->write_domain = obj->pending_write_domain;
2156 2156 }
2157 2157
2158 2158 exec_offset = exec_list[args.buffer_count - 1].offset;
2159 2159
2160 2160 /* Exec the batchbuffer */
2161 2161 ret = i915_dispatch_gem_execbuffer(dev, &args, exec_offset);
2162 2162 if (ret) {
2163 2163 DRM_ERROR("dispatch failed %d\n", ret);
2164 2164 goto err;
2165 2165 }
2166 2166
2167 2167 /*
2168 2168 * Ensure that the commands in the batch buffer are
2169 2169 * finished before the interrupt fires
2170 2170 */
2171 2171 flush_domains = i915_retire_commands(dev);
2172 2172
2173 2173 /*
2174 2174 * Get a seqno representing the execution of the current buffer,
2175 2175 * which we can wait on. We would like to mitigate these interrupts,
2176 2176 * likely by only creating seqnos occasionally (so that we have
2177 2177 * *some* interrupts representing completion of buffers that we can
2178 2178 * wait on when trying to clear up gtt space).
2179 2179 */
2180 2180 seqno = i915_add_request(dev, flush_domains);
2181 2181 ASSERT(!(seqno == 0));
2182 2182 i915_file_priv->mm.last_gem_seqno = seqno;
2183 2183 for (i = 0; i < args.buffer_count; i++) {
2184 2184 struct drm_gem_object *obj = object_list[i];
2185 2185 i915_gem_object_move_to_active(obj, seqno);
2186 2186 DRM_DEBUG("%s: move to exec list %p\n", __func__, obj);
2187 2187 }
2188 2188
2189 2189 err:
2190 2190 if (object_list != NULL) {
2191 2191 for (i = 0; i < pinned; i++)
2192 2192 i915_gem_object_unpin(object_list[i]);
2193 2193
2194 2194 for (i = 0; i < args.buffer_count; i++) {
2195 2195 if (object_list[i]) {
2196 2196 obj_priv = object_list[i]->driver_private;
2197 2197 obj_priv->in_execbuffer = 0;
2198 2198 }
2199 2199 drm_gem_object_unreference(object_list[i]);
2200 2200 }
2201 2201 }
2202 2202 spin_unlock(&dev->struct_mutex);
2203 2203
2204 2204 if (!ret) {
2205 2205 /* Copy the new buffer offsets back to the user's exec list. */
2206 2206 ret = DRM_COPY_TO_USER((struct drm_i915_relocation_entry __user *)
2207 2207 (uintptr_t) args.buffers_ptr,
2208 2208 exec_list,
2209 2209 sizeof(*exec_list) * args.buffer_count);
2210 2210 if (ret)
2211 2211 DRM_ERROR("failed to copy %d exec entries "
2212 2212 "back to user (%d)\n",
2213 2213 args.buffer_count, ret);
2214 2214 }
2215 2215
2216 2216 pre_mutex_err:
2217 2217 drm_free(object_list, sizeof(*object_list) * args.buffer_count,
2218 2218 DRM_MEM_DRIVER);
2219 2219 drm_free(exec_list, sizeof(*exec_list) * args.buffer_count,
2220 2220 DRM_MEM_DRIVER);
2221 2221
2222 2222 return ret;
2223 2223 }
2224 2224
2225 2225 int
2226 2226 i915_gem_object_pin(struct drm_gem_object *obj, uint32_t alignment)
2227 2227 {
2228 2228 struct drm_device *dev = obj->dev;
2229 2229 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2230 2230 int ret;
2231 2231
2232 2232 if (obj_priv->gtt_space == NULL) {
2233 2233 ret = i915_gem_object_bind_to_gtt(obj, alignment);
2234 2234 if (ret != 0) {
2235 2235 DRM_ERROR("Failure to bind: %d", ret);
2236 2236 return ret;
2237 2237 }
2238 2238 }
2239 2239 obj_priv->pin_count++;
2240 2240
2241 2241 /* If the object is not active and not pending a flush,
2242 2242 * remove it from the inactive list
2243 2243 */
2244 2244 if (obj_priv->pin_count == 1) {
2245 2245 atomic_inc(&dev->pin_count);
2246 2246 atomic_add(obj->size, &dev->pin_memory);
2247 2247 if (!obj_priv->active &&
2248 2248 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
2249 2249 I915_GEM_DOMAIN_GTT)) == 0 &&
2250 2250 !list_empty(&obj_priv->list))
2251 2251 list_del_init(&obj_priv->list);
2252 2252 }
2253 2253 return 0;
2254 2254 }
2255 2255
2256 2256 void
2257 2257 i915_gem_object_unpin(struct drm_gem_object *obj)
2258 2258 {
2259 2259 struct drm_device *dev = obj->dev;
2260 2260 drm_i915_private_t *dev_priv = dev->dev_private;
2261 2261 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2262 2262 obj_priv->pin_count--;
2263 2263 ASSERT(!(obj_priv->pin_count < 0));
2264 2264 ASSERT(!(obj_priv->gtt_space == NULL));
2265 2265
2266 2266 /* If the object is no longer pinned, and is
2267 2267 * neither active nor being flushed, then stick it on
2268 2268 * the inactive list
2269 2269 */
2270 2270 if (obj_priv->pin_count == 0) {
2271 2271 if (!obj_priv->active &&
2272 2272 (obj->write_domain & ~(I915_GEM_DOMAIN_CPU |
2273 2273 I915_GEM_DOMAIN_GTT)) == 0)
2274 2274 list_move_tail(&obj_priv->list,
2275 2275 &dev_priv->mm.inactive_list, (caddr_t)obj_priv);
2276 2276 atomic_dec(&dev->pin_count);
2277 2277 atomic_sub(obj->size, &dev->pin_memory);
2278 2278 }
2279 2279 }
2280 2280
2281 2281 /*ARGSUSED*/
2282 2282 int
2283 2283 i915_gem_pin_ioctl(DRM_IOCTL_ARGS)
2284 2284 {
2285 2285 DRM_DEVICE;
2286 2286 struct drm_i915_gem_pin args;
2287 2287 struct drm_gem_object *obj;
2288 2288 struct drm_i915_gem_object *obj_priv;
2289 2289 int ret;
2290 2290
2291 2291 if (dev->driver->use_gem != 1)
2292 2292 return ENODEV;
2293 2293
2294 2294 DRM_COPYFROM_WITH_RETURN(&args,
2295 2295 (struct drm_i915_gem_pin __user *) data, sizeof(args));
2296 2296
2297 2297 spin_lock(&dev->struct_mutex);
2298 2298
2299 2299 obj = drm_gem_object_lookup(fpriv, args.handle);
2300 2300 if (obj == NULL) {
2301 2301 DRM_ERROR("Bad handle in i915_gem_pin_ioctl(): %d\n",
2302 2302 args.handle);
2303 2303 spin_unlock(&dev->struct_mutex);
2304 2304 return EBADF;
2305 2305 }
2306 2306 DRM_DEBUG("i915_gem_pin_ioctl obj->name %d", obj->name);
2307 2307 obj_priv = obj->driver_private;
2308 2308
2309 2309 if (obj_priv->pin_filp != NULL && obj_priv->pin_filp != fpriv) {
2310 2310 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
2311 2311 args.handle);
2312 2312 drm_gem_object_unreference(obj);
2313 2313 spin_unlock(&dev->struct_mutex);
2314 2314 return EINVAL;
2315 2315 }
2316 2316
2317 2317 obj_priv->user_pin_count++;
2318 2318 obj_priv->pin_filp = fpriv;
2319 2319 if (obj_priv->user_pin_count == 1) {
2320 2320 ret = i915_gem_object_pin(obj, args.alignment);
2321 2321 if (ret != 0) {
2322 2322 drm_gem_object_unreference(obj);
2323 2323 spin_unlock(&dev->struct_mutex);
2324 2324 return ret;
2325 2325 }
2326 2326 }
2327 2327
2328 2328 /* XXX - flush the CPU caches for pinned objects
2329 2329 * as the X server doesn't manage domains yet
2330 2330 */
2331 2331 i915_gem_object_flush_cpu_write_domain(obj);
2332 2332 args.offset = obj_priv->gtt_offset;
2333 2333
2334 2334 ret = DRM_COPY_TO_USER((struct drm_i915_gem_pin __user *) data, &args, sizeof(args));
2335 2335 if ( ret != 0)
2336 2336 DRM_ERROR(" gem pin ioctl error! %d", ret);
2337 2337
2338 2338 drm_gem_object_unreference(obj);
2339 2339 spin_unlock(&dev->struct_mutex);
2340 2340
2341 2341 return 0;
2342 2342 }
2343 2343
2344 2344 /*ARGSUSED*/
2345 2345 int
2346 2346 i915_gem_unpin_ioctl(DRM_IOCTL_ARGS)
2347 2347 {
2348 2348 DRM_DEVICE;
2349 2349 struct drm_i915_gem_pin args;
2350 2350 struct drm_gem_object *obj;
2351 2351 struct drm_i915_gem_object *obj_priv;
2352 2352
2353 2353 if (dev->driver->use_gem != 1)
2354 2354 return ENODEV;
2355 2355
2356 2356 DRM_COPYFROM_WITH_RETURN(&args,
2357 2357 (struct drm_i915_gem_pin __user *) data, sizeof(args));
2358 2358
2359 2359 spin_lock(&dev->struct_mutex);
2360 2360
2361 2361 obj = drm_gem_object_lookup(fpriv, args.handle);
2362 2362 if (obj == NULL) {
2363 2363 DRM_ERROR("Bad handle in i915_gem_unpin_ioctl(): %d\n",
2364 2364 args.handle);
2365 2365 spin_unlock(&dev->struct_mutex);
2366 2366 return EBADF;
2367 2367 }
2368 2368 obj_priv = obj->driver_private;
2369 2369 DRM_DEBUG("i915_gem_unpin_ioctl, obj->name %d", obj->name);
2370 2370 if (obj_priv->pin_filp != fpriv) {
2371 2371 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
2372 2372 args.handle);
2373 2373 drm_gem_object_unreference(obj);
2374 2374 spin_unlock(&dev->struct_mutex);
2375 2375 return EINVAL;
2376 2376 }
2377 2377 obj_priv->user_pin_count--;
2378 2378 if (obj_priv->user_pin_count == 0) {
2379 2379 obj_priv->pin_filp = NULL;
2380 2380 i915_gem_object_unpin(obj);
2381 2381 }
2382 2382 drm_gem_object_unreference(obj);
2383 2383 spin_unlock(&dev->struct_mutex);
2384 2384 return 0;
2385 2385 }
2386 2386
2387 2387 /*ARGSUSED*/
2388 2388 int
2389 2389 i915_gem_busy_ioctl(DRM_IOCTL_ARGS)
2390 2390 {
2391 2391 DRM_DEVICE;
2392 2392 struct drm_i915_gem_busy args;
2393 2393 struct drm_gem_object *obj;
2394 2394 struct drm_i915_gem_object *obj_priv;
2395 2395 int ret;
2396 2396
2397 2397 if (dev->driver->use_gem != 1)
2398 2398 return ENODEV;
2399 2399
2400 2400 DRM_COPYFROM_WITH_RETURN(&args,
2401 2401 (struct drm_i915_gem_busy __user *) data, sizeof(args));
2402 2402
2403 2403 spin_lock(&dev->struct_mutex);
2404 2404 obj = drm_gem_object_lookup(fpriv, args.handle);
2405 2405 if (obj == NULL) {
2406 2406 DRM_ERROR("Bad handle in i915_gem_busy_ioctl(): %d\n",
2407 2407 args.handle);
2408 2408 spin_unlock(&dev->struct_mutex);
2409 2409 return EBADF;
2410 2410 }
2411 2411
2412 2412 obj_priv = obj->driver_private;
2413 2413 /* Don't count being on the flushing list against the object being
2414 2414 * done. Otherwise, a buffer left on the flushing list but not getting
2415 2415 * flushed (because nobody's flushing that domain) won't ever return
2416 2416 * unbusy and get reused by libdrm's bo cache. The other expected
2417 2417 * consumer of this interface, OpenGL's occlusion queries, also specs
2418 2418 * that the objects get unbusy "eventually" without any interference.
2419 2419 */
2420 2420 args.busy = obj_priv->active && obj_priv->last_rendering_seqno != 0;
2421 2421 DRM_DEBUG("i915_gem_busy_ioctl call obj->name %d busy %d", obj->name, args.busy);
2422 2422
2423 2423 ret = DRM_COPY_TO_USER((struct drm_i915_gem_busy __user *) data, &args, sizeof(args));
2424 2424 if ( ret != 0)
2425 2425 DRM_ERROR(" gem busy error! %d", ret);
2426 2426
2427 2427 drm_gem_object_unreference(obj);
2428 2428 spin_unlock(&dev->struct_mutex);
2429 2429 return 0;
2430 2430 }
2431 2431
2432 2432 /*ARGSUSED*/
2433 2433 int
2434 2434 i915_gem_throttle_ioctl(DRM_IOCTL_ARGS)
2435 2435 {
2436 2436 DRM_DEVICE;
2437 2437
2438 2438 if (dev->driver->use_gem != 1)
2439 2439 return ENODEV;
2440 2440
2441 2441 return i915_gem_ring_throttle(dev, fpriv);
2442 2442 }
2443 2443
2444 2444 static int
2445 2445 i915_gem_object_get_page_list(struct drm_gem_object *obj)
|
↓ open down ↓ |
2445 lines elided |
↑ open up ↑ |
2446 2446 {
2447 2447 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2448 2448 caddr_t va;
2449 2449 long i;
2450 2450
2451 2451 if (obj_priv->page_list)
2452 2452 return 0;
2453 2453 pgcnt_t np = btop(obj->size);
2454 2454
2455 2455 obj_priv->page_list = kmem_zalloc(np * sizeof(caddr_t), KM_SLEEP);
2456 - if (obj_priv->page_list == NULL) {
2457 - DRM_ERROR("Faled to allocate page list\n");
2458 - return ENOMEM;
2459 - }
2460 2456
2461 2457 for (i = 0, va = obj->kaddr; i < np; i++, va += PAGESIZE) {
2462 2458 obj_priv->page_list[i] = va;
2463 2459 }
2464 2460 return 0;
2465 2461 }
2466 2462
2467 2463
2468 2464 int i915_gem_init_object(struct drm_gem_object *obj)
2469 2465 {
2470 2466 struct drm_i915_gem_object *obj_priv;
2471 2467
2472 2468 obj_priv = drm_calloc(1, sizeof(*obj_priv), DRM_MEM_DRIVER);
2473 2469 if (obj_priv == NULL)
2474 2470 return ENOMEM;
2475 2471
2476 2472 /*
2477 2473 * We've just allocated pages from the kernel,
2478 2474 * so they've just been written by the CPU with
2479 2475 * zeros. They'll need to be clflushed before we
2480 2476 * use them with the GPU.
2481 2477 */
2482 2478 obj->write_domain = I915_GEM_DOMAIN_CPU;
2483 2479 obj->read_domains = I915_GEM_DOMAIN_CPU;
2484 2480
2485 2481 obj->driver_private = obj_priv;
2486 2482 obj_priv->obj = obj;
2487 2483 INIT_LIST_HEAD(&obj_priv->list);
2488 2484 return 0;
2489 2485 }
2490 2486
2491 2487 void i915_gem_free_object(struct drm_gem_object *obj)
2492 2488 {
2493 2489 struct drm_i915_gem_object *obj_priv = obj->driver_private;
2494 2490
2495 2491 while (obj_priv->pin_count > 0)
2496 2492 i915_gem_object_unpin(obj);
2497 2493
2498 2494 DRM_DEBUG("%s: obj %d",__func__, obj->name);
2499 2495
2500 2496 (void) i915_gem_object_unbind(obj, 1);
2501 2497 if (obj_priv->page_cpu_valid != NULL)
2502 2498 drm_free(obj_priv->page_cpu_valid, obj->size / PAGE_SIZE, DRM_MEM_DRIVER);
2503 2499 drm_free(obj->driver_private, sizeof(*obj_priv), DRM_MEM_DRIVER);
2504 2500 }
2505 2501
2506 2502 /** Unbinds all objects that are on the given buffer list. */
2507 2503 static int
2508 2504 i915_gem_evict_from_list(struct drm_device *dev, struct list_head *head, uint32_t type)
2509 2505 {
2510 2506 struct drm_gem_object *obj;
2511 2507 struct drm_i915_gem_object *obj_priv;
2512 2508 int ret;
2513 2509
2514 2510 while (!list_empty(head)) {
2515 2511 obj_priv = list_entry(head->next,
2516 2512 struct drm_i915_gem_object,
2517 2513 list);
2518 2514 obj = obj_priv->obj;
2519 2515
2520 2516 if (obj_priv->pin_count != 0) {
2521 2517 DRM_ERROR("Pinned object in unbind list\n");
2522 2518 spin_unlock(&dev->struct_mutex);
2523 2519 return EINVAL;
2524 2520 }
2525 2521 DRM_DEBUG("%s: obj %d type %d",__func__, obj->name, type);
2526 2522 ret = i915_gem_object_unbind(obj, type);
2527 2523 if (ret != 0) {
2528 2524 DRM_ERROR("Error unbinding object in LeaveVT: %d\n",
2529 2525 ret);
2530 2526 spin_unlock(&dev->struct_mutex);
2531 2527 return ret;
2532 2528 }
2533 2529 }
2534 2530
2535 2531
2536 2532 return 0;
2537 2533 }
2538 2534
2539 2535 static int
2540 2536 i915_gem_idle(struct drm_device *dev, uint32_t type)
2541 2537 {
2542 2538 drm_i915_private_t *dev_priv = dev->dev_private;
2543 2539 uint32_t seqno, cur_seqno, last_seqno;
2544 2540 int stuck, ret;
2545 2541
2546 2542 spin_lock(&dev->struct_mutex);
2547 2543
2548 2544 if (dev_priv->mm.suspended || dev_priv->ring.ring_obj == NULL) {
2549 2545 spin_unlock(&dev->struct_mutex);
2550 2546 return 0;
2551 2547 }
2552 2548
2553 2549 /* Hack! Don't let anybody do execbuf while we don't control the chip.
2554 2550 * We need to replace this with a semaphore, or something.
2555 2551 */
2556 2552 dev_priv->mm.suspended = 1;
2557 2553
2558 2554 /* Cancel the retire work handler, wait for it to finish if running
2559 2555 */
2560 2556 if (worktimer_id != NULL) {
2561 2557 (void) untimeout(worktimer_id);
2562 2558 worktimer_id = NULL;
2563 2559 }
2564 2560
2565 2561 i915_kernel_lost_context(dev);
2566 2562
2567 2563 /* Flush the GPU along with all non-CPU write domains
2568 2564 */
2569 2565 i915_gem_flush(dev, ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT),
2570 2566 ~(I915_GEM_DOMAIN_CPU|I915_GEM_DOMAIN_GTT));
2571 2567 seqno = i915_add_request(dev, ~(I915_GEM_DOMAIN_CPU |
2572 2568 I915_GEM_DOMAIN_GTT));
2573 2569 if (seqno == 0) {
2574 2570 spin_unlock(&dev->struct_mutex);
2575 2571 return ENOMEM;
2576 2572 }
2577 2573
2578 2574 dev_priv->mm.waiting_gem_seqno = seqno;
2579 2575 last_seqno = 0;
2580 2576 stuck = 0;
2581 2577 for (;;) {
2582 2578 cur_seqno = i915_get_gem_seqno(dev);
2583 2579 if (i915_seqno_passed(cur_seqno, seqno))
2584 2580 break;
2585 2581 if (last_seqno == cur_seqno) {
2586 2582 if (stuck++ > 100) {
2587 2583 DRM_ERROR("hardware wedged\n");
2588 2584 dev_priv->mm.wedged = 1;
2589 2585 DRM_WAKEUP(&dev_priv->irq_queue);
2590 2586 break;
2591 2587 }
2592 2588 }
2593 2589 DRM_UDELAY(10);
2594 2590 last_seqno = cur_seqno;
2595 2591 }
2596 2592 dev_priv->mm.waiting_gem_seqno = 0;
2597 2593
2598 2594 i915_gem_retire_requests(dev);
2599 2595
2600 2596 /* Empty the active and flushing lists to inactive. If there's
2601 2597 * anything left at this point, it means that we're wedged and
2602 2598 * nothing good's going to happen by leaving them there. So strip
2603 2599 * the GPU domains and just stuff them onto inactive.
2604 2600 */
2605 2601 while (!list_empty(&dev_priv->mm.active_list)) {
2606 2602 struct drm_i915_gem_object *obj_priv;
2607 2603
2608 2604 obj_priv = list_entry(dev_priv->mm.active_list.next,
2609 2605 struct drm_i915_gem_object,
2610 2606 list);
2611 2607 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
2612 2608 i915_gem_object_move_to_inactive(obj_priv->obj);
2613 2609 }
2614 2610
2615 2611 while (!list_empty(&dev_priv->mm.flushing_list)) {
2616 2612 struct drm_i915_gem_object *obj_priv;
2617 2613
2618 2614 obj_priv = list_entry(dev_priv->mm.flushing_list.next,
2619 2615 struct drm_i915_gem_object,
2620 2616 list);
2621 2617 obj_priv->obj->write_domain &= ~I915_GEM_GPU_DOMAINS;
2622 2618 i915_gem_object_move_to_inactive(obj_priv->obj);
2623 2619 }
2624 2620
2625 2621 /* Move all inactive buffers out of the GTT. */
2626 2622 ret = i915_gem_evict_from_list(dev, &dev_priv->mm.inactive_list, type);
2627 2623 ASSERT(list_empty(&dev_priv->mm.inactive_list));
2628 2624 if (ret) {
2629 2625 spin_unlock(&dev->struct_mutex);
2630 2626 return ret;
2631 2627 }
2632 2628
2633 2629 i915_gem_cleanup_ringbuffer(dev);
2634 2630 spin_unlock(&dev->struct_mutex);
2635 2631
2636 2632 return 0;
2637 2633 }
2638 2634
2639 2635 static int
2640 2636 i915_gem_init_hws(struct drm_device *dev)
2641 2637 {
2642 2638 drm_i915_private_t *dev_priv = dev->dev_private;
2643 2639 struct drm_gem_object *obj;
2644 2640 struct drm_i915_gem_object *obj_priv;
2645 2641 int ret;
2646 2642
2647 2643 /* If we need a physical address for the status page, it's already
2648 2644 * initialized at driver load time.
2649 2645 */
2650 2646 if (!I915_NEED_GFX_HWS(dev))
2651 2647 return 0;
2652 2648
2653 2649
2654 2650 obj = drm_gem_object_alloc(dev, 4096);
2655 2651 if (obj == NULL) {
2656 2652 DRM_ERROR("Failed to allocate status page\n");
2657 2653 return ENOMEM;
2658 2654 }
2659 2655
2660 2656 obj_priv = obj->driver_private;
2661 2657
2662 2658 ret = i915_gem_object_pin(obj, 4096);
2663 2659 if (ret != 0) {
2664 2660 drm_gem_object_unreference(obj);
2665 2661 return ret;
2666 2662 }
2667 2663
2668 2664 dev_priv->status_gfx_addr = obj_priv->gtt_offset;
2669 2665 dev_priv->hws_map.offset = dev->agp->agp_info.agpi_aperbase + obj_priv->gtt_offset;
2670 2666 dev_priv->hws_map.size = 4096;
2671 2667 dev_priv->hws_map.type = 0;
2672 2668 dev_priv->hws_map.flags = 0;
2673 2669 dev_priv->hws_map.mtrr = 0;
2674 2670
2675 2671 drm_core_ioremap(&dev_priv->hws_map, dev);
2676 2672 if (dev_priv->hws_map.handle == NULL) {
2677 2673 DRM_ERROR("Failed to map status page.\n");
2678 2674 (void) memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
2679 2675 drm_gem_object_unreference(obj);
2680 2676 return EINVAL;
2681 2677 }
2682 2678
2683 2679 dev_priv->hws_obj = obj;
2684 2680
2685 2681 dev_priv->hw_status_page = dev_priv->hws_map.handle;
2686 2682
2687 2683 (void) memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
2688 2684 I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
2689 2685 (void) I915_READ(HWS_PGA); /* posting read */
2690 2686 DRM_DEBUG("hws offset: 0x%08x\n", dev_priv->status_gfx_addr);
2691 2687
2692 2688 return 0;
2693 2689 }
2694 2690
2695 2691 static void
2696 2692 i915_gem_cleanup_hws(struct drm_device *dev)
2697 2693 {
2698 2694 drm_i915_private_t *dev_priv = dev->dev_private;
2699 2695 struct drm_gem_object *obj;
2700 2696
2701 2697 if (dev_priv->hws_obj == NULL)
2702 2698 return;
2703 2699
2704 2700 obj = dev_priv->hws_obj;
2705 2701
2706 2702 drm_core_ioremapfree(&dev_priv->hws_map, dev);
2707 2703 i915_gem_object_unpin(obj);
2708 2704 drm_gem_object_unreference(obj);
2709 2705 dev_priv->hws_obj = NULL;
2710 2706
2711 2707 (void) memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
2712 2708 dev_priv->hw_status_page = NULL;
2713 2709
2714 2710 /* Write high address into HWS_PGA when disabling. */
2715 2711 I915_WRITE(HWS_PGA, 0x1ffff000);
2716 2712 }
2717 2713
2718 2714 int
2719 2715 i915_gem_init_ringbuffer(struct drm_device *dev)
2720 2716 {
2721 2717 drm_i915_private_t *dev_priv = dev->dev_private;
2722 2718 struct drm_gem_object *obj;
2723 2719 struct drm_i915_gem_object *obj_priv;
2724 2720 int ret;
2725 2721 u32 head;
2726 2722
2727 2723 ret = i915_gem_init_hws(dev);
2728 2724 if (ret != 0)
2729 2725 return ret;
2730 2726 obj = drm_gem_object_alloc(dev, 128 * 1024);
2731 2727 if (obj == NULL) {
2732 2728 DRM_ERROR("Failed to allocate ringbuffer\n");
2733 2729 i915_gem_cleanup_hws(dev);
2734 2730 return ENOMEM;
2735 2731 }
2736 2732
2737 2733 obj_priv = obj->driver_private;
2738 2734 ret = i915_gem_object_pin(obj, 4096);
2739 2735 if (ret != 0) {
2740 2736 drm_gem_object_unreference(obj);
2741 2737 i915_gem_cleanup_hws(dev);
2742 2738 return ret;
2743 2739 }
2744 2740
2745 2741 /* Set up the kernel mapping for the ring. */
2746 2742 dev_priv->ring.Size = obj->size;
2747 2743 dev_priv->ring.tail_mask = obj->size - 1;
2748 2744
2749 2745 dev_priv->ring.map.offset = dev->agp->agp_info.agpi_aperbase + obj_priv->gtt_offset;
2750 2746 dev_priv->ring.map.size = obj->size;
2751 2747 dev_priv->ring.map.type = 0;
2752 2748 dev_priv->ring.map.flags = 0;
2753 2749 dev_priv->ring.map.mtrr = 0;
2754 2750
2755 2751 drm_core_ioremap(&dev_priv->ring.map, dev);
2756 2752 if (dev_priv->ring.map.handle == NULL) {
2757 2753 DRM_ERROR("Failed to map ringbuffer.\n");
2758 2754 (void) memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
2759 2755 drm_gem_object_unreference(obj);
2760 2756 i915_gem_cleanup_hws(dev);
2761 2757 return EINVAL;
2762 2758 }
2763 2759
2764 2760 dev_priv->ring.ring_obj = obj;
2765 2761
2766 2762 dev_priv->ring.virtual_start = (u8 *) dev_priv->ring.map.handle;
2767 2763
2768 2764 /* Stop the ring if it's running. */
2769 2765 I915_WRITE(PRB0_CTL, 0);
2770 2766 I915_WRITE(PRB0_HEAD, 0);
2771 2767 I915_WRITE(PRB0_TAIL, 0);
2772 2768
2773 2769
2774 2770 /* Initialize the ring. */
2775 2771 I915_WRITE(PRB0_START, obj_priv->gtt_offset);
2776 2772 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
2777 2773
2778 2774 /* G45 ring initialization fails to reset head to zero */
2779 2775 if (head != 0) {
2780 2776 DRM_ERROR("Ring head not reset to zero "
2781 2777 "ctl %08x head %08x tail %08x start %08x\n",
2782 2778 I915_READ(PRB0_CTL),
2783 2779 I915_READ(PRB0_HEAD),
2784 2780 I915_READ(PRB0_TAIL),
2785 2781 I915_READ(PRB0_START));
2786 2782 I915_WRITE(PRB0_HEAD, 0);
2787 2783
2788 2784 DRM_ERROR("Ring head forced to zero "
2789 2785 "ctl %08x head %08x tail %08x start %08x\n",
2790 2786 I915_READ(PRB0_CTL),
2791 2787 I915_READ(PRB0_HEAD),
2792 2788 I915_READ(PRB0_TAIL),
2793 2789 I915_READ(PRB0_START));
2794 2790 }
2795 2791
2796 2792 I915_WRITE(PRB0_CTL,
2797 2793 ((obj->size - 4096) & RING_NR_PAGES) |
2798 2794 RING_NO_REPORT |
2799 2795 RING_VALID);
2800 2796
2801 2797 head = I915_READ(PRB0_HEAD) & HEAD_ADDR;
2802 2798
2803 2799 /* If the head is still not zero, the ring is dead */
2804 2800 if (head != 0) {
2805 2801 DRM_ERROR("Ring initialization failed "
2806 2802 "ctl %08x head %08x tail %08x start %08x\n",
2807 2803 I915_READ(PRB0_CTL),
2808 2804 I915_READ(PRB0_HEAD),
2809 2805 I915_READ(PRB0_TAIL),
2810 2806 I915_READ(PRB0_START));
2811 2807 return EIO;
2812 2808 }
2813 2809
2814 2810 /* Update our cache of the ring state */
2815 2811 i915_kernel_lost_context(dev);
2816 2812
2817 2813 return 0;
2818 2814 }
2819 2815
2820 2816 static void
2821 2817 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
2822 2818 {
2823 2819 drm_i915_private_t *dev_priv = dev->dev_private;
2824 2820
2825 2821 if (dev_priv->ring.ring_obj == NULL)
2826 2822 return;
2827 2823
2828 2824 drm_core_ioremapfree(&dev_priv->ring.map, dev);
2829 2825
2830 2826 i915_gem_object_unpin(dev_priv->ring.ring_obj);
2831 2827 drm_gem_object_unreference(dev_priv->ring.ring_obj);
2832 2828 dev_priv->ring.ring_obj = NULL;
2833 2829 (void) memset(&dev_priv->ring, 0, sizeof(dev_priv->ring));
2834 2830 i915_gem_cleanup_hws(dev);
2835 2831 }
2836 2832
2837 2833 /*ARGSUSED*/
2838 2834 int
2839 2835 i915_gem_entervt_ioctl(DRM_IOCTL_ARGS)
2840 2836 {
2841 2837 DRM_DEVICE;
2842 2838 drm_i915_private_t *dev_priv = dev->dev_private;
2843 2839 int ret;
2844 2840
2845 2841 if (dev->driver->use_gem != 1)
2846 2842 return ENODEV;
2847 2843
2848 2844 if (dev_priv->mm.wedged) {
2849 2845 DRM_ERROR("Reenabling wedged hardware, good luck\n");
2850 2846 dev_priv->mm.wedged = 0;
2851 2847 }
2852 2848 /* Set up the kernel mapping for the ring. */
2853 2849 dev_priv->mm.gtt_mapping.offset = dev->agp->agp_info.agpi_aperbase;
2854 2850 dev_priv->mm.gtt_mapping.size = dev->agp->agp_info.agpi_apersize;
2855 2851 dev_priv->mm.gtt_mapping.type = 0;
2856 2852 dev_priv->mm.gtt_mapping.flags = 0;
2857 2853 dev_priv->mm.gtt_mapping.mtrr = 0;
2858 2854
2859 2855 drm_core_ioremap(&dev_priv->mm.gtt_mapping, dev);
2860 2856
2861 2857 spin_lock(&dev->struct_mutex);
2862 2858 dev_priv->mm.suspended = 0;
2863 2859 ret = i915_gem_init_ringbuffer(dev);
2864 2860 if (ret != 0)
2865 2861 return ret;
2866 2862
2867 2863 spin_unlock(&dev->struct_mutex);
2868 2864
2869 2865 (void) drm_irq_install(dev);
2870 2866
2871 2867 return 0;
2872 2868 }
2873 2869
2874 2870 /*ARGSUSED*/
2875 2871 int
2876 2872 i915_gem_leavevt_ioctl(DRM_IOCTL_ARGS)
2877 2873 {
2878 2874 DRM_DEVICE;
2879 2875 drm_i915_private_t *dev_priv = dev->dev_private;
2880 2876 int ret;
2881 2877
2882 2878 if (dev->driver->use_gem != 1)
2883 2879 return ENODEV;
2884 2880
2885 2881 ret = i915_gem_idle(dev, 0);
2886 2882 (void) drm_irq_uninstall(dev);
2887 2883
2888 2884 drm_core_ioremapfree(&dev_priv->mm.gtt_mapping, dev);
2889 2885 return ret;
2890 2886 }
2891 2887
2892 2888 void
2893 2889 i915_gem_lastclose(struct drm_device *dev)
2894 2890 {
2895 2891 drm_i915_private_t *dev_priv = dev->dev_private;
2896 2892 int ret;
2897 2893
2898 2894 ret = i915_gem_idle(dev, 1);
2899 2895 if (ret)
2900 2896 DRM_ERROR("failed to idle hardware: %d\n", ret);
2901 2897
2902 2898 drm_mm_clean_ml(&dev_priv->mm.gtt_space);
2903 2899 }
2904 2900
2905 2901 void
2906 2902 i915_gem_load(struct drm_device *dev)
2907 2903 {
2908 2904 drm_i915_private_t *dev_priv = dev->dev_private;
2909 2905
2910 2906 INIT_LIST_HEAD(&dev_priv->mm.active_list);
2911 2907 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
2912 2908 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
2913 2909 INIT_LIST_HEAD(&dev_priv->mm.request_list);
2914 2910 dev_priv->mm.next_gem_seqno = 1;
2915 2911
2916 2912 i915_gem_detect_bit_6_swizzle(dev);
2917 2913
2918 2914 }
2919 2915
|
↓ open down ↓ |
450 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX