patch nuke-the-dbuf-hash patch make-the-merge-easy
1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 23 * Copyright 2013 Nexenta Systems, Inc. All rights reserved. 24 * Copyright (c) 2012, 2014 by Delphix. All rights reserved. 25 * Copyright (c) 2013 by Saso Kiselkov. All rights reserved. 26 * Copyright (c) 2013, Joyent, Inc. All rights reserved. 27 */ 28 29 #include <sys/zfs_context.h> 30 #include <sys/dmu.h> 31 #include <sys/dmu_send.h> 32 #include <sys/dmu_impl.h> 33 #include <sys/dbuf.h> 34 #include <sys/dmu_objset.h> 35 #include <sys/dsl_dataset.h> 36 #include <sys/dsl_dir.h> 37 #include <sys/dmu_tx.h> 38 #include <sys/spa.h> 39 #include <sys/spa_impl.h> 40 #include <sys/zio.h> 41 #include <sys/dmu_zfetch.h> 42 #include <sys/sa.h> 43 #include <sys/sa_impl.h> 44 #include <sys/zfeature.h> 45 #include <sys/blkptr.h> 46 #include <sys/range_tree.h> 47 48 /* 49 * Number of times that zfs_free_range() took the slow path while doing 50 * a zfs receive. A nonzero value indicates a potential performance problem. 51 */ 52 uint64_t zfs_free_range_recv_miss; 53 54 static void dbuf_destroy(dmu_buf_impl_t *db); 55 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx); 56 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx); 57 58 /* 59 * Global data structures and functions for the dbuf cache. 60 */ 61 static kmem_cache_t *dbuf_cache; 62 63 /* ARGSUSED */ 64 static int 65 dbuf_cons(void *vdb, void *unused, int kmflag) 66 { 67 dmu_buf_impl_t *db = vdb; 68 bzero(db, sizeof (dmu_buf_impl_t)); 69 70 mutex_init(&db->db_mtx, NULL, MUTEX_DEFAULT, NULL); 71 cv_init(&db->db_changed, NULL, CV_DEFAULT, NULL); 72 refcount_create(&db->db_holds); 73 74 return (0); 75 } 76 77 /* ARGSUSED */ 78 static void 79 dbuf_dest(void *vdb, void *unused) 80 { 81 dmu_buf_impl_t *db = vdb; 82 mutex_destroy(&db->db_mtx); 83 cv_destroy(&db->db_changed); 84 refcount_destroy(&db->db_holds); 85 } 86 87 /* 88 * dbuf hash table routines 89 */ 90 #pragma align 64(dbuf_hash_table) 91 static dbuf_hash_table_t dbuf_hash_table; 92 93 static uint64_t dbuf_hash_count; 94 95 static uint64_t 96 dbuf_hash(void *os, uint64_t obj, uint8_t lvl, uint64_t blkid) 97 { 98 uintptr_t osv = (uintptr_t)os; 99 uint64_t crc = -1ULL; 100 101 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 102 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (lvl)) & 0xFF]; 103 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (osv >> 6)) & 0xFF]; 104 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 0)) & 0xFF]; 105 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (obj >> 8)) & 0xFF]; 106 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 0)) & 0xFF]; 107 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (blkid >> 8)) & 0xFF]; 108 109 crc ^= (osv>>14) ^ (obj>>16) ^ (blkid>>16); 110 111 return (crc); 112 } 113 114 #define DBUF_HASH(os, obj, level, blkid) dbuf_hash(os, obj, level, blkid); 115 116 #define DBUF_EQUAL(dbuf, os, obj, level, blkid) \ 117 ((dbuf)->db.db_object == (obj) && \ 118 (dbuf)->db_objset == (os) && \ 119 (dbuf)->db_level == (level) && \ 120 (dbuf)->db_blkid == (blkid)) 121 122 dmu_buf_impl_t * 123 dbuf_find(dnode_t *dn, uint8_t level, uint64_t blkid) 124 { 125 dbuf_hash_table_t *h = &dbuf_hash_table; 126 objset_t *os = dn->dn_objset; 127 uint64_t obj = dn->dn_object; 128 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 129 uint64_t idx = hv & h->hash_table_mask; 130 dmu_buf_impl_t *db; 131 132 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 133 for (db = h->hash_table[idx]; db != NULL; db = db->db_hash_next) { 134 if (DBUF_EQUAL(db, os, obj, level, blkid)) { 135 mutex_enter(&db->db_mtx); 136 if (db->db_state != DB_EVICTING) { 137 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 138 return (db); 139 } 140 mutex_exit(&db->db_mtx); 141 } 142 } 143 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 144 return (NULL); 145 } 146 147 /* 148 * Insert an entry into the hash table. If there is already an element 149 * equal to elem in the hash table, then the already existing element 150 * will be returned and the new element will not be inserted. 151 * Otherwise returns NULL. 152 */ 153 static dmu_buf_impl_t * 154 dbuf_hash_insert(dmu_buf_impl_t *db) 155 { 156 dbuf_hash_table_t *h = &dbuf_hash_table; 157 objset_t *os = db->db_objset; 158 uint64_t obj = db->db.db_object; 159 int level = db->db_level; 160 uint64_t blkid = db->db_blkid; 161 uint64_t hv = DBUF_HASH(os, obj, level, blkid); 162 uint64_t idx = hv & h->hash_table_mask; 163 dmu_buf_impl_t *dbf; 164 165 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 166 for (dbf = h->hash_table[idx]; dbf != NULL; dbf = dbf->db_hash_next) { 167 if (DBUF_EQUAL(dbf, os, obj, level, blkid)) { 168 mutex_enter(&dbf->db_mtx); 169 if (dbf->db_state != DB_EVICTING) { 170 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 171 return (dbf); 172 } 173 mutex_exit(&dbf->db_mtx); 174 } 175 } 176 177 mutex_enter(&db->db_mtx); 178 db->db_hash_next = h->hash_table[idx]; 179 h->hash_table[idx] = db; 180 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 181 atomic_inc_64(&dbuf_hash_count); 182 183 return (NULL); 184 } 185 186 /* 187 * Remove an entry from the hash table. It must be in the EVICTING state. 188 */ 189 static void 190 dbuf_hash_remove(dmu_buf_impl_t *db) 191 { 192 dbuf_hash_table_t *h = &dbuf_hash_table; 193 uint64_t hv = DBUF_HASH(db->db_objset, db->db.db_object, 194 db->db_level, db->db_blkid); 195 uint64_t idx = hv & h->hash_table_mask; 196 dmu_buf_impl_t *dbf, **dbp; 197 198 /* 199 * We musn't hold db_mtx to maintain lock ordering: 200 * DBUF_HASH_MUTEX > db_mtx. 201 */ 202 ASSERT(refcount_is_zero(&db->db_holds)); 203 ASSERT(db->db_state == DB_EVICTING); 204 ASSERT(!MUTEX_HELD(&db->db_mtx)); 205 206 mutex_enter(DBUF_HASH_MUTEX(h, idx)); 207 dbp = &h->hash_table[idx]; 208 while ((dbf = *dbp) != db) { 209 dbp = &dbf->db_hash_next; 210 ASSERT(dbf != NULL); 211 } 212 *dbp = db->db_hash_next; 213 db->db_hash_next = NULL; 214 mutex_exit(DBUF_HASH_MUTEX(h, idx)); 215 atomic_dec_64(&dbuf_hash_count); 216 } 217 218 static arc_evict_func_t dbuf_do_evict; 219 220 static void 221 dbuf_evict_user(dmu_buf_impl_t *db) 222 { 223 ASSERT(MUTEX_HELD(&db->db_mtx)); 224 225 if (db->db_level != 0 || db->db_evict_func == NULL) 226 return; 227 228 if (db->db_user_data_ptr_ptr) 229 *db->db_user_data_ptr_ptr = db->db.db_data; 230 db->db_evict_func(&db->db, db->db_user_ptr); 231 db->db_user_ptr = NULL; 232 db->db_user_data_ptr_ptr = NULL; 233 db->db_evict_func = NULL; 234 } 235 236 boolean_t 237 dbuf_is_metadata(dmu_buf_impl_t *db) 238 { 239 if (db->db_level > 0) { 240 return (B_TRUE); 241 } else { 242 boolean_t is_metadata; 243 244 DB_DNODE_ENTER(db); 245 is_metadata = DMU_OT_IS_METADATA(DB_DNODE(db)->dn_type); 246 DB_DNODE_EXIT(db); 247 248 return (is_metadata); 249 } 250 } 251 252 void 253 dbuf_evict(dmu_buf_impl_t *db) 254 { 255 ASSERT(MUTEX_HELD(&db->db_mtx)); 256 ASSERT(db->db_buf == NULL); 257 ASSERT(db->db_data_pending == NULL); 258 259 dbuf_clear(db); 260 dbuf_destroy(db); 261 } 262 263 void 264 dbuf_init(void) 265 { 266 uint64_t hsize = 1ULL << 16; 267 dbuf_hash_table_t *h = &dbuf_hash_table; 268 int i; 269 270 /* 271 * The hash table is big enough to fill all of physical memory 272 * with an average 4K block size. The table will take up 273 * totalmem*sizeof(void*)/4K (i.e. 2MB/GB with 8-byte pointers). 274 */ 275 while (hsize * 4096 < physmem * PAGESIZE) 276 hsize <<= 1; 277 278 retry: 279 h->hash_table_mask = hsize - 1; 280 h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); 281 if (h->hash_table == NULL) { 282 /* XXX - we should really return an error instead of assert */ 283 ASSERT(hsize > (1ULL << 10)); 284 hsize >>= 1; 285 goto retry; 286 } 287 288 dbuf_cache = kmem_cache_create("dmu_buf_impl_t", 289 sizeof (dmu_buf_impl_t), 290 0, dbuf_cons, dbuf_dest, NULL, NULL, NULL, 0); 291 292 for (i = 0; i < DBUF_MUTEXES; i++) 293 mutex_init(DBUF_HASH_MUTEX(h, i), NULL, MUTEX_DEFAULT, NULL); 294 } 295 296 void 297 dbuf_fini(void) 298 { 299 dbuf_hash_table_t *h = &dbuf_hash_table; 300 int i; 301 302 for (i = 0; i < DBUF_MUTEXES; i++) 303 mutex_destroy(DBUF_HASH_MUTEX(h, i)); 304 kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *)); 305 kmem_cache_destroy(dbuf_cache); 306 } 307 308 /* 309 * Other stuff. 310 */ 311 312 #ifdef ZFS_DEBUG 313 static void 314 dbuf_verify(dmu_buf_impl_t *db) 315 { 316 dnode_t *dn; 317 dbuf_dirty_record_t *dr; 318 319 ASSERT(MUTEX_HELD(&db->db_mtx)); 320 321 if (!(zfs_flags & ZFS_DEBUG_DBUF_VERIFY)) 322 return; 323 324 ASSERT(db->db_objset != NULL); 325 DB_DNODE_ENTER(db); 326 dn = DB_DNODE(db); 327 if (dn == NULL) { 328 ASSERT(db->db_parent == NULL); 329 ASSERT(db->db_blkptr == NULL); 330 } else { 331 ASSERT3U(db->db.db_object, ==, dn->dn_object); 332 ASSERT3P(db->db_objset, ==, dn->dn_objset); 333 ASSERT3U(db->db_level, <, dn->dn_nlevels); 334 ASSERT(db->db_blkid == DMU_BONUS_BLKID || 335 db->db_blkid == DMU_SPILL_BLKID || 336 !avl_is_empty(&dn->dn_dbufs)); 337 } 338 if (db->db_blkid == DMU_BONUS_BLKID) { 339 ASSERT(dn != NULL); 340 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 341 ASSERT3U(db->db.db_offset, ==, DMU_BONUS_BLKID); 342 } else if (db->db_blkid == DMU_SPILL_BLKID) { 343 ASSERT(dn != NULL); 344 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 345 ASSERT0(db->db.db_offset); 346 } else { 347 ASSERT3U(db->db.db_offset, ==, db->db_blkid * db->db.db_size); 348 } 349 350 for (dr = db->db_data_pending; dr != NULL; dr = dr->dr_next) 351 ASSERT(dr->dr_dbuf == db); 352 353 for (dr = db->db_last_dirty; dr != NULL; dr = dr->dr_next) 354 ASSERT(dr->dr_dbuf == db); 355 356 /* 357 * We can't assert that db_size matches dn_datablksz because it 358 * can be momentarily different when another thread is doing 359 * dnode_set_blksz(). 360 */ 361 if (db->db_level == 0 && db->db.db_object == DMU_META_DNODE_OBJECT) { 362 dr = db->db_data_pending; 363 /* 364 * It should only be modified in syncing context, so 365 * make sure we only have one copy of the data. 366 */ 367 ASSERT(dr == NULL || dr->dt.dl.dr_data == db->db_buf); 368 } 369 370 /* verify db->db_blkptr */ 371 if (db->db_blkptr) { 372 if (db->db_parent == dn->dn_dbuf) { 373 /* db is pointed to by the dnode */ 374 /* ASSERT3U(db->db_blkid, <, dn->dn_nblkptr); */ 375 if (DMU_OBJECT_IS_SPECIAL(db->db.db_object)) 376 ASSERT(db->db_parent == NULL); 377 else 378 ASSERT(db->db_parent != NULL); 379 if (db->db_blkid != DMU_SPILL_BLKID) 380 ASSERT3P(db->db_blkptr, ==, 381 &dn->dn_phys->dn_blkptr[db->db_blkid]); 382 } else { 383 /* db is pointed to by an indirect block */ 384 int epb = db->db_parent->db.db_size >> SPA_BLKPTRSHIFT; 385 ASSERT3U(db->db_parent->db_level, ==, db->db_level+1); 386 ASSERT3U(db->db_parent->db.db_object, ==, 387 db->db.db_object); 388 /* 389 * dnode_grow_indblksz() can make this fail if we don't 390 * have the struct_rwlock. XXX indblksz no longer 391 * grows. safe to do this now? 392 */ 393 if (RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 394 ASSERT3P(db->db_blkptr, ==, 395 ((blkptr_t *)db->db_parent->db.db_data + 396 db->db_blkid % epb)); 397 } 398 } 399 } 400 if ((db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr)) && 401 (db->db_buf == NULL || db->db_buf->b_data) && 402 db->db.db_data && db->db_blkid != DMU_BONUS_BLKID && 403 db->db_state != DB_FILL && !dn->dn_free_txg) { 404 /* 405 * If the blkptr isn't set but they have nonzero data, 406 * it had better be dirty, otherwise we'll lose that 407 * data when we evict this buffer. 408 */ 409 if (db->db_dirtycnt == 0) { 410 uint64_t *buf = db->db.db_data; 411 int i; 412 413 for (i = 0; i < db->db.db_size >> 3; i++) { 414 ASSERT(buf[i] == 0); 415 } 416 } 417 } 418 DB_DNODE_EXIT(db); 419 } 420 #endif 421 422 static void 423 dbuf_update_data(dmu_buf_impl_t *db) 424 { 425 ASSERT(MUTEX_HELD(&db->db_mtx)); 426 if (db->db_level == 0 && db->db_user_data_ptr_ptr) { 427 ASSERT(!refcount_is_zero(&db->db_holds)); 428 *db->db_user_data_ptr_ptr = db->db.db_data; 429 } 430 } 431 432 static void 433 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf) 434 { 435 ASSERT(MUTEX_HELD(&db->db_mtx)); 436 db->db_buf = buf; 437 if (buf != NULL) { 438 ASSERT(buf->b_data != NULL); 439 db->db.db_data = buf->b_data; 440 if (!arc_released(buf)) 441 arc_set_callback(buf, dbuf_do_evict, db); 442 dbuf_update_data(db); 443 } else { 444 dbuf_evict_user(db); 445 db->db.db_data = NULL; 446 if (db->db_state != DB_NOFILL) 447 db->db_state = DB_UNCACHED; 448 } 449 } 450 451 /* 452 * Loan out an arc_buf for read. Return the loaned arc_buf. 453 */ 454 arc_buf_t * 455 dbuf_loan_arcbuf(dmu_buf_impl_t *db) 456 { 457 arc_buf_t *abuf; 458 459 mutex_enter(&db->db_mtx); 460 if (arc_released(db->db_buf) || refcount_count(&db->db_holds) > 1) { 461 int blksz = db->db.db_size; 462 spa_t *spa = db->db_objset->os_spa; 463 464 mutex_exit(&db->db_mtx); 465 abuf = arc_loan_buf(spa, blksz); 466 bcopy(db->db.db_data, abuf->b_data, blksz); 467 } else { 468 abuf = db->db_buf; 469 arc_loan_inuse_buf(abuf, db); 470 dbuf_set_data(db, NULL); 471 mutex_exit(&db->db_mtx); 472 } 473 return (abuf); 474 } 475 476 uint64_t 477 dbuf_whichblock(dnode_t *dn, uint64_t offset) 478 { 479 if (dn->dn_datablkshift) { 480 return (offset >> dn->dn_datablkshift); 481 } else { 482 ASSERT3U(offset, <, dn->dn_datablksz); 483 return (0); 484 } 485 } 486 487 static void 488 dbuf_read_done(zio_t *zio, arc_buf_t *buf, void *vdb) 489 { 490 dmu_buf_impl_t *db = vdb; 491 492 mutex_enter(&db->db_mtx); 493 ASSERT3U(db->db_state, ==, DB_READ); 494 /* 495 * All reads are synchronous, so we must have a hold on the dbuf 496 */ 497 ASSERT(refcount_count(&db->db_holds) > 0); 498 ASSERT(db->db_buf == NULL); 499 ASSERT(db->db.db_data == NULL); 500 if (db->db_level == 0 && db->db_freed_in_flight) { 501 /* we were freed in flight; disregard any error */ 502 arc_release(buf, db); 503 bzero(buf->b_data, db->db.db_size); 504 arc_buf_freeze(buf); 505 db->db_freed_in_flight = FALSE; 506 dbuf_set_data(db, buf); 507 db->db_state = DB_CACHED; 508 } else if (zio == NULL || zio->io_error == 0) { 509 dbuf_set_data(db, buf); 510 db->db_state = DB_CACHED; 511 } else { 512 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 513 ASSERT3P(db->db_buf, ==, NULL); 514 VERIFY(arc_buf_remove_ref(buf, db)); 515 db->db_state = DB_UNCACHED; 516 } 517 cv_broadcast(&db->db_changed); 518 dbuf_rele_and_unlock(db, NULL); 519 } 520 521 static void 522 dbuf_read_impl(dmu_buf_impl_t *db, zio_t *zio, uint32_t *flags) 523 { 524 dnode_t *dn; 525 zbookmark_phys_t zb; 526 uint32_t aflags = ARC_NOWAIT; 527 528 DB_DNODE_ENTER(db); 529 dn = DB_DNODE(db); 530 ASSERT(!refcount_is_zero(&db->db_holds)); 531 /* We need the struct_rwlock to prevent db_blkptr from changing. */ 532 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 533 ASSERT(MUTEX_HELD(&db->db_mtx)); 534 ASSERT(db->db_state == DB_UNCACHED); 535 ASSERT(db->db_buf == NULL); 536 537 if (db->db_blkid == DMU_BONUS_BLKID) { 538 int bonuslen = MIN(dn->dn_bonuslen, dn->dn_phys->dn_bonuslen); 539 540 ASSERT3U(bonuslen, <=, db->db.db_size); 541 db->db.db_data = zio_buf_alloc(DN_MAX_BONUSLEN); 542 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 543 if (bonuslen < DN_MAX_BONUSLEN) 544 bzero(db->db.db_data, DN_MAX_BONUSLEN); 545 if (bonuslen) 546 bcopy(DN_BONUS(dn->dn_phys), db->db.db_data, bonuslen); 547 DB_DNODE_EXIT(db); 548 dbuf_update_data(db); 549 db->db_state = DB_CACHED; 550 mutex_exit(&db->db_mtx); 551 return; 552 } 553 554 /* 555 * Recheck BP_IS_HOLE() after dnode_block_freed() in case dnode_sync() 556 * processes the delete record and clears the bp while we are waiting 557 * for the dn_mtx (resulting in a "no" from block_freed). 558 */ 559 if (db->db_blkptr == NULL || BP_IS_HOLE(db->db_blkptr) || 560 (db->db_level == 0 && (dnode_block_freed(dn, db->db_blkid) || 561 BP_IS_HOLE(db->db_blkptr)))) { 562 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 563 564 DB_DNODE_EXIT(db); 565 dbuf_set_data(db, arc_buf_alloc(db->db_objset->os_spa, 566 db->db.db_size, db, type)); 567 bzero(db->db.db_data, db->db.db_size); 568 db->db_state = DB_CACHED; 569 *flags |= DB_RF_CACHED; 570 mutex_exit(&db->db_mtx); 571 return; 572 } 573 574 DB_DNODE_EXIT(db); 575 576 db->db_state = DB_READ; 577 mutex_exit(&db->db_mtx); 578 579 if (DBUF_IS_L2CACHEABLE(db)) 580 aflags |= ARC_L2CACHE; 581 if (DBUF_IS_L2COMPRESSIBLE(db)) 582 aflags |= ARC_L2COMPRESS; 583 584 SET_BOOKMARK(&zb, db->db_objset->os_dsl_dataset ? 585 db->db_objset->os_dsl_dataset->ds_object : DMU_META_OBJSET, 586 db->db.db_object, db->db_level, db->db_blkid); 587 588 dbuf_add_ref(db, NULL); 589 590 (void) arc_read(zio, db->db_objset->os_spa, db->db_blkptr, 591 dbuf_read_done, db, ZIO_PRIORITY_SYNC_READ, 592 (*flags & DB_RF_CANFAIL) ? ZIO_FLAG_CANFAIL : ZIO_FLAG_MUSTSUCCEED, 593 &aflags, &zb); 594 if (aflags & ARC_CACHED) 595 *flags |= DB_RF_CACHED; 596 } 597 598 int 599 dbuf_read(dmu_buf_impl_t *db, zio_t *zio, uint32_t flags) 600 { 601 int err = 0; 602 boolean_t havepzio = (zio != NULL); 603 boolean_t prefetch; 604 dnode_t *dn; 605 606 /* 607 * We don't have to hold the mutex to check db_state because it 608 * can't be freed while we have a hold on the buffer. 609 */ 610 ASSERT(!refcount_is_zero(&db->db_holds)); 611 612 if (db->db_state == DB_NOFILL) 613 return (SET_ERROR(EIO)); 614 615 DB_DNODE_ENTER(db); 616 dn = DB_DNODE(db); 617 if ((flags & DB_RF_HAVESTRUCT) == 0) 618 rw_enter(&dn->dn_struct_rwlock, RW_READER); 619 620 prefetch = db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 621 (flags & DB_RF_NOPREFETCH) == 0 && dn != NULL && 622 DBUF_IS_CACHEABLE(db); 623 624 mutex_enter(&db->db_mtx); 625 if (db->db_state == DB_CACHED) { 626 mutex_exit(&db->db_mtx); 627 if (prefetch) 628 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 629 db->db.db_size, TRUE); 630 if ((flags & DB_RF_HAVESTRUCT) == 0) 631 rw_exit(&dn->dn_struct_rwlock); 632 DB_DNODE_EXIT(db); 633 } else if (db->db_state == DB_UNCACHED) { 634 spa_t *spa = dn->dn_objset->os_spa; 635 636 if (zio == NULL) 637 zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 638 dbuf_read_impl(db, zio, &flags); 639 640 /* dbuf_read_impl has dropped db_mtx for us */ 641 642 if (prefetch) 643 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 644 db->db.db_size, flags & DB_RF_CACHED); 645 646 if ((flags & DB_RF_HAVESTRUCT) == 0) 647 rw_exit(&dn->dn_struct_rwlock); 648 DB_DNODE_EXIT(db); 649 650 if (!havepzio) 651 err = zio_wait(zio); 652 } else { 653 /* 654 * Another reader came in while the dbuf was in flight 655 * between UNCACHED and CACHED. Either a writer will finish 656 * writing the buffer (sending the dbuf to CACHED) or the 657 * first reader's request will reach the read_done callback 658 * and send the dbuf to CACHED. Otherwise, a failure 659 * occurred and the dbuf went to UNCACHED. 660 */ 661 mutex_exit(&db->db_mtx); 662 if (prefetch) 663 dmu_zfetch(&dn->dn_zfetch, db->db.db_offset, 664 db->db.db_size, TRUE); 665 if ((flags & DB_RF_HAVESTRUCT) == 0) 666 rw_exit(&dn->dn_struct_rwlock); 667 DB_DNODE_EXIT(db); 668 669 /* Skip the wait per the caller's request. */ 670 mutex_enter(&db->db_mtx); 671 if ((flags & DB_RF_NEVERWAIT) == 0) { 672 while (db->db_state == DB_READ || 673 db->db_state == DB_FILL) { 674 ASSERT(db->db_state == DB_READ || 675 (flags & DB_RF_HAVESTRUCT) == 0); 676 cv_wait(&db->db_changed, &db->db_mtx); 677 } 678 if (db->db_state == DB_UNCACHED) 679 err = SET_ERROR(EIO); 680 } 681 mutex_exit(&db->db_mtx); 682 } 683 684 ASSERT(err || havepzio || db->db_state == DB_CACHED); 685 return (err); 686 } 687 688 static void 689 dbuf_noread(dmu_buf_impl_t *db) 690 { 691 ASSERT(!refcount_is_zero(&db->db_holds)); 692 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 693 mutex_enter(&db->db_mtx); 694 while (db->db_state == DB_READ || db->db_state == DB_FILL) 695 cv_wait(&db->db_changed, &db->db_mtx); 696 if (db->db_state == DB_UNCACHED) { 697 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 698 spa_t *spa = db->db_objset->os_spa; 699 700 ASSERT(db->db_buf == NULL); 701 ASSERT(db->db.db_data == NULL); 702 dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type)); 703 db->db_state = DB_FILL; 704 } else if (db->db_state == DB_NOFILL) { 705 dbuf_set_data(db, NULL); 706 } else { 707 ASSERT3U(db->db_state, ==, DB_CACHED); 708 } 709 mutex_exit(&db->db_mtx); 710 } 711 712 /* 713 * This is our just-in-time copy function. It makes a copy of 714 * buffers, that have been modified in a previous transaction 715 * group, before we modify them in the current active group. 716 * 717 * This function is used in two places: when we are dirtying a 718 * buffer for the first time in a txg, and when we are freeing 719 * a range in a dnode that includes this buffer. 720 * 721 * Note that when we are called from dbuf_free_range() we do 722 * not put a hold on the buffer, we just traverse the active 723 * dbuf list for the dnode. 724 */ 725 static void 726 dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg) 727 { 728 dbuf_dirty_record_t *dr = db->db_last_dirty; 729 730 ASSERT(MUTEX_HELD(&db->db_mtx)); 731 ASSERT(db->db.db_data != NULL); 732 ASSERT(db->db_level == 0); 733 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT); 734 735 if (dr == NULL || 736 (dr->dt.dl.dr_data != 737 ((db->db_blkid == DMU_BONUS_BLKID) ? db->db.db_data : db->db_buf))) 738 return; 739 740 /* 741 * If the last dirty record for this dbuf has not yet synced 742 * and its referencing the dbuf data, either: 743 * reset the reference to point to a new copy, 744 * or (if there a no active holders) 745 * just null out the current db_data pointer. 746 */ 747 ASSERT(dr->dr_txg >= txg - 2); 748 if (db->db_blkid == DMU_BONUS_BLKID) { 749 /* Note that the data bufs here are zio_bufs */ 750 dr->dt.dl.dr_data = zio_buf_alloc(DN_MAX_BONUSLEN); 751 arc_space_consume(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 752 bcopy(db->db.db_data, dr->dt.dl.dr_data, DN_MAX_BONUSLEN); 753 } else if (refcount_count(&db->db_holds) > db->db_dirtycnt) { 754 int size = db->db.db_size; 755 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 756 spa_t *spa = db->db_objset->os_spa; 757 758 dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type); 759 bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size); 760 } else { 761 dbuf_set_data(db, NULL); 762 } 763 } 764 765 void 766 dbuf_unoverride(dbuf_dirty_record_t *dr) 767 { 768 dmu_buf_impl_t *db = dr->dr_dbuf; 769 blkptr_t *bp = &dr->dt.dl.dr_overridden_by; 770 uint64_t txg = dr->dr_txg; 771 772 ASSERT(MUTEX_HELD(&db->db_mtx)); 773 ASSERT(dr->dt.dl.dr_override_state != DR_IN_DMU_SYNC); 774 ASSERT(db->db_level == 0); 775 776 if (db->db_blkid == DMU_BONUS_BLKID || 777 dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN) 778 return; 779 780 ASSERT(db->db_data_pending != dr); 781 782 /* free this block */ 783 if (!BP_IS_HOLE(bp) && !dr->dt.dl.dr_nopwrite) 784 zio_free(db->db_objset->os_spa, txg, bp); 785 786 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 787 dr->dt.dl.dr_nopwrite = B_FALSE; 788 789 /* 790 * Release the already-written buffer, so we leave it in 791 * a consistent dirty state. Note that all callers are 792 * modifying the buffer, so they will immediately do 793 * another (redundant) arc_release(). Therefore, leave 794 * the buf thawed to save the effort of freezing & 795 * immediately re-thawing it. 796 */ 797 arc_release(dr->dt.dl.dr_data, db); 798 } 799 800 /* 801 * Evict (if its unreferenced) or clear (if its referenced) any level-0 802 * data blocks in the free range, so that any future readers will find 803 * empty blocks. 804 * 805 * This is a no-op if the dataset is in the middle of an incremental 806 * receive; see comment below for details. 807 */ 808 void 809 dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid, 810 dmu_tx_t *tx) 811 { 812 dmu_buf_impl_t *db, *db_next, db_search; 813 uint64_t txg = tx->tx_txg; 814 avl_index_t where; 815 816 if (end_blkid > dn->dn_maxblkid && (end_blkid != DMU_SPILL_BLKID)) 817 end_blkid = dn->dn_maxblkid; 818 dprintf_dnode(dn, "start=%llu end=%llu\n", start_blkid, end_blkid); 819 820 db_search.db_level = 0; 821 db_search.db_blkid = start_blkid; 822 db_search.db_state = DB_SEARCH; 823 824 mutex_enter(&dn->dn_dbufs_mtx); 825 if (start_blkid >= dn->dn_unlisted_l0_blkid) { 826 /* There can't be any dbufs in this range; no need to search. */ 827 #ifdef DEBUG 828 db = avl_find(&dn->dn_dbufs, &db_search, &where); 829 ASSERT3P(db, ==, NULL); 830 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); 831 ASSERT(db == NULL || db->db_level > 0); 832 #endif 833 mutex_exit(&dn->dn_dbufs_mtx); 834 return; 835 } else if (dmu_objset_is_receiving(dn->dn_objset)) { 836 /* 837 * If we are receiving, we expect there to be no dbufs in 838 * the range to be freed, because receive modifies each 839 * block at most once, and in offset order. If this is 840 * not the case, it can lead to performance problems, 841 * so note that we unexpectedly took the slow path. 842 */ 843 atomic_inc_64(&zfs_free_range_recv_miss); 844 } 845 846 db = avl_find(&dn->dn_dbufs, &db_search, &where); 847 ASSERT3P(db, ==, NULL); 848 db = avl_nearest(&dn->dn_dbufs, where, AVL_AFTER); 849 850 for (; db != NULL; db = db_next) { 851 db_next = AVL_NEXT(&dn->dn_dbufs, db); 852 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 853 854 if (db->db_level != 0 || db->db_blkid > end_blkid) { 855 break; 856 } 857 ASSERT3U(db->db_blkid, >=, start_blkid); 858 859 /* found a level 0 buffer in the range */ 860 mutex_enter(&db->db_mtx); 861 if (dbuf_undirty(db, tx)) { 862 /* mutex has been dropped and dbuf destroyed */ 863 continue; 864 } 865 866 if (db->db_state == DB_UNCACHED || 867 db->db_state == DB_NOFILL || 868 db->db_state == DB_EVICTING) { 869 ASSERT(db->db.db_data == NULL); 870 mutex_exit(&db->db_mtx); 871 continue; 872 } 873 if (db->db_state == DB_READ || db->db_state == DB_FILL) { 874 /* will be handled in dbuf_read_done or dbuf_rele */ 875 db->db_freed_in_flight = TRUE; 876 mutex_exit(&db->db_mtx); 877 continue; 878 } 879 if (refcount_count(&db->db_holds) == 0) { 880 ASSERT(db->db_buf); 881 dbuf_clear(db); 882 continue; 883 } 884 /* The dbuf is referenced */ 885 886 if (db->db_last_dirty != NULL) { 887 dbuf_dirty_record_t *dr = db->db_last_dirty; 888 889 if (dr->dr_txg == txg) { 890 /* 891 * This buffer is "in-use", re-adjust the file 892 * size to reflect that this buffer may 893 * contain new data when we sync. 894 */ 895 if (db->db_blkid != DMU_SPILL_BLKID && 896 db->db_blkid > dn->dn_maxblkid) 897 dn->dn_maxblkid = db->db_blkid; 898 dbuf_unoverride(dr); 899 } else { 900 /* 901 * This dbuf is not dirty in the open context. 902 * Either uncache it (if its not referenced in 903 * the open context) or reset its contents to 904 * empty. 905 */ 906 dbuf_fix_old_data(db, txg); 907 } 908 } 909 /* clear the contents if its cached */ 910 if (db->db_state == DB_CACHED) { 911 ASSERT(db->db.db_data != NULL); 912 arc_release(db->db_buf, db); 913 bzero(db->db.db_data, db->db.db_size); 914 arc_buf_freeze(db->db_buf); 915 } 916 917 mutex_exit(&db->db_mtx); 918 } 919 mutex_exit(&dn->dn_dbufs_mtx); 920 } 921 922 static int 923 dbuf_block_freeable(dmu_buf_impl_t *db) 924 { 925 dsl_dataset_t *ds = db->db_objset->os_dsl_dataset; 926 uint64_t birth_txg = 0; 927 928 /* 929 * We don't need any locking to protect db_blkptr: 930 * If it's syncing, then db_last_dirty will be set 931 * so we'll ignore db_blkptr. 932 * 933 * This logic ensures that only block births for 934 * filled blocks are considered. 935 */ 936 ASSERT(MUTEX_HELD(&db->db_mtx)); 937 if (db->db_last_dirty && (db->db_blkptr == NULL || 938 !BP_IS_HOLE(db->db_blkptr))) { 939 birth_txg = db->db_last_dirty->dr_txg; 940 } else if (db->db_blkptr != NULL && !BP_IS_HOLE(db->db_blkptr)) { 941 birth_txg = db->db_blkptr->blk_birth; 942 } 943 944 /* 945 * If this block don't exist or is in a snapshot, it can't be freed. 946 * Don't pass the bp to dsl_dataset_block_freeable() since we 947 * are holding the db_mtx lock and might deadlock if we are 948 * prefetching a dedup-ed block. 949 */ 950 if (birth_txg != 0) 951 return (ds == NULL || 952 dsl_dataset_block_freeable(ds, NULL, birth_txg)); 953 else 954 return (B_FALSE); 955 } 956 957 void 958 dbuf_new_size(dmu_buf_impl_t *db, int size, dmu_tx_t *tx) 959 { 960 arc_buf_t *buf, *obuf; 961 int osize = db->db.db_size; 962 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 963 dnode_t *dn; 964 965 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 966 967 DB_DNODE_ENTER(db); 968 dn = DB_DNODE(db); 969 970 /* XXX does *this* func really need the lock? */ 971 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 972 973 /* 974 * This call to dmu_buf_will_dirty() with the dn_struct_rwlock held 975 * is OK, because there can be no other references to the db 976 * when we are changing its size, so no concurrent DB_FILL can 977 * be happening. 978 */ 979 /* 980 * XXX we should be doing a dbuf_read, checking the return 981 * value and returning that up to our callers 982 */ 983 dmu_buf_will_dirty(&db->db, tx); 984 985 /* create the data buffer for the new block */ 986 buf = arc_buf_alloc(dn->dn_objset->os_spa, size, db, type); 987 988 /* copy old block data to the new block */ 989 obuf = db->db_buf; 990 bcopy(obuf->b_data, buf->b_data, MIN(osize, size)); 991 /* zero the remainder */ 992 if (size > osize) 993 bzero((uint8_t *)buf->b_data + osize, size - osize); 994 995 mutex_enter(&db->db_mtx); 996 dbuf_set_data(db, buf); 997 VERIFY(arc_buf_remove_ref(obuf, db)); 998 db->db.db_size = size; 999 1000 if (db->db_level == 0) { 1001 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 1002 db->db_last_dirty->dt.dl.dr_data = buf; 1003 } 1004 mutex_exit(&db->db_mtx); 1005 1006 dnode_willuse_space(dn, size-osize, tx); 1007 DB_DNODE_EXIT(db); 1008 } 1009 1010 void 1011 dbuf_release_bp(dmu_buf_impl_t *db) 1012 { 1013 objset_t *os = db->db_objset; 1014 1015 ASSERT(dsl_pool_sync_context(dmu_objset_pool(os))); 1016 ASSERT(arc_released(os->os_phys_buf) || 1017 list_link_active(&os->os_dsl_dataset->ds_synced_link)); 1018 ASSERT(db->db_parent == NULL || arc_released(db->db_parent->db_buf)); 1019 1020 (void) arc_release(db->db_buf, db); 1021 } 1022 1023 dbuf_dirty_record_t * 1024 dbuf_dirty_sc(dmu_buf_impl_t *db, dmu_tx_t *tx, boolean_t usesc) 1025 { 1026 dnode_t *dn; 1027 objset_t *os; 1028 dbuf_dirty_record_t **drp, *dr; 1029 int drop_struct_lock = FALSE; 1030 boolean_t do_free_accounting = B_FALSE; 1031 int txgoff = tx->tx_txg & TXG_MASK; 1032 1033 ASSERT(tx->tx_txg != 0); 1034 ASSERT(!refcount_is_zero(&db->db_holds)); 1035 DMU_TX_DIRTY_BUF(tx, db); 1036 1037 DB_DNODE_ENTER(db); 1038 dn = DB_DNODE(db); 1039 /* 1040 * Shouldn't dirty a regular buffer in syncing context. Private 1041 * objects may be dirtied in syncing context, but only if they 1042 * were already pre-dirtied in open context. 1043 */ 1044 ASSERT(!dmu_tx_is_syncing(tx) || 1045 BP_IS_HOLE(dn->dn_objset->os_rootbp) || 1046 DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1047 dn->dn_objset->os_dsl_dataset == NULL); 1048 /* 1049 * We make this assert for private objects as well, but after we 1050 * check if we're already dirty. They are allowed to re-dirty 1051 * in syncing context. 1052 */ 1053 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1054 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1055 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1056 1057 mutex_enter(&db->db_mtx); 1058 /* 1059 * XXX make this true for indirects too? The problem is that 1060 * transactions created with dmu_tx_create_assigned() from 1061 * syncing context don't bother holding ahead. 1062 */ 1063 ASSERT(db->db_level != 0 || 1064 db->db_state == DB_CACHED || db->db_state == DB_FILL || 1065 db->db_state == DB_NOFILL); 1066 1067 mutex_enter(&dn->dn_mtx); 1068 /* 1069 * Don't set dirtyctx to SYNC if we're just modifying this as we 1070 * initialize the objset. 1071 */ 1072 if (dn->dn_dirtyctx == DN_UNDIRTIED && 1073 !BP_IS_HOLE(dn->dn_objset->os_rootbp)) { 1074 dn->dn_dirtyctx = 1075 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN); 1076 ASSERT(dn->dn_dirtyctx_firstset == NULL); 1077 dn->dn_dirtyctx_firstset = kmem_alloc(1, KM_SLEEP); 1078 } 1079 mutex_exit(&dn->dn_mtx); 1080 1081 if (db->db_blkid == DMU_SPILL_BLKID) 1082 dn->dn_have_spill = B_TRUE; 1083 1084 /* 1085 * If this buffer is already dirty, we're done. 1086 */ 1087 drp = &db->db_last_dirty; 1088 ASSERT(*drp == NULL || (*drp)->dr_txg <= tx->tx_txg || 1089 db->db.db_object == DMU_META_DNODE_OBJECT); 1090 while ((dr = *drp) != NULL && dr->dr_txg > tx->tx_txg) 1091 drp = &dr->dr_next; 1092 if (dr && dr->dr_txg == tx->tx_txg) { 1093 DB_DNODE_EXIT(db); 1094 1095 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID) { 1096 /* 1097 * If this buffer has already been written out, 1098 * we now need to reset its state. 1099 */ 1100 dbuf_unoverride(dr); 1101 if (db->db.db_object != DMU_META_DNODE_OBJECT && 1102 db->db_state != DB_NOFILL) 1103 arc_buf_thaw(db->db_buf); 1104 } 1105 1106 /* 1107 * Special class usage of dirty dbuf could be changed, 1108 * update the dirty entry. 1109 */ 1110 dr->dr_usesc = usesc; 1111 mutex_exit(&db->db_mtx); 1112 return (dr); 1113 } 1114 1115 /* 1116 * Only valid if not already dirty. 1117 */ 1118 ASSERT(dn->dn_object == 0 || 1119 dn->dn_dirtyctx == DN_UNDIRTIED || dn->dn_dirtyctx == 1120 (dmu_tx_is_syncing(tx) ? DN_DIRTY_SYNC : DN_DIRTY_OPEN)); 1121 1122 ASSERT3U(dn->dn_nlevels, >, db->db_level); 1123 ASSERT((dn->dn_phys->dn_nlevels == 0 && db->db_level == 0) || 1124 dn->dn_phys->dn_nlevels > db->db_level || 1125 dn->dn_next_nlevels[txgoff] > db->db_level || 1126 dn->dn_next_nlevels[(tx->tx_txg-1) & TXG_MASK] > db->db_level || 1127 dn->dn_next_nlevels[(tx->tx_txg-2) & TXG_MASK] > db->db_level); 1128 1129 /* 1130 * We should only be dirtying in syncing context if it's the 1131 * mos or we're initializing the os or it's a special object. 1132 * However, we are allowed to dirty in syncing context provided 1133 * we already dirtied it in open context. Hence we must make 1134 * this assertion only if we're not already dirty. 1135 */ 1136 os = dn->dn_objset; 1137 ASSERT(!dmu_tx_is_syncing(tx) || DMU_OBJECT_IS_SPECIAL(dn->dn_object) || 1138 os->os_dsl_dataset == NULL || BP_IS_HOLE(os->os_rootbp)); 1139 ASSERT(db->db.db_size != 0); 1140 1141 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1142 1143 if (db->db_blkid != DMU_BONUS_BLKID) { 1144 /* 1145 * Update the accounting. 1146 * Note: we delay "free accounting" until after we drop 1147 * the db_mtx. This keeps us from grabbing other locks 1148 * (and possibly deadlocking) in bp_get_dsize() while 1149 * also holding the db_mtx. 1150 */ 1151 dnode_willuse_space(dn, db->db.db_size, tx); 1152 do_free_accounting = dbuf_block_freeable(db); 1153 } 1154 1155 /* 1156 * If this buffer is dirty in an old transaction group we need 1157 * to make a copy of it so that the changes we make in this 1158 * transaction group won't leak out when we sync the older txg. 1159 */ 1160 dr = kmem_zalloc(sizeof (dbuf_dirty_record_t), KM_SLEEP); 1161 if (db->db_level == 0) { 1162 void *data_old = db->db_buf; 1163 1164 if (db->db_state != DB_NOFILL) { 1165 if (db->db_blkid == DMU_BONUS_BLKID) { 1166 dbuf_fix_old_data(db, tx->tx_txg); 1167 data_old = db->db.db_data; 1168 } else if (db->db.db_object != DMU_META_DNODE_OBJECT) { 1169 /* 1170 * Release the data buffer from the cache so 1171 * that we can modify it without impacting 1172 * possible other users of this cached data 1173 * block. Note that indirect blocks and 1174 * private objects are not released until the 1175 * syncing state (since they are only modified 1176 * then). 1177 */ 1178 arc_release(db->db_buf, db); 1179 dbuf_fix_old_data(db, tx->tx_txg); 1180 data_old = db->db_buf; 1181 } 1182 ASSERT(data_old != NULL); 1183 } 1184 dr->dt.dl.dr_data = data_old; 1185 } else { 1186 mutex_init(&dr->dt.di.dr_mtx, NULL, MUTEX_DEFAULT, NULL); 1187 list_create(&dr->dt.di.dr_children, 1188 sizeof (dbuf_dirty_record_t), 1189 offsetof(dbuf_dirty_record_t, dr_dirty_node)); 1190 } 1191 if (db->db_blkid != DMU_BONUS_BLKID && os->os_dsl_dataset != NULL) 1192 dr->dr_accounted = db->db.db_size; 1193 dr->dr_dbuf = db; 1194 dr->dr_txg = tx->tx_txg; 1195 dr->dr_next = *drp; 1196 dr->dr_usesc = usesc; 1197 *drp = dr; 1198 1199 /* 1200 * We could have been freed_in_flight between the dbuf_noread 1201 * and dbuf_dirty. We win, as though the dbuf_noread() had 1202 * happened after the free. 1203 */ 1204 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1205 db->db_blkid != DMU_SPILL_BLKID) { 1206 mutex_enter(&dn->dn_mtx); 1207 if (dn->dn_free_ranges[txgoff] != NULL) { 1208 range_tree_clear(dn->dn_free_ranges[txgoff], 1209 db->db_blkid, 1); 1210 } 1211 mutex_exit(&dn->dn_mtx); 1212 db->db_freed_in_flight = FALSE; 1213 } 1214 1215 /* 1216 * This buffer is now part of this txg 1217 */ 1218 dbuf_add_ref(db, (void *)(uintptr_t)tx->tx_txg); 1219 db->db_dirtycnt += 1; 1220 ASSERT3U(db->db_dirtycnt, <=, 3); 1221 1222 mutex_exit(&db->db_mtx); 1223 1224 if (db->db_blkid == DMU_BONUS_BLKID || 1225 db->db_blkid == DMU_SPILL_BLKID) { 1226 mutex_enter(&dn->dn_mtx); 1227 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1228 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1229 mutex_exit(&dn->dn_mtx); 1230 dnode_setdirty_sc(dn, tx, usesc); 1231 DB_DNODE_EXIT(db); 1232 return (dr); 1233 } else if (do_free_accounting) { 1234 blkptr_t *bp = db->db_blkptr; 1235 int64_t willfree = (bp && !BP_IS_HOLE(bp)) ? 1236 bp_get_dsize(os->os_spa, bp) : db->db.db_size; 1237 /* 1238 * This is only a guess -- if the dbuf is dirty 1239 * in a previous txg, we don't know how much 1240 * space it will use on disk yet. We should 1241 * really have the struct_rwlock to access 1242 * db_blkptr, but since this is just a guess, 1243 * it's OK if we get an odd answer. 1244 */ 1245 ddt_prefetch(os->os_spa, bp); 1246 dnode_willuse_space(dn, -willfree, tx); 1247 } 1248 1249 if (!RW_WRITE_HELD(&dn->dn_struct_rwlock)) { 1250 rw_enter(&dn->dn_struct_rwlock, RW_READER); 1251 drop_struct_lock = TRUE; 1252 } 1253 1254 if (db->db_level == 0) { 1255 dnode_new_blkid(dn, db->db_blkid, tx, usesc, drop_struct_lock); 1256 ASSERT(dn->dn_maxblkid >= db->db_blkid); 1257 } 1258 1259 if (db->db_level+1 < dn->dn_nlevels) { 1260 dmu_buf_impl_t *parent = db->db_parent; 1261 dbuf_dirty_record_t *di; 1262 int parent_held = FALSE; 1263 1264 if (db->db_parent == NULL || db->db_parent == dn->dn_dbuf) { 1265 int epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1266 1267 parent = dbuf_hold_level(dn, db->db_level+1, 1268 db->db_blkid >> epbs, FTAG); 1269 ASSERT(parent != NULL); 1270 parent_held = TRUE; 1271 } 1272 if (drop_struct_lock) 1273 rw_exit(&dn->dn_struct_rwlock); 1274 ASSERT3U(db->db_level+1, ==, parent->db_level); 1275 di = dbuf_dirty_sc(parent, tx, usesc); 1276 if (parent_held) 1277 dbuf_rele(parent, FTAG); 1278 1279 mutex_enter(&db->db_mtx); 1280 /* 1281 * Since we've dropped the mutex, it's possible that 1282 * dbuf_undirty() might have changed this out from under us. 1283 */ 1284 if (db->db_last_dirty == dr || 1285 dn->dn_object == DMU_META_DNODE_OBJECT) { 1286 mutex_enter(&di->dt.di.dr_mtx); 1287 ASSERT3U(di->dr_txg, ==, tx->tx_txg); 1288 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1289 list_insert_tail(&di->dt.di.dr_children, dr); 1290 mutex_exit(&di->dt.di.dr_mtx); 1291 dr->dr_parent = di; 1292 } 1293 1294 /* 1295 * Special class usage of dirty dbuf could be changed, 1296 * update the dirty entry. 1297 */ 1298 dr->dr_usesc = usesc; 1299 mutex_exit(&db->db_mtx); 1300 } else { 1301 ASSERT(db->db_level+1 == dn->dn_nlevels); 1302 ASSERT(db->db_blkid < dn->dn_nblkptr); 1303 ASSERT(db->db_parent == NULL || db->db_parent == dn->dn_dbuf); 1304 mutex_enter(&dn->dn_mtx); 1305 ASSERT(!list_link_active(&dr->dr_dirty_node)); 1306 list_insert_tail(&dn->dn_dirty_records[txgoff], dr); 1307 mutex_exit(&dn->dn_mtx); 1308 if (drop_struct_lock) 1309 rw_exit(&dn->dn_struct_rwlock); 1310 } 1311 1312 dnode_setdirty_sc(dn, tx, usesc); 1313 DB_DNODE_EXIT(db); 1314 return (dr); 1315 } 1316 1317 dbuf_dirty_record_t * 1318 dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1319 { 1320 spa_t *spa; 1321 1322 ASSERT(db->db_objset != NULL); 1323 spa = db->db_objset->os_spa; 1324 1325 return (dbuf_dirty_sc(db, tx, spa->spa_usesc)); 1326 } 1327 1328 /* 1329 * Undirty a buffer in the transaction group referenced by the given 1330 * transaction. Return whether this evicted the dbuf. 1331 */ 1332 static boolean_t 1333 dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx) 1334 { 1335 dnode_t *dn; 1336 uint64_t txg = tx->tx_txg; 1337 dbuf_dirty_record_t *dr, **drp; 1338 1339 ASSERT(txg != 0); 1340 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1341 ASSERT0(db->db_level); 1342 ASSERT(MUTEX_HELD(&db->db_mtx)); 1343 1344 /* 1345 * If this buffer is not dirty, we're done. 1346 */ 1347 for (drp = &db->db_last_dirty; (dr = *drp) != NULL; drp = &dr->dr_next) 1348 if (dr->dr_txg <= txg) 1349 break; 1350 if (dr == NULL || dr->dr_txg < txg) 1351 return (B_FALSE); 1352 ASSERT(dr->dr_txg == txg); 1353 ASSERT(dr->dr_dbuf == db); 1354 1355 DB_DNODE_ENTER(db); 1356 dn = DB_DNODE(db); 1357 1358 dprintf_dbuf(db, "size=%llx\n", (u_longlong_t)db->db.db_size); 1359 1360 ASSERT(db->db.db_size != 0); 1361 1362 /* 1363 * Any space we accounted for in dp_dirty_* will be cleaned up by 1364 * dsl_pool_sync(). This is relatively rare so the discrepancy 1365 * is not a big deal. 1366 */ 1367 1368 *drp = dr->dr_next; 1369 1370 /* 1371 * Note that there are three places in dbuf_dirty() 1372 * where this dirty record may be put on a list. 1373 * Make sure to do a list_remove corresponding to 1374 * every one of those list_insert calls. 1375 */ 1376 if (dr->dr_parent) { 1377 mutex_enter(&dr->dr_parent->dt.di.dr_mtx); 1378 list_remove(&dr->dr_parent->dt.di.dr_children, dr); 1379 mutex_exit(&dr->dr_parent->dt.di.dr_mtx); 1380 } else if (db->db_blkid == DMU_SPILL_BLKID || 1381 db->db_level+1 == dn->dn_nlevels) { 1382 ASSERT(db->db_blkptr == NULL || db->db_parent == dn->dn_dbuf); 1383 mutex_enter(&dn->dn_mtx); 1384 list_remove(&dn->dn_dirty_records[txg & TXG_MASK], dr); 1385 mutex_exit(&dn->dn_mtx); 1386 } 1387 DB_DNODE_EXIT(db); 1388 1389 if (db->db_state != DB_NOFILL) { 1390 dbuf_unoverride(dr); 1391 1392 ASSERT(db->db_buf != NULL); 1393 ASSERT(dr->dt.dl.dr_data != NULL); 1394 if (dr->dt.dl.dr_data != db->db_buf) 1395 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, db)); 1396 } 1397 1398 if (db->db_level != 0) { 1399 mutex_destroy(&dr->dt.di.dr_mtx); 1400 list_destroy(&dr->dt.di.dr_children); 1401 } 1402 1403 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 1404 1405 ASSERT(db->db_dirtycnt > 0); 1406 db->db_dirtycnt -= 1; 1407 1408 if (refcount_remove(&db->db_holds, (void *)(uintptr_t)txg) == 0) { 1409 arc_buf_t *buf = db->db_buf; 1410 1411 ASSERT(db->db_state == DB_NOFILL || arc_released(buf)); 1412 dbuf_set_data(db, NULL); 1413 VERIFY(arc_buf_remove_ref(buf, db)); 1414 dbuf_evict(db); 1415 return (B_TRUE); 1416 } 1417 1418 return (B_FALSE); 1419 } 1420 1421 void 1422 dmu_buf_will_dirty(dmu_buf_t *db_fake, dmu_tx_t *tx) 1423 { 1424 dmu_buf_will_dirty_sc(db_fake, tx, B_TRUE); 1425 } 1426 1427 void 1428 dmu_buf_will_dirty_sc(dmu_buf_t *db_fake, dmu_tx_t *tx, boolean_t usesc) 1429 { 1430 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1431 int rf = DB_RF_MUST_SUCCEED | DB_RF_NOPREFETCH; 1432 1433 ASSERT(tx->tx_txg != 0); 1434 ASSERT(!refcount_is_zero(&db->db_holds)); 1435 1436 DB_DNODE_ENTER(db); 1437 if (RW_WRITE_HELD(&DB_DNODE(db)->dn_struct_rwlock)) 1438 rf |= DB_RF_HAVESTRUCT; 1439 DB_DNODE_EXIT(db); 1440 (void) dbuf_read(db, NULL, rf); 1441 (void) dbuf_dirty_sc(db, tx, usesc); 1442 } 1443 1444 1445 void 1446 dmu_buf_will_not_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1447 { 1448 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1449 1450 db->db_state = DB_NOFILL; 1451 1452 dmu_buf_will_fill(db_fake, tx); 1453 } 1454 1455 void 1456 dmu_buf_will_fill(dmu_buf_t *db_fake, dmu_tx_t *tx) 1457 { 1458 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 1459 1460 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1461 ASSERT(tx->tx_txg != 0); 1462 ASSERT(db->db_level == 0); 1463 ASSERT(!refcount_is_zero(&db->db_holds)); 1464 1465 ASSERT(db->db.db_object != DMU_META_DNODE_OBJECT || 1466 dmu_tx_private_ok(tx)); 1467 1468 dbuf_noread(db); 1469 (void) dbuf_dirty(db, tx); 1470 } 1471 1472 #pragma weak dmu_buf_fill_done = dbuf_fill_done 1473 /* ARGSUSED */ 1474 void 1475 dbuf_fill_done(dmu_buf_impl_t *db, dmu_tx_t *tx) 1476 { 1477 mutex_enter(&db->db_mtx); 1478 DBUF_VERIFY(db); 1479 1480 if (db->db_state == DB_FILL) { 1481 if (db->db_level == 0 && db->db_freed_in_flight) { 1482 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1483 /* we were freed while filling */ 1484 /* XXX dbuf_undirty? */ 1485 bzero(db->db.db_data, db->db.db_size); 1486 db->db_freed_in_flight = FALSE; 1487 } 1488 db->db_state = DB_CACHED; 1489 cv_broadcast(&db->db_changed); 1490 } 1491 mutex_exit(&db->db_mtx); 1492 } 1493 1494 void 1495 dmu_buf_write_embedded(dmu_buf_t *dbuf, void *data, 1496 bp_embedded_type_t etype, enum zio_compress comp, 1497 int uncompressed_size, int compressed_size, int byteorder, 1498 dmu_tx_t *tx) 1499 { 1500 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 1501 struct dirty_leaf *dl; 1502 dmu_object_type_t type; 1503 1504 DB_DNODE_ENTER(db); 1505 type = DB_DNODE(db)->dn_type; 1506 DB_DNODE_EXIT(db); 1507 1508 ASSERT0(db->db_level); 1509 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1510 1511 dmu_buf_will_not_fill(dbuf, tx); 1512 1513 ASSERT3U(db->db_last_dirty->dr_txg, ==, tx->tx_txg); 1514 dl = &db->db_last_dirty->dt.dl; 1515 encode_embedded_bp_compressed(&dl->dr_overridden_by, 1516 data, comp, uncompressed_size, compressed_size); 1517 BPE_SET_ETYPE(&dl->dr_overridden_by, etype); 1518 BP_SET_TYPE(&dl->dr_overridden_by, type); 1519 BP_SET_LEVEL(&dl->dr_overridden_by, 0); 1520 BP_SET_BYTEORDER(&dl->dr_overridden_by, byteorder); 1521 1522 dl->dr_override_state = DR_OVERRIDDEN; 1523 dl->dr_overridden_by.blk_birth = db->db_last_dirty->dr_txg; 1524 } 1525 1526 /* 1527 * Directly assign a provided arc buf to a given dbuf if it's not referenced 1528 * by anybody except our caller. Otherwise copy arcbuf's contents to dbuf. 1529 */ 1530 void 1531 dbuf_assign_arcbuf(dmu_buf_impl_t *db, arc_buf_t *buf, dmu_tx_t *tx) 1532 { 1533 ASSERT(!refcount_is_zero(&db->db_holds)); 1534 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 1535 ASSERT(db->db_level == 0); 1536 ASSERT(DBUF_GET_BUFC_TYPE(db) == ARC_BUFC_DATA); 1537 ASSERT(buf != NULL); 1538 ASSERT(arc_buf_size(buf) == db->db.db_size); 1539 ASSERT(tx->tx_txg != 0); 1540 1541 arc_return_buf(buf, db); 1542 ASSERT(arc_released(buf)); 1543 1544 mutex_enter(&db->db_mtx); 1545 1546 while (db->db_state == DB_READ || db->db_state == DB_FILL) 1547 cv_wait(&db->db_changed, &db->db_mtx); 1548 1549 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_UNCACHED); 1550 1551 if (db->db_state == DB_CACHED && 1552 refcount_count(&db->db_holds) - 1 > db->db_dirtycnt) { 1553 mutex_exit(&db->db_mtx); 1554 (void) dbuf_dirty(db, tx); 1555 bcopy(buf->b_data, db->db.db_data, db->db.db_size); 1556 VERIFY(arc_buf_remove_ref(buf, db)); 1557 xuio_stat_wbuf_copied(); 1558 return; 1559 } 1560 1561 xuio_stat_wbuf_nocopy(); 1562 if (db->db_state == DB_CACHED) { 1563 dbuf_dirty_record_t *dr = db->db_last_dirty; 1564 1565 ASSERT(db->db_buf != NULL); 1566 if (dr != NULL && dr->dr_txg == tx->tx_txg) { 1567 ASSERT(dr->dt.dl.dr_data == db->db_buf); 1568 if (!arc_released(db->db_buf)) { 1569 ASSERT(dr->dt.dl.dr_override_state == 1570 DR_OVERRIDDEN); 1571 arc_release(db->db_buf, db); 1572 } 1573 dr->dt.dl.dr_data = buf; 1574 VERIFY(arc_buf_remove_ref(db->db_buf, db)); 1575 } else if (dr == NULL || dr->dt.dl.dr_data != db->db_buf) { 1576 arc_release(db->db_buf, db); 1577 VERIFY(arc_buf_remove_ref(db->db_buf, db)); 1578 } 1579 db->db_buf = NULL; 1580 } 1581 ASSERT(db->db_buf == NULL); 1582 dbuf_set_data(db, buf); 1583 db->db_state = DB_FILL; 1584 mutex_exit(&db->db_mtx); 1585 (void) dbuf_dirty(db, tx); 1586 dmu_buf_fill_done(&db->db, tx); 1587 } 1588 1589 /* 1590 * "Clear" the contents of this dbuf. This will mark the dbuf 1591 * EVICTING and clear *most* of its references. Unfortunately, 1592 * when we are not holding the dn_dbufs_mtx, we can't clear the 1593 * entry in the dn_dbufs list. We have to wait until dbuf_destroy() 1594 * in this case. For callers from the DMU we will usually see: 1595 * dbuf_clear()->arc_clear_callback()->dbuf_do_evict()->dbuf_destroy() 1596 * For the arc callback, we will usually see: 1597 * dbuf_do_evict()->dbuf_clear();dbuf_destroy() 1598 * Sometimes, though, we will get a mix of these two: 1599 * DMU: dbuf_clear()->arc_clear_callback() 1600 * ARC: dbuf_do_evict()->dbuf_destroy() 1601 * 1602 * This routine will dissociate the dbuf from the arc, by calling 1603 * arc_clear_callback(), but will not evict the data from the ARC. 1604 */ 1605 void 1606 dbuf_clear(dmu_buf_impl_t *db) 1607 { 1608 dnode_t *dn; 1609 dmu_buf_impl_t *parent = db->db_parent; 1610 dmu_buf_impl_t *dndb; 1611 boolean_t dbuf_gone = B_FALSE; 1612 1613 ASSERT(MUTEX_HELD(&db->db_mtx)); 1614 ASSERT(refcount_is_zero(&db->db_holds)); 1615 1616 dbuf_evict_user(db); 1617 1618 if (db->db_state == DB_CACHED) { 1619 ASSERT(db->db.db_data != NULL); 1620 if (db->db_blkid == DMU_BONUS_BLKID) { 1621 zio_buf_free(db->db.db_data, DN_MAX_BONUSLEN); 1622 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 1623 } 1624 db->db.db_data = NULL; 1625 db->db_state = DB_UNCACHED; 1626 } 1627 1628 ASSERT(db->db_state == DB_UNCACHED || db->db_state == DB_NOFILL); 1629 ASSERT(db->db_data_pending == NULL); 1630 1631 db->db_state = DB_EVICTING; 1632 db->db_blkptr = NULL; 1633 1634 DB_DNODE_ENTER(db); 1635 dn = DB_DNODE(db); 1636 dndb = dn->dn_dbuf; 1637 if (db->db_blkid != DMU_BONUS_BLKID && MUTEX_HELD(&dn->dn_dbufs_mtx)) { 1638 avl_remove(&dn->dn_dbufs, db); 1639 atomic_dec_32(&dn->dn_dbufs_count); 1640 membar_producer(); 1641 DB_DNODE_EXIT(db); 1642 /* 1643 * Decrementing the dbuf count means that the hold corresponding 1644 * to the removed dbuf is no longer discounted in dnode_move(), 1645 * so the dnode cannot be moved until after we release the hold. 1646 * The membar_producer() ensures visibility of the decremented 1647 * value in dnode_move(), since DB_DNODE_EXIT doesn't actually 1648 * release any lock. 1649 */ 1650 dnode_rele(dn, db); 1651 db->db_dnode_handle = NULL; 1652 } else { 1653 DB_DNODE_EXIT(db); 1654 } 1655 1656 if (db->db_buf) 1657 dbuf_gone = arc_clear_callback(db->db_buf); 1658 1659 if (!dbuf_gone) 1660 mutex_exit(&db->db_mtx); 1661 1662 /* 1663 * If this dbuf is referenced from an indirect dbuf, 1664 * decrement the ref count on the indirect dbuf. 1665 */ 1666 if (parent && parent != dndb) 1667 dbuf_rele(parent, db); 1668 } 1669 1670 static int 1671 dbuf_findbp(dnode_t *dn, int level, uint64_t blkid, int fail_sparse, 1672 dmu_buf_impl_t **parentp, blkptr_t **bpp) 1673 { 1674 int nlevels, epbs; 1675 1676 *parentp = NULL; 1677 *bpp = NULL; 1678 1679 ASSERT(blkid != DMU_BONUS_BLKID); 1680 1681 if (blkid == DMU_SPILL_BLKID) { 1682 mutex_enter(&dn->dn_mtx); 1683 if (dn->dn_have_spill && 1684 (dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR)) 1685 *bpp = &dn->dn_phys->dn_spill; 1686 else 1687 *bpp = NULL; 1688 dbuf_add_ref(dn->dn_dbuf, NULL); 1689 *parentp = dn->dn_dbuf; 1690 mutex_exit(&dn->dn_mtx); 1691 return (0); 1692 } 1693 1694 if (dn->dn_phys->dn_nlevels == 0) 1695 nlevels = 1; 1696 else 1697 nlevels = dn->dn_phys->dn_nlevels; 1698 1699 epbs = dn->dn_indblkshift - SPA_BLKPTRSHIFT; 1700 1701 ASSERT3U(level * epbs, <, 64); 1702 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1703 if (level >= nlevels || 1704 (blkid > (dn->dn_phys->dn_maxblkid >> (level * epbs)))) { 1705 /* the buffer has no parent yet */ 1706 return (SET_ERROR(ENOENT)); 1707 } else if (level < nlevels-1) { 1708 /* this block is referenced from an indirect block */ 1709 int err = dbuf_hold_impl(dn, level+1, 1710 blkid >> epbs, fail_sparse, NULL, parentp); 1711 if (err) 1712 return (err); 1713 err = dbuf_read(*parentp, NULL, 1714 (DB_RF_HAVESTRUCT | DB_RF_NOPREFETCH | DB_RF_CANFAIL)); 1715 if (err) { 1716 dbuf_rele(*parentp, NULL); 1717 *parentp = NULL; 1718 return (err); 1719 } 1720 *bpp = ((blkptr_t *)(*parentp)->db.db_data) + 1721 (blkid & ((1ULL << epbs) - 1)); 1722 return (0); 1723 } else { 1724 /* the block is referenced from the dnode */ 1725 ASSERT3U(level, ==, nlevels-1); 1726 ASSERT(dn->dn_phys->dn_nblkptr == 0 || 1727 blkid < dn->dn_phys->dn_nblkptr); 1728 if (dn->dn_dbuf) { 1729 dbuf_add_ref(dn->dn_dbuf, NULL); 1730 *parentp = dn->dn_dbuf; 1731 } 1732 *bpp = &dn->dn_phys->dn_blkptr[blkid]; 1733 return (0); 1734 } 1735 } 1736 1737 static dmu_buf_impl_t * 1738 dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, 1739 dmu_buf_impl_t *parent, blkptr_t *blkptr) 1740 { 1741 objset_t *os = dn->dn_objset; 1742 dmu_buf_impl_t *db, *odb; 1743 1744 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1745 ASSERT(dn->dn_type != DMU_OT_NONE); 1746 1747 db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); 1748 1749 db->db_objset = os; 1750 db->db.db_object = dn->dn_object; 1751 db->db_level = level; 1752 db->db_blkid = blkid; 1753 db->db_last_dirty = NULL; 1754 db->db_dirtycnt = 0; 1755 db->db_dnode_handle = dn->dn_handle; 1756 db->db_parent = parent; 1757 db->db_blkptr = blkptr; 1758 1759 db->db_user_ptr = NULL; 1760 db->db_user_data_ptr_ptr = NULL; 1761 db->db_evict_func = NULL; 1762 db->db_immediate_evict = 0; 1763 db->db_freed_in_flight = 0; 1764 1765 if (blkid == DMU_BONUS_BLKID) { 1766 ASSERT3P(parent, ==, dn->dn_dbuf); 1767 db->db.db_size = DN_MAX_BONUSLEN - 1768 (dn->dn_nblkptr-1) * sizeof (blkptr_t); 1769 ASSERT3U(db->db.db_size, >=, dn->dn_bonuslen); 1770 db->db.db_offset = DMU_BONUS_BLKID; 1771 db->db_state = DB_UNCACHED; 1772 /* the bonus dbuf is not placed in the hash table */ 1773 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1774 return (db); 1775 } else if (blkid == DMU_SPILL_BLKID) { 1776 db->db.db_size = (blkptr != NULL) ? 1777 BP_GET_LSIZE(blkptr) : SPA_MINBLOCKSIZE; 1778 db->db.db_offset = 0; 1779 } else { 1780 int blocksize = 1781 db->db_level ? 1 << dn->dn_indblkshift : dn->dn_datablksz; 1782 db->db.db_size = blocksize; 1783 db->db.db_offset = db->db_blkid * blocksize; 1784 } 1785 1786 /* 1787 * Hold the dn_dbufs_mtx while we get the new dbuf 1788 * in the hash table *and* added to the dbufs list. 1789 * This prevents a possible deadlock with someone 1790 * trying to look up this dbuf before its added to the 1791 * dn_dbufs list. 1792 */ 1793 mutex_enter(&dn->dn_dbufs_mtx); 1794 db->db_state = DB_EVICTING; 1795 if ((odb = dbuf_hash_insert(db)) != NULL) { 1796 /* someone else inserted it first */ 1797 kmem_cache_free(dbuf_cache, db); 1798 mutex_exit(&dn->dn_dbufs_mtx); 1799 return (odb); 1800 } 1801 avl_add(&dn->dn_dbufs, db); 1802 if (db->db_level == 0 && db->db_blkid >= 1803 dn->dn_unlisted_l0_blkid) 1804 dn->dn_unlisted_l0_blkid = db->db_blkid + 1; 1805 db->db_state = DB_UNCACHED; 1806 mutex_exit(&dn->dn_dbufs_mtx); 1807 arc_space_consume(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1808 1809 if (parent && parent != dn->dn_dbuf) 1810 dbuf_add_ref(parent, db); 1811 1812 ASSERT(dn->dn_object == DMU_META_DNODE_OBJECT || 1813 refcount_count(&dn->dn_holds) > 0); 1814 (void) refcount_add(&dn->dn_holds, db); 1815 atomic_inc_32(&dn->dn_dbufs_count); 1816 1817 dprintf_dbuf(db, "db=%p\n", db); 1818 1819 return (db); 1820 } 1821 1822 static int 1823 dbuf_do_evict(void *private) 1824 { 1825 dmu_buf_impl_t *db = private; 1826 1827 if (!MUTEX_HELD(&db->db_mtx)) 1828 mutex_enter(&db->db_mtx); 1829 1830 ASSERT(refcount_is_zero(&db->db_holds)); 1831 1832 if (db->db_state != DB_EVICTING) { 1833 ASSERT(db->db_state == DB_CACHED); 1834 DBUF_VERIFY(db); 1835 db->db_buf = NULL; 1836 dbuf_evict(db); 1837 } else { 1838 mutex_exit(&db->db_mtx); 1839 dbuf_destroy(db); 1840 } 1841 return (0); 1842 } 1843 1844 static void 1845 dbuf_destroy(dmu_buf_impl_t *db) 1846 { 1847 ASSERT(refcount_is_zero(&db->db_holds)); 1848 1849 if (db->db_blkid != DMU_BONUS_BLKID) { 1850 /* 1851 * If this dbuf is still on the dn_dbufs list, 1852 * remove it from that list. 1853 */ 1854 if (db->db_dnode_handle != NULL) { 1855 dnode_t *dn; 1856 1857 DB_DNODE_ENTER(db); 1858 dn = DB_DNODE(db); 1859 mutex_enter(&dn->dn_dbufs_mtx); 1860 avl_remove(&dn->dn_dbufs, db); 1861 atomic_dec_32(&dn->dn_dbufs_count); 1862 mutex_exit(&dn->dn_dbufs_mtx); 1863 DB_DNODE_EXIT(db); 1864 /* 1865 * Decrementing the dbuf count means that the hold 1866 * corresponding to the removed dbuf is no longer 1867 * discounted in dnode_move(), so the dnode cannot be 1868 * moved until after we release the hold. 1869 */ 1870 dnode_rele(dn, db); 1871 db->db_dnode_handle = NULL; 1872 } 1873 dbuf_hash_remove(db); 1874 } 1875 db->db_parent = NULL; 1876 db->db_buf = NULL; 1877 1878 ASSERT(db->db.db_data == NULL); 1879 ASSERT(db->db_hash_next == NULL); 1880 ASSERT(db->db_blkptr == NULL); 1881 ASSERT(db->db_data_pending == NULL); 1882 1883 kmem_cache_free(dbuf_cache, db); 1884 arc_space_return(sizeof (dmu_buf_impl_t), ARC_SPACE_OTHER); 1885 } 1886 1887 void 1888 dbuf_prefetch(dnode_t *dn, uint64_t blkid, zio_priority_t prio) 1889 { 1890 dmu_buf_impl_t *db = NULL; 1891 blkptr_t *bp = NULL; 1892 1893 ASSERT(blkid != DMU_BONUS_BLKID); 1894 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1895 1896 if (dnode_block_freed(dn, blkid)) 1897 return; 1898 1899 /* dbuf_find() returns with db_mtx held */ 1900 if (db = dbuf_find(dn, 0, blkid)) { 1901 /* 1902 * This dbuf is already in the cache. We assume that 1903 * it is already CACHED, or else about to be either 1904 * read or filled. 1905 */ 1906 mutex_exit(&db->db_mtx); 1907 return; 1908 } 1909 1910 if (dbuf_findbp(dn, 0, blkid, TRUE, &db, &bp) == 0) { 1911 if (bp && !BP_IS_HOLE(bp) && !BP_IS_EMBEDDED(bp)) { 1912 dsl_dataset_t *ds = dn->dn_objset->os_dsl_dataset; 1913 uint32_t aflags = ARC_NOWAIT | ARC_PREFETCH; 1914 zbookmark_phys_t zb; 1915 1916 SET_BOOKMARK(&zb, ds ? ds->ds_object : DMU_META_OBJSET, 1917 dn->dn_object, 0, blkid); 1918 1919 (void) arc_read(NULL, dn->dn_objset->os_spa, 1920 bp, NULL, NULL, prio, 1921 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, 1922 &aflags, &zb); 1923 } 1924 if (db) 1925 dbuf_rele(db, NULL); 1926 } 1927 } 1928 1929 /* 1930 * Returns with db_holds incremented, and db_mtx not held. 1931 * Note: dn_struct_rwlock must be held. 1932 */ 1933 int 1934 dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, 1935 void *tag, dmu_buf_impl_t **dbp) 1936 { 1937 dmu_buf_impl_t *db, *parent = NULL; 1938 1939 ASSERT(blkid != DMU_BONUS_BLKID); 1940 ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); 1941 ASSERT3U(dn->dn_nlevels, >, level); 1942 1943 *dbp = NULL; 1944 top: 1945 /* dbuf_find() returns with db_mtx held */ 1946 db = dbuf_find(dn, level, blkid); 1947 1948 if (db == NULL) { 1949 blkptr_t *bp = NULL; 1950 int err; 1951 1952 ASSERT3P(parent, ==, NULL); 1953 err = dbuf_findbp(dn, level, blkid, fail_sparse, &parent, &bp); 1954 if (fail_sparse) { 1955 if (err == 0 && bp && BP_IS_HOLE(bp)) 1956 err = SET_ERROR(ENOENT); 1957 if (err) { 1958 if (parent) 1959 dbuf_rele(parent, NULL); 1960 return (err); 1961 } 1962 } 1963 if (err && err != ENOENT) 1964 return (err); 1965 db = dbuf_create(dn, level, blkid, parent, bp); 1966 } 1967 1968 if (db->db_buf && refcount_is_zero(&db->db_holds)) { 1969 arc_buf_add_ref(db->db_buf, db); 1970 if (db->db_buf->b_data == NULL) { 1971 dbuf_clear(db); 1972 if (parent) { 1973 dbuf_rele(parent, NULL); 1974 parent = NULL; 1975 } 1976 goto top; 1977 } 1978 ASSERT3P(db->db.db_data, ==, db->db_buf->b_data); 1979 } 1980 1981 ASSERT(db->db_buf == NULL || arc_referenced(db->db_buf)); 1982 1983 /* 1984 * If this buffer is currently syncing out, and we are are 1985 * still referencing it from db_data, we need to make a copy 1986 * of it in case we decide we want to dirty it again in this txg. 1987 */ 1988 if (db->db_level == 0 && db->db_blkid != DMU_BONUS_BLKID && 1989 dn->dn_object != DMU_META_DNODE_OBJECT && 1990 db->db_state == DB_CACHED && db->db_data_pending) { 1991 dbuf_dirty_record_t *dr = db->db_data_pending; 1992 1993 if (dr->dt.dl.dr_data == db->db_buf) { 1994 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 1995 1996 dbuf_set_data(db, 1997 arc_buf_alloc(dn->dn_objset->os_spa, 1998 db->db.db_size, db, type)); 1999 bcopy(dr->dt.dl.dr_data->b_data, db->db.db_data, 2000 db->db.db_size); 2001 } 2002 } 2003 2004 (void) refcount_add(&db->db_holds, tag); 2005 dbuf_update_data(db); 2006 DBUF_VERIFY(db); 2007 mutex_exit(&db->db_mtx); 2008 2009 /* NOTE: we can't rele the parent until after we drop the db_mtx */ 2010 if (parent) 2011 dbuf_rele(parent, NULL); 2012 2013 ASSERT3P(DB_DNODE(db), ==, dn); 2014 ASSERT3U(db->db_blkid, ==, blkid); 2015 ASSERT3U(db->db_level, ==, level); 2016 *dbp = db; 2017 2018 return (0); 2019 } 2020 2021 dmu_buf_impl_t * 2022 dbuf_hold(dnode_t *dn, uint64_t blkid, void *tag) 2023 { 2024 dmu_buf_impl_t *db; 2025 int err = dbuf_hold_impl(dn, 0, blkid, FALSE, tag, &db); 2026 return (err ? NULL : db); 2027 } 2028 2029 dmu_buf_impl_t * 2030 dbuf_hold_level(dnode_t *dn, int level, uint64_t blkid, void *tag) 2031 { 2032 dmu_buf_impl_t *db; 2033 int err = dbuf_hold_impl(dn, level, blkid, FALSE, tag, &db); 2034 return (err ? NULL : db); 2035 } 2036 2037 void 2038 dbuf_create_bonus(dnode_t *dn) 2039 { 2040 ASSERT(RW_WRITE_HELD(&dn->dn_struct_rwlock)); 2041 2042 ASSERT(dn->dn_bonus == NULL); 2043 dn->dn_bonus = dbuf_create(dn, 0, DMU_BONUS_BLKID, dn->dn_dbuf, NULL); 2044 } 2045 2046 int 2047 dbuf_spill_set_blksz(dmu_buf_t *db_fake, uint64_t blksz, dmu_tx_t *tx) 2048 { 2049 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2050 dnode_t *dn; 2051 2052 if (db->db_blkid != DMU_SPILL_BLKID) 2053 return (SET_ERROR(ENOTSUP)); 2054 if (blksz == 0) 2055 blksz = SPA_MINBLOCKSIZE; 2056 if (blksz > SPA_MAXBLOCKSIZE) 2057 blksz = SPA_MAXBLOCKSIZE; 2058 else 2059 blksz = P2ROUNDUP(blksz, SPA_MINBLOCKSIZE); 2060 2061 DB_DNODE_ENTER(db); 2062 dn = DB_DNODE(db); 2063 rw_enter(&dn->dn_struct_rwlock, RW_WRITER); 2064 dbuf_new_size(db, blksz, tx); 2065 rw_exit(&dn->dn_struct_rwlock); 2066 DB_DNODE_EXIT(db); 2067 2068 return (0); 2069 } 2070 2071 void 2072 dbuf_rm_spill(dnode_t *dn, dmu_tx_t *tx) 2073 { 2074 dbuf_free_range(dn, DMU_SPILL_BLKID, DMU_SPILL_BLKID, tx); 2075 } 2076 2077 #pragma weak dmu_buf_add_ref = dbuf_add_ref 2078 void 2079 dbuf_add_ref(dmu_buf_impl_t *db, void *tag) 2080 { 2081 int64_t holds = refcount_add(&db->db_holds, tag); 2082 ASSERT(holds > 1); 2083 } 2084 2085 /* 2086 * If you call dbuf_rele() you had better not be referencing the dnode handle 2087 * unless you have some other direct or indirect hold on the dnode. (An indirect 2088 * hold is a hold on one of the dnode's dbufs, including the bonus buffer.) 2089 * Without that, the dbuf_rele() could lead to a dnode_rele() followed by the 2090 * dnode's parent dbuf evicting its dnode handles. 2091 */ 2092 void 2093 dbuf_rele(dmu_buf_impl_t *db, void *tag) 2094 { 2095 mutex_enter(&db->db_mtx); 2096 dbuf_rele_and_unlock(db, tag); 2097 } 2098 2099 void 2100 dmu_buf_rele(dmu_buf_t *db, void *tag) 2101 { 2102 dbuf_rele((dmu_buf_impl_t *)db, tag); 2103 } 2104 2105 /* 2106 * dbuf_rele() for an already-locked dbuf. This is necessary to allow 2107 * db_dirtycnt and db_holds to be updated atomically. 2108 */ 2109 void 2110 dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag) 2111 { 2112 int64_t holds; 2113 2114 ASSERT(MUTEX_HELD(&db->db_mtx)); 2115 DBUF_VERIFY(db); 2116 2117 /* 2118 * Remove the reference to the dbuf before removing its hold on the 2119 * dnode so we can guarantee in dnode_move() that a referenced bonus 2120 * buffer has a corresponding dnode hold. 2121 */ 2122 holds = refcount_remove(&db->db_holds, tag); 2123 ASSERT(holds >= 0); 2124 2125 /* 2126 * We can't freeze indirects if there is a possibility that they 2127 * may be modified in the current syncing context. 2128 */ 2129 if (db->db_buf && holds == (db->db_level == 0 ? db->db_dirtycnt : 0)) 2130 arc_buf_freeze(db->db_buf); 2131 2132 if (holds == db->db_dirtycnt && 2133 db->db_level == 0 && db->db_immediate_evict) 2134 dbuf_evict_user(db); 2135 2136 if (holds == 0) { 2137 if (db->db_blkid == DMU_BONUS_BLKID) { 2138 mutex_exit(&db->db_mtx); 2139 2140 /* 2141 * If the dnode moves here, we cannot cross this barrier 2142 * until the move completes. 2143 */ 2144 DB_DNODE_ENTER(db); 2145 atomic_dec_32(&DB_DNODE(db)->dn_dbufs_count); 2146 DB_DNODE_EXIT(db); 2147 /* 2148 * The bonus buffer's dnode hold is no longer discounted 2149 * in dnode_move(). The dnode cannot move until after 2150 * the dnode_rele(). 2151 */ 2152 dnode_rele(DB_DNODE(db), db); 2153 } else if (db->db_buf == NULL) { 2154 /* 2155 * This is a special case: we never associated this 2156 * dbuf with any data allocated from the ARC. 2157 */ 2158 ASSERT(db->db_state == DB_UNCACHED || 2159 db->db_state == DB_NOFILL); 2160 dbuf_evict(db); 2161 } else if (arc_released(db->db_buf)) { 2162 arc_buf_t *buf = db->db_buf; 2163 /* 2164 * This dbuf has anonymous data associated with it. 2165 */ 2166 dbuf_set_data(db, NULL); 2167 VERIFY(arc_buf_remove_ref(buf, db)); 2168 dbuf_evict(db); 2169 } else { 2170 VERIFY(!arc_buf_remove_ref(db->db_buf, db)); 2171 2172 /* 2173 * A dbuf will be eligible for eviction if either the 2174 * 'primarycache' property is set or a duplicate 2175 * copy of this buffer is already cached in the arc. 2176 * 2177 * In the case of the 'primarycache' a buffer 2178 * is considered for eviction if it matches the 2179 * criteria set in the property. 2180 * 2181 * To decide if our buffer is considered a 2182 * duplicate, we must call into the arc to determine 2183 * if multiple buffers are referencing the same 2184 * block on-disk. If so, then we simply evict 2185 * ourselves. 2186 */ 2187 if (!DBUF_IS_CACHEABLE(db)) { 2188 if (db->db_blkptr != NULL && 2189 !BP_IS_HOLE(db->db_blkptr) && 2190 !BP_IS_EMBEDDED(db->db_blkptr)) { 2191 spa_t *spa = 2192 dmu_objset_spa(db->db_objset); 2193 blkptr_t bp = *db->db_blkptr; 2194 dbuf_clear(db); 2195 arc_freed(spa, &bp); 2196 } else { 2197 dbuf_clear(db); 2198 } 2199 } else if (arc_buf_eviction_needed(db->db_buf)) { 2200 dbuf_clear(db); 2201 } else { 2202 mutex_exit(&db->db_mtx); 2203 } 2204 } 2205 } else { 2206 mutex_exit(&db->db_mtx); 2207 } 2208 } 2209 2210 #pragma weak dmu_buf_refcount = dbuf_refcount 2211 uint64_t 2212 dbuf_refcount(dmu_buf_impl_t *db) 2213 { 2214 return (refcount_count(&db->db_holds)); 2215 } 2216 2217 void * 2218 dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2219 dmu_buf_evict_func_t *evict_func) 2220 { 2221 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2222 user_data_ptr_ptr, evict_func)); 2223 } 2224 2225 void * 2226 dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr, void *user_data_ptr_ptr, 2227 dmu_buf_evict_func_t *evict_func) 2228 { 2229 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2230 2231 db->db_immediate_evict = TRUE; 2232 return (dmu_buf_update_user(db_fake, NULL, user_ptr, 2233 user_data_ptr_ptr, evict_func)); 2234 } 2235 2236 void * 2237 dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr, 2238 void *user_data_ptr_ptr, dmu_buf_evict_func_t *evict_func) 2239 { 2240 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2241 ASSERT(db->db_level == 0); 2242 2243 ASSERT((user_ptr == NULL) == (evict_func == NULL)); 2244 2245 mutex_enter(&db->db_mtx); 2246 2247 if (db->db_user_ptr == old_user_ptr) { 2248 db->db_user_ptr = user_ptr; 2249 db->db_user_data_ptr_ptr = user_data_ptr_ptr; 2250 db->db_evict_func = evict_func; 2251 2252 dbuf_update_data(db); 2253 } else { 2254 old_user_ptr = db->db_user_ptr; 2255 } 2256 2257 mutex_exit(&db->db_mtx); 2258 return (old_user_ptr); 2259 } 2260 2261 void * 2262 dmu_buf_get_user(dmu_buf_t *db_fake) 2263 { 2264 dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake; 2265 ASSERT(!refcount_is_zero(&db->db_holds)); 2266 2267 return (db->db_user_ptr); 2268 } 2269 2270 boolean_t 2271 dmu_buf_freeable(dmu_buf_t *dbuf) 2272 { 2273 boolean_t res = B_FALSE; 2274 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbuf; 2275 2276 if (db->db_blkptr) 2277 res = dsl_dataset_block_freeable(db->db_objset->os_dsl_dataset, 2278 db->db_blkptr, db->db_blkptr->blk_birth); 2279 2280 return (res); 2281 } 2282 2283 blkptr_t * 2284 dmu_buf_get_blkptr(dmu_buf_t *db) 2285 { 2286 dmu_buf_impl_t *dbi = (dmu_buf_impl_t *)db; 2287 return (dbi->db_blkptr); 2288 } 2289 2290 static void 2291 dbuf_check_blkptr(dnode_t *dn, dmu_buf_impl_t *db) 2292 { 2293 /* ASSERT(dmu_tx_is_syncing(tx) */ 2294 ASSERT(MUTEX_HELD(&db->db_mtx)); 2295 2296 if (db->db_blkptr != NULL) 2297 return; 2298 2299 if (db->db_blkid == DMU_SPILL_BLKID) { 2300 db->db_blkptr = &dn->dn_phys->dn_spill; 2301 BP_ZERO(db->db_blkptr); 2302 return; 2303 } 2304 if (db->db_level == dn->dn_phys->dn_nlevels-1) { 2305 /* 2306 * This buffer was allocated at a time when there was 2307 * no available blkptrs from the dnode, or it was 2308 * inappropriate to hook it in (i.e., nlevels mis-match). 2309 */ 2310 ASSERT(db->db_blkid < dn->dn_phys->dn_nblkptr); 2311 ASSERT(db->db_parent == NULL); 2312 db->db_parent = dn->dn_dbuf; 2313 db->db_blkptr = &dn->dn_phys->dn_blkptr[db->db_blkid]; 2314 DBUF_VERIFY(db); 2315 } else { 2316 dmu_buf_impl_t *parent = db->db_parent; 2317 int epbs = dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2318 2319 ASSERT(dn->dn_phys->dn_nlevels > 1); 2320 if (parent == NULL) { 2321 mutex_exit(&db->db_mtx); 2322 rw_enter(&dn->dn_struct_rwlock, RW_READER); 2323 (void) dbuf_hold_impl(dn, db->db_level+1, 2324 db->db_blkid >> epbs, FALSE, db, &parent); 2325 rw_exit(&dn->dn_struct_rwlock); 2326 mutex_enter(&db->db_mtx); 2327 db->db_parent = parent; 2328 } 2329 db->db_blkptr = (blkptr_t *)parent->db.db_data + 2330 (db->db_blkid & ((1ULL << epbs) - 1)); 2331 DBUF_VERIFY(db); 2332 } 2333 } 2334 2335 static void 2336 dbuf_sync_indirect(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2337 { 2338 dmu_buf_impl_t *db = dr->dr_dbuf; 2339 dnode_t *dn; 2340 zio_t *zio; 2341 2342 ASSERT(dmu_tx_is_syncing(tx)); 2343 2344 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2345 2346 mutex_enter(&db->db_mtx); 2347 2348 ASSERT(db->db_level > 0); 2349 DBUF_VERIFY(db); 2350 2351 /* Read the block if it hasn't been read yet. */ 2352 if (db->db_buf == NULL) { 2353 mutex_exit(&db->db_mtx); 2354 (void) dbuf_read(db, NULL, DB_RF_MUST_SUCCEED); 2355 mutex_enter(&db->db_mtx); 2356 } 2357 ASSERT3U(db->db_state, ==, DB_CACHED); 2358 ASSERT(db->db_buf != NULL); 2359 2360 DB_DNODE_ENTER(db); 2361 dn = DB_DNODE(db); 2362 /* Indirect block size must match what the dnode thinks it is. */ 2363 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2364 dbuf_check_blkptr(dn, db); 2365 DB_DNODE_EXIT(db); 2366 2367 /* Provide the pending dirty record to child dbufs */ 2368 db->db_data_pending = dr; 2369 2370 mutex_exit(&db->db_mtx); 2371 dbuf_write(dr, db->db_buf, tx); 2372 2373 zio = dr->dr_zio; 2374 mutex_enter(&dr->dt.di.dr_mtx); 2375 dbuf_sync_list(&dr->dt.di.dr_children, tx); 2376 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2377 mutex_exit(&dr->dt.di.dr_mtx); 2378 zio_nowait(zio); 2379 } 2380 2381 static void 2382 dbuf_sync_leaf(dbuf_dirty_record_t *dr, dmu_tx_t *tx) 2383 { 2384 arc_buf_t **datap = &dr->dt.dl.dr_data; 2385 dmu_buf_impl_t *db = dr->dr_dbuf; 2386 dnode_t *dn; 2387 objset_t *os; 2388 uint64_t txg = tx->tx_txg; 2389 2390 ASSERT(dmu_tx_is_syncing(tx)); 2391 2392 dprintf_dbuf_bp(db, db->db_blkptr, "blkptr=%p", db->db_blkptr); 2393 2394 mutex_enter(&db->db_mtx); 2395 /* 2396 * To be synced, we must be dirtied. But we 2397 * might have been freed after the dirty. 2398 */ 2399 if (db->db_state == DB_UNCACHED) { 2400 /* This buffer has been freed since it was dirtied */ 2401 ASSERT(db->db.db_data == NULL); 2402 } else if (db->db_state == DB_FILL) { 2403 /* This buffer was freed and is now being re-filled */ 2404 ASSERT(db->db.db_data != dr->dt.dl.dr_data); 2405 } else { 2406 ASSERT(db->db_state == DB_CACHED || db->db_state == DB_NOFILL); 2407 } 2408 DBUF_VERIFY(db); 2409 2410 DB_DNODE_ENTER(db); 2411 dn = DB_DNODE(db); 2412 2413 if (db->db_blkid == DMU_SPILL_BLKID) { 2414 mutex_enter(&dn->dn_mtx); 2415 dn->dn_phys->dn_flags |= DNODE_FLAG_SPILL_BLKPTR; 2416 mutex_exit(&dn->dn_mtx); 2417 } 2418 2419 /* 2420 * If this is a bonus buffer, simply copy the bonus data into the 2421 * dnode. It will be written out when the dnode is synced (and it 2422 * will be synced, since it must have been dirty for dbuf_sync to 2423 * be called). 2424 */ 2425 if (db->db_blkid == DMU_BONUS_BLKID) { 2426 dbuf_dirty_record_t **drp; 2427 2428 ASSERT(*datap != NULL); 2429 ASSERT0(db->db_level); 2430 ASSERT3U(dn->dn_phys->dn_bonuslen, <=, DN_MAX_BONUSLEN); 2431 bcopy(*datap, DN_BONUS(dn->dn_phys), dn->dn_phys->dn_bonuslen); 2432 DB_DNODE_EXIT(db); 2433 2434 if (*datap != db->db.db_data) { 2435 zio_buf_free(*datap, DN_MAX_BONUSLEN); 2436 arc_space_return(DN_MAX_BONUSLEN, ARC_SPACE_OTHER); 2437 } 2438 db->db_data_pending = NULL; 2439 drp = &db->db_last_dirty; 2440 while (*drp != dr) 2441 drp = &(*drp)->dr_next; 2442 ASSERT(dr->dr_next == NULL); 2443 ASSERT(dr->dr_dbuf == db); 2444 *drp = dr->dr_next; 2445 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2446 ASSERT(db->db_dirtycnt > 0); 2447 db->db_dirtycnt -= 1; 2448 dbuf_rele_and_unlock(db, (void *)(uintptr_t)txg); 2449 return; 2450 } 2451 2452 os = dn->dn_objset; 2453 2454 /* 2455 * This function may have dropped the db_mtx lock allowing a dmu_sync 2456 * operation to sneak in. As a result, we need to ensure that we 2457 * don't check the dr_override_state until we have returned from 2458 * dbuf_check_blkptr. 2459 */ 2460 dbuf_check_blkptr(dn, db); 2461 2462 /* 2463 * If this buffer is in the middle of an immediate write, 2464 * wait for the synchronous IO to complete. 2465 */ 2466 while (dr->dt.dl.dr_override_state == DR_IN_DMU_SYNC) { 2467 ASSERT(dn->dn_object != DMU_META_DNODE_OBJECT); 2468 cv_wait(&db->db_changed, &db->db_mtx); 2469 ASSERT(dr->dt.dl.dr_override_state != DR_NOT_OVERRIDDEN); 2470 } 2471 2472 if (db->db_state != DB_NOFILL && 2473 dn->dn_object != DMU_META_DNODE_OBJECT && 2474 refcount_count(&db->db_holds) > 1 && 2475 dr->dt.dl.dr_override_state != DR_OVERRIDDEN && 2476 *datap == db->db_buf) { 2477 /* 2478 * If this buffer is currently "in use" (i.e., there 2479 * are active holds and db_data still references it), 2480 * then make a copy before we start the write so that 2481 * any modifications from the open txg will not leak 2482 * into this write. 2483 * 2484 * NOTE: this copy does not need to be made for 2485 * objects only modified in the syncing context (e.g. 2486 * DNONE_DNODE blocks). 2487 */ 2488 int blksz = arc_buf_size(*datap); 2489 arc_buf_contents_t type = DBUF_GET_BUFC_TYPE(db); 2490 *datap = arc_buf_alloc(os->os_spa, blksz, db, type); 2491 bcopy(db->db.db_data, (*datap)->b_data, blksz); 2492 } 2493 db->db_data_pending = dr; 2494 2495 mutex_exit(&db->db_mtx); 2496 2497 dbuf_write(dr, *datap, tx); 2498 2499 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2500 if (dn->dn_object == DMU_META_DNODE_OBJECT) { 2501 list_insert_tail(&dn->dn_dirty_records[txg&TXG_MASK], dr); 2502 DB_DNODE_EXIT(db); 2503 } else { 2504 /* 2505 * Although zio_nowait() does not "wait for an IO", it does 2506 * initiate the IO. If this is an empty write it seems plausible 2507 * that the IO could actually be completed before the nowait 2508 * returns. We need to DB_DNODE_EXIT() first in case 2509 * zio_nowait() invalidates the dbuf. 2510 */ 2511 DB_DNODE_EXIT(db); 2512 zio_nowait(dr->dr_zio); 2513 } 2514 } 2515 2516 void 2517 dbuf_sync_list(list_t *list, dmu_tx_t *tx) 2518 { 2519 dbuf_dirty_record_t *dr; 2520 2521 while (dr = list_head(list)) { 2522 if (dr->dr_zio != NULL) { 2523 /* 2524 * If we find an already initialized zio then we 2525 * are processing the meta-dnode, and we have finished. 2526 * The dbufs for all dnodes are put back on the list 2527 * during processing, so that we can zio_wait() 2528 * these IOs after initiating all child IOs. 2529 */ 2530 ASSERT3U(dr->dr_dbuf->db.db_object, ==, 2531 DMU_META_DNODE_OBJECT); 2532 break; 2533 } 2534 list_remove(list, dr); 2535 if (dr->dr_dbuf->db_level > 0) 2536 dbuf_sync_indirect(dr, tx); 2537 else 2538 dbuf_sync_leaf(dr, tx); 2539 } 2540 } 2541 2542 /* ARGSUSED */ 2543 static void 2544 dbuf_write_ready(zio_t *zio, arc_buf_t *buf, void *vdb) 2545 { 2546 dmu_buf_impl_t *db = vdb; 2547 dnode_t *dn; 2548 blkptr_t *bp = zio->io_bp; 2549 blkptr_t *bp_orig = &zio->io_bp_orig; 2550 spa_t *spa = zio->io_spa; 2551 int64_t delta; 2552 uint64_t fill = 0; 2553 int i; 2554 2555 ASSERT3P(db->db_blkptr, ==, bp); 2556 2557 DB_DNODE_ENTER(db); 2558 dn = DB_DNODE(db); 2559 delta = bp_get_dsize_sync(spa, bp) - bp_get_dsize_sync(spa, bp_orig); 2560 dnode_diduse_space(dn, delta - zio->io_prev_space_delta); 2561 zio->io_prev_space_delta = delta; 2562 2563 if (bp->blk_birth != 0) { 2564 ASSERT((db->db_blkid != DMU_SPILL_BLKID && 2565 BP_GET_TYPE(bp) == dn->dn_type) || 2566 (db->db_blkid == DMU_SPILL_BLKID && 2567 BP_GET_TYPE(bp) == dn->dn_bonustype) || 2568 BP_IS_EMBEDDED(bp)); 2569 ASSERT(BP_GET_LEVEL(bp) == db->db_level); 2570 } 2571 2572 mutex_enter(&db->db_mtx); 2573 2574 #ifdef ZFS_DEBUG 2575 if (db->db_blkid == DMU_SPILL_BLKID) { 2576 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 2577 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 2578 db->db_blkptr == &dn->dn_phys->dn_spill); 2579 } 2580 #endif 2581 2582 if (db->db_level == 0) { 2583 mutex_enter(&dn->dn_mtx); 2584 if (db->db_blkid > dn->dn_phys->dn_maxblkid && 2585 db->db_blkid != DMU_SPILL_BLKID) 2586 dn->dn_phys->dn_maxblkid = db->db_blkid; 2587 mutex_exit(&dn->dn_mtx); 2588 2589 if (dn->dn_type == DMU_OT_DNODE) { 2590 dnode_phys_t *dnp = db->db.db_data; 2591 for (i = db->db.db_size >> DNODE_SHIFT; i > 0; 2592 i--, dnp++) { 2593 if (dnp->dn_type != DMU_OT_NONE) 2594 fill++; 2595 } 2596 } else { 2597 if (BP_IS_HOLE(bp)) { 2598 fill = 0; 2599 } else { 2600 fill = 1; 2601 } 2602 } 2603 } else { 2604 blkptr_t *ibp = db->db.db_data; 2605 ASSERT3U(db->db.db_size, ==, 1<<dn->dn_phys->dn_indblkshift); 2606 for (i = db->db.db_size >> SPA_BLKPTRSHIFT; i > 0; i--, ibp++) { 2607 if (BP_IS_HOLE(ibp)) 2608 continue; 2609 fill += BP_GET_FILL(ibp); 2610 } 2611 } 2612 DB_DNODE_EXIT(db); 2613 2614 if (!BP_IS_EMBEDDED(bp)) 2615 bp->blk_fill = fill; 2616 2617 mutex_exit(&db->db_mtx); 2618 } 2619 2620 /* 2621 * The SPA will call this callback several times for each zio - once 2622 * for every physical child i/o (zio->io_phys_children times). This 2623 * allows the DMU to monitor the progress of each logical i/o. For example, 2624 * there may be 2 copies of an indirect block, or many fragments of a RAID-Z 2625 * block. There may be a long delay before all copies/fragments are completed, 2626 * so this callback allows us to retire dirty space gradually, as the physical 2627 * i/os complete. 2628 */ 2629 /* ARGSUSED */ 2630 static void 2631 dbuf_write_physdone(zio_t *zio, arc_buf_t *buf, void *arg) 2632 { 2633 dmu_buf_impl_t *db = arg; 2634 objset_t *os = db->db_objset; 2635 dsl_pool_t *dp = dmu_objset_pool(os); 2636 dbuf_dirty_record_t *dr; 2637 int delta = 0; 2638 2639 dr = db->db_data_pending; 2640 ASSERT3U(dr->dr_txg, ==, zio->io_txg); 2641 2642 /* 2643 * The callback will be called io_phys_children times. Retire one 2644 * portion of our dirty space each time we are called. Any rounding 2645 * error will be cleaned up by dsl_pool_sync()'s call to 2646 * dsl_pool_undirty_space(). 2647 */ 2648 delta = dr->dr_accounted / zio->io_phys_children; 2649 dsl_pool_undirty_space(dp, delta, zio->io_txg); 2650 } 2651 2652 /* ARGSUSED */ 2653 static void 2654 dbuf_write_done(zio_t *zio, arc_buf_t *buf, void *vdb) 2655 { 2656 dmu_buf_impl_t *db = vdb; 2657 blkptr_t *bp_orig = &zio->io_bp_orig; 2658 blkptr_t *bp = db->db_blkptr; 2659 objset_t *os = db->db_objset; 2660 dmu_tx_t *tx = os->os_synctx; 2661 dbuf_dirty_record_t **drp, *dr; 2662 2663 ASSERT0(zio->io_error); 2664 ASSERT(db->db_blkptr == bp); 2665 2666 /* 2667 * For nopwrites and rewrites we ensure that the bp matches our 2668 * original and bypass all the accounting. 2669 */ 2670 if (zio->io_flags & (ZIO_FLAG_IO_REWRITE | ZIO_FLAG_NOPWRITE)) { 2671 ASSERT(BP_EQUAL(bp, bp_orig)); 2672 } else { 2673 dsl_dataset_t *ds = os->os_dsl_dataset; 2674 (void) dsl_dataset_block_kill(ds, bp_orig, tx, B_TRUE); 2675 dsl_dataset_block_born(ds, bp, tx); 2676 } 2677 2678 mutex_enter(&db->db_mtx); 2679 2680 DBUF_VERIFY(db); 2681 2682 drp = &db->db_last_dirty; 2683 while ((dr = *drp) != db->db_data_pending) 2684 drp = &dr->dr_next; 2685 ASSERT(!list_link_active(&dr->dr_dirty_node)); 2686 ASSERT(dr->dr_dbuf == db); 2687 ASSERT(dr->dr_next == NULL); 2688 *drp = dr->dr_next; 2689 2690 #ifdef ZFS_DEBUG 2691 if (db->db_blkid == DMU_SPILL_BLKID) { 2692 dnode_t *dn; 2693 2694 DB_DNODE_ENTER(db); 2695 dn = DB_DNODE(db); 2696 ASSERT(dn->dn_phys->dn_flags & DNODE_FLAG_SPILL_BLKPTR); 2697 ASSERT(!(BP_IS_HOLE(db->db_blkptr)) && 2698 db->db_blkptr == &dn->dn_phys->dn_spill); 2699 DB_DNODE_EXIT(db); 2700 } 2701 #endif 2702 2703 if (db->db_level == 0) { 2704 ASSERT(db->db_blkid != DMU_BONUS_BLKID); 2705 ASSERT(dr->dt.dl.dr_override_state == DR_NOT_OVERRIDDEN); 2706 if (db->db_state != DB_NOFILL) { 2707 if (dr->dt.dl.dr_data != db->db_buf) 2708 VERIFY(arc_buf_remove_ref(dr->dt.dl.dr_data, 2709 db)); 2710 else if (!arc_released(db->db_buf)) 2711 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2712 } 2713 } else { 2714 dnode_t *dn; 2715 2716 DB_DNODE_ENTER(db); 2717 dn = DB_DNODE(db); 2718 ASSERT(list_head(&dr->dt.di.dr_children) == NULL); 2719 ASSERT3U(db->db.db_size, ==, 1 << dn->dn_phys->dn_indblkshift); 2720 if (!BP_IS_HOLE(db->db_blkptr)) { 2721 int epbs = 2722 dn->dn_phys->dn_indblkshift - SPA_BLKPTRSHIFT; 2723 ASSERT3U(db->db_blkid, <=, 2724 dn->dn_phys->dn_maxblkid >> (db->db_level * epbs)); 2725 ASSERT3U(BP_GET_LSIZE(db->db_blkptr), ==, 2726 db->db.db_size); 2727 if (!arc_released(db->db_buf)) 2728 arc_set_callback(db->db_buf, dbuf_do_evict, db); 2729 } 2730 DB_DNODE_EXIT(db); 2731 mutex_destroy(&dr->dt.di.dr_mtx); 2732 list_destroy(&dr->dt.di.dr_children); 2733 } 2734 kmem_free(dr, sizeof (dbuf_dirty_record_t)); 2735 2736 cv_broadcast(&db->db_changed); 2737 ASSERT(db->db_dirtycnt > 0); 2738 db->db_dirtycnt -= 1; 2739 db->db_data_pending = NULL; 2740 dbuf_rele_and_unlock(db, (void *)(uintptr_t)tx->tx_txg); 2741 } 2742 2743 static void 2744 dbuf_write_nofill_ready(zio_t *zio) 2745 { 2746 dbuf_write_ready(zio, NULL, zio->io_private); 2747 } 2748 2749 static void 2750 dbuf_write_nofill_done(zio_t *zio) 2751 { 2752 dbuf_write_done(zio, NULL, zio->io_private); 2753 } 2754 2755 static void 2756 dbuf_write_override_ready(zio_t *zio) 2757 { 2758 dbuf_dirty_record_t *dr = zio->io_private; 2759 dmu_buf_impl_t *db = dr->dr_dbuf; 2760 2761 dbuf_write_ready(zio, NULL, db); 2762 } 2763 2764 static void 2765 dbuf_write_override_done(zio_t *zio) 2766 { 2767 dbuf_dirty_record_t *dr = zio->io_private; 2768 dmu_buf_impl_t *db = dr->dr_dbuf; 2769 blkptr_t *obp = &dr->dt.dl.dr_overridden_by; 2770 2771 mutex_enter(&db->db_mtx); 2772 if (!BP_EQUAL(zio->io_bp, obp)) { 2773 if (!BP_IS_HOLE(obp)) 2774 dsl_free(spa_get_dsl(zio->io_spa), zio->io_txg, obp); 2775 arc_release(dr->dt.dl.dr_data, db); 2776 } 2777 mutex_exit(&db->db_mtx); 2778 2779 dbuf_write_done(zio, NULL, db); 2780 } 2781 2782 /* Issue I/O to commit a dirty buffer to disk. */ 2783 static void 2784 dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx) 2785 { 2786 dmu_buf_impl_t *db = dr->dr_dbuf; 2787 dnode_t *dn; 2788 objset_t *os; 2789 dmu_buf_impl_t *parent = db->db_parent; 2790 uint64_t txg = tx->tx_txg; 2791 zbookmark_phys_t zb; 2792 zio_prop_t zp; 2793 zio_t *zio; 2794 int wp_flag = 0; 2795 2796 DB_DNODE_ENTER(db); 2797 dn = DB_DNODE(db); 2798 os = dn->dn_objset; 2799 2800 if (db->db_state != DB_NOFILL) { 2801 if (db->db_level > 0 || dn->dn_type == DMU_OT_DNODE) { 2802 /* 2803 * Private object buffers are released here rather 2804 * than in dbuf_dirty() since they are only modified 2805 * in the syncing context and we don't want the 2806 * overhead of making multiple copies of the data. 2807 */ 2808 if (BP_IS_HOLE(db->db_blkptr)) { 2809 arc_buf_thaw(data); 2810 } else { 2811 dbuf_release_bp(db); 2812 } 2813 } 2814 } 2815 2816 if (parent != dn->dn_dbuf) { 2817 /* Our parent is an indirect block. */ 2818 /* We have a dirty parent that has been scheduled for write. */ 2819 ASSERT(parent && parent->db_data_pending); 2820 /* Our parent's buffer is one level closer to the dnode. */ 2821 ASSERT(db->db_level == parent->db_level-1); 2822 /* 2823 * We're about to modify our parent's db_data by modifying 2824 * our block pointer, so the parent must be released. 2825 */ 2826 ASSERT(arc_released(parent->db_buf)); 2827 zio = parent->db_data_pending->dr_zio; 2828 } else { 2829 /* Our parent is the dnode itself. */ 2830 ASSERT((db->db_level == dn->dn_phys->dn_nlevels-1 && 2831 db->db_blkid != DMU_SPILL_BLKID) || 2832 (db->db_blkid == DMU_SPILL_BLKID && db->db_level == 0)); 2833 if (db->db_blkid != DMU_SPILL_BLKID) 2834 ASSERT3P(db->db_blkptr, ==, 2835 &dn->dn_phys->dn_blkptr[db->db_blkid]); 2836 zio = dn->dn_zio; 2837 } 2838 2839 ASSERT(db->db_level == 0 || data == db->db_buf); 2840 ASSERT3U(db->db_blkptr->blk_birth, <=, txg); 2841 ASSERT(zio); 2842 2843 SET_BOOKMARK(&zb, os->os_dsl_dataset ? 2844 os->os_dsl_dataset->ds_object : DMU_META_OBJSET, 2845 db->db.db_object, db->db_level, db->db_blkid); 2846 2847 if (db->db_blkid == DMU_SPILL_BLKID) 2848 wp_flag = WP_SPILL; 2849 wp_flag |= (db->db_state == DB_NOFILL) ? WP_NOFILL : 0; 2850 WP_SET_SPECIALCLASS(wp_flag, dr->dr_usesc); 2851 2852 dmu_write_policy(os, dn, db->db_level, wp_flag, &zp); 2853 DB_DNODE_EXIT(db); 2854 2855 if (db->db_level == 0 && 2856 dr->dt.dl.dr_override_state == DR_OVERRIDDEN) { 2857 /* 2858 * The BP for this block has been provided by open context 2859 * (by dmu_sync() or dmu_buf_write_embedded()). 2860 */ 2861 void *contents = (data != NULL) ? data->b_data : NULL; 2862 2863 dr->dr_zio = zio_write(zio, os->os_spa, txg, 2864 db->db_blkptr, contents, db->db.db_size, &zp, 2865 dbuf_write_override_ready, NULL, dbuf_write_override_done, 2866 dr, ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2867 mutex_enter(&db->db_mtx); 2868 dr->dt.dl.dr_override_state = DR_NOT_OVERRIDDEN; 2869 zio_write_override(dr->dr_zio, &dr->dt.dl.dr_overridden_by, 2870 dr->dt.dl.dr_copies, dr->dt.dl.dr_nopwrite); 2871 mutex_exit(&db->db_mtx); 2872 } else if (db->db_state == DB_NOFILL) { 2873 ASSERT(zp.zp_checksum == ZIO_CHECKSUM_OFF || 2874 zp.zp_checksum == ZIO_CHECKSUM_NOPARITY); 2875 dr->dr_zio = zio_write(zio, os->os_spa, txg, 2876 db->db_blkptr, NULL, db->db.db_size, &zp, 2877 dbuf_write_nofill_ready, NULL, dbuf_write_nofill_done, db, 2878 ZIO_PRIORITY_ASYNC_WRITE, 2879 ZIO_FLAG_MUSTSUCCEED | ZIO_FLAG_NODATA, &zb); 2880 } else { 2881 ASSERT(arc_released(data)); 2882 dr->dr_zio = arc_write(zio, os->os_spa, txg, 2883 db->db_blkptr, data, DBUF_IS_L2CACHEABLE(db), 2884 DBUF_IS_L2COMPRESSIBLE(db), &zp, dbuf_write_ready, 2885 dbuf_write_physdone, dbuf_write_done, db, 2886 ZIO_PRIORITY_ASYNC_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 2887 } 2888 } --- EOF ---