1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  24  * Use is subject to license terms.
  25  */
  26 
  27 /*
  28  * Vnode operations for the High Sierra filesystem
  29  */
  30 
  31 #include <sys/types.h>
  32 #include <sys/t_lock.h>
  33 #include <sys/param.h>
  34 #include <sys/time.h>
  35 #include <sys/systm.h>
  36 #include <sys/sysmacros.h>
  37 #include <sys/resource.h>
  38 #include <sys/signal.h>
  39 #include <sys/cred.h>
  40 #include <sys/user.h>
  41 #include <sys/buf.h>
  42 #include <sys/vfs.h>
  43 #include <sys/vfs_opreg.h>
  44 #include <sys/stat.h>
  45 #include <sys/vnode.h>
  46 #include <sys/mode.h>
  47 #include <sys/proc.h>
  48 #include <sys/disp.h>
  49 #include <sys/file.h>
  50 #include <sys/fcntl.h>
  51 #include <sys/flock.h>
  52 #include <sys/kmem.h>
  53 #include <sys/uio.h>
  54 #include <sys/conf.h>
  55 #include <sys/errno.h>
  56 #include <sys/mman.h>
  57 #include <sys/pathname.h>
  58 #include <sys/debug.h>
  59 #include <sys/vmsystm.h>
  60 #include <sys/cmn_err.h>
  61 #include <sys/fbuf.h>
  62 #include <sys/dirent.h>
  63 #include <sys/errno.h>
  64 #include <sys/dkio.h>
  65 #include <sys/cmn_err.h>
  66 #include <sys/atomic.h>
  67 
  68 #include <vm/hat.h>
  69 #include <vm/page.h>
  70 #include <vm/pvn.h>
  71 #include <vm/as.h>
  72 #include <vm/seg.h>
  73 #include <vm/seg_map.h>
  74 #include <vm/seg_kmem.h>
  75 #include <vm/seg_vn.h>
  76 #include <vm/rm.h>
  77 #include <vm/page.h>
  78 #include <sys/swap.h>
  79 #include <sys/avl.h>
  80 #include <sys/sunldi.h>
  81 #include <sys/ddi.h>
  82 #include <sys/sunddi.h>
  83 #include <sys/sdt.h>
  84 
  85 /*
  86  * For struct modlinkage
  87  */
  88 #include <sys/modctl.h>
  89 
  90 #include <sys/fs/hsfs_spec.h>
  91 #include <sys/fs/hsfs_node.h>
  92 #include <sys/fs/hsfs_impl.h>
  93 #include <sys/fs/hsfs_susp.h>
  94 #include <sys/fs/hsfs_rrip.h>
  95 
  96 #include <fs/fs_subr.h>
  97 
  98 /* # of contiguous requests to detect sequential access pattern */
  99 static int seq_contig_requests = 2;
 100 
 101 /*
 102  * This is the max number os taskq threads that will be created
 103  * if required. Since we are using a Dynamic TaskQ by default only
 104  * one thread is created initially.
 105  *
 106  * NOTE: In the usual hsfs use case this per fs instance number
 107  * of taskq threads should not place any undue load on a system.
 108  * Even on an unusual system with say 100 CDROM drives, 800 threads
 109  * will not be created unless all the drives are loaded and all
 110  * of them are saturated with I/O at the same time! If there is at
 111  * all a complaint of system load due to such an unusual case it
 112  * should be easy enough to change to one per-machine Dynamic TaskQ
 113  * for all hsfs mounts with a nthreads of say 32.
 114  */
 115 static int hsfs_taskq_nthreads = 8;     /* # of taskq threads per fs */
 116 
 117 /* Min count of adjacent bufs that will avoid buf coalescing */
 118 static int hsched_coalesce_min = 2;
 119 
 120 /*
 121  * Kmem caches for heavily used small allocations. Using these kmem
 122  * caches provides a factor of 3 reduction in system time and greatly
 123  * aids overall throughput esp. on SPARC.
 124  */
 125 struct kmem_cache *hio_cache;
 126 struct kmem_cache *hio_info_cache;
 127 
 128 /*
 129  * This tunable allows us to ignore inode numbers from rrip-1.12.
 130  * In this case, we fall back to our default inode algorithm.
 131  */
 132 extern int use_rrip_inodes;
 133 
 134 /*
 135  * Free behind logic from UFS to tame our thirst for
 136  * the page cache.
 137  * See usr/src/uts/common/fs/ufs/ufs_vnops.c for more
 138  * explanation.
 139  */
 140 static int      freebehind = 1;
 141 static int      smallfile = 0;
 142 static int      cache_read_ahead = 0;
 143 static u_offset_t smallfile64 = 32 * 1024;
 144 #define SMALLFILE1_D 1000
 145 #define SMALLFILE2_D 10
 146 static u_offset_t smallfile1 = 32 * 1024;
 147 static u_offset_t smallfile2 = 32 * 1024;
 148 static clock_t smallfile_update = 0; /* when to recompute */
 149 static uint_t smallfile1_d = SMALLFILE1_D;
 150 static uint_t smallfile2_d = SMALLFILE2_D;
 151 
 152 static int hsched_deadline_compare(const void *x1, const void *x2);
 153 static int hsched_offset_compare(const void *x1, const void *x2);
 154 static void hsched_enqueue_io(struct hsfs *fsp, struct hio *hsio, int ra);
 155 int hsched_invoke_strategy(struct hsfs *fsp);
 156 
 157 /* ARGSUSED */
 158 static int
 159 hsfs_fsync(vnode_t *cp,
 160         int syncflag,
 161         cred_t *cred,
 162         caller_context_t *ct)
 163 {
 164         return (0);
 165 }
 166 
 167 
 168 /*ARGSUSED*/
 169 static int
 170 hsfs_read(struct vnode *vp,
 171         struct uio *uiop,
 172         int ioflag,
 173         struct cred *cred,
 174         struct caller_context *ct)
 175 {
 176         caddr_t base;
 177         offset_t diff;
 178         int error;
 179         struct hsnode *hp;
 180         uint_t filesize;
 181         int dofree;
 182 
 183         hp = VTOH(vp);
 184         /*
 185          * if vp is of type VDIR, make sure dirent
 186          * is filled up with all info (because of ptbl)
 187          */
 188         if (vp->v_type == VDIR) {
 189                 if (hp->hs_dirent.ext_size == 0)
 190                         hs_filldirent(vp, &hp->hs_dirent);
 191         }
 192         filesize = hp->hs_dirent.ext_size;
 193 
 194         /* Sanity checks. */
 195         if (uiop->uio_resid == 0 ||          /* No data wanted. */
 196             uiop->uio_loffset > HS_MAXFILEOFF ||  /* Offset too big. */
 197             uiop->uio_loffset >= filesize)        /* Past EOF. */
 198                 return (0);
 199 
 200         do {
 201                 /*
 202                  * We want to ask for only the "right" amount of data.
 203                  * In this case that means:-
 204                  *
 205                  * We can't get data from beyond our EOF. If asked,
 206                  * we will give a short read.
 207                  *
 208                  * segmap_getmapflt returns buffers of MAXBSIZE bytes.
 209                  * These buffers are always MAXBSIZE aligned.
 210                  * If our starting offset is not MAXBSIZE aligned,
 211                  * we can only ask for less than MAXBSIZE bytes.
 212                  *
 213                  * If our requested offset and length are such that
 214                  * they belong in different MAXBSIZE aligned slots
 215                  * then we'll be making more than one call on
 216                  * segmap_getmapflt.
 217                  *
 218                  * This diagram shows the variables we use and their
 219                  * relationships.
 220                  *
 221                  * |<-----MAXBSIZE----->|
 222                  * +--------------------------...+
 223                  * |.....mapon->|<--n-->|....*...|EOF
 224                  * +--------------------------...+
 225                  * uio_loffset->|
 226                  * uio_resid....|<---------->|
 227                  * diff.........|<-------------->|
 228                  *
 229                  * So, in this case our offset is not aligned
 230                  * and our request takes us outside of the
 231                  * MAXBSIZE window. We will break this up into
 232                  * two segmap_getmapflt calls.
 233                  */
 234                 size_t nbytes;
 235                 offset_t mapon;
 236                 size_t n;
 237                 uint_t flags;
 238 
 239                 mapon = uiop->uio_loffset & MAXBOFFSET;
 240                 diff = filesize - uiop->uio_loffset;
 241                 nbytes = (size_t)MIN(MAXBSIZE - mapon, uiop->uio_resid);
 242                 n = MIN(diff, nbytes);
 243                 if (n <= 0) {
 244                         /* EOF or request satisfied. */
 245                         return (0);
 246                 }
 247 
 248                 /*
 249                  * Freebehind computation taken from:
 250                  * usr/src/uts/common/fs/ufs/ufs_vnops.c
 251                  */
 252                 if (drv_hztousec(ddi_get_lbolt()) >= smallfile_update) {
 253                         uint64_t percpufreeb;
 254                         if (smallfile1_d == 0) smallfile1_d = SMALLFILE1_D;
 255                         if (smallfile2_d == 0) smallfile2_d = SMALLFILE2_D;
 256                         percpufreeb = ptob((uint64_t)freemem) / ncpus_online;
 257                         smallfile1 = percpufreeb / smallfile1_d;
 258                         smallfile2 = percpufreeb / smallfile2_d;
 259                         smallfile1 = MAX(smallfile1, smallfile);
 260                         smallfile1 = MAX(smallfile1, smallfile64);
 261                         smallfile2 = MAX(smallfile1, smallfile2);
 262                         smallfile_update = drv_hztousec(ddi_get_lbolt())
 263                             + 1000000;
 264                 }
 265 
 266                 dofree = freebehind &&
 267                     hp->hs_prev_offset == uiop->uio_loffset &&
 268                     hp->hs_ra_bytes > 0;
 269 
 270                 base = segmap_getmapflt(segkmap, vp,
 271                     (u_offset_t)uiop->uio_loffset, n, 1, S_READ);
 272 
 273                 error = uiomove(base + mapon, n, UIO_READ, uiop);
 274 
 275                 if (error == 0) {
 276                         /*
 277                          * if read a whole block, or read to eof,
 278                          *  won't need this buffer again soon.
 279                          */
 280                         if (n + mapon == MAXBSIZE ||
 281                             uiop->uio_loffset == filesize)
 282                                 flags = SM_DONTNEED;
 283                         else
 284                                 flags = 0;
 285 
 286                         if (dofree) {
 287                                 flags = SM_FREE | SM_ASYNC;
 288                                 if ((cache_read_ahead == 0) &&
 289                                     uiop->uio_loffset > smallfile2)
 290                                         flags |=  SM_DONTNEED;
 291                         }
 292 
 293                         error = segmap_release(segkmap, base, flags);
 294                 } else
 295                         (void) segmap_release(segkmap, base, 0);
 296         } while (error == 0 && uiop->uio_resid > 0);
 297 
 298         return (error);
 299 }
 300 
 301 /*ARGSUSED2*/
 302 static int
 303 hsfs_getattr(
 304         struct vnode *vp,
 305         struct vattr *vap,
 306         int flags,
 307         struct cred *cred,
 308         caller_context_t *ct)
 309 {
 310         struct hsnode *hp;
 311         struct vfs *vfsp;
 312         struct hsfs *fsp;
 313 
 314         hp = VTOH(vp);
 315         fsp = VFS_TO_HSFS(vp->v_vfsp);
 316         vfsp = vp->v_vfsp;
 317 
 318         if ((hp->hs_dirent.ext_size == 0) && (vp->v_type == VDIR)) {
 319                 hs_filldirent(vp, &hp->hs_dirent);
 320         }
 321         vap->va_type = IFTOVT(hp->hs_dirent.mode);
 322         vap->va_mode = hp->hs_dirent.mode;
 323         vap->va_uid = hp->hs_dirent.uid;
 324         vap->va_gid = hp->hs_dirent.gid;
 325 
 326         vap->va_fsid = vfsp->vfs_dev;
 327         vap->va_nodeid = (ino64_t)hp->hs_nodeid;
 328         vap->va_nlink = hp->hs_dirent.nlink;
 329         vap->va_size =       (offset_t)hp->hs_dirent.ext_size;
 330 
 331         vap->va_atime.tv_sec = hp->hs_dirent.adate.tv_sec;
 332         vap->va_atime.tv_nsec = hp->hs_dirent.adate.tv_usec*1000;
 333         vap->va_mtime.tv_sec = hp->hs_dirent.mdate.tv_sec;
 334         vap->va_mtime.tv_nsec = hp->hs_dirent.mdate.tv_usec*1000;
 335         vap->va_ctime.tv_sec = hp->hs_dirent.cdate.tv_sec;
 336         vap->va_ctime.tv_nsec = hp->hs_dirent.cdate.tv_usec*1000;
 337         if (vp->v_type == VCHR || vp->v_type == VBLK)
 338                 vap->va_rdev = hp->hs_dirent.r_dev;
 339         else
 340                 vap->va_rdev = 0;
 341         vap->va_blksize = vfsp->vfs_bsize;
 342         /* no. of blocks = no. of data blocks + no. of xar blocks */
 343         vap->va_nblocks = (fsblkcnt64_t)howmany(vap->va_size + (u_longlong_t)
 344             (hp->hs_dirent.xar_len << fsp->hsfs_vol.lbn_shift), DEV_BSIZE);
 345         vap->va_seq = hp->hs_seq;
 346         return (0);
 347 }
 348 
 349 /*ARGSUSED*/
 350 static int
 351 hsfs_readlink(struct vnode *vp,
 352         struct uio *uiop,
 353         struct cred *cred,
 354         caller_context_t *ct)
 355 {
 356         struct hsnode *hp;
 357 
 358         if (vp->v_type != VLNK)
 359                 return (EINVAL);
 360 
 361         hp = VTOH(vp);
 362 
 363         if (hp->hs_dirent.sym_link == (char *)NULL)
 364                 return (ENOENT);
 365 
 366         return (uiomove(hp->hs_dirent.sym_link,
 367             (size_t)MIN(hp->hs_dirent.ext_size,
 368             uiop->uio_resid), UIO_READ, uiop));
 369 }
 370 
 371 /*ARGSUSED*/
 372 static void
 373 hsfs_inactive(struct vnode *vp,
 374         struct cred *cred,
 375         caller_context_t *ct)
 376 {
 377         struct hsnode *hp;
 378         struct hsfs *fsp;
 379 
 380         int nopage;
 381 
 382         hp = VTOH(vp);
 383         fsp = VFS_TO_HSFS(vp->v_vfsp);
 384         /*
 385          * Note: acquiring and holding v_lock for quite a while
 386          * here serializes on the vnode; this is unfortunate, but
 387          * likely not to overly impact performance, as the underlying
 388          * device (CDROM drive) is quite slow.
 389          */
 390         rw_enter(&fsp->hsfs_hash_lock, RW_WRITER);
 391         mutex_enter(&hp->hs_contents_lock);
 392         mutex_enter(&vp->v_lock);
 393 
 394         if (vp->v_count < 1) {
 395                 panic("hsfs_inactive: v_count < 1");
 396                 /*NOTREACHED*/
 397         }
 398 
 399         if (vp->v_count > 1 || (hp->hs_flags & HREF) == 0) {
 400                 vp->v_count--;       /* release hold from vn_rele */
 401                 mutex_exit(&vp->v_lock);
 402                 mutex_exit(&hp->hs_contents_lock);
 403                 rw_exit(&fsp->hsfs_hash_lock);
 404                 return;
 405         }
 406         vp->v_count--;       /* release hold from vn_rele */
 407         if (vp->v_count == 0) {
 408                 /*
 409                  * Free the hsnode.
 410                  * If there are no pages associated with the
 411                  * hsnode, give it back to the kmem_cache,
 412                  * else put at the end of this file system's
 413                  * internal free list.
 414                  */
 415                 nopage = !vn_has_cached_data(vp);
 416                 hp->hs_flags = 0;
 417                 /*
 418                  * exit these locks now, since hs_freenode may
 419                  * kmem_free the hsnode and embedded vnode
 420                  */
 421                 mutex_exit(&vp->v_lock);
 422                 mutex_exit(&hp->hs_contents_lock);
 423                 hs_freenode(vp, fsp, nopage);
 424         } else {
 425                 mutex_exit(&vp->v_lock);
 426                 mutex_exit(&hp->hs_contents_lock);
 427         }
 428         rw_exit(&fsp->hsfs_hash_lock);
 429 }
 430 
 431 
 432 /*ARGSUSED*/
 433 static int
 434 hsfs_lookup(
 435         struct vnode *dvp,
 436         char *nm,
 437         struct vnode **vpp,
 438         struct pathname *pnp,
 439         int flags,
 440         struct vnode *rdir,
 441         struct cred *cred,
 442         caller_context_t *ct,
 443         int *direntflags,
 444         pathname_t *realpnp)
 445 {
 446         int error;
 447         int namelen = (int)strlen(nm);
 448 
 449         if (*nm == '\0') {
 450                 VN_HOLD(dvp);
 451                 *vpp = dvp;
 452                 return (0);
 453         }
 454 
 455         /*
 456          * If we're looking for ourself, life is simple.
 457          */
 458         if (namelen == 1 && *nm == '.') {
 459                 if (error = hs_access(dvp, (mode_t)VEXEC, cred))
 460                         return (error);
 461                 VN_HOLD(dvp);
 462                 *vpp = dvp;
 463                 return (0);
 464         }
 465 
 466         return (hs_dirlook(dvp, nm, namelen, vpp, cred));
 467 }
 468 
 469 
 470 /*ARGSUSED*/
 471 static int
 472 hsfs_readdir(
 473         struct vnode            *vp,
 474         struct uio              *uiop,
 475         struct cred             *cred,
 476         int                     *eofp,
 477         caller_context_t        *ct,
 478         int                     flags)
 479 {
 480         struct hsnode   *dhp;
 481         struct hsfs     *fsp;
 482         struct hs_direntry hd;
 483         struct dirent64 *nd;
 484         int             error;
 485         uint_t          offset;         /* real offset in directory */
 486         uint_t          dirsiz;         /* real size of directory */
 487         uchar_t         *blkp;
 488         int             hdlen;          /* length of hs directory entry */
 489         long            ndlen;          /* length of dirent entry */
 490         int             bytes_wanted;
 491         size_t          bufsize;        /* size of dirent buffer */
 492         char            *outbuf;        /* ptr to dirent buffer */
 493         char            *dname;
 494         int             dnamelen;
 495         size_t          dname_size;
 496         struct fbuf     *fbp;
 497         uint_t          last_offset;    /* last index into current dir block */
 498         ino64_t         dirino; /* temporary storage before storing in dirent */
 499         off_t           diroff;
 500 
 501         dhp = VTOH(vp);
 502         fsp = VFS_TO_HSFS(vp->v_vfsp);
 503         if (dhp->hs_dirent.ext_size == 0)
 504                 hs_filldirent(vp, &dhp->hs_dirent);
 505         dirsiz = dhp->hs_dirent.ext_size;
 506         if (uiop->uio_loffset >= dirsiz) {        /* at or beyond EOF */
 507                 if (eofp)
 508                         *eofp = 1;
 509                 return (0);
 510         }
 511         ASSERT(uiop->uio_loffset <= HS_MAXFILEOFF);
 512         offset = uiop->uio_loffset;
 513 
 514         dname_size = fsp->hsfs_namemax + 1;  /* 1 for the ending NUL */
 515         dname = kmem_alloc(dname_size, KM_SLEEP);
 516         bufsize = uiop->uio_resid + sizeof (struct dirent64);
 517 
 518         outbuf = kmem_alloc(bufsize, KM_SLEEP);
 519         nd = (struct dirent64 *)outbuf;
 520 
 521         while (offset < dirsiz) {
 522                 bytes_wanted = MIN(MAXBSIZE, dirsiz - (offset & MAXBMASK));
 523 
 524                 error = fbread(vp, (offset_t)(offset & MAXBMASK),
 525                     (unsigned int)bytes_wanted, S_READ, &fbp);
 526                 if (error)
 527                         goto done;
 528 
 529                 blkp = (uchar_t *)fbp->fb_addr;
 530                 last_offset = (offset & MAXBMASK) + fbp->fb_count;
 531 
 532 #define rel_offset(offset) ((offset) & MAXBOFFSET)  /* index into blkp */
 533 
 534                 while (offset < last_offset) {
 535                         /*
 536                          * Very similar validation code is found in
 537                          * process_dirblock(), hsfs_node.c.
 538                          * For an explanation, see there.
 539                          * It may make sense for the future to
 540                          * "consolidate" the code in hs_parsedir(),
 541                          * process_dirblock() and hsfs_readdir() into
 542                          * a single utility function.
 543                          */
 544                         hdlen = (int)((uchar_t)
 545                             HDE_DIR_LEN(&blkp[rel_offset(offset)]));
 546                         if (hdlen < HDE_ROOT_DIR_REC_SIZE ||
 547                             offset + hdlen > last_offset) {
 548                                 /*
 549                                  * advance to next sector boundary
 550                                  */
 551                                 offset = roundup(offset + 1, HS_SECTOR_SIZE);
 552                                 if (hdlen)
 553                                         hs_log_bogus_disk_warning(fsp,
 554                                             HSFS_ERR_TRAILING_JUNK, 0);
 555 
 556                                 continue;
 557                         }
 558 
 559                         bzero(&hd, sizeof (hd));
 560 
 561                         /*
 562                          * Just ignore invalid directory entries.
 563                          * XXX - maybe hs_parsedir() will detect EXISTENCE bit
 564                          */
 565                         if (!hs_parsedir(fsp, &blkp[rel_offset(offset)],
 566                             &hd, dname, &dnamelen, last_offset - offset)) {
 567                                 /*
 568                                  * Determine if there is enough room
 569                                  */
 570                                 ndlen = (long)DIRENT64_RECLEN((dnamelen));
 571 
 572                                 if ((ndlen + ((char *)nd - outbuf)) >
 573                                     uiop->uio_resid) {
 574                                         fbrelse(fbp, S_READ);
 575                                         goto done; /* output buffer full */
 576                                 }
 577 
 578                                 diroff = offset + hdlen;
 579                                 /*
 580                                  * If the media carries rrip-v1.12 or newer,
 581                                  * and we trust the inodes from the rrip data
 582                                  * (use_rrip_inodes != 0), use that data. If the
 583                                  * media has been created by a recent mkisofs
 584                                  * version, we may trust all numbers in the
 585                                  * starting extent number; otherwise, we cannot
 586                                  * do this for zero sized files and symlinks,
 587                                  * because if we did we'd end up mapping all of
 588                                  * them to the same node. We use HS_DUMMY_INO
 589                                  * in this case and make sure that we will not
 590                                  * map all files to the same meta data.
 591                                  */
 592                                 if (hd.inode != 0 && use_rrip_inodes) {
 593                                         dirino = hd.inode;
 594                                 } else if ((hd.ext_size == 0 ||
 595                                     hd.sym_link != (char *)NULL) &&
 596                                     (fsp->hsfs_flags & HSFSMNT_INODE) == 0) {
 597                                         dirino = HS_DUMMY_INO;
 598                                 } else {
 599                                         dirino = hd.ext_lbn;
 600                                 }
 601 
 602                                 /* strncpy(9f) will zero uninitialized bytes */
 603 
 604                                 ASSERT(strlen(dname) + 1 <=
 605                                     DIRENT64_NAMELEN(ndlen));
 606                                 (void) strncpy(nd->d_name, dname,
 607                                     DIRENT64_NAMELEN(ndlen));
 608                                 nd->d_reclen = (ushort_t)ndlen;
 609                                 nd->d_off = (offset_t)diroff;
 610                                 nd->d_ino = dirino;
 611                                 nd = (struct dirent64 *)((char *)nd + ndlen);
 612 
 613                                 /*
 614                                  * free up space allocated for symlink
 615                                  */
 616                                 if (hd.sym_link != (char *)NULL) {
 617                                         kmem_free(hd.sym_link,
 618                                             (size_t)(hd.ext_size+1));
 619                                         hd.sym_link = (char *)NULL;
 620                                 }
 621                         }
 622                         offset += hdlen;
 623                 }
 624                 fbrelse(fbp, S_READ);
 625         }
 626 
 627         /*
 628          * Got here for one of the following reasons:
 629          *      1) outbuf is full (error == 0)
 630          *      2) end of directory reached (error == 0)
 631          *      3) error reading directory sector (error != 0)
 632          *      4) directory entry crosses sector boundary (error == 0)
 633          *
 634          * If any directory entries have been copied, don't report
 635          * case 4.  Instead, return the valid directory entries.
 636          *
 637          * If no entries have been copied, report the error.
 638          * If case 4, this will be indistiguishable from EOF.
 639          */
 640 done:
 641         ndlen = ((char *)nd - outbuf);
 642         if (ndlen != 0) {
 643                 error = uiomove(outbuf, (size_t)ndlen, UIO_READ, uiop);
 644                 uiop->uio_loffset = offset;
 645         }
 646         kmem_free(dname, dname_size);
 647         kmem_free(outbuf, bufsize);
 648         if (eofp && error == 0)
 649                 *eofp = (uiop->uio_loffset >= dirsiz);
 650         return (error);
 651 }
 652 
 653 /*ARGSUSED2*/
 654 static int
 655 hsfs_fid(struct vnode *vp, struct fid *fidp, caller_context_t *ct)
 656 {
 657         struct hsnode *hp;
 658         struct hsfid *fid;
 659 
 660         if (fidp->fid_len < (sizeof (*fid) - sizeof (fid->hf_len))) {
 661                 fidp->fid_len = sizeof (*fid) - sizeof (fid->hf_len);
 662                 return (ENOSPC);
 663         }
 664 
 665         fid = (struct hsfid *)fidp;
 666         fid->hf_len = sizeof (*fid) - sizeof (fid->hf_len);
 667         hp = VTOH(vp);
 668         mutex_enter(&hp->hs_contents_lock);
 669         fid->hf_dir_lbn = hp->hs_dir_lbn;
 670         fid->hf_dir_off = (ushort_t)hp->hs_dir_off;
 671         fid->hf_ino = hp->hs_nodeid;
 672         mutex_exit(&hp->hs_contents_lock);
 673         return (0);
 674 }
 675 
 676 /*ARGSUSED*/
 677 static int
 678 hsfs_open(struct vnode **vpp,
 679         int flag,
 680         struct cred *cred,
 681         caller_context_t *ct)
 682 {
 683         return (0);
 684 }
 685 
 686 /*ARGSUSED*/
 687 static int
 688 hsfs_close(
 689         struct vnode *vp,
 690         int flag,
 691         int count,
 692         offset_t offset,
 693         struct cred *cred,
 694         caller_context_t *ct)
 695 {
 696         (void) cleanlocks(vp, ttoproc(curthread)->p_pid, 0);
 697         cleanshares(vp, ttoproc(curthread)->p_pid);
 698         return (0);
 699 }
 700 
 701 /*ARGSUSED2*/
 702 static int
 703 hsfs_access(struct vnode *vp,
 704         int mode,
 705         int flags,
 706         cred_t *cred,
 707         caller_context_t *ct)
 708 {
 709         return (hs_access(vp, (mode_t)mode, cred));
 710 }
 711 
 712 /*
 713  * the seek time of a CD-ROM is very slow, and data transfer
 714  * rate is even worse (max. 150K per sec).  The design
 715  * decision is to reduce access to cd-rom as much as possible,
 716  * and to transfer a sizable block (read-ahead) of data at a time.
 717  * UFS style of read ahead one block at a time is not appropriate,
 718  * and is not supported
 719  */
 720 
 721 /*
 722  * KLUSTSIZE should be a multiple of PAGESIZE and <= MAXPHYS.
 723  */
 724 #define KLUSTSIZE       (56 * 1024)
 725 /* we don't support read ahead */
 726 int hsfs_lostpage;      /* no. of times we lost original page */
 727 
 728 /*
 729  * Used to prevent biodone() from releasing buf resources that
 730  * we didn't allocate in quite the usual way.
 731  */
 732 /*ARGSUSED*/
 733 int
 734 hsfs_iodone(struct buf *bp)
 735 {
 736         sema_v(&bp->b_io);
 737         return (0);
 738 }
 739 
 740 /*
 741  * The taskq thread that invokes the scheduling function to ensure
 742  * that all readaheads are complete and cleans up the associated
 743  * memory and releases the page lock.
 744  */
 745 void
 746 hsfs_ra_task(void *arg)
 747 {
 748         struct hio_info *info = arg;
 749         uint_t count;
 750         struct buf *wbuf;
 751 
 752         ASSERT(info->pp != NULL);
 753 
 754         for (count = 0; count < info->bufsused; count++) {
 755                 wbuf = &(info->bufs[count]);
 756 
 757                 DTRACE_PROBE1(hsfs_io_wait_ra, struct buf *, wbuf);
 758                 while (sema_tryp(&(info->sema[count])) == 0) {
 759                         if (hsched_invoke_strategy(info->fsp)) {
 760                                 sema_p(&(info->sema[count]));
 761                                 break;
 762                         }
 763                 }
 764                 sema_destroy(&(info->sema[count]));
 765                 DTRACE_PROBE1(hsfs_io_done_ra, struct buf *, wbuf);
 766                 biofini(&(info->bufs[count]));
 767         }
 768         for (count = 0; count < info->bufsused; count++) {
 769                 if (info->vas[count] != NULL) {
 770                         ppmapout(info->vas[count]);
 771                 }
 772         }
 773         kmem_free(info->vas, info->bufcnt * sizeof (caddr_t));
 774         kmem_free(info->bufs, info->bufcnt * sizeof (struct buf));
 775         kmem_free(info->sema, info->bufcnt * sizeof (ksema_t));
 776 
 777         pvn_read_done(info->pp, 0);
 778         kmem_cache_free(hio_info_cache, info);
 779 }
 780 
 781 /*
 782  * Submit asynchronous readahead requests to the I/O scheduler
 783  * depending on the number of pages to read ahead. These requests
 784  * are asynchronous to the calling thread but I/O requests issued
 785  * subsequently by other threads with higher LBNs must wait for
 786  * these readaheads to complete since we have a single ordered
 787  * I/O pipeline. Thus these readaheads are semi-asynchronous.
 788  * A TaskQ handles waiting for the readaheads to complete.
 789  *
 790  * This function is mostly a copy of hsfs_getapage but somewhat
 791  * simpler. A readahead request is aborted if page allocation
 792  * fails.
 793  */
 794 /*ARGSUSED*/
 795 static int
 796 hsfs_getpage_ra(
 797         struct vnode *vp,
 798         u_offset_t off,
 799         struct seg *seg,
 800         caddr_t addr,
 801         struct hsnode *hp,
 802         struct hsfs *fsp,
 803         int     xarsiz,
 804         offset_t        bof,
 805         int     chunk_lbn_count,
 806         int     chunk_data_bytes)
 807 {
 808         struct buf *bufs;
 809         caddr_t *vas;
 810         caddr_t va;
 811         struct page *pp, *searchp, *lastp;
 812         struct vnode *devvp;
 813         ulong_t byte_offset;
 814         size_t  io_len_tmp;
 815         uint_t  io_off, io_len;
 816         uint_t  xlen;
 817         uint_t  filsiz;
 818         uint_t  secsize;
 819         uint_t  bufcnt;
 820         uint_t  bufsused;
 821         uint_t  count;
 822         uint_t  io_end;
 823         uint_t  which_chunk_lbn;
 824         uint_t  offset_lbn;
 825         uint_t  offset_extra;
 826         offset_t        offset_bytes;
 827         uint_t  remaining_bytes;
 828         uint_t  extension;
 829         int     remainder;      /* must be signed */
 830         diskaddr_t driver_block;
 831         u_offset_t io_off_tmp;
 832         ksema_t *fio_done;
 833         struct hio_info *info;
 834         size_t len;
 835 
 836         ASSERT(fsp->hqueue != NULL);
 837 
 838         if (addr >= seg->s_base + seg->s_size) {
 839                 return (-1);
 840         }
 841 
 842         devvp = fsp->hsfs_devvp;
 843         secsize = fsp->hsfs_vol.lbn_size;  /* bytes per logical block */
 844 
 845         /* file data size */
 846         filsiz = hp->hs_dirent.ext_size;
 847 
 848         if (off >= filsiz)
 849                 return (0);
 850 
 851         extension = 0;
 852         pp = NULL;
 853 
 854         extension += hp->hs_ra_bytes;
 855 
 856         /*
 857          * Some CD writers (e.g. Kodak Photo CD writers)
 858          * create CDs in TAO mode and reserve tracks that
 859          * are not completely written. Some sectors remain
 860          * unreadable for this reason and give I/O errors.
 861          * Also, there's no point in reading sectors
 862          * we'll never look at.  So, if we're asked to go
 863          * beyond the end of a file, truncate to the length
 864          * of that file.
 865          *
 866          * Additionally, this behaviour is required by section
 867          * 6.4.5 of ISO 9660:1988(E).
 868          */
 869         len = MIN(extension ? extension : PAGESIZE, filsiz - off);
 870 
 871         /* A little paranoia */
 872         if (len <= 0)
 873                 return (-1);
 874 
 875         /*
 876          * After all that, make sure we're asking for things in units
 877          * that bdev_strategy() will understand (see bug 4202551).
 878          */
 879         len = roundup(len, DEV_BSIZE);
 880 
 881         pp = pvn_read_kluster(vp, off, seg, addr, &io_off_tmp,
 882             &io_len_tmp, off, len, 1);
 883 
 884         if (pp == NULL) {
 885                 hp->hs_num_contig = 0;
 886                 hp->hs_ra_bytes = 0;
 887                 hp->hs_prev_offset = 0;
 888                 return (-1);
 889         }
 890 
 891         io_off = (uint_t)io_off_tmp;
 892         io_len = (uint_t)io_len_tmp;
 893 
 894         /* check for truncation */
 895         /*
 896          * xxx Clean up and return EIO instead?
 897          * xxx Ought to go to u_offset_t for everything, but we
 898          * xxx call lots of things that want uint_t arguments.
 899          */
 900         ASSERT(io_off == io_off_tmp);
 901 
 902         /*
 903          * get enough buffers for worst-case scenario
 904          * (i.e., no coalescing possible).
 905          */
 906         bufcnt = (len + secsize - 1) / secsize;
 907         bufs = kmem_alloc(bufcnt * sizeof (struct buf), KM_SLEEP);
 908         vas = kmem_alloc(bufcnt * sizeof (caddr_t), KM_SLEEP);
 909 
 910         /*
 911          * Allocate a array of semaphores since we are doing I/O
 912          * scheduling.
 913          */
 914         fio_done = kmem_alloc(bufcnt * sizeof (ksema_t), KM_SLEEP);
 915 
 916         /*
 917          * If our filesize is not an integer multiple of PAGESIZE,
 918          * we zero that part of the last page that's between EOF and
 919          * the PAGESIZE boundary.
 920          */
 921         xlen = io_len & PAGEOFFSET;
 922         if (xlen != 0)
 923                 pagezero(pp->p_prev, xlen, PAGESIZE - xlen);
 924 
 925         DTRACE_PROBE2(hsfs_readahead, struct vnode *, vp, uint_t, io_len);
 926 
 927         va = NULL;
 928         lastp = NULL;
 929         searchp = pp;
 930         io_end = io_off + io_len;
 931         for (count = 0, byte_offset = io_off;
 932             byte_offset < io_end;
 933             count++) {
 934                 ASSERT(count < bufcnt);
 935 
 936                 bioinit(&bufs[count]);
 937                 bufs[count].b_edev = devvp->v_rdev;
 938                 bufs[count].b_dev = cmpdev(devvp->v_rdev);
 939                 bufs[count].b_flags = B_NOCACHE|B_BUSY|B_READ;
 940                 bufs[count].b_iodone = hsfs_iodone;
 941                 bufs[count].b_vp = vp;
 942                 bufs[count].b_file = vp;
 943 
 944                 /* Compute disk address for interleaving. */
 945 
 946                 /* considered without skips */
 947                 which_chunk_lbn = byte_offset / chunk_data_bytes;
 948 
 949                 /* factor in skips */
 950                 offset_lbn = which_chunk_lbn * chunk_lbn_count;
 951 
 952                 /* convert to physical byte offset for lbn */
 953                 offset_bytes = LBN_TO_BYTE(offset_lbn, vp->v_vfsp);
 954 
 955                 /* don't forget offset into lbn */
 956                 offset_extra = byte_offset % chunk_data_bytes;
 957 
 958                 /* get virtual block number for driver */
 959                 driver_block = lbtodb(bof + xarsiz
 960                     + offset_bytes + offset_extra);
 961 
 962                 if (lastp != searchp) {
 963                         /* this branch taken first time through loop */
 964                         va = vas[count] = ppmapin(searchp, PROT_WRITE,
 965                             (caddr_t)-1);
 966                         /* ppmapin() guarantees not to return NULL */
 967                 } else {
 968                         vas[count] = NULL;
 969                 }
 970 
 971                 bufs[count].b_un.b_addr = va + byte_offset % PAGESIZE;
 972                 bufs[count].b_offset =
 973                     (offset_t)(byte_offset - io_off + off);
 974 
 975                 /*
 976                  * We specifically use the b_lblkno member here
 977                  * as even in the 32 bit world driver_block can
 978                  * get very large in line with the ISO9660 spec.
 979                  */
 980 
 981                 bufs[count].b_lblkno = driver_block;
 982 
 983                 remaining_bytes = ((which_chunk_lbn + 1) * chunk_data_bytes)
 984                     - byte_offset;
 985 
 986                 /*
 987                  * remaining_bytes can't be zero, as we derived
 988                  * which_chunk_lbn directly from byte_offset.
 989                  */
 990                 if ((remaining_bytes + byte_offset) < (off + len)) {
 991                         /* coalesce-read the rest of the chunk */
 992                         bufs[count].b_bcount = remaining_bytes;
 993                 } else {
 994                         /* get the final bits */
 995                         bufs[count].b_bcount = off + len - byte_offset;
 996                 }
 997 
 998                 remainder = PAGESIZE - (byte_offset % PAGESIZE);
 999                 if (bufs[count].b_bcount > remainder) {
1000                         bufs[count].b_bcount = remainder;
1001                 }
1002 
1003                 bufs[count].b_bufsize = bufs[count].b_bcount;
1004                 if (((offset_t)byte_offset + bufs[count].b_bcount) >
1005                     HS_MAXFILEOFF) {
1006                         break;
1007                 }
1008                 byte_offset += bufs[count].b_bcount;
1009 
1010                 /*
1011                  * We are scheduling I/O so we need to enqueue
1012                  * requests rather than calling bdev_strategy
1013                  * here. A later invocation of the scheduling
1014                  * function will take care of doing the actual
1015                  * I/O as it selects requests from the queue as
1016                  * per the scheduling logic.
1017                  */
1018                 struct hio *hsio = kmem_cache_alloc(hio_cache,
1019                     KM_SLEEP);
1020 
1021                 sema_init(&fio_done[count], 0, NULL,
1022                     SEMA_DEFAULT, NULL);
1023                 hsio->bp = &bufs[count];
1024                 hsio->sema = &fio_done[count];
1025                 hsio->io_lblkno = bufs[count].b_lblkno;
1026                 hsio->nblocks = howmany(hsio->bp->b_bcount,
1027                     DEV_BSIZE);
1028 
1029                 /* used for deadline */
1030                 hsio->io_timestamp = drv_hztousec(ddi_get_lbolt());
1031 
1032                 /* for I/O coalescing */
1033                 hsio->contig_chain = NULL;
1034                 hsched_enqueue_io(fsp, hsio, 1);
1035 
1036                 lwp_stat_update(LWP_STAT_INBLK, 1);
1037                 lastp = searchp;
1038                 if ((remainder - bufs[count].b_bcount) < 1) {
1039                         searchp = searchp->p_next;
1040                 }
1041         }
1042 
1043         bufsused = count;
1044         info = kmem_cache_alloc(hio_info_cache, KM_SLEEP);
1045         info->bufs = bufs;
1046         info->vas = vas;
1047         info->sema = fio_done;
1048         info->bufsused = bufsused;
1049         info->bufcnt = bufcnt;
1050         info->fsp = fsp;
1051         info->pp = pp;
1052 
1053         (void) taskq_dispatch(fsp->hqueue->ra_task,
1054             hsfs_ra_task, info, KM_SLEEP);
1055         /*
1056          * The I/O locked pages are unlocked in our taskq thread.
1057          */
1058         return (0);
1059 }
1060 
1061 /*
1062  * Each file may have a different interleaving on disk.  This makes
1063  * things somewhat interesting.  The gist is that there are some
1064  * number of contiguous data sectors, followed by some other number
1065  * of contiguous skip sectors.  The sum of those two sets of sectors
1066  * defines the interleave size.  Unfortunately, it means that we generally
1067  * can't simply read N sectors starting at a given offset to satisfy
1068  * any given request.
1069  *
1070  * What we do is get the relevant memory pages via pvn_read_kluster(),
1071  * then stride through the interleaves, setting up a buf for each
1072  * sector that needs to be brought in.  Instead of kmem_alloc'ing
1073  * space for the sectors, though, we just point at the appropriate
1074  * spot in the relevant page for each of them.  This saves us a bunch
1075  * of copying.
1076  *
1077  * NOTICE: The code below in hsfs_getapage is mostly same as the code
1078  *         in hsfs_getpage_ra above (with some omissions). If you are
1079  *         making any change to this function, please also look at
1080  *         hsfs_getpage_ra.
1081  */
1082 /*ARGSUSED*/
1083 static int
1084 hsfs_getapage(
1085         struct vnode *vp,
1086         u_offset_t off,
1087         size_t len,
1088         uint_t *protp,
1089         struct page *pl[],
1090         size_t plsz,
1091         struct seg *seg,
1092         caddr_t addr,
1093         enum seg_rw rw,
1094         struct cred *cred)
1095 {
1096         struct hsnode *hp;
1097         struct hsfs *fsp;
1098         int     err;
1099         struct buf *bufs;
1100         caddr_t *vas;
1101         caddr_t va;
1102         struct page *pp, *searchp, *lastp;
1103         page_t  *pagefound;
1104         offset_t        bof;
1105         struct vnode *devvp;
1106         ulong_t byte_offset;
1107         size_t  io_len_tmp;
1108         uint_t  io_off, io_len;
1109         uint_t  xlen;
1110         uint_t  filsiz;
1111         uint_t  secsize;
1112         uint_t  bufcnt;
1113         uint_t  bufsused;
1114         uint_t  count;
1115         uint_t  io_end;
1116         uint_t  which_chunk_lbn;
1117         uint_t  offset_lbn;
1118         uint_t  offset_extra;
1119         offset_t        offset_bytes;
1120         uint_t  remaining_bytes;
1121         uint_t  extension;
1122         int     remainder;      /* must be signed */
1123         int     chunk_lbn_count;
1124         int     chunk_data_bytes;
1125         int     xarsiz;
1126         diskaddr_t driver_block;
1127         u_offset_t io_off_tmp;
1128         ksema_t *fio_done;
1129         int     calcdone;
1130 
1131         /*
1132          * We don't support asynchronous operation at the moment, so
1133          * just pretend we did it.  If the pages are ever actually
1134          * needed, they'll get brought in then.
1135          */
1136         if (pl == NULL)
1137                 return (0);
1138 
1139         hp = VTOH(vp);
1140         fsp = VFS_TO_HSFS(vp->v_vfsp);
1141         devvp = fsp->hsfs_devvp;
1142         secsize = fsp->hsfs_vol.lbn_size;  /* bytes per logical block */
1143 
1144         /* file data size */
1145         filsiz = hp->hs_dirent.ext_size;
1146 
1147         /* disk addr for start of file */
1148         bof = LBN_TO_BYTE((offset_t)hp->hs_dirent.ext_lbn, vp->v_vfsp);
1149 
1150         /* xarsiz byte must be skipped for data */
1151         xarsiz = hp->hs_dirent.xar_len << fsp->hsfs_vol.lbn_shift;
1152 
1153         /* how many logical blocks in an interleave (data+skip) */
1154         chunk_lbn_count = hp->hs_dirent.intlf_sz + hp->hs_dirent.intlf_sk;
1155 
1156         if (chunk_lbn_count == 0) {
1157                 chunk_lbn_count = 1;
1158         }
1159 
1160         /*
1161          * Convert interleaving size into bytes.  The zero case
1162          * (no interleaving) optimization is handled as a side-
1163          * effect of the read-ahead logic.
1164          */
1165         if (hp->hs_dirent.intlf_sz == 0) {
1166                 chunk_data_bytes = LBN_TO_BYTE(1, vp->v_vfsp);
1167                 /*
1168                  * Optimization: If our pagesize is a multiple of LBN
1169                  * bytes, we can avoid breaking up a page into individual
1170                  * lbn-sized requests.
1171                  */
1172                 if (PAGESIZE % chunk_data_bytes == 0) {
1173                         chunk_lbn_count = BYTE_TO_LBN(PAGESIZE, vp->v_vfsp);
1174                         chunk_data_bytes = PAGESIZE;
1175                 }
1176         } else {
1177                 chunk_data_bytes =
1178                     LBN_TO_BYTE(hp->hs_dirent.intlf_sz, vp->v_vfsp);
1179         }
1180 
1181 reread:
1182         err = 0;
1183         pagefound = 0;
1184         calcdone = 0;
1185 
1186         /*
1187          * Do some read-ahead.  This mostly saves us a bit of
1188          * system cpu time more than anything else when doing
1189          * sequential reads.  At some point, could do the
1190          * read-ahead asynchronously which might gain us something
1191          * on wall time, but it seems unlikely....
1192          *
1193          * We do the easy case here, which is to read through
1194          * the end of the chunk, minus whatever's at the end that
1195          * won't exactly fill a page.
1196          */
1197         if (hp->hs_ra_bytes > 0 && chunk_data_bytes != PAGESIZE) {
1198                 which_chunk_lbn = (off + len) / chunk_data_bytes;
1199                 extension = ((which_chunk_lbn + 1) * chunk_data_bytes) - off;
1200                 extension -= (extension % PAGESIZE);
1201         } else {
1202                 extension = roundup(len, PAGESIZE);
1203         }
1204 
1205         atomic_inc_64(&fsp->total_pages_requested);
1206 
1207         pp = NULL;
1208 again:
1209         /* search for page in buffer */
1210         if ((pagefound = page_exists(vp, off)) == 0) {
1211                 /*
1212                  * Need to really do disk IO to get the page.
1213                  */
1214                 if (!calcdone) {
1215                         extension += hp->hs_ra_bytes;
1216 
1217                         /*
1218                          * Some cd writers don't write sectors that aren't
1219                          * used. Also, there's no point in reading sectors
1220                          * we'll never look at.  So, if we're asked to go
1221                          * beyond the end of a file, truncate to the length
1222                          * of that file.
1223                          *
1224                          * Additionally, this behaviour is required by section
1225                          * 6.4.5 of ISO 9660:1988(E).
1226                          */
1227                         len = MIN(extension ? extension : PAGESIZE,
1228                             filsiz - off);
1229 
1230                         /* A little paranoia. */
1231                         ASSERT(len > 0);
1232 
1233                         /*
1234                          * After all that, make sure we're asking for things
1235                          * in units that bdev_strategy() will understand
1236                          * (see bug 4202551).
1237                          */
1238                         len = roundup(len, DEV_BSIZE);
1239                         calcdone = 1;
1240                 }
1241 
1242                 pp = pvn_read_kluster(vp, off, seg, addr, &io_off_tmp,
1243                     &io_len_tmp, off, len, 0);
1244 
1245                 if (pp == NULL) {
1246                         /*
1247                          * Pressure on memory, roll back readahead
1248                          */
1249                         hp->hs_num_contig = 0;
1250                         hp->hs_ra_bytes = 0;
1251                         hp->hs_prev_offset = 0;
1252                         goto again;
1253                 }
1254 
1255                 io_off = (uint_t)io_off_tmp;
1256                 io_len = (uint_t)io_len_tmp;
1257 
1258                 /* check for truncation */
1259                 /*
1260                  * xxx Clean up and return EIO instead?
1261                  * xxx Ought to go to u_offset_t for everything, but we
1262                  * xxx call lots of things that want uint_t arguments.
1263                  */
1264                 ASSERT(io_off == io_off_tmp);
1265 
1266                 /*
1267                  * get enough buffers for worst-case scenario
1268                  * (i.e., no coalescing possible).
1269                  */
1270                 bufcnt = (len + secsize - 1) / secsize;
1271                 bufs = kmem_zalloc(bufcnt * sizeof (struct buf), KM_SLEEP);
1272                 vas = kmem_alloc(bufcnt * sizeof (caddr_t), KM_SLEEP);
1273 
1274                 /*
1275                  * Allocate a array of semaphores if we are doing I/O
1276                  * scheduling.
1277                  */
1278                 if (fsp->hqueue != NULL)
1279                         fio_done = kmem_alloc(bufcnt * sizeof (ksema_t),
1280                             KM_SLEEP);
1281                 for (count = 0; count < bufcnt; count++) {
1282                         bioinit(&bufs[count]);
1283                         bufs[count].b_edev = devvp->v_rdev;
1284                         bufs[count].b_dev = cmpdev(devvp->v_rdev);
1285                         bufs[count].b_flags = B_NOCACHE|B_BUSY|B_READ;
1286                         bufs[count].b_iodone = hsfs_iodone;
1287                         bufs[count].b_vp = vp;
1288                         bufs[count].b_file = vp;
1289                 }
1290 
1291                 /*
1292                  * If our filesize is not an integer multiple of PAGESIZE,
1293                  * we zero that part of the last page that's between EOF and
1294                  * the PAGESIZE boundary.
1295                  */
1296                 xlen = io_len & PAGEOFFSET;
1297                 if (xlen != 0)
1298                         pagezero(pp->p_prev, xlen, PAGESIZE - xlen);
1299 
1300                 va = NULL;
1301                 lastp = NULL;
1302                 searchp = pp;
1303                 io_end = io_off + io_len;
1304                 for (count = 0, byte_offset = io_off;
1305                     byte_offset < io_end; count++) {
1306                         ASSERT(count < bufcnt);
1307 
1308                         /* Compute disk address for interleaving. */
1309 
1310                         /* considered without skips */
1311                         which_chunk_lbn = byte_offset / chunk_data_bytes;
1312 
1313                         /* factor in skips */
1314                         offset_lbn = which_chunk_lbn * chunk_lbn_count;
1315 
1316                         /* convert to physical byte offset for lbn */
1317                         offset_bytes = LBN_TO_BYTE(offset_lbn, vp->v_vfsp);
1318 
1319                         /* don't forget offset into lbn */
1320                         offset_extra = byte_offset % chunk_data_bytes;
1321 
1322                         /* get virtual block number for driver */
1323                         driver_block =
1324                             lbtodb(bof + xarsiz + offset_bytes + offset_extra);
1325 
1326                         if (lastp != searchp) {
1327                                 /* this branch taken first time through loop */
1328                                 va = vas[count] =
1329                                     ppmapin(searchp, PROT_WRITE, (caddr_t)-1);
1330                                 /* ppmapin() guarantees not to return NULL */
1331                         } else {
1332                                 vas[count] = NULL;
1333                         }
1334 
1335                         bufs[count].b_un.b_addr = va + byte_offset % PAGESIZE;
1336                         bufs[count].b_offset =
1337                             (offset_t)(byte_offset - io_off + off);
1338 
1339                         /*
1340                          * We specifically use the b_lblkno member here
1341                          * as even in the 32 bit world driver_block can
1342                          * get very large in line with the ISO9660 spec.
1343                          */
1344 
1345                         bufs[count].b_lblkno = driver_block;
1346 
1347                         remaining_bytes =
1348                             ((which_chunk_lbn + 1) * chunk_data_bytes)
1349                             - byte_offset;
1350 
1351                         /*
1352                          * remaining_bytes can't be zero, as we derived
1353                          * which_chunk_lbn directly from byte_offset.
1354                          */
1355                         if ((remaining_bytes + byte_offset) < (off + len)) {
1356                                 /* coalesce-read the rest of the chunk */
1357                                 bufs[count].b_bcount = remaining_bytes;
1358                         } else {
1359                                 /* get the final bits */
1360                                 bufs[count].b_bcount = off + len - byte_offset;
1361                         }
1362 
1363                         /*
1364                          * It would be nice to do multiple pages'
1365                          * worth at once here when the opportunity
1366                          * arises, as that has been shown to improve
1367                          * our wall time.  However, to do that
1368                          * requires that we use the pageio subsystem,
1369                          * which doesn't mix well with what we're
1370                          * already using here.  We can't use pageio
1371                          * all the time, because that subsystem
1372                          * assumes that a page is stored in N
1373                          * contiguous blocks on the device.
1374                          * Interleaving violates that assumption.
1375                          *
1376                          * Update: This is now not so big a problem
1377                          * because of the I/O scheduler sitting below
1378                          * that can re-order and coalesce I/O requests.
1379                          */
1380 
1381                         remainder = PAGESIZE - (byte_offset % PAGESIZE);
1382                         if (bufs[count].b_bcount > remainder) {
1383                                 bufs[count].b_bcount = remainder;
1384                         }
1385 
1386                         bufs[count].b_bufsize = bufs[count].b_bcount;
1387                         if (((offset_t)byte_offset + bufs[count].b_bcount) >
1388                             HS_MAXFILEOFF) {
1389                                 break;
1390                         }
1391                         byte_offset += bufs[count].b_bcount;
1392 
1393                         if (fsp->hqueue == NULL) {
1394                                 (void) bdev_strategy(&bufs[count]);
1395 
1396                         } else {
1397                                 /*
1398                                  * We are scheduling I/O so we need to enqueue
1399                                  * requests rather than calling bdev_strategy
1400                                  * here. A later invocation of the scheduling
1401                                  * function will take care of doing the actual
1402                                  * I/O as it selects requests from the queue as
1403                                  * per the scheduling logic.
1404                                  */
1405                                 struct hio *hsio = kmem_cache_alloc(hio_cache,
1406                                     KM_SLEEP);
1407 
1408                                 sema_init(&fio_done[count], 0, NULL,
1409                                     SEMA_DEFAULT, NULL);
1410                                 hsio->bp = &bufs[count];
1411                                 hsio->sema = &fio_done[count];
1412                                 hsio->io_lblkno = bufs[count].b_lblkno;
1413                                 hsio->nblocks = howmany(hsio->bp->b_bcount,
1414                                     DEV_BSIZE);
1415 
1416                                 /* used for deadline */
1417                                 hsio->io_timestamp =
1418                                     drv_hztousec(ddi_get_lbolt());
1419 
1420                                 /* for I/O coalescing */
1421                                 hsio->contig_chain = NULL;
1422                                 hsched_enqueue_io(fsp, hsio, 0);
1423                         }
1424 
1425                         lwp_stat_update(LWP_STAT_INBLK, 1);
1426                         lastp = searchp;
1427                         if ((remainder - bufs[count].b_bcount) < 1) {
1428                                 searchp = searchp->p_next;
1429                         }
1430                 }
1431 
1432                 bufsused = count;
1433                 /* Now wait for everything to come in */
1434                 if (fsp->hqueue == NULL) {
1435                         for (count = 0; count < bufsused; count++) {
1436                                 if (err == 0) {
1437                                         err = biowait(&bufs[count]);
1438                                 } else
1439                                         (void) biowait(&bufs[count]);
1440                         }
1441                 } else {
1442                         for (count = 0; count < bufsused; count++) {
1443                                 struct buf *wbuf;
1444 
1445                                 /*
1446                                  * Invoke scheduling function till our buf
1447                                  * is processed. In doing this it might
1448                                  * process bufs enqueued by other threads
1449                                  * which is good.
1450                                  */
1451                                 wbuf = &bufs[count];
1452                                 DTRACE_PROBE1(hsfs_io_wait, struct buf *, wbuf);
1453                                 while (sema_tryp(&fio_done[count]) == 0) {
1454                                         /*
1455                                          * hsched_invoke_strategy will return 1
1456                                          * if the I/O queue is empty. This means
1457                                          * that there is another thread who has
1458                                          * issued our buf and is waiting. So we
1459                                          * just block instead of spinning.
1460                                          */
1461                                         if (hsched_invoke_strategy(fsp)) {
1462                                                 sema_p(&fio_done[count]);
1463                                                 break;
1464                                         }
1465                                 }
1466                                 sema_destroy(&fio_done[count]);
1467                                 DTRACE_PROBE1(hsfs_io_done, struct buf *, wbuf);
1468 
1469                                 if (err == 0) {
1470                                         err = geterror(wbuf);
1471                                 }
1472                         }
1473                         kmem_free(fio_done, bufcnt * sizeof (ksema_t));
1474                 }
1475 
1476                 /* Don't leak resources */
1477                 for (count = 0; count < bufcnt; count++) {
1478                         biofini(&bufs[count]);
1479                         if (count < bufsused && vas[count] != NULL) {
1480                                 ppmapout(vas[count]);
1481                         }
1482                 }
1483 
1484                 kmem_free(vas, bufcnt * sizeof (caddr_t));
1485                 kmem_free(bufs, bufcnt * sizeof (struct buf));
1486         }
1487 
1488         if (err) {
1489                 pvn_read_done(pp, B_ERROR);
1490                 return (err);
1491         }
1492 
1493         /*
1494          * Lock the requested page, and the one after it if possible.
1495          * Don't bother if our caller hasn't given us a place to stash
1496          * the page pointers, since otherwise we'd lock pages that would
1497          * never get unlocked.
1498          */
1499         if (pagefound) {
1500                 int index;
1501                 ulong_t soff;
1502 
1503                 /*
1504                  * Make sure it's in memory before we say it's here.
1505                  */
1506                 if ((pp = page_lookup(vp, off, SE_SHARED)) == NULL) {
1507                         hsfs_lostpage++;
1508                         goto reread;
1509                 }
1510 
1511                 pl[0] = pp;
1512                 index = 1;
1513                 atomic_inc_64(&fsp->cache_read_pages);
1514 
1515                 /*
1516                  * Try to lock the next page, if it exists, without
1517                  * blocking.
1518                  */
1519                 plsz -= PAGESIZE;
1520                 /* LINTED (plsz is unsigned) */
1521                 for (soff = off + PAGESIZE; plsz > 0;
1522                     soff += PAGESIZE, plsz -= PAGESIZE) {
1523                         pp = page_lookup_nowait(vp, (u_offset_t)soff,
1524                             SE_SHARED);
1525                         if (pp == NULL)
1526                                 break;
1527                         pl[index++] = pp;
1528                 }
1529                 pl[index] = NULL;
1530 
1531                 /*
1532                  * Schedule a semi-asynchronous readahead if we are
1533                  * accessing the last cached page for the current
1534                  * file.
1535                  *
1536                  * Doing this here means that readaheads will be
1537                  * issued only if cache-hits occur. This is an advantage
1538                  * since cache-hits would mean that readahead is giving
1539                  * the desired benefit. If cache-hits do not occur there
1540                  * is no point in reading ahead of time - the system
1541                  * is loaded anyway.
1542                  */
1543                 if (fsp->hqueue != NULL &&
1544                     hp->hs_prev_offset - off == PAGESIZE &&
1545                     hp->hs_prev_offset < filsiz &&
1546                     hp->hs_ra_bytes > 0 &&
1547                     !page_exists(vp, hp->hs_prev_offset)) {
1548                         (void) hsfs_getpage_ra(vp, hp->hs_prev_offset, seg,
1549                             addr + PAGESIZE, hp, fsp, xarsiz, bof,
1550                             chunk_lbn_count, chunk_data_bytes);
1551                 }
1552 
1553                 return (0);
1554         }
1555 
1556         if (pp != NULL) {
1557                 pvn_plist_init(pp, pl, plsz, off, io_len, rw);
1558         }
1559 
1560         return (err);
1561 }
1562 
1563 /*ARGSUSED*/
1564 static int
1565 hsfs_getpage(
1566         struct vnode *vp,
1567         offset_t off,
1568         size_t len,
1569         uint_t *protp,
1570         struct page *pl[],
1571         size_t plsz,
1572         struct seg *seg,
1573         caddr_t addr,
1574         enum seg_rw rw,
1575         struct cred *cred,
1576         caller_context_t *ct)
1577 {
1578         int err;
1579         uint_t filsiz;
1580         struct hsfs *fsp;
1581         struct hsnode *hp;
1582 
1583         fsp = VFS_TO_HSFS(vp->v_vfsp);
1584         hp = VTOH(vp);
1585 
1586         /* does not support write */
1587         if (rw == S_WRITE) {
1588                 return (EROFS);
1589         }
1590 
1591         if (vp->v_flag & VNOMAP) {
1592                 return (ENOSYS);
1593         }
1594 
1595         ASSERT(off <= HS_MAXFILEOFF);
1596 
1597         /*
1598          * Determine file data size for EOF check.
1599          */
1600         filsiz = hp->hs_dirent.ext_size;
1601         if ((off + len) > (offset_t)(filsiz + PAGEOFFSET) && seg != segkmap)
1602                 return (EFAULT);        /* beyond EOF */
1603 
1604         /*
1605          * Async Read-ahead computation.
1606          * This attempts to detect sequential access pattern and
1607          * enables reading extra pages ahead of time.
1608          */
1609         if (fsp->hqueue != NULL) {
1610                 /*
1611                  * This check for sequential access also takes into
1612                  * account segmap weirdness when reading in chunks
1613                  * less than the segmap size of 8K.
1614                  */
1615                 if (hp->hs_prev_offset == off || (off <
1616                     hp->hs_prev_offset && off + MAX(len, PAGESIZE)
1617                     >= hp->hs_prev_offset)) {
1618                         if (hp->hs_num_contig <
1619                             (seq_contig_requests - 1)) {
1620                                 hp->hs_num_contig++;
1621 
1622                         } else {
1623                                 /*
1624                                  * We increase readahead quantum till
1625                                  * a predefined max. max_readahead_bytes
1626                                  * is a multiple of PAGESIZE.
1627                                  */
1628                                 if (hp->hs_ra_bytes <
1629                                     fsp->hqueue->max_ra_bytes) {
1630                                         hp->hs_ra_bytes += PAGESIZE;
1631                                 }
1632                         }
1633                 } else {
1634                         /*
1635                          * Not contiguous so reduce read ahead counters.
1636                          */
1637                         if (hp->hs_ra_bytes > 0)
1638                                 hp->hs_ra_bytes -= PAGESIZE;
1639 
1640                         if (hp->hs_ra_bytes <= 0) {
1641                                 hp->hs_ra_bytes = 0;
1642                                 if (hp->hs_num_contig > 0)
1643                                         hp->hs_num_contig--;
1644                         }
1645                 }
1646                 /*
1647                  * Length must be rounded up to page boundary.
1648                  * since we read in units of pages.
1649                  */
1650                 hp->hs_prev_offset = off + roundup(len, PAGESIZE);
1651                 DTRACE_PROBE1(hsfs_compute_ra, struct hsnode *, hp);
1652         }
1653         if (protp != NULL)
1654                 *protp = PROT_ALL;
1655 
1656         if (len <= PAGESIZE)
1657                 err = hsfs_getapage(vp, (u_offset_t)off, len, protp, pl, plsz,
1658                     seg, addr, rw, cred);
1659         else
1660                 err = pvn_getpages(hsfs_getapage, vp, off, len, protp,
1661                     pl, plsz, seg, addr, rw, cred);
1662 
1663         return (err);
1664 }
1665 
1666 
1667 
1668 /*
1669  * This function should never be called. We need to have it to pass
1670  * it as an argument to other functions.
1671  */
1672 /*ARGSUSED*/
1673 int
1674 hsfs_putapage(
1675         vnode_t         *vp,
1676         page_t          *pp,
1677         u_offset_t      *offp,
1678         size_t          *lenp,
1679         int             flags,
1680         cred_t          *cr)
1681 {
1682         /* should never happen - just destroy it */
1683         cmn_err(CE_NOTE, "hsfs_putapage: dirty HSFS page");
1684         pvn_write_done(pp, B_ERROR | B_WRITE | B_INVAL | B_FORCE | flags);
1685         return (0);
1686 }
1687 
1688 
1689 /*
1690  * The only flags we support are B_INVAL, B_FREE and B_DONTNEED.
1691  * B_INVAL is set by:
1692  *
1693  *      1) the MC_SYNC command of memcntl(2) to support the MS_INVALIDATE flag.
1694  *      2) the MC_ADVISE command of memcntl(2) with the MADV_DONTNEED advice
1695  *         which translates to an MC_SYNC with the MS_INVALIDATE flag.
1696  *
1697  * The B_FREE (as well as the B_DONTNEED) flag is set when the
1698  * MADV_SEQUENTIAL advice has been used. VOP_PUTPAGE is invoked
1699  * from SEGVN to release pages behind a pagefault.
1700  */
1701 /*ARGSUSED*/
1702 static int
1703 hsfs_putpage(
1704         struct vnode            *vp,
1705         offset_t                off,
1706         size_t                  len,
1707         int                     flags,
1708         struct cred             *cr,
1709         caller_context_t        *ct)
1710 {
1711         int error = 0;
1712 
1713         if (vp->v_count == 0) {
1714                 panic("hsfs_putpage: bad v_count");
1715                 /*NOTREACHED*/
1716         }
1717 
1718         if (vp->v_flag & VNOMAP)
1719                 return (ENOSYS);
1720 
1721         ASSERT(off <= HS_MAXFILEOFF);
1722 
1723         if (!vn_has_cached_data(vp))    /* no pages mapped */
1724                 return (0);
1725 
1726         if (len == 0) {         /* from 'off' to EOF */
1727                 error = pvn_vplist_dirty(vp, off, hsfs_putapage, flags, cr);
1728         } else {
1729                 offset_t end_off = off + len;
1730                 offset_t file_size = VTOH(vp)->hs_dirent.ext_size;
1731                 offset_t io_off;
1732 
1733                 file_size = (file_size + PAGESIZE - 1) & PAGEMASK;
1734                 if (end_off > file_size)
1735                         end_off = file_size;
1736 
1737                 for (io_off = off; io_off < end_off; io_off += PAGESIZE) {
1738                         page_t *pp;
1739 
1740                         /*
1741                          * We insist on getting the page only if we are
1742                          * about to invalidate, free or write it and
1743                          * the B_ASYNC flag is not set.
1744                          */
1745                         if ((flags & B_INVAL) || ((flags & B_ASYNC) == 0)) {
1746                                 pp = page_lookup(vp, io_off,
1747                                     (flags & (B_INVAL | B_FREE)) ?
1748                                     SE_EXCL : SE_SHARED);
1749                         } else {
1750                                 pp = page_lookup_nowait(vp, io_off,
1751                                     (flags & B_FREE) ? SE_EXCL : SE_SHARED);
1752                         }
1753 
1754                         if (pp == NULL)
1755                                 continue;
1756 
1757                         /*
1758                          * Normally pvn_getdirty() should return 0, which
1759                          * impies that it has done the job for us.
1760                          * The shouldn't-happen scenario is when it returns 1.
1761                          * This means that the page has been modified and
1762                          * needs to be put back.
1763                          * Since we can't write on a CD, we fake a failed
1764                          * I/O and force pvn_write_done() to destroy the page.
1765                          */
1766                         if (pvn_getdirty(pp, flags) == 1) {
1767                                 cmn_err(CE_NOTE,
1768                                     "hsfs_putpage: dirty HSFS page");
1769                                 pvn_write_done(pp, flags |
1770                                     B_ERROR | B_WRITE | B_INVAL | B_FORCE);
1771                         }
1772                 }
1773         }
1774         return (error);
1775 }
1776 
1777 
1778 /*ARGSUSED*/
1779 static int
1780 hsfs_map(
1781         struct vnode *vp,
1782         offset_t off,
1783         struct as *as,
1784         caddr_t *addrp,
1785         size_t len,
1786         uchar_t prot,
1787         uchar_t maxprot,
1788         uint_t flags,
1789         struct cred *cred,
1790         caller_context_t *ct)
1791 {
1792         struct segvn_crargs vn_a;
1793         int error;
1794 
1795         /* VFS_RECORD(vp->v_vfsp, VS_MAP, VS_CALL); */
1796 
1797         if (vp->v_flag & VNOMAP)
1798                 return (ENOSYS);
1799 
1800         if ((prot & PROT_WRITE) && (flags & MAP_SHARED))
1801                 return (ENOSYS);
1802 
1803         if (off > HS_MAXFILEOFF || off < 0 ||
1804             (off + len) < 0 || (off + len) > HS_MAXFILEOFF)
1805                 return (ENXIO);
1806 
1807         if (vp->v_type != VREG) {
1808                 return (ENODEV);
1809         }
1810 
1811         /*
1812          * If file is being locked, disallow mapping.
1813          */
1814         if (vn_has_mandatory_locks(vp, VTOH(vp)->hs_dirent.mode))
1815                 return (EAGAIN);
1816 
1817         as_rangelock(as);
1818         error = choose_addr(as, addrp, len, off, ADDR_VACALIGN, flags);
1819         if (error != 0) {
1820                 as_rangeunlock(as);
1821                 return (error);
1822         }
1823 
1824         vn_a.vp = vp;
1825         vn_a.offset = off;
1826         vn_a.type = flags & MAP_TYPE;
1827         vn_a.prot = prot;
1828         vn_a.maxprot = maxprot;
1829         vn_a.flags = flags & ~MAP_TYPE;
1830         vn_a.cred = cred;
1831         vn_a.amp = NULL;
1832         vn_a.szc = 0;
1833         vn_a.lgrp_mem_policy_flags = 0;
1834 
1835         error = as_map(as, *addrp, len, segvn_create, &vn_a);
1836         as_rangeunlock(as);
1837         return (error);
1838 }
1839 
1840 /* ARGSUSED */
1841 static int
1842 hsfs_addmap(
1843         struct vnode *vp,
1844         offset_t off,
1845         struct as *as,
1846         caddr_t addr,
1847         size_t len,
1848         uchar_t prot,
1849         uchar_t maxprot,
1850         uint_t flags,
1851         struct cred *cr,
1852         caller_context_t *ct)
1853 {
1854         struct hsnode *hp;
1855 
1856         if (vp->v_flag & VNOMAP)
1857                 return (ENOSYS);
1858 
1859         hp = VTOH(vp);
1860         mutex_enter(&hp->hs_contents_lock);
1861         hp->hs_mapcnt += btopr(len);
1862         mutex_exit(&hp->hs_contents_lock);
1863         return (0);
1864 }
1865 
1866 /*ARGSUSED*/
1867 static int
1868 hsfs_delmap(
1869         struct vnode *vp,
1870         offset_t off,
1871         struct as *as,
1872         caddr_t addr,
1873         size_t len,
1874         uint_t prot,
1875         uint_t maxprot,
1876         uint_t flags,
1877         struct cred *cr,
1878         caller_context_t *ct)
1879 {
1880         struct hsnode *hp;
1881 
1882         if (vp->v_flag & VNOMAP)
1883                 return (ENOSYS);
1884 
1885         hp = VTOH(vp);
1886         mutex_enter(&hp->hs_contents_lock);
1887         hp->hs_mapcnt -= btopr(len); /* Count released mappings */
1888         ASSERT(hp->hs_mapcnt >= 0);
1889         mutex_exit(&hp->hs_contents_lock);
1890         return (0);
1891 }
1892 
1893 /* ARGSUSED */
1894 static int
1895 hsfs_seek(
1896         struct vnode *vp,
1897         offset_t ooff,
1898         offset_t *noffp,
1899         caller_context_t *ct)
1900 {
1901         return ((*noffp < 0 || *noffp > MAXOFFSET_T) ? EINVAL : 0);
1902 }
1903 
1904 /* ARGSUSED */
1905 static int
1906 hsfs_frlock(
1907         struct vnode *vp,
1908         int cmd,
1909         struct flock64 *bfp,
1910         int flag,
1911         offset_t offset,
1912         struct flk_callback *flk_cbp,
1913         cred_t *cr,
1914         caller_context_t *ct)
1915 {
1916         struct hsnode *hp = VTOH(vp);
1917 
1918         /*
1919          * If the file is being mapped, disallow fs_frlock.
1920          * We are not holding the hs_contents_lock while checking
1921          * hs_mapcnt because the current locking strategy drops all
1922          * locks before calling fs_frlock.
1923          * So, hs_mapcnt could change before we enter fs_frlock making
1924          * it meaningless to have held hs_contents_lock in the first place.
1925          */
1926         if (hp->hs_mapcnt > 0 && MANDLOCK(vp, hp->hs_dirent.mode))
1927                 return (EAGAIN);
1928 
1929         return (fs_frlock(vp, cmd, bfp, flag, offset, flk_cbp, cr, ct));
1930 }
1931 
1932 static int
1933 hsched_deadline_compare(const void *x1, const void *x2)
1934 {
1935         const struct hio *h1 = x1;
1936         const struct hio *h2 = x2;
1937 
1938         if (h1->io_timestamp < h2->io_timestamp)
1939                 return (-1);
1940         if (h1->io_timestamp > h2->io_timestamp)
1941                 return (1);
1942 
1943         if (h1->io_lblkno < h2->io_lblkno)
1944                 return (-1);
1945         if (h1->io_lblkno > h2->io_lblkno)
1946                 return (1);
1947 
1948         if (h1 < h2)
1949                 return (-1);
1950         if (h1 > h2)
1951                 return (1);
1952 
1953         return (0);
1954 }
1955 
1956 static int
1957 hsched_offset_compare(const void *x1, const void *x2)
1958 {
1959         const struct hio *h1 = x1;
1960         const struct hio *h2 = x2;
1961 
1962         if (h1->io_lblkno < h2->io_lblkno)
1963                 return (-1);
1964         if (h1->io_lblkno > h2->io_lblkno)
1965                 return (1);
1966 
1967         if (h1 < h2)
1968                 return (-1);
1969         if (h1 > h2)
1970                 return (1);
1971 
1972         return (0);
1973 }
1974 
1975 void
1976 hsched_init_caches(void)
1977 {
1978         hio_cache = kmem_cache_create("hsfs_hio_cache",
1979             sizeof (struct hio), 0, NULL,
1980             NULL, NULL, NULL, NULL, 0);
1981 
1982         hio_info_cache = kmem_cache_create("hsfs_hio_info_cache",
1983             sizeof (struct hio_info), 0, NULL,
1984             NULL, NULL, NULL, NULL, 0);
1985 }
1986 
1987 void
1988 hsched_fini_caches(void)
1989 {
1990         kmem_cache_destroy(hio_cache);
1991         kmem_cache_destroy(hio_info_cache);
1992 }
1993 
1994 /*
1995  * Initialize I/O scheduling structures. This is called via hsfs_mount
1996  */
1997 void
1998 hsched_init(struct hsfs *fsp, int fsid, struct modlinkage *modlinkage)
1999 {
2000         struct hsfs_queue *hqueue = fsp->hqueue;
2001         struct vnode *vp = fsp->hsfs_devvp;
2002 
2003         /* TaskQ name of the form: hsched_task_ + stringof(int) */
2004         char namebuf[23];
2005         int error, err;
2006         struct dk_cinfo info;
2007         ldi_handle_t lh;
2008         ldi_ident_t li;
2009 
2010         /*
2011          * Default maxtransfer = 16k chunk
2012          */
2013         hqueue->dev_maxtransfer = 16384;
2014 
2015         /*
2016          * Try to fetch the maximum device transfer size. This is used to
2017          * ensure that a coalesced block does not exceed the maxtransfer.
2018          */
2019         err  = ldi_ident_from_mod(modlinkage, &li);
2020         if (err) {
2021                 cmn_err(CE_NOTE, "hsched_init: Querying device failed");
2022                 cmn_err(CE_NOTE, "hsched_init: ldi_ident_from_mod err=%d\n",
2023                     err);
2024                 goto set_ra;
2025         }
2026 
2027         err = ldi_open_by_dev(&(vp->v_rdev), OTYP_CHR, FREAD, CRED(), &lh, li);
2028         ldi_ident_release(li);
2029         if (err) {
2030                 cmn_err(CE_NOTE, "hsched_init: Querying device failed");
2031                 cmn_err(CE_NOTE, "hsched_init: ldi_open err=%d\n", err);
2032                 goto set_ra;
2033         }
2034 
2035         error = ldi_ioctl(lh, DKIOCINFO, (intptr_t)&info, FKIOCTL,
2036             CRED(), &err);
2037         err = ldi_close(lh, FREAD, CRED());
2038         if (err) {
2039                 cmn_err(CE_NOTE, "hsched_init: Querying device failed");
2040                 cmn_err(CE_NOTE, "hsched_init: ldi_close err=%d\n", err);
2041         }
2042 
2043         if (error == 0) {
2044                 hqueue->dev_maxtransfer = ldbtob(info.dki_maxtransfer);
2045         }
2046 
2047 set_ra:
2048         /*
2049          * Max size of data to read ahead for sequential access pattern.
2050          * Conservative to avoid letting the underlying CD drive to spin
2051          * down, in case the application is reading slowly.
2052          * We read ahead upto a max of 4 pages.
2053          */
2054         hqueue->max_ra_bytes = PAGESIZE * 8;
2055 
2056         mutex_init(&(hqueue->hsfs_queue_lock), NULL, MUTEX_DEFAULT, NULL);
2057         mutex_init(&(hqueue->strategy_lock), NULL, MUTEX_DEFAULT, NULL);
2058         avl_create(&(hqueue->read_tree), hsched_offset_compare,
2059             sizeof (struct hio), offsetof(struct hio, io_offset_node));
2060         avl_create(&(hqueue->deadline_tree), hsched_deadline_compare,
2061             sizeof (struct hio), offsetof(struct hio, io_deadline_node));
2062 
2063         (void) snprintf(namebuf, sizeof (namebuf), "hsched_task_%d", fsid);
2064         hqueue->ra_task = taskq_create(namebuf, hsfs_taskq_nthreads,
2065             minclsyspri + 2, 1, 104857600 / PAGESIZE, TASKQ_DYNAMIC);
2066 
2067         hqueue->next = NULL;
2068         hqueue->nbuf = kmem_zalloc(sizeof (struct buf), KM_SLEEP);
2069 }
2070 
2071 void
2072 hsched_fini(struct hsfs_queue *hqueue)
2073 {
2074         if (hqueue != NULL) {
2075                 /*
2076                  * Remove the sentinel if there was one.
2077                  */
2078                 if (hqueue->next != NULL) {
2079                         avl_remove(&hqueue->read_tree, hqueue->next);
2080                         kmem_cache_free(hio_cache, hqueue->next);
2081                 }
2082                 avl_destroy(&(hqueue->read_tree));
2083                 avl_destroy(&(hqueue->deadline_tree));
2084                 mutex_destroy(&(hqueue->hsfs_queue_lock));
2085                 mutex_destroy(&(hqueue->strategy_lock));
2086 
2087                 /*
2088                  * If there are any existing readahead threads running
2089                  * taskq_destroy will wait for them to finish.
2090                  */
2091                 taskq_destroy(hqueue->ra_task);
2092                 kmem_free(hqueue->nbuf, sizeof (struct buf));
2093         }
2094 }
2095 
2096 /*
2097  * Determine if two I/O requests are adjacent to each other so
2098  * that they can coalesced.
2099  */
2100 #define IS_ADJACENT(io, nio) \
2101         (((io)->io_lblkno + (io)->nblocks == (nio)->io_lblkno) && \
2102         (io)->bp->b_edev == (nio)->bp->b_edev)
2103 
2104 /*
2105  * This performs the actual I/O scheduling logic. We use the Circular
2106  * Look algorithm here. Sort the I/O requests in ascending order of
2107  * logical block number and process them starting with the lowest
2108  * numbered block and progressing towards higher block numbers in the
2109  * queue. Once there are no more higher numbered blocks, start again
2110  * with the lowest one. This is good for CD/DVD as you keep moving
2111  * the head in one direction along the outward spiral track and avoid
2112  * too many seeks as much as possible. The re-ordering also allows
2113  * us to coalesce adjacent requests into one larger request.
2114  * This is thus essentially a 1-way Elevator with front merging.
2115  *
2116  * In addition each read request here has a deadline and will be
2117  * processed out of turn if the deadline (500ms) expires.
2118  *
2119  * This function is necessarily serialized via hqueue->strategy_lock.
2120  * This function sits just below hsfs_getapage and processes all read
2121  * requests orginating from that function.
2122  */
2123 int
2124 hsched_invoke_strategy(struct hsfs *fsp)
2125 {
2126         struct hsfs_queue *hqueue;
2127         struct buf *nbuf;
2128         struct hio *fio, *nio, *tio, *prev, *last;
2129         size_t bsize, soffset, offset, data;
2130         int bioret, bufcount;
2131         struct vnode *fvp;
2132         ksema_t *io_done;
2133         caddr_t iodata;
2134 
2135         hqueue = fsp->hqueue;
2136         mutex_enter(&hqueue->strategy_lock);
2137         mutex_enter(&hqueue->hsfs_queue_lock);
2138 
2139         /*
2140          * Check for Deadline expiration first
2141          */
2142         fio = avl_first(&hqueue->deadline_tree);
2143 
2144         /*
2145          * Paranoid check for empty I/O queue. Both deadline
2146          * and read trees contain same data sorted in different
2147          * ways. So empty deadline tree = empty read tree.
2148          */
2149         if (fio == NULL) {
2150                 /*
2151                  * Remove the sentinel if there was one.
2152                  */
2153                 if (hqueue->next != NULL) {
2154                         avl_remove(&hqueue->read_tree, hqueue->next);
2155                         kmem_cache_free(hio_cache, hqueue->next);
2156                         hqueue->next = NULL;
2157                 }
2158                 mutex_exit(&hqueue->hsfs_queue_lock);
2159                 mutex_exit(&hqueue->strategy_lock);
2160                 return (1);
2161         }
2162 
2163         if (drv_hztousec(ddi_get_lbolt()) - fio->io_timestamp
2164             < HSFS_READ_DEADLINE) {
2165                 /*
2166                  * Apply standard scheduling logic. This uses the
2167                  * C-LOOK approach. Process I/O requests in ascending
2168                  * order of logical block address till no subsequent
2169                  * higher numbered block request remains. Then start
2170                  * again from the lowest numbered block in the queue.
2171                  *
2172                  * We do this cheaply here by means of a sentinel.
2173                  * The last processed I/O structure from the previous
2174                  * invocation of this func, is left dangling in the
2175                  * read_tree so that we can easily scan to the next
2176                  * higher numbered request and remove the sentinel.
2177                  */
2178                 fio = NULL;
2179                 if (hqueue->next != NULL) {
2180                         fio = AVL_NEXT(&hqueue->read_tree, hqueue->next);
2181                         avl_remove(&hqueue->read_tree, hqueue->next);
2182                         kmem_cache_free(hio_cache, hqueue->next);
2183                         hqueue->next = NULL;
2184                 }
2185                 if (fio == NULL) {
2186                         fio = avl_first(&hqueue->read_tree);
2187                 }
2188         } else if (hqueue->next != NULL) {
2189                 DTRACE_PROBE1(hsfs_deadline_expiry, struct hio *, fio);
2190 
2191                 avl_remove(&hqueue->read_tree, hqueue->next);
2192                 kmem_cache_free(hio_cache, hqueue->next);
2193                 hqueue->next = NULL;
2194         }
2195 
2196         /*
2197          * In addition we try to coalesce contiguous
2198          * requests into one bigger request.
2199          */
2200         bufcount = 1;
2201         bsize = ldbtob(fio->nblocks);
2202         fvp = fio->bp->b_file;
2203         nio = AVL_NEXT(&hqueue->read_tree, fio);
2204         tio = fio;
2205         while (nio != NULL && IS_ADJACENT(tio, nio) &&
2206             bsize < hqueue->dev_maxtransfer) {
2207                 avl_remove(&hqueue->deadline_tree, tio);
2208                 avl_remove(&hqueue->read_tree, tio);
2209                 tio->contig_chain = nio;
2210                 bsize += ldbtob(nio->nblocks);
2211                 prev = tio;
2212                 tio = nio;
2213 
2214                 /*
2215                  * This check is required to detect the case where
2216                  * we are merging adjacent buffers belonging to
2217                  * different files. fvp is used to set the b_file
2218                  * parameter in the coalesced buf. b_file is used
2219                  * by DTrace so we do not want DTrace to accrue
2220                  * requests to two different files to any one file.
2221                  */
2222                 if (fvp && tio->bp->b_file != fvp) {
2223                         fvp = NULL;
2224                 }
2225 
2226                 nio = AVL_NEXT(&hqueue->read_tree, nio);
2227                 bufcount++;
2228         }
2229 
2230         /*
2231          * tio is not removed from the read_tree as it serves as a sentinel
2232          * to cheaply allow us to scan to the next higher numbered I/O
2233          * request.
2234          */
2235         hqueue->next = tio;
2236         avl_remove(&hqueue->deadline_tree, tio);
2237         mutex_exit(&hqueue->hsfs_queue_lock);
2238         DTRACE_PROBE3(hsfs_io_dequeued, struct hio *, fio, int, bufcount,
2239             size_t, bsize);
2240 
2241         /*
2242          * The benefit of coalescing occurs if the the savings in I/O outweighs
2243          * the cost of doing the additional work below.
2244          * It was observed that coalescing 2 buffers results in diminishing
2245          * returns, so we do coalescing if we have >2 adjacent bufs.
2246          */
2247         if (bufcount > hsched_coalesce_min) {
2248                 /*
2249                  * We have coalesced blocks. First allocate mem and buf for
2250                  * the entire coalesced chunk.
2251                  * Since we are guaranteed single-threaded here we pre-allocate
2252                  * one buf at mount time and that is re-used every time. This
2253                  * is a synthesized buf structure that uses kmem_alloced chunk.
2254                  * Not quite a normal buf attached to pages.
2255                  */
2256                 fsp->coalesced_bytes += bsize;
2257                 nbuf = hqueue->nbuf;
2258                 bioinit(nbuf);
2259                 nbuf->b_edev = fio->bp->b_edev;
2260                 nbuf->b_dev = fio->bp->b_dev;
2261                 nbuf->b_flags = fio->bp->b_flags;
2262                 nbuf->b_iodone = fio->bp->b_iodone;
2263                 iodata = kmem_alloc(bsize, KM_SLEEP);
2264                 nbuf->b_un.b_addr = iodata;
2265                 nbuf->b_lblkno = fio->bp->b_lblkno;
2266                 nbuf->b_vp = fvp;
2267                 nbuf->b_file = fvp;
2268                 nbuf->b_bcount = bsize;
2269                 nbuf->b_bufsize = bsize;
2270 
2271                 DTRACE_PROBE3(hsfs_coalesced_io_start, struct hio *, fio, int,
2272                     bufcount, size_t, bsize);
2273 
2274                 /*
2275                  * Perform I/O for the coalesced block.
2276                  */
2277                 (void) bdev_strategy(nbuf);
2278 
2279                 /*
2280                  * Duplicate the last IO node to leave the sentinel alone.
2281                  * The sentinel is freed in the next invocation of this
2282                  * function.
2283                  */
2284                 prev->contig_chain = kmem_cache_alloc(hio_cache, KM_SLEEP);
2285                 prev->contig_chain->bp = tio->bp;
2286                 prev->contig_chain->sema = tio->sema;
2287                 tio = prev->contig_chain;
2288                 tio->contig_chain = NULL;
2289                 soffset = ldbtob(fio->bp->b_lblkno);
2290                 nio = fio;
2291 
2292                 bioret = biowait(nbuf);
2293                 data = bsize - nbuf->b_resid;
2294                 biofini(nbuf);
2295                 mutex_exit(&hqueue->strategy_lock);
2296 
2297                 /*
2298                  * We use the b_resid parameter to detect how much
2299                  * data was succesfully transferred. We will signal
2300                  * a success to all the fully retrieved actual bufs
2301                  * before coalescing, rest is signaled as error,
2302                  * if any.
2303                  */
2304                 tio = nio;
2305                 DTRACE_PROBE3(hsfs_coalesced_io_done, struct hio *, nio,
2306                     int, bioret, size_t, data);
2307 
2308                 /*
2309                  * Copy data and signal success to all the bufs
2310                  * which can be fully satisfied from b_resid.
2311                  */
2312                 while (nio != NULL && data >= nio->bp->b_bcount) {
2313                         offset = ldbtob(nio->bp->b_lblkno) - soffset;
2314                         bcopy(iodata + offset, nio->bp->b_un.b_addr,
2315                             nio->bp->b_bcount);
2316                         data -= nio->bp->b_bcount;
2317                         bioerror(nio->bp, 0);
2318                         biodone(nio->bp);
2319                         sema_v(nio->sema);
2320                         tio = nio;
2321                         nio = nio->contig_chain;
2322                         kmem_cache_free(hio_cache, tio);
2323                 }
2324 
2325                 /*
2326                  * Signal error to all the leftover bufs (if any)
2327                  * after b_resid data is exhausted.
2328                  */
2329                 while (nio != NULL) {
2330                         nio->bp->b_resid = nio->bp->b_bcount - data;
2331                         bzero(nio->bp->b_un.b_addr + data, nio->bp->b_resid);
2332                         bioerror(nio->bp, bioret);
2333                         biodone(nio->bp);
2334                         sema_v(nio->sema);
2335                         tio = nio;
2336                         nio = nio->contig_chain;
2337                         kmem_cache_free(hio_cache, tio);
2338                         data = 0;
2339                 }
2340                 kmem_free(iodata, bsize);
2341         } else {
2342 
2343                 nbuf = tio->bp;
2344                 io_done = tio->sema;
2345                 nio = fio;
2346                 last = tio;
2347 
2348                 while (nio != NULL) {
2349                         (void) bdev_strategy(nio->bp);
2350                         nio = nio->contig_chain;
2351                 }
2352                 nio = fio;
2353                 mutex_exit(&hqueue->strategy_lock);
2354 
2355                 while (nio != NULL) {
2356                         if (nio == last) {
2357                                 (void) biowait(nbuf);
2358                                 sema_v(io_done);
2359                                 break;
2360                                 /* sentinel last not freed. See above. */
2361                         } else {
2362                                 (void) biowait(nio->bp);
2363                                 sema_v(nio->sema);
2364                         }
2365                         tio = nio;
2366                         nio = nio->contig_chain;
2367                         kmem_cache_free(hio_cache, tio);
2368                 }
2369         }
2370         return (0);
2371 }
2372 
2373 /*
2374  * Insert an I/O request in the I/O scheduler's pipeline
2375  * Using AVL tree makes it easy to reorder the I/O request
2376  * based on logical block number.
2377  */
2378 static void
2379 hsched_enqueue_io(struct hsfs *fsp, struct hio *hsio, int ra)
2380 {
2381         struct hsfs_queue *hqueue = fsp->hqueue;
2382 
2383         mutex_enter(&hqueue->hsfs_queue_lock);
2384 
2385         fsp->physical_read_bytes += hsio->bp->b_bcount;
2386         if (ra)
2387                 fsp->readahead_bytes += hsio->bp->b_bcount;
2388 
2389         avl_add(&hqueue->deadline_tree, hsio);
2390         avl_add(&hqueue->read_tree, hsio);
2391 
2392         DTRACE_PROBE3(hsfs_io_enqueued, struct hio *, hsio,
2393             struct hsfs_queue *, hqueue, int, ra);
2394 
2395         mutex_exit(&hqueue->hsfs_queue_lock);
2396 }
2397 
2398 /* ARGSUSED */
2399 static int
2400 hsfs_pathconf(struct vnode *vp,
2401         int cmd,
2402         ulong_t *valp,
2403         struct cred *cr,
2404         caller_context_t *ct)
2405 {
2406         struct hsfs     *fsp;
2407 
2408         int             error = 0;
2409 
2410         switch (cmd) {
2411 
2412         case _PC_NAME_MAX:
2413                 fsp = VFS_TO_HSFS(vp->v_vfsp);
2414                 *valp = fsp->hsfs_namemax;
2415                 break;
2416 
2417         case _PC_FILESIZEBITS:
2418                 *valp = 33;     /* Without multi extent support: 4 GB - 2k */
2419                 break;
2420 
2421         case _PC_TIMESTAMP_RESOLUTION:
2422                 /*
2423                  * HSFS keeps, at best, 1/100 second timestamp resolution.
2424                  */
2425                 *valp = 10000000L;
2426                 break;
2427 
2428         default:
2429                 error = fs_pathconf(vp, cmd, valp, cr, ct);
2430                 break;
2431         }
2432 
2433         return (error);
2434 }
2435 
2436 
2437 
2438 const fs_operation_def_t hsfs_vnodeops_template[] = {
2439         VOPNAME_OPEN,           { .vop_open = hsfs_open },
2440         VOPNAME_CLOSE,          { .vop_close = hsfs_close },
2441         VOPNAME_READ,           { .vop_read = hsfs_read },
2442         VOPNAME_GETATTR,        { .vop_getattr = hsfs_getattr },
2443         VOPNAME_ACCESS,         { .vop_access = hsfs_access },
2444         VOPNAME_LOOKUP,         { .vop_lookup = hsfs_lookup },
2445         VOPNAME_READDIR,        { .vop_readdir = hsfs_readdir },
2446         VOPNAME_READLINK,       { .vop_readlink = hsfs_readlink },
2447         VOPNAME_FSYNC,          { .vop_fsync = hsfs_fsync },
2448         VOPNAME_INACTIVE,       { .vop_inactive = hsfs_inactive },
2449         VOPNAME_FID,            { .vop_fid = hsfs_fid },
2450         VOPNAME_SEEK,           { .vop_seek = hsfs_seek },
2451         VOPNAME_FRLOCK,         { .vop_frlock = hsfs_frlock },
2452         VOPNAME_GETPAGE,        { .vop_getpage = hsfs_getpage },
2453         VOPNAME_PUTPAGE,        { .vop_putpage = hsfs_putpage },
2454         VOPNAME_MAP,            { .vop_map = hsfs_map },
2455         VOPNAME_ADDMAP,         { .vop_addmap = hsfs_addmap },
2456         VOPNAME_DELMAP,         { .vop_delmap = hsfs_delmap },
2457         VOPNAME_PATHCONF,       { .vop_pathconf = hsfs_pathconf },
2458         NULL,                   NULL
2459 };
2460 
2461 struct vnodeops *hsfs_vnodeops;