/*        $NetBSD: ufs_rename.c,v 1.13 2016/10/28 20:38:12 jdolecek Exp $        */
      
      /*-
       * Copyright (c) 2012 The NetBSD Foundation, Inc.
       * All rights reserved.
       *
       * This code is derived from software contributed to The NetBSD Foundation
       * by Taylor R Campbell.
       *
       * Redistribution and use in source and binary forms, with or without
       * modification, are permitted provided that the following conditions
       * are met:
       * 1. Redistributions of source code must retain the above copyright
       *    notice, this list of conditions and the following disclaimer.
       * 2. Redistributions in binary form must reproduce the above copyright
       *    notice, this list of conditions and the following disclaimer in the
       *    documentation and/or other materials provided with the distribution.
       *
       * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
       * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
       * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
       * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
       * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
       * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
       * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
       * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
       * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
       * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
       * POSSIBILITY OF SUCH DAMAGE.
       */
      
      /*
       * UFS Rename
       */
      
      #include <sys/cdefs.h>
      __KERNEL_RCSID(0, "$NetBSD: ufs_rename.c,v 1.13 2016/10/28 20:38:12 jdolecek Exp $");
      
      #include <sys/param.h>
      #include <sys/buf.h>
      #include <sys/errno.h>
      #include <sys/kauth.h>
      #include <sys/mount.h>
      #include <sys/namei.h>
      #include <sys/pool.h>
      #include <sys/vnode.h>
      #include <sys/vnode_if.h>
      #include <sys/wapbl.h>
      
      #include <miscfs/genfs/genfs.h>
      
      #include <ufs/ufs/dir.h>
      #include <ufs/ufs/inode.h>
      #include <ufs/ufs/ufs_bswap.h>
      #include <ufs/ufs/ufs_extern.h>
      #include <ufs/ufs/ufs_wapbl.h>
      #include <ufs/ufs/ufsmount.h>
      
      /*
       * Forward declarations
       */
      
      static int ufs_sane_rename(struct vnode *, struct componentname *,
          struct vnode *, struct componentname *,
          kauth_cred_t, bool);
      static bool ufs_rename_ulr_overlap_p(const struct ufs_lookup_results *,
          const struct ufs_lookup_results *);
      static int ufs_rename_recalculate_fulr(struct vnode *,
          struct ufs_lookup_results *, const struct ufs_lookup_results *,
          const struct componentname *);
      static int ufs_direct_namlen(const struct direct *, const struct vnode *);
      static int ufs_read_dotdot(struct vnode *, kauth_cred_t, ino_t *);
      static int ufs_dirbuf_dotdot_namlen(const struct dirtemplate *,
          const struct vnode *);
      
      static const struct genfs_rename_ops ufs_genfs_rename_ops;
      
      /*
       * ufs_sane_rename: The hairiest vop, with the saner API.
       *
       * Arguments:
       *
       * . fdvp (from directory vnode),
       * . fcnp (from component name),
       * . tdvp (to directory vnode),
       * . tcnp (to component name),
       * . cred (credentials structure), and
       * . posixly_correct (flag for behaviour if target & source link same file).
       *
       * fdvp and tdvp may be the same, and must be referenced and unlocked.
       */
      static int
      ufs_sane_rename(
          struct vnode *fdvp, struct componentname *fcnp,
          struct vnode *tdvp, struct componentname *tcnp,
          kauth_cred_t cred, bool posixly_correct)
      {
   24         struct ufs_lookup_results fulr, tulr;
      
              return genfs_sane_rename(&ufs_genfs_rename_ops,
                  fdvp, fcnp, &fulr, tdvp, tcnp, &tulr,
                  cred, posixly_correct);
      }
      
      /*
       * ufs_rename: The hairiest vop, with the insanest API.  Defer to
       * genfs_insane_rename immediately.
       */
      int
      ufs_rename(void *v)
      {
      
   24         return genfs_insane_rename(v, &ufs_sane_rename);
      }
      
      /*
       * ufs_gro_directory_empty_p: Return true if the directory vp is
       * empty.  dvp is its parent.
       *
       * vp and dvp must be locked and referenced.
       */
      bool
      ufs_gro_directory_empty_p(struct mount *mp, kauth_cred_t cred,
          struct vnode *vp, struct vnode *dvp)
      {
      
              (void)mp;
    2         KASSERT(mp != NULL);
    2         KASSERT(vp != NULL);
    2         KASSERT(dvp != NULL);
    2         KASSERT(vp != dvp);
    2         KASSERT(vp->v_mount == mp);
    2         KASSERT(dvp->v_mount == mp);
    2         KASSERT(VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
    2         KASSERT(VOP_ISLOCKED(dvp) == LK_EXCLUSIVE);
      
    2         return ufs_dirempty(VTOI(vp), VTOI(dvp)->i_number, cred);
      }
      
      /*
       * ufs_gro_rename_check_possible: Check whether a rename is possible
       * independent of credentials.
       */
      int
      ufs_gro_rename_check_possible(struct mount *mp,
          struct vnode *fdvp, struct vnode *fvp,
          struct vnode *tdvp, struct vnode *tvp)
      {
      
              (void)mp;
   19         KASSERT(mp != NULL);
   19         KASSERT(fdvp != NULL);
   19         KASSERT(fvp != NULL);
   19         KASSERT(tdvp != NULL);
   19         KASSERT(fdvp != fvp);
   19         KASSERT(fdvp != tvp);
   19         KASSERT(tdvp != fvp);
   19         KASSERT(tdvp != tvp);
   19         KASSERT(fvp != tvp);
   19         KASSERT(fdvp->v_type == VDIR);
   19         KASSERT(tdvp->v_type == VDIR);
   19         KASSERT(fdvp->v_mount == mp);
   19         KASSERT(fvp->v_mount == mp);
   19         KASSERT(tdvp->v_mount == mp);
   19         KASSERT((tvp == NULL) || (tvp->v_mount == mp));
   19         KASSERT(VOP_ISLOCKED(fdvp) == LK_EXCLUSIVE);
   19         KASSERT(VOP_ISLOCKED(fvp) == LK_EXCLUSIVE);
   19         KASSERT(VOP_ISLOCKED(tdvp) == LK_EXCLUSIVE);
   19         KASSERT((tvp == NULL) || (VOP_ISLOCKED(tvp) == LK_EXCLUSIVE));
      
              return genfs_ufslike_rename_check_possible(
                  VTOI(fdvp)->i_flags, VTOI(fvp)->i_flags,
   19             VTOI(tdvp)->i_flags, (tvp? VTOI(tvp)->i_flags : 0),
                  (tvp != NULL),
                  IMMUTABLE, APPEND);
      }
      
      /*
       * ufs_gro_rename_check_permitted: Check whether a rename is permitted
       * given our credentials.
       */
      int
      ufs_gro_rename_check_permitted(struct mount *mp, kauth_cred_t cred,
          struct vnode *fdvp, struct vnode *fvp,
          struct vnode *tdvp, struct vnode *tvp)
      {
      
              (void)mp;
   19         KASSERT(mp != NULL);
   19         KASSERT(fdvp != NULL);
   19         KASSERT(fvp != NULL);
   19         KASSERT(tdvp != NULL);
   19         KASSERT(fdvp != fvp);
   19         KASSERT(fdvp != tvp);
   19         KASSERT(tdvp != fvp);
   19         KASSERT(tdvp != tvp);
   19         KASSERT(fvp != tvp);
   19         KASSERT(fdvp->v_type == VDIR);
   19         KASSERT(tdvp->v_type == VDIR);
   19         KASSERT(fdvp->v_mount == mp);
   19         KASSERT(fvp->v_mount == mp);
   19         KASSERT(tdvp->v_mount == mp);
   19         KASSERT((tvp == NULL) || (tvp->v_mount == mp));
   19         KASSERT(VOP_ISLOCKED(fdvp) == LK_EXCLUSIVE);
   19         KASSERT(VOP_ISLOCKED(fvp) == LK_EXCLUSIVE);
   19         KASSERT(VOP_ISLOCKED(tdvp) == LK_EXCLUSIVE);
   19         KASSERT((tvp == NULL) || (VOP_ISLOCKED(tvp) == LK_EXCLUSIVE));
      
              return genfs_ufslike_rename_check_permitted(cred,
                  fdvp, VTOI(fdvp)->i_mode, VTOI(fdvp)->i_uid,
                  fvp, VTOI(fvp)->i_uid,
   19             tdvp, VTOI(tdvp)->i_mode, VTOI(tdvp)->i_uid,
    7             tvp, (tvp? VTOI(tvp)->i_uid : 0));
      }
      
      /*
       * ufs_gro_remove_check_possible: Check whether a remove is possible
       * independent of credentials.
       */
      int
      ufs_gro_remove_check_possible(struct mount *mp,
          struct vnode *dvp, struct vnode *vp)
      {
      
              (void)mp;
    2         KASSERT(mp != NULL);
    2         KASSERT(dvp != NULL);
    2         KASSERT(vp != NULL);
    2         KASSERT(dvp != vp);
    2         KASSERT(dvp->v_type == VDIR);
    2         KASSERT(vp->v_type != VDIR);
    2         KASSERT(dvp->v_mount == mp);
    2         KASSERT(vp->v_mount == mp);
    2         KASSERT(VOP_ISLOCKED(dvp) == LK_EXCLUSIVE);
    2         KASSERT(VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
      
              return genfs_ufslike_remove_check_possible(
    2             VTOI(dvp)->i_flags, VTOI(vp)->i_flags,
                  IMMUTABLE, APPEND);
      }
      
      /*
       * ufs_gro_remove_check_permitted: Check whether a remove is permitted
       * given our credentials.
       */
      int
      ufs_gro_remove_check_permitted(struct mount *mp, kauth_cred_t cred,
          struct vnode *dvp, struct vnode *vp)
      {
      
              (void)mp;
    2         KASSERT(mp != NULL);
    2         KASSERT(dvp != NULL);
    2         KASSERT(vp != NULL);
    2         KASSERT(dvp != vp);
    2         KASSERT(dvp->v_type == VDIR);
    2         KASSERT(vp->v_type != VDIR);
    2         KASSERT(dvp->v_mount == mp);
    2         KASSERT(vp->v_mount == mp);
    2         KASSERT(VOP_ISLOCKED(dvp) == LK_EXCLUSIVE);
    2         KASSERT(VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
      
              return genfs_ufslike_remove_check_permitted(cred,
    2             dvp, VTOI(dvp)->i_mode, VTOI(dvp)->i_uid, vp, VTOI(vp)->i_uid);
      }
      
      /*
       * A virgin directory (no blushing please).
       *
       * XXX Copypasta from ufs_vnops.c.  Kill!
       */
      static const struct dirtemplate mastertemplate = {
              0,        12,                        DT_DIR,        1,        ".",
              0,        UFS_DIRBLKSIZ - 12,        DT_DIR,        2,        ".."
      };
      
      /*
       * ufs_gro_rename: Actually perform the rename operation.
       */
      int
      ufs_gro_rename(struct mount *mp, kauth_cred_t cred,
          struct vnode *fdvp, struct componentname *fcnp,
          void *fde, struct vnode *fvp,
          struct vnode *tdvp, struct componentname *tcnp,
          void *tde, struct vnode *tvp)
      {
              struct ufs_lookup_results *fulr = fde;
              struct ufs_lookup_results *tulr = tde;
              bool directory_p, reparent_p;
              struct direct *newdir;
              int error;
      
   18         KASSERT(mp != NULL);
   18         KASSERT(fdvp != NULL);
   18         KASSERT(fcnp != NULL);
   18         KASSERT(fulr != NULL);
   18         KASSERT(fvp != NULL);
   18         KASSERT(tdvp != NULL);
   18         KASSERT(tcnp != NULL);
   18         KASSERT(tulr != NULL);
   18         KASSERT(fulr != tulr);
   18         KASSERT(fdvp != fvp);
   18         KASSERT(fdvp != tvp);
   18         KASSERT(tdvp != fvp);
   18         KASSERT(tdvp != tvp);
   18         KASSERT(fvp != tvp);
   18         KASSERT(fdvp->v_mount == mp);
   18         KASSERT(fvp->v_mount == mp);
   18         KASSERT(tdvp->v_mount == mp);
   18         KASSERT((tvp == NULL) || (tvp->v_mount == mp));
   18         KASSERT(VOP_ISLOCKED(fdvp) == LK_EXCLUSIVE);
   18         KASSERT(VOP_ISLOCKED(fvp) == LK_EXCLUSIVE);
   18         KASSERT(VOP_ISLOCKED(tdvp) == LK_EXCLUSIVE);
   18         KASSERT((tvp == NULL) || (VOP_ISLOCKED(tvp) == LK_EXCLUSIVE));
      
              /*
               * We shall need to temporarily bump the link count, so make
               * sure there is room to do so.
               */
   18         if ((nlink_t)VTOI(fvp)->i_nlink >= LINK_MAX)
                      return EMLINK;
      
   18         directory_p = (fvp->v_type == VDIR);
    6         KASSERT(directory_p == ((VTOI(fvp)->i_mode & IFMT) == IFDIR));
   18         KASSERT((tvp == NULL) || (directory_p == (tvp->v_type == VDIR)));
    6         KASSERT((tvp == NULL) || (directory_p ==
                      ((VTOI(tvp)->i_mode & IFMT) == IFDIR)));
      
   18         reparent_p = (fdvp != tdvp);
              KASSERT(reparent_p == (VTOI(fdvp)->i_number != VTOI(tdvp)->i_number));
      
              /*
               * Commence hacking of the data on disk.
               */
      
   18         error = UFS_WAPBL_BEGIN(mp);
              if (error)
                      goto ihateyou;
      
              /*
               * 1) Bump link count while we're moving stuff
               *    around.  If we crash somewhere before
               *    completing our work, the link count
               *    may be wrong, but correctable.
               */
      
   18         KASSERT((nlink_t)VTOI(fvp)->i_nlink < LINK_MAX);
   18         VTOI(fvp)->i_nlink++;
   18         DIP_ASSIGN(VTOI(fvp), nlink, VTOI(fvp)->i_nlink);
   18         VTOI(fvp)->i_flag |= IN_CHANGE;
              error = UFS_UPDATE(fvp, NULL, NULL, UPDATE_DIROP);
              if (error)
                      goto whymustithurtsomuch;
      
              /*
               * 2) If target doesn't exist, link the target
               *    to the source and unlink the source.
               *    Otherwise, rewrite the target directory
               *    entry to reference the source inode and
               *    expunge the original entry's existence.
               */
      
   18         if (tvp == NULL) {
                      /*
                       * Account for ".." in new directory.
                       * When source and destination have the same
                       * parent we don't fool with the link count.
                       */
   12                 if (directory_p && reparent_p) {
    2                         if ((nlink_t)VTOI(tdvp)->i_nlink >= LINK_MAX) {
                                      error = EMLINK;
                                      goto whymustithurtsomuch;
                              }
                              KASSERT((nlink_t)VTOI(tdvp)->i_nlink < LINK_MAX);
    2                         VTOI(tdvp)->i_nlink++;
    2                         DIP_ASSIGN(VTOI(tdvp), nlink, VTOI(tdvp)->i_nlink);
    2                         VTOI(tdvp)->i_flag |= IN_CHANGE;
                              error = UFS_UPDATE(tdvp, NULL, NULL, UPDATE_DIROP);
                              if (error) {
                                      /*
                                       * Link count update didn't take --
                                       * back out the in-memory link count.
                                       */
                                      KASSERT(0 < VTOI(tdvp)->i_nlink);
                                      VTOI(tdvp)->i_nlink--;
                                      DIP_ASSIGN(VTOI(tdvp), nlink,
                                          VTOI(tdvp)->i_nlink);
                                      VTOI(tdvp)->i_flag |= IN_CHANGE;
                                      goto whymustithurtsomuch;
                              }
                      }
      
   12                 newdir = pool_cache_get(ufs_direct_cache, PR_WAITOK);
                      ufs_makedirentry(VTOI(fvp), tcnp, newdir);
                      error = ufs_direnter(tdvp, tulr, NULL, newdir, tcnp, NULL);
                      pool_cache_put(ufs_direct_cache, newdir);
                      if (error) {
                              if (directory_p && reparent_p) {
                                      /*
                                       * Directory update didn't take, but
                                       * the link count update did -- back
                                       * out the in-memory link count and the
                                       * on-disk link count.
                                       */
                                      KASSERT(0 < VTOI(tdvp)->i_nlink);
                                      VTOI(tdvp)->i_nlink--;
                                      DIP_ASSIGN(VTOI(tdvp), nlink,
                                          VTOI(tdvp)->i_nlink);
                                      VTOI(tdvp)->i_flag |= IN_CHANGE;
                                      (void)UFS_UPDATE(tdvp, NULL, NULL,
                                          UPDATE_WAIT | UPDATE_DIROP);
                              }
                              goto whymustithurtsomuch;
                      }
              } else {
                      if (directory_p)
                              /* XXX WTF?  Why purge here?  Why not purge others?  */
    2                         cache_purge(tdvp);
      
                      /*
                       * Make the target directory's entry for tcnp point at
                       * the source node.
                       *
                       * XXX ufs_dirrewrite decrements tvp's link count, but
                       * doesn't touch the link count of the new inode.  Go
                       * figure.
                       */
                      error = ufs_dirrewrite(VTOI(tdvp), tulr->ulr_offset,
    6                     VTOI(tvp), VTOI(fvp)->i_number, IFTODT(VTOI(fvp)->i_mode),
                          ((directory_p && reparent_p) ? reparent_p : directory_p),
                          IN_CHANGE | IN_UPDATE);
                      if (error)
                              goto whymustithurtsomuch;
      
                      /*
                       * If the source and target are directories, and the
                       * target is in the same directory as the source,
                       * decrement the link count of the common parent
                       * directory, since we are removing the target from
                       * that directory.
                       */
    1                 if (directory_p && !reparent_p) {
                              KASSERT(fdvp == tdvp);
                              /* XXX check, don't kassert */
    1                         KASSERT(0 < VTOI(tdvp)->i_nlink);
    1                         VTOI(tdvp)->i_nlink--;
    1                         DIP_ASSIGN(VTOI(tdvp), nlink, VTOI(tdvp)->i_nlink);
    1                         VTOI(tdvp)->i_flag |= IN_CHANGE;
                              UFS_WAPBL_UPDATE(tdvp, NULL, NULL, 0);
                      }
      
                      if (directory_p) {
                              /*
                               * XXX I don't understand the following comment
                               * from ufs_rename -- in particular, the part
                               * about `there may be other hard links'.
                               *
                               * Truncate inode. The only stuff left in the directory
                               * is "." and "..". The "." reference is inconsequential
                               * since we are quashing it. We have removed the "."
                               * reference and the reference in the parent directory,
                               * but there may be other hard links.
                               *
                               * XXX The ufs_dirempty call earlier does
                               * not guarantee anything about nlink.
                               */
    2                         if (VTOI(tvp)->i_nlink != 1)
                                      ufs_dirbad(VTOI(tvp), (doff_t)0,
                                          "hard-linked directory");
    2                         VTOI(tvp)->i_nlink = 0;
    2                         DIP_ASSIGN(VTOI(tvp), nlink, 0);
    2                         (void) UFS_TRUNCATE(tvp, (off_t)0, IO_SYNC, cred);
                      }
              }
      
              /*
               * If the source is a directory with a new parent, the link
               * count of the old parent directory must be decremented and
               * ".." set to point to the new parent.
               *
               * XXX ufs_dirrewrite updates the link count of fdvp, but not
               * the link count of fvp or the link count of tdvp.  Go figure.
               */
    8         if (directory_p && reparent_p) {
                      error = ufs_dirrewrite(VTOI(fvp), mastertemplate.dot_reclen,
    3                     VTOI(fdvp), VTOI(tdvp)->i_number, DT_DIR, 0, IN_CHANGE);
      #if 0                /* XXX This branch was not in ufs_rename! */
                      if (error)
                              goto whymustithurtsomuch;
      #endif
      
                      /* XXX WTF?  Why purge here?  Why not purge others?  */
                      cache_purge(fdvp);
              }
      
              /*
               * 3) Unlink the source.
               */
      
              /*
               * ufs_direnter may compact the directory in the process of
               * inserting a new entry.  That may invalidate fulr, which we
               * need in order to remove the old entry.  In that case, we
               * need to recalculate what fulr should be.
               */
    8         if (!reparent_p && (tvp == NULL) &&
    9             ufs_rename_ulr_overlap_p(fulr, tulr)) {
    8                 error = ufs_rename_recalculate_fulr(fdvp, fulr, tulr, fcnp);
      #if 0                                /* XXX */
                      if (error)        /* XXX Try to back out changes?  */
                              goto whymustithurtsomuch;
      #endif
              }
      
              /*
               * XXX 0 means !isrmdir.  But can't this be an rmdir?
               * XXX Well, turns out that argument to ufs_dirremove is ignored...
               * XXX And it turns out ufs_dirremove updates the link count of fvp.
               * XXX But it doesn't update the link count of fdvp.  Go figure.
               * XXX fdvp's link count is updated in ufs_dirrewrite instead.
               * XXX Actually, sometimes it doesn't update fvp's link count.
               * XXX I hate the world.
               */
   18         error = ufs_dirremove(fdvp, fulr, VTOI(fvp), fcnp->cn_flags, 0);
              if (error)
      #if 0                                /* XXX */
                      goto whymustithurtsomuch;
      #endif
                      goto arghmybrainhurts;
      
              /*
               * XXX Perhaps this should go at the top, in case the file
               * system is modified but incompletely so because of an
               * intermediate error.
               */
   18         genfs_rename_knote(fdvp, fvp, tdvp, tvp,
    9             ((tvp != NULL) && (VTOI(tvp)->i_nlink == 0)));
      #if 0                                /* XXX */
              genfs_rename_cache_purge(fdvp, fvp, tdvp, tvp);
      #endif
              goto arghmybrainhurts;
      
      whymustithurtsomuch:
              KASSERT(0 < VTOI(fvp)->i_nlink);
              VTOI(fvp)->i_nlink--;
              DIP_ASSIGN(VTOI(fvp), nlink, VTOI(fvp)->i_nlink);
              VTOI(fvp)->i_flag |= IN_CHANGE;
              UFS_WAPBL_UPDATE(fvp, NULL, NULL, 0);
      
      arghmybrainhurts:
   18         UFS_WAPBL_END(mp);
      
      ihateyou:
              return error;
      }
      
      /*
       * ufs_rename_ulr_overlap_p: True iff tulr overlaps with fulr so that
       * entering a directory entry at tulr may move fulr.
       */
      static bool
      ufs_rename_ulr_overlap_p(const struct ufs_lookup_results *fulr,
          const struct ufs_lookup_results *tulr)
      {
              doff_t from_prev_start, from_prev_end, to_start, to_end;
      
    9         KASSERT(fulr != NULL);
    9         KASSERT(tulr != NULL);
    9         KASSERT(fulr != tulr);
      
              /*
               * fulr is from a DELETE lookup, so fulr->ulr_count is the size
               * of the preceding entry (d_reclen).
               */
    9         from_prev_end = fulr->ulr_offset;
              KASSERT(fulr->ulr_count <= from_prev_end);
    9         from_prev_start = (from_prev_end - fulr->ulr_count);
      
              /*
               * tulr is from a RENAME lookup, so tulr->ulr_count is the size
               * of the free space for an entry that we are about to fill.
               */
              to_start = tulr->ulr_offset;
              KASSERT(tulr->ulr_count < (UFS_MAXDIRSIZE - to_start));
    9         to_end = (to_start + tulr->ulr_count);
      
              return
    9             (((to_start <= from_prev_start) && (from_prev_start < to_end)) ||
    9                 ((to_start <= from_prev_end) && (from_prev_end < to_end)));
      }
      
      /*
       * ufs_rename_recalculate_fulr: If we have just entered a directory into
       * dvp at tulr, and we were about to remove one at fulr for an entry
       * named fcnp, fulr may be invalid.  So, if necessary, recalculate it.
       */
      static int
      ufs_rename_recalculate_fulr(struct vnode *dvp,
          struct ufs_lookup_results *fulr, const struct ufs_lookup_results *tulr,
          const struct componentname *fcnp)
      {
              struct mount *mp;
              struct ufsmount *ump;
              int needswap;
              /* XXX int is a silly type for this; blame ufsmount::um_dirblksiz.  */
              int dirblksiz;
              doff_t search_start, search_end;
              doff_t offset;                /* Offset of entry we're examining.  */
    8         struct buf *bp;                /* I/O block we're examining.  */
              char *dirbuf;                /* Pointer into directory at search_start.  */
              struct direct *ep;        /* Pointer to the entry we're examining.  */
              /* XXX direct::d_reclen is 16-bit;
               * ufs_lookup_results::ulr_reclen is 32-bit.  Blah.  */
              uint32_t reclen;        /* Length of the entry we're examining.  */
              uint32_t prev_reclen;        /* Length of the preceding entry.  */
              int error;
      
              KASSERT(dvp != NULL);
    8         KASSERT(dvp->v_mount != NULL);
    8         KASSERT(VTOI(dvp) != NULL);
    8         KASSERT(fulr != NULL);
    8         KASSERT(tulr != NULL);
    8         KASSERT(fulr != tulr);
    8         KASSERT(ufs_rename_ulr_overlap_p(fulr, tulr));
      
    8         mp = dvp->v_mount;
              ump = VFSTOUFS(mp);
              KASSERT(ump != NULL);
    8         KASSERT(ump == VTOI(dvp)->i_ump);
      
    8         needswap = UFS_MPNEEDSWAP(ump);
      
              dirblksiz = ump->um_dirblksiz;
              KASSERT(0 < dirblksiz);
    8         KASSERT((dirblksiz & (dirblksiz - 1)) == 0);
      
              /* A directory block may not span across multiple I/O blocks.  */
    8         KASSERT(dirblksiz <= mp->mnt_stat.f_iosize);
      
              /* Find the bounds of the search.  */
    8         search_start = tulr->ulr_offset;
              KASSERT(fulr->ulr_reclen < (UFS_MAXDIRSIZE - fulr->ulr_offset));
    8         search_end = (fulr->ulr_offset + fulr->ulr_reclen);
      
              /* Compaction must happen only within a directory block. (*)  */
              KASSERT(search_start <= search_end);
    8         KASSERT((search_end - (search_start &~ (dirblksiz - 1))) <= dirblksiz);
      
    8         dirbuf = NULL;
              bp = NULL;
              error = ufs_blkatoff(dvp, (off_t)search_start, &dirbuf, &bp, false);
              if (error)
                      return error;
    8         KASSERT(dirbuf != NULL);
    8         KASSERT(bp != NULL);
      
              /*
               * Guarantee we sha'n't go past the end of the buffer we got.
               * dirbuf is bp->b_data + (search_start & (iosize - 1)), and
               * the valid range is [bp->b_data, bp->b_data + bp->b_bcount).
               */
    8         KASSERT((search_end - search_start) <=
                  (bp->b_bcount - (search_start & (mp->mnt_stat.f_iosize - 1))));
      
              prev_reclen = fulr->ulr_count;
              offset = search_start;
      
              /*
               * Search from search_start to search_end for the entry matching
               * fcnp, which must be there because we found it before and it
               * should only at most have moved earlier.
               */
              for (;;) {
                      KASSERT(search_start <= offset);
    8                 KASSERT(offset < search_end);
      
                      /*
                       * Examine the directory entry at offset.
                       */
    8                 ep = (struct direct *)(dirbuf + (offset - search_start));
    8                 reclen = ufs_rw16(ep->d_reclen, needswap);
      
                      if (ep->d_ino == 0)
                              goto next;        /* Entry is unused.  */
      
    8                 if (ufs_rw32(ep->d_ino, needswap) == UFS_WINO)
                              goto next;        /* Entry is whiteout.  */
      
    8                 if (fcnp->cn_namelen != ufs_direct_namlen(ep, dvp))
                              goto next;        /* Wrong name length.  */
      
    8                 if (memcmp(ep->d_name, fcnp->cn_nameptr, fcnp->cn_namelen))
                              goto next;        /* Wrong name.  */
      
                      /* Got it!  */
                      break;
      
    1 next:
                      if (! ((reclen < search_end) &&
    1                         (offset < (search_end - reclen)))) {
                              brelse(bp, 0);
                              return EIO;        /* XXX Panic?  What?  */
                      }
      
                      /* We may not move past the search end.  */
                      KASSERT(reclen < search_end);
                      KASSERT(offset < (search_end - reclen));
      
                      /*
                       * We may not move across a directory block boundary;
                       * see (*) above.
                       */
    1                 KASSERT((offset &~ (dirblksiz - 1)) ==
                          ((offset + reclen) &~ (dirblksiz - 1)));
      
                      prev_reclen = reclen;
    1                 offset += reclen;
              }
      
              /*
               * Found the entry.  Record where.
               */
    8         fulr->ulr_offset = offset;
              fulr->ulr_reclen = reclen;
      
              /*
               * Record the preceding record length, but not if we're at the
               * start of a directory block.
               */
    8         fulr->ulr_count = ((offset & (dirblksiz - 1))? prev_reclen : 0);
      
              brelse(bp, 0);
    8         return 0;
      }
      
      /*
       * ufs_direct_namlen: Return the namlen of the directory entry ep from
       * the directory vp.
       */
      static int                        /* XXX int?  uint8_t?  */
      ufs_direct_namlen(const struct direct *ep, const struct vnode *vp)
      {
              bool swap;
      
              KASSERT(ep != NULL);
    8         KASSERT(vp != NULL);
    8         KASSERT(VTOI(vp) != NULL);
    8         KASSERT(VTOI(vp)->i_ump != NULL);
      
      #if (BYTE_ORDER == LITTLE_ENDIAN)
    8         swap = (UFS_MPNEEDSWAP(VTOI(vp)->i_ump) == 0);
      #else
              swap = (UFS_MPNEEDSWAP(VTOI(vp)->i_ump) != 0);
      #endif
      
    8         return ((FSFMT(vp) && swap)? ep->d_type : ep->d_namlen);
      }
      
      /*
       * ufs_gro_remove: Rename an object over another link to itself,
       * effectively removing just the original link.
       */
      int
      ufs_gro_remove(struct mount *mp, kauth_cred_t cred,
          struct vnode *dvp, struct componentname *cnp, void *de, struct vnode *vp)
      {
              struct ufs_lookup_results *ulr = de;
              int error;
      
    2         KASSERT(mp != NULL);
    2         KASSERT(dvp != NULL);
    2         KASSERT(cnp != NULL);
    2         KASSERT(ulr != NULL);
    2         KASSERT(vp != NULL);
    2         KASSERT(dvp != vp);
    2         KASSERT(dvp->v_mount == mp);
    2         KASSERT(vp->v_mount == mp);
    2         KASSERT(dvp->v_type == VDIR);
    2         KASSERT(vp->v_type != VDIR);
    2         KASSERT(VOP_ISLOCKED(dvp) == LK_EXCLUSIVE);
    2         KASSERT(VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
    2         KASSERT(cnp->cn_nameiop == DELETE);
      
    2         error = UFS_WAPBL_BEGIN(mp);
              if (error)
                      goto out0;
      
              /* XXX ufs_dirremove decrements vp's link count for us.  */
    2         error = ufs_dirremove(dvp, ulr, VTOI(vp), cnp->cn_flags, 0);
              if (error)
                      goto out1;
      
    2         VN_KNOTE(dvp, NOTE_WRITE);
    2         VN_KNOTE(vp, (VTOI(vp)->i_nlink? NOTE_LINK : NOTE_DELETE));
      
    2 out1:        UFS_WAPBL_END(mp);
      out0:
    2         return error;
      }
      
      /*
       * ufs_gro_lookup: Look up and save the lookup results.
       */
      int
      ufs_gro_lookup(struct mount *mp, struct vnode *dvp,
          struct componentname *cnp, void *de_ret, struct vnode **vp_ret)
      {
              struct ufs_lookup_results *ulr_ret = de_ret;
   24         struct vnode *vp = NULL;
              int error;
      
              (void)mp;
              KASSERT(mp != NULL);
   24         KASSERT(dvp != NULL);
   24         KASSERT(cnp != NULL);
   24         KASSERT(ulr_ret != NULL);
   24         KASSERT(vp_ret != NULL);
   24         KASSERT(VOP_ISLOCKED(dvp) == LK_EXCLUSIVE);
      
              /* Kludge cargo-culted from dholland's ufs_rename.  */
   24         cnp->cn_flags &=~ MODMASK;
              cnp->cn_flags |= (LOCKPARENT | LOCKLEAF);
      
              error = relookup(dvp, &vp, cnp, 0 /* dummy */);
   24         if ((error == 0) && (vp == NULL)) {
                      error = ENOENT;
                      goto out;
              } else if (error) {
                      return error;
              }
      
              /*
               * Thanks to VFS insanity, relookup locks vp, which screws us
               * in various ways.
               */
              KASSERT(vp != NULL);
   24         VOP_UNLOCK(vp);
      
   24 out:        *ulr_ret = VTOI(dvp)->i_crap;
              *vp_ret = vp;
   24         return error;
      }
      
      /*
       * ufs_rmdired_p: Check whether the directory vp has been rmdired.
       *
       * vp must be locked and referenced.
       */
      static bool
      ufs_rmdired_p(struct vnode *vp)
      {
      
   24         KASSERT(vp != NULL);
   24         KASSERT(VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
   24         KASSERT(vp->v_type == VDIR);
      
              /* XXX Is this correct?  */
   24         return (VTOI(vp)->i_size == 0);
      }
      
      /*
       * ufs_read_dotdot: Store in *ino_ret the inode number of the parent
       * of the directory vp.
       */
      static int
      ufs_read_dotdot(struct vnode *vp, kauth_cred_t cred, ino_t *ino_ret)
      {
    8         struct dirtemplate dirbuf;
              int error;
      
              KASSERT(vp != NULL);
              KASSERT(ino_ret != NULL);
    8         KASSERT(vp->v_type == VDIR);
      
    8         error = ufs_bufio(UIO_READ, vp, &dirbuf, sizeof dirbuf, (off_t)0,
                  IO_NODELOCKED, cred, NULL, NULL);
              if (error)
                      return error;
      
    8         if (ufs_dirbuf_dotdot_namlen(&dirbuf, vp) != 2 ||
    8             dirbuf.dotdot_name[0] != '.' ||
                  dirbuf.dotdot_name[1] != '.')
                      /* XXX Panic?  Print warning?  */
                      return ENOTDIR;
      
    8         *ino_ret = ufs_rw32(dirbuf.dotdot_ino,
    8             UFS_MPNEEDSWAP(VTOI(vp)->i_ump));
              return 0;
      }
      
      /*
       * ufs_dirbuf_dotdot_namlen: Return the namlen of the directory buffer
       * dirbuf that came from the directory vp.  Swap byte order if
       * necessary.
       */
      static int                        /* XXX int?  uint8_t?  */
      ufs_dirbuf_dotdot_namlen(const struct dirtemplate *dirbuf,
          const struct vnode *vp)
      {
              bool swap;
      
              KASSERT(dirbuf != NULL);
    8         KASSERT(vp != NULL);
    8         KASSERT(VTOI(vp) != NULL);
    8         KASSERT(VTOI(vp)->i_ump != NULL);
      
      #if (BYTE_ORDER == LITTLE_ENDIAN)
    8         swap = (UFS_MPNEEDSWAP(VTOI(vp)->i_ump) == 0);
      #else
              swap = (UFS_MPNEEDSWAP(VTOI(vp)->i_ump) != 0);
      #endif
      
              return ((FSFMT(vp) && swap)?
    8             dirbuf->dotdot_type : dirbuf->dotdot_namlen);
      }
      
      /*
       * ufs_gro_genealogy: Analyze the genealogy of the source and target
       * directories.
       */
      int
      ufs_gro_genealogy(struct mount *mp, kauth_cred_t cred,
          struct vnode *fdvp, struct vnode *tdvp,
          struct vnode **intermediate_node_ret)
      {
   10         struct vnode *vp, *dvp;
              ino_t dotdot_ino = 0;        /* XXX: gcc */
              int error;
      
              KASSERT(mp != NULL);
   10         KASSERT(fdvp != NULL);
   10         KASSERT(tdvp != NULL);
   10         KASSERT(fdvp != tdvp);
   10         KASSERT(intermediate_node_ret != NULL);
   10         KASSERT(fdvp->v_mount == mp);
   10         KASSERT(tdvp->v_mount == mp);
   10         KASSERT(fdvp->v_type == VDIR);
   10         KASSERT(tdvp->v_type == VDIR);
      
              /*
               * We need to provisionally lock tdvp to keep rmdir from
               * deleting it -- or any ancestor -- at an inopportune moment.
               */
   10         error = ufs_gro_lock_directory(mp, tdvp);
   10         if (error)
                      return error;
      
              vp = tdvp;
   10         vref(vp);
      
              for (;;) {
   10                 KASSERT(vp != NULL);
   10                 KASSERT(VOP_ISLOCKED(vp) == LK_EXCLUSIVE);
   10                 KASSERT(vp->v_mount == mp);
   10                 KASSERT(vp->v_type == VDIR);
   10                 KASSERT(!ufs_rmdired_p(vp));
      
                      /* Did we hit the root without finding fdvp?  */
   10                 if (VTOI(vp)->i_number == UFS_ROOTINO) {
    5                         vput(vp);
                              *intermediate_node_ret = NULL;
                              return 0;
                      }
      
    8                 error = ufs_read_dotdot(vp, cred, &dotdot_ino);
                      if (error) {
                              vput(vp);
                              return error;
                      }
      
                      /* Did we find that fdvp is an ancestor of tdvp?  */
                      if (VTOI(fdvp)->i_number == dotdot_ino) {
                              /* Unlock vp, but keep it referenced.  */
    5                         VOP_UNLOCK(vp);
                              *intermediate_node_ret = vp;
                              return 0;
                      }
      
                      /* Neither -- keep ascending the family tree.  */
    6                 error = vcache_get(mp, &dotdot_ino, sizeof(dotdot_ino), &dvp);
                      vput(vp);
                      if (error)
                              return error;
    6                 error = vn_lock(dvp, LK_EXCLUSIVE);
                      if (error) {
                              vrele(dvp);
                              return error;
                      }
      
    6                 KASSERT(dvp != NULL);
    6                 KASSERT(VOP_ISLOCKED(dvp) == LK_EXCLUSIVE);
    6                 vp = dvp;
      
                      if (vp->v_type != VDIR) {
                              /*
                               * XXX Panic?  Print a warning?  Can this
                               * happen if we lose the race I suspect to
                               * exist above, and the `..' inode number has
                               * been recycled?
                               */
                              vput(vp);
                              return ENOTDIR;
                      }
      
    6                 if (ufs_rmdired_p(vp)) {
                              vput(vp);
                              return ENOENT;
                      }
              }
      }
      
      /*
       * ufs_gro_lock_directory: Lock the directory vp, but fail if it has
       * been rmdir'd.
       */
      int
      ufs_gro_lock_directory(struct mount *mp, struct vnode *vp)
      {
      
              (void)mp;
   24         KASSERT(mp != NULL);
   24         KASSERT(vp != NULL);
   24         KASSERT(vp->v_mount == mp);
      
   24         vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
      
   24         if (ufs_rmdired_p(vp)) {
                      VOP_UNLOCK(vp);
                      return ENOENT;
              }
      
              return 0;
      }
      
      static const struct genfs_rename_ops ufs_genfs_rename_ops = {
              .gro_directory_empty_p                = ufs_gro_directory_empty_p,
              .gro_rename_check_possible        = ufs_gro_rename_check_possible,
              .gro_rename_check_permitted        = ufs_gro_rename_check_permitted,
              .gro_remove_check_possible        = ufs_gro_remove_check_possible,
              .gro_remove_check_permitted        = ufs_gro_remove_check_permitted,
              .gro_rename                        = ufs_gro_rename,
              .gro_remove                        = ufs_gro_remove,
              .gro_lookup                        = ufs_gro_lookup,
              .gro_genealogy                        = ufs_gro_genealogy,
              .gro_lock_directory                = ufs_gro_lock_directory,
      };
      /*        $NetBSD: tty_ptm.c,v 1.42 2020/05/23 22:16:17 ad Exp $        */
      
      /*-
       * Copyright (c) 2004, 2020 The NetBSD Foundation, Inc.
       * All rights reserved.
       *
       * Redistribution and use in source and binary forms, with or without
       * modification, are permitted provided that the following conditions
       * are met:
       * 1. Redistributions of source code must retain the above copyright
       *    notice, this list of conditions and the following disclaimer.
       * 2. Redistributions in binary form must reproduce the above copyright
       *    notice, this list of conditions and the following disclaimer in the
       *    documentation and/or other materials provided with the distribution.
       *
       * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
       * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
       * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
       * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
       * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
       * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
       * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
       * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
       * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
       * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
       * POSSIBILITY OF SUCH DAMAGE.
       */
      
      #include <sys/cdefs.h>
      __KERNEL_RCSID(0, "$NetBSD: tty_ptm.c,v 1.42 2020/05/23 22:16:17 ad Exp $");
      
      #ifdef _KERNEL_OPT
      #include "opt_compat_netbsd.h"
      #include "opt_ptm.h"
      #endif
      
      /* pty multiplexor driver /dev/ptm{,x} */
      
      #include <sys/param.h>
      #include <sys/systm.h>
      #include <sys/ioctl.h>
      #include <sys/proc.h>
      #include <sys/tty.h>
      #include <sys/stat.h>
      #include <sys/file.h>
      #include <sys/uio.h>
      #include <sys/kernel.h>
      #include <sys/vnode.h>
      #include <sys/namei.h>
      #include <sys/signalvar.h>
      #include <sys/filedesc.h>
      #include <sys/conf.h>
      #include <sys/poll.h>
      #include <sys/pty.h>
      #include <sys/kauth.h>
      #include <sys/compat_stub.h>
      
      #include <miscfs/specfs/specdev.h>
      
      #include <compat/sys/ttycom.h>
      
      #include "ioconf.h"
      
      #ifdef DEBUG_PTM
      #define DPRINTF(a)        printf a
      #else
      #define DPRINTF(a)
      #endif
      
      #ifdef NO_DEV_PTM
      const struct cdevsw ptm_cdevsw = {
              .d_open = noopen,
              .d_close = noclose,
              .d_read = noread,
              .d_write = nowrite,
              .d_ioctl = noioctl,
              .d_stop = nostop,
              .d_tty = notty,
              .d_poll = nopoll,
              .d_mmap = nommap,
              .d_kqfilter = nokqfilter,
              .d_discard = nodiscard,
              .d_flag = D_TTY
      };
      #else
      
      int pts_major, ptc_major;
      
      static dev_t pty_getfree(void);
      static int pty_alloc_master(struct lwp *, int *, dev_t *, struct mount *);
      static int pty_alloc_slave(struct lwp *, int *, dev_t, struct mount *);
      static int pty_vn_open(struct vnode *, struct lwp *);
      
      int
      pty_getmp(struct lwp *l, struct mount **mpp)
      {
              if (ptm == NULL)
                      return EOPNOTSUPP;
      
              return (*ptm->getmp)(l, mpp);
      }
      
      dev_t
      pty_makedev(char ms, int minor)
      {
              return makedev(ms == 't' ? pts_major : ptc_major, minor);
      }
      
      
      static dev_t
      pty_getfree(void)
      {
              extern kmutex_t pt_softc_mutex;
              int i;
      
              mutex_enter(&pt_softc_mutex);
              for (i = 0; i < npty; i++) {
                      if (pty_isfree(i, 0))
                              break;
              }
              mutex_exit(&pt_softc_mutex);
              return pty_makedev('t', i);
      }
      
      /*
       * Hacked up version of vn_open. We _only_ handle ptys and only open
       * them with FREAD|FWRITE and never deal with creat or stuff like that.
       *
       * We need it because we have to fake up root credentials to open the pty.
       */
      int
      pty_vn_open(struct vnode *vp, struct lwp *l)
      {
              int error;
      
              if (vp->v_type != VCHR) {
                      vput(vp);
                      return EINVAL;
              }
      
              error = VOP_OPEN(vp, FREAD|FWRITE, lwp0.l_cred);
      
              if (error) {
                      vput(vp);
                      return error;
              }
      
              mutex_enter(vp->v_interlock);
              vp->v_writecount++;
              mutex_exit(vp->v_interlock);
      
              return 0;
      }
      
      static int
      pty_alloc_master(struct lwp *l, int *fd, dev_t *dev, struct mount *mp)
      {
              int error;
              struct file *fp;
              struct vnode *vp;
              int md;
      
              if ((error = fd_allocfile(&fp, fd)) != 0) {
                      DPRINTF(("fd_allocfile %d\n", error));
                      return error;
              }
      retry:
              /* Find and open a free master pty. */
              *dev = pty_getfree();
              md = minor(*dev);
              if ((error = pty_check(md)) != 0) {
                      DPRINTF(("pty_check %d\n", error));
                      goto bad;
              }
              if (ptm == NULL) {
                      DPRINTF(("no ptm\n"));
                      error = EOPNOTSUPP;
                      goto bad;
              }
              if ((error = (*ptm->allocvp)(mp, l, &vp, *dev, 'p')) != 0) {
                      DPRINTF(("pty_allocvp %d\n", error));
                      goto bad;
              }
      
              if ((error = pty_vn_open(vp, l)) != 0) {
                      DPRINTF(("pty_vn_open %d\n", error));
                      /*
                       * Check if the master open failed because we lost
                       * the race to grab it.
                       */
                      if (error != EIO)
                              goto bad;
                      error = !pty_isfree(md, 1);
                      DPRINTF(("pty_isfree %d\n", error));
                      if (error)
                              goto retry;
                      else
                              goto bad;
              }
              fp->f_flag = FREAD|FWRITE;
              fp->f_type = DTYPE_VNODE;
              fp->f_ops = &vnops;
              fp->f_vnode = vp;
              VOP_UNLOCK(vp);
              fd_affix(curproc, fp, *fd);
              return 0;
      bad:
              fd_abort(curproc, fp, *fd);
              return error;
      }
      
      int
      pty_grant_slave(struct lwp *l, dev_t dev, struct mount *mp)
      {
              int error;
              struct vnode *vp;
      
              /*
               * Open the slave.
               * namei -> setattr -> unlock -> revoke -> vrele ->
               * namei -> open -> unlock
               * Three stage rocket:
               * 1. Change the owner and permissions on the slave.
               * 2. Revoke all the users of the slave.
               * 3. open the slave.
               */
              if (ptm == NULL)
                      return EOPNOTSUPP;
              if ((error = (*ptm->allocvp)(mp, l, &vp, dev, 't')) != 0)
                      return error;
      
              if ((vp->v_mount->mnt_flag & MNT_RDONLY) == 0) {
                      struct vattr vattr;
                      (*ptm->getvattr)(mp, l, &vattr);
                      /* Do the VOP_SETATTR() as root. */
                      error = VOP_SETATTR(vp, &vattr, lwp0.l_cred);
                      if (error) {
                              DPRINTF(("setattr %d\n", error));
                              vput(vp);
                              return error;
                      }
              }
              VOP_UNLOCK(vp);
              VOP_REVOKE(vp, REVOKEALL);
      
              /*
               * The vnode is useless after the revoke, we need to get it again.
               */
              vrele(vp);
              return 0;
      }
      
      static int
      pty_alloc_slave(struct lwp *l, int *fd, dev_t dev, struct mount *mp)
      {
              int error;
              struct file *fp;
              struct vnode *vp;
      
              /* Grab a filedescriptor for the slave */
              if ((error = fd_allocfile(&fp, fd)) != 0) {
                      DPRINTF(("fd_allocfile %d\n", error));
                      return error;
              }
      
              if (ptm == NULL) {
                      error = EOPNOTSUPP;
                      goto bad;
              }
      
              if ((error = (*ptm->allocvp)(mp, l, &vp, dev, 't')) != 0)
                      goto bad;
              if ((error = pty_vn_open(vp, l)) != 0)
                      goto bad;
      
              fp->f_flag = FREAD|FWRITE;
              fp->f_type = DTYPE_VNODE;
              fp->f_ops = &vnops;
              fp->f_vnode = vp;
              VOP_UNLOCK(vp);
              fd_affix(curproc, fp, *fd);
              return 0;
      bad:
              fd_abort(curproc, fp, *fd);
              return error;
      }
      
      struct ptm_pty *
      pty_sethandler(struct ptm_pty *nptm)
      {
              struct ptm_pty *optm = ptm;
              ptm = nptm;
              return optm;
      }
      
      int
      pty_fill_ptmget(struct lwp *l, dev_t dev, int cfd, int sfd, void *data, struct mount *mp)
      {
              struct ptmget *ptmg = data;
              int error;
      
              if (ptm == NULL)
                      return EOPNOTSUPP;
      
              ptmg->cfd = cfd == -1 ? minor(dev) : cfd;
              ptmg->sfd = sfd == -1 ? minor(dev) : sfd;
      
              error = (*ptm->makename)(mp, l, ptmg->cn, sizeof(ptmg->cn), dev, 'p');
              if (error)
                      return error;
      
              return (*ptm->makename)(mp, l, ptmg->sn, sizeof(ptmg->sn), dev, 't');
      }
      
      void
      /*ARGSUSED*/
      ptmattach(int n)
      {
              extern const struct cdevsw pts_cdevsw, ptc_cdevsw;
              /* find the major and minor of the pty devices */
              if ((pts_major = cdevsw_lookup_major(&pts_cdevsw)) == -1)
                      panic("ptmattach: Can't find pty slave in cdevsw");
              if ((ptc_major = cdevsw_lookup_major(&ptc_cdevsw)) == -1)
                      panic("ptmattach: Can't find pty master in cdevsw");
      #ifdef COMPAT_BSDPTY
              ptm = &ptm_bsdpty;
      #endif
      }
      
      static int
      /*ARGSUSED*/
      ptmopen(dev_t dev, int flag, int mode, struct lwp *l)
      {
              int error;
    1         int fd;
              dev_t ttydev;
              struct mount *mp;
      
              switch(minor(dev)) {
              case 0:                /* /dev/ptmx */
              case 2:                /* /emul/linux/dev/ptmx */
                      if ((error = pty_getmp(l, &mp)) != 0)
                              return error;
                      if ((error = pty_alloc_master(l, &fd, &ttydev, mp)) != 0)
                              return error;
                      if (minor(dev) == 2) {
                              /*
                               * Linux ptyfs grants the pty right here.
                               * Handle this case here, instead of writing
                               * a new linux module.
                               */
                              if ((error = pty_grant_slave(l, ttydev, mp)) != 0) {
                                      file_t *fp = fd_getfile(fd);
                                      if (fp != NULL) {
                                              fd_close(fd);
                                      }
                                      return error;
                              }
                      }
                      curlwp->l_dupfd = fd;
                      return EMOVEFD;
              case 1:                /* /dev/ptm */
                      return 0;
              default:
    1                 return ENODEV;
              }
      }
      
      static int
      /*ARGSUSED*/
      ptmclose(dev_t dev, int flag, int mode, struct lwp *l)
      {
      
              return (0);
      }
      
      static int
      /*ARGSUSED*/
      ptmioctl(dev_t dev, u_long cmd, void *data, int flag, struct lwp *l)
      {
              int error;
              dev_t newdev;
              int cfd, sfd;
              file_t *fp;
              struct mount *mp;
      
              error = 0;
              switch (cmd) {
              case TIOCPTMGET:
                      if ((error = pty_getmp(l, &mp)) != 0)
                              return error;
      
                      if ((error = pty_alloc_master(l, &cfd, &newdev, mp)) != 0)
                              return error;
      
                      if ((error = pty_grant_slave(l, newdev, mp)) != 0)
                              goto bad;
      
                      if ((error = pty_alloc_slave(l, &sfd, newdev, mp)) != 0)
                              goto bad;
      
                      /* now, put the indices and names into struct ptmget */
                      if ((error = pty_fill_ptmget(l, newdev, cfd, sfd, data, mp)) != 0)
                              goto bad2;
                      return 0;
              default:
                      MODULE_HOOK_CALL(tty_ptmioctl_60_hook,
                          (dev, cmd, data, flag, l), EPASSTHROUGH, error);
                      if (error != EPASSTHROUGH)
                              return error;
                      DPRINTF(("ptmioctl EINVAL\n"));
                      return EINVAL;
              }
      bad2:
              fp = fd_getfile(sfd);
              if (fp != NULL) {
                      fd_close(sfd);
              }
       bad:
              fp = fd_getfile(cfd);
              if (fp != NULL) {
                      fd_close(cfd);
              }
              return error;
      }
      
      const struct cdevsw ptm_cdevsw = {
              .d_open = ptmopen,
              .d_close = ptmclose,
              .d_read = noread,
              .d_write = nowrite,
              .d_ioctl = ptmioctl,
              .d_stop = nullstop,
              .d_tty = notty,
              .d_poll = nopoll,
              .d_mmap = nommap,
              .d_kqfilter = nokqfilter,
              .d_discard = nodiscard,
              .d_flag = D_TTY
      };
      #endif
      /*        $NetBSD: uvm_fault_i.h,v 1.33 2020/02/23 15:46:43 ad Exp $        */
      
      /*
       * Copyright (c) 1997 Charles D. Cranor and Washington University.
       * All rights reserved.
       *
       * Redistribution and use in source and binary forms, with or without
       * modification, are permitted provided that the following conditions
       * are met:
       * 1. Redistributions of source code must retain the above copyright
       *    notice, this list of conditions and the following disclaimer.
       * 2. Redistributions in binary form must reproduce the above copyright
       *    notice, this list of conditions and the following disclaimer in the
       *    documentation and/or other materials provided with the distribution.
       *
       * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
       * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
       * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
       * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
       * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
       * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
       * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
       * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
       * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
       * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
       *
       * from: Id: uvm_fault_i.h,v 1.1.6.1 1997/12/08 16:07:12 chuck Exp
       */
      
      #ifndef _UVM_UVM_FAULT_I_H_
      #define _UVM_UVM_FAULT_I_H_
      
      /*
       * uvm_fault_i.h: fault inline functions
       */
      void uvmfault_update_stats(struct uvm_faultinfo *);
      
      
      /*
       * uvmfault_unlockmaps: unlock the maps
       */
      
      static __inline void
  552 uvmfault_unlockmaps(struct uvm_faultinfo *ufi, bool write_locked)
      {
              /*
               * ufi can be NULL when this isn't really a fault,
               * but merely paging in anon data.
               */
      
    1         if (ufi == NULL) {
                      return;
              }
      
      #ifndef __HAVE_NO_PMAP_STATS
  561         uvmfault_update_stats(ufi);
      #endif
              if (write_locked) {
                      vm_map_unlock(ufi->map);
              } else {
    1                 vm_map_unlock_read(ufi->map);
              }
      }
      
      /*
       * uvmfault_unlockall: unlock everything passed in.
       *
       * => maps must be read-locked (not write-locked).
       */
      
      static __inline void
      uvmfault_unlockall(struct uvm_faultinfo *ufi, struct vm_amap *amap,
          struct uvm_object *uobj)
      {
      
              if (uobj)
  241                 rw_exit(uobj->vmobjlock);
  477         if (amap)
  320                 amap_unlock(amap);
  486         uvmfault_unlockmaps(ufi, false);
      }
      
      /*
       * uvmfault_lookup: lookup a virtual address in a map
       *
       * => caller must provide a uvm_faultinfo structure with the IN
       *        params properly filled in
       * => we will lookup the map entry (handling submaps) as we go
       * => if the lookup is a success we will return with the maps locked
       * => if "write_lock" is true, we write_lock the map, otherwise we only
       *        get a read lock.
       * => note that submaps can only appear in the kernel and they are
       *        required to use the same virtual addresses as the map they
       *        are referenced by (thus address translation between the main
       *        map and the submap is unnecessary).
       */
      
      static __inline bool
   79 uvmfault_lookup(struct uvm_faultinfo *ufi, bool write_lock)
      {
              struct vm_map *tmpmap;
      
              /*
               * init ufi values for lookup.
               */
      
  561         ufi->map = ufi->orig_map;
              ufi->size = ufi->orig_size;
      
              /*
               * keep going down levels until we are done.   note that there can
               * only be two levels so we won't loop very long.
               */
      
              for (;;) {
                      /*
                       * lock map
                       */
                      if (write_lock) {
  235                         vm_map_lock(ufi->map);
                      } else {
  561                         vm_map_lock_read(ufi->map);
                      }
      
                      /*
                       * lookup
                       */
                      if (!uvm_map_lookup_entry(ufi->map, ufi->orig_rvaddr,
                          &ufi->entry)) {
   79                         uvmfault_unlockmaps(ufi, write_lock);
                              return(false);
                      }
      
                      /*
                       * reduce size if necessary
                       */
  504                 if (ufi->entry->end - ufi->orig_rvaddr < ufi->size)
                              ufi->size = ufi->entry->end - ufi->orig_rvaddr;
      
                      /*
                       * submap?    replace map with the submap and lookup again.
                       * note: VAs in submaps must match VAs in main map.
                       */
  504                 if (UVM_ET_ISSUBMAP(ufi->entry)) {
                              tmpmap = ufi->entry->object.sub_map;
                              if (write_lock) {
                                      vm_map_unlock(ufi->map);
                              } else {
                                      vm_map_unlock_read(ufi->map);
                              }
                              ufi->map = tmpmap;
                              continue;
                      }
      
                      /*
                       * got it!
                       */
      
  504                 ufi->mapv = ufi->map->timestamp;
                      return(true);
      
              }        /* while loop */
      
              /*NOTREACHED*/
      }
      
      /*
       * uvmfault_relock: attempt to relock the same version of the map
       *
       * => fault data structures should be unlocked before calling.
       * => if a success (true) maps will be locked after call.
       */
      
      static __inline bool
   17 uvmfault_relock(struct uvm_faultinfo *ufi)
      {
              /*
               * ufi can be NULL when this isn't really a fault,
               * but merely paging in anon data.
               */
      
              if (ufi == NULL) {
                      return true;
              }
      
   17         cpu_count(CPU_COUNT_FLTRELCK, 1);
      
              /*
               * relock map.   fail if version mismatch (in which case nothing
               * gets locked).
               */
      
              vm_map_lock_read(ufi->map);
              if (ufi->mapv != ufi->map->timestamp) {
    1                 vm_map_unlock_read(ufi->map);
                      return(false);
              }
      
   17         cpu_count(CPU_COUNT_FLTRELCKOK, 1);
              return(true);
      }
      
      #endif /* _UVM_UVM_FAULT_I_H_ */
      /*        $NetBSD: ld.c,v 1.111 2020/08/02 01:17:56 riastradh Exp $        */
      
      /*-
       * Copyright (c) 1998, 2000 The NetBSD Foundation, Inc.
       * All rights reserved.
       *
       * This code is derived from software contributed to The NetBSD Foundation
       * by Andrew Doran and Charles M. Hannum.
       *
       * Redistribution and use in source and binary forms, with or without
       * modification, are permitted provided that the following conditions
       * are met:
       * 1. Redistributions of source code must retain the above copyright
       *    notice, this list of conditions and the following disclaimer.
       * 2. Redistributions in binary form must reproduce the above copyright
       *    notice, this list of conditions and the following disclaimer in the
       *    documentation and/or other materials provided with the distribution.
       *
       * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
       * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
       * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
       * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
       * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
       * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
       * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
       * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
       * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
       * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
       * POSSIBILITY OF SUCH DAMAGE.
       */
      
      /*
       * Disk driver for use by RAID controllers.
       */
      
      #include <sys/cdefs.h>
      __KERNEL_RCSID(0, "$NetBSD: ld.c,v 1.111 2020/08/02 01:17:56 riastradh Exp $");
      
      #include <sys/param.h>
      #include <sys/systm.h>
      #include <sys/kernel.h>
      #include <sys/device.h>
      #include <sys/queue.h>
      #include <sys/proc.h>
      #include <sys/buf.h>
      #include <sys/bufq.h>
      #include <sys/endian.h>
      #include <sys/disklabel.h>
      #include <sys/disk.h>
      #include <sys/dkio.h>
      #include <sys/stat.h>
      #include <sys/conf.h>
      #include <sys/fcntl.h>
      #include <sys/vnode.h>
      #include <sys/syslog.h>
      #include <sys/mutex.h>
      #include <sys/module.h>
      #include <sys/reboot.h>
      
      #include <dev/ldvar.h>
      
      #include "ioconf.h"
      
      static void        ldminphys(struct buf *bp);
      static bool        ld_suspend(device_t, const pmf_qual_t *);
      static bool        ld_shutdown(device_t, int);
      static int        ld_diskstart(device_t, struct buf *bp);
      static void        ld_iosize(device_t, int *);
      static int        ld_dumpblocks(device_t, void *, daddr_t, int);
      static void        ld_fake_geometry(struct ld_softc *);
      static void        ld_set_geometry(struct ld_softc *);
      static void        ld_config_interrupts (device_t);
      static int        ld_lastclose(device_t);
      static int        ld_discard(device_t, off_t, off_t);
      static int        ld_flush(device_t, bool);
      
      static dev_type_open(ldopen);
      static dev_type_close(ldclose);
      static dev_type_read(ldread);
      static dev_type_write(ldwrite);
      static dev_type_ioctl(ldioctl);
      static dev_type_strategy(ldstrategy);
      static dev_type_dump(lddump);
      static dev_type_size(ldsize);
      static dev_type_discard(lddiscard);
      
      const struct bdevsw ld_bdevsw = {
              .d_open = ldopen,
              .d_close = ldclose,
              .d_strategy = ldstrategy,
              .d_ioctl = ldioctl,
              .d_dump = lddump,
              .d_psize = ldsize,
              .d_discard = lddiscard,
              .d_flag = D_DISK | D_MPSAFE
      };
      
      const struct cdevsw ld_cdevsw = {
              .d_open = ldopen,
              .d_close = ldclose,
              .d_read = ldread,
              .d_write = ldwrite,
              .d_ioctl = ldioctl,
              .d_stop = nostop,
              .d_tty = notty,
              .d_poll = nopoll,
              .d_mmap = nommap,
              .d_kqfilter = nokqfilter,
              .d_discard = lddiscard,
              .d_flag = D_DISK | D_MPSAFE
      };
      
      static const struct        dkdriver lddkdriver = {
              .d_open = ldopen,
              .d_close = ldclose,
              .d_strategy = ldstrategy,
              .d_iosize = ld_iosize,
              .d_minphys  = ldminphys,
              .d_diskstart = ld_diskstart,
              .d_dumpblocks = ld_dumpblocks,
              .d_lastclose = ld_lastclose,
              .d_discard = ld_discard
      };
      
      void
      ldattach(struct ld_softc *sc, const char *default_strategy)
      {
              device_t self = sc->sc_dv;
              struct dk_softc *dksc = &sc->sc_dksc;
      
              mutex_init(&sc->sc_mutex, MUTEX_DEFAULT, IPL_VM);
              cv_init(&sc->sc_drain, "lddrain");
      
              if ((sc->sc_flags & LDF_ENABLED) == 0) {
                      return;
              }
      
              /* don't attach a disk that we cannot handle */
              if (sc->sc_secsize < DEV_BSIZE) {
                      sc->sc_flags &= ~LDF_ENABLED;
                      return;
              }
      
              /* Initialise dk and disk structure. */
              dk_init(dksc, self, DKTYPE_LD);
              disk_init(&dksc->sc_dkdev, dksc->sc_xname, &lddkdriver);
      
              if (sc->sc_maxxfer > MAXPHYS)
                      sc->sc_maxxfer = MAXPHYS;
      
              /* Build synthetic geometry if necessary. */
              if (sc->sc_nheads == 0 || sc->sc_nsectors == 0 ||
                  sc->sc_ncylinders == 0)
                  ld_fake_geometry(sc);
      
              sc->sc_disksize512 = sc->sc_secperunit * sc->sc_secsize / DEV_BSIZE;
      
              if (sc->sc_flags & LDF_NO_RND)
                      dksc->sc_flags |= DKF_NO_RND;
      
              /* Attach dk and disk subsystems */
              dk_attach(dksc);
              disk_attach(&dksc->sc_dkdev);
              ld_set_geometry(sc);
      
              bufq_alloc(&dksc->sc_bufq, default_strategy, BUFQ_SORT_RAWBLOCK);
      
              /* Register with PMF */
              if (!pmf_device_register1(dksc->sc_dev, ld_suspend, NULL, ld_shutdown))
                      aprint_error_dev(dksc->sc_dev,
                          "couldn't establish power handler\n");
      
              /* Discover wedges on this disk. */
              config_interrupts(sc->sc_dv, ld_config_interrupts);
      }
      
      int
      ldadjqparam(struct ld_softc *sc, int xmax)
      {
      
              mutex_enter(&sc->sc_mutex);
              sc->sc_maxqueuecnt = xmax;
              mutex_exit(&sc->sc_mutex);
      
              return (0);
      }
      
      int
      ldbegindetach(struct ld_softc *sc, int flags)
      {
              struct dk_softc *dksc = &sc->sc_dksc;
              int error;
      
              /* If we never attached properly, no problem with detaching.  */
              if ((sc->sc_flags & LDF_ENABLED) == 0)
                      return 0;
      
              /*
               * If the disk is still open, back out before we commit to
               * detaching.
               */
              error = disk_begindetach(&dksc->sc_dkdev, ld_lastclose, dksc->sc_dev,
                  flags);
              if (error)
                      return error;
      
              /* We are now committed to detaching.  Prevent new xfers.  */
              ldadjqparam(sc, 0);
      
              return 0;
      }
      
      void
      ldenddetach(struct ld_softc *sc)
      {
              struct dk_softc *dksc = &sc->sc_dksc;
              int bmaj, cmaj, i, mn;
      
              if ((sc->sc_flags & LDF_ENABLED) == 0)
                      return;
      
              /* Wait for commands queued with the hardware to complete. */
              mutex_enter(&sc->sc_mutex);
              while (sc->sc_queuecnt > 0) {
                      if (cv_timedwait(&sc->sc_drain, &sc->sc_mutex, 30 * hz)) {
                              /*
                               * XXX This seems like a recipe for crashing on
                               * use after free...
                               */
                              printf("%s: not drained\n", dksc->sc_xname);
                              break;
                      }
              }
              mutex_exit(&sc->sc_mutex);
      
              /* Kill off any queued buffers. */
              dk_drain(dksc);
              bufq_free(dksc->sc_bufq);
      
              /* Locate the major numbers. */
              bmaj = bdevsw_lookup_major(&ld_bdevsw);
              cmaj = cdevsw_lookup_major(&ld_cdevsw);
      
              /* Nuke the vnodes for any open instances. */
              for (i = 0; i < MAXPARTITIONS; i++) {
                      mn = DISKMINOR(device_unit(dksc->sc_dev), i);
                      vdevgone(bmaj, mn, mn, VBLK);
                      vdevgone(cmaj, mn, mn, VCHR);
              }
      
              /* Delete all of our wedges. */
              dkwedge_delall(&dksc->sc_dkdev);
      
              /* Detach from the disk list. */
              disk_detach(&dksc->sc_dkdev);
              disk_destroy(&dksc->sc_dkdev);
      
              dk_detach(dksc);
      
              /* Deregister with PMF */
              pmf_device_deregister(dksc->sc_dev);
      
              /*
               * XXX We can't really flush the cache here, because the
               * XXX device may already be non-existent from the controller's
               * XXX perspective.
               */
      #if 0
              ld_flush(dksc->sc_dev, false);
      #endif
              cv_destroy(&sc->sc_drain);
              mutex_destroy(&sc->sc_mutex);
      }
      
      /* ARGSUSED */
      static bool
      ld_suspend(device_t dev, const pmf_qual_t *qual)
      {
              return ld_shutdown(dev, 0);
      }
      
      /* ARGSUSED */
      static bool
      ld_shutdown(device_t dev, int flags)
      {
              if ((flags & RB_NOSYNC) == 0 && ld_flush(dev, true) != 0)
                      return false;
      
              return true;
      }
      
      /* ARGSUSED */
      static int
      ldopen(dev_t dev, int flags, int fmt, struct lwp *l)
      {
              struct ld_softc *sc;
              struct dk_softc *dksc;
              int unit;
      
    2         unit = DISKUNIT(dev);
              if ((sc = device_lookup_private(&ld_cd, unit)) == NULL)
                      return (ENXIO);
      
              if ((sc->sc_flags & LDF_ENABLED) == 0)
                      return (ENODEV);
      
              dksc = &sc->sc_dksc;
      
    2         return dk_open(dksc, dev, flags, fmt, l);
      }
      
      static int
      ld_lastclose(device_t self)
      {
              ld_flush(self, false);
      
              return 0;
      }
      
      /* ARGSUSED */
      static int
      ldclose(dev_t dev, int flags, int fmt, struct lwp *l)
      {
              struct ld_softc *sc;
              struct dk_softc *dksc;
              int unit;
      
              unit = DISKUNIT(dev);
              sc = device_lookup_private(&ld_cd, unit);
              dksc = &sc->sc_dksc;
      
              return dk_close(dksc, dev, flags, fmt, l);
      }
      
      /* ARGSUSED */
      static int
      ldread(dev_t dev, struct uio *uio, int ioflag)
      {
      
              return (physio(ldstrategy, NULL, dev, B_READ, ldminphys, uio));
      }
      
      /* ARGSUSED */
      static int
      ldwrite(dev_t dev, struct uio *uio, int ioflag)
      {
      
              return (physio(ldstrategy, NULL, dev, B_WRITE, ldminphys, uio));
      }
      
      /* ARGSUSED */
      static int
      ldioctl(dev_t dev, u_long cmd, void *addr, int32_t flag, struct lwp *l)
      {
              struct ld_softc *sc;
              struct dk_softc *dksc;
              int unit, error;
      
              unit = DISKUNIT(dev);
              sc = device_lookup_private(&ld_cd, unit);
              dksc = &sc->sc_dksc;
      
              error = 0;
      
              /*
               * Some common checks so that individual attachments wouldn't need
               * to duplicate them.
               */
              switch (cmd) {
              case DIOCCACHESYNC:
                      /*
                       * XXX Do we really need to care about having a writable
                       * file descriptor here?
                       */
                      if ((flag & FWRITE) == 0)
                              error = EBADF;
                      else
                              error = 0;
                      break;
              }
      
              if (error != 0)
                      return (error);
      
              if (sc->sc_ioctl) {
                      if ((sc->sc_flags & LDF_MPSAFE) == 0)
                              KERNEL_LOCK(1, curlwp);
                      error = (*sc->sc_ioctl)(sc, cmd, addr, flag, 0);
                      if ((sc->sc_flags & LDF_MPSAFE) == 0)
                              KERNEL_UNLOCK_ONE(curlwp);
                      if (error != EPASSTHROUGH)
                              return (error);
              }
      
              /* something not handled by the attachment */
              return dk_ioctl(dksc, dev, cmd, addr, flag, l);
      }
      
      /*
       * Flush the device's cache.
       */
      static int
      ld_flush(device_t self, bool poll)
      {
              int error = 0;
              struct ld_softc *sc = device_private(self);
      
              if (sc->sc_ioctl) {
                      if ((sc->sc_flags & LDF_MPSAFE) == 0)
                              KERNEL_LOCK(1, curlwp);
                      error = (*sc->sc_ioctl)(sc, DIOCCACHESYNC, NULL, 0, poll);
                      if ((sc->sc_flags & LDF_MPSAFE) == 0)
                              KERNEL_UNLOCK_ONE(curlwp);
                      if (error != 0)
                              device_printf(self, "unable to flush cache\n");
              }
      
              return error;
      }
      
      static void
      ldstrategy(struct buf *bp)
      {
              struct ld_softc *sc;
              struct dk_softc *dksc;
              int unit;
      
              unit = DISKUNIT(bp->b_dev);
              sc = device_lookup_private(&ld_cd, unit);
              dksc = &sc->sc_dksc;
      
              dk_strategy(dksc, bp);
      }
      
      static int
      ld_diskstart(device_t dev, struct buf *bp)
      {
              struct ld_softc *sc = device_private(dev);
              int error;
      
              if (sc->sc_queuecnt >= sc->sc_maxqueuecnt)
                      return EAGAIN;
      
              if ((sc->sc_flags & LDF_MPSAFE) == 0)
                      KERNEL_LOCK(1, curlwp);
      
              mutex_enter(&sc->sc_mutex);
      
              if (sc->sc_queuecnt >= sc->sc_maxqueuecnt)
                      error = EAGAIN;
              else {
                      error = (*sc->sc_start)(sc, bp);
                      if (error == 0)
                              sc->sc_queuecnt++;
              }
      
              mutex_exit(&sc->sc_mutex);
      
              if ((sc->sc_flags & LDF_MPSAFE) == 0)
                      KERNEL_UNLOCK_ONE(curlwp);
      
              return error;
      }
      
      void
      lddone(struct ld_softc *sc, struct buf *bp)
      {
              struct dk_softc *dksc = &sc->sc_dksc;
      
              dk_done(dksc, bp);
      
              mutex_enter(&sc->sc_mutex);
              if (--sc->sc_queuecnt <= sc->sc_maxqueuecnt) {
                      cv_broadcast(&sc->sc_drain);
                      mutex_exit(&sc->sc_mutex);
                      dk_start(dksc, NULL);
              } else
                      mutex_exit(&sc->sc_mutex);
      }
      
      static int
      ldsize(dev_t dev)
      {
              struct ld_softc *sc;
              struct dk_softc *dksc;
              int unit;
      
              unit = DISKUNIT(dev);
              if ((sc = device_lookup_private(&ld_cd, unit)) == NULL)
                      return (-1);
              dksc = &sc->sc_dksc;
      
              if ((sc->sc_flags & LDF_ENABLED) == 0)
                      return (-1);
      
              return dk_size(dksc, dev);
      }
      
      /*
       * Take a dump.
       */
      static int
      lddump(dev_t dev, daddr_t blkno, void *va, size_t size)
      {
              struct ld_softc *sc;
              struct dk_softc *dksc;
              int unit;
      
              unit = DISKUNIT(dev);
              if ((sc = device_lookup_private(&ld_cd, unit)) == NULL)
                      return (ENXIO);
              dksc = &sc->sc_dksc;
      
              if ((sc->sc_flags & LDF_ENABLED) == 0)
                      return (ENODEV);
      
              return dk_dump(dksc, dev, blkno, va, size, 0);
      }
      
      static int
      ld_dumpblocks(device_t dev, void *va, daddr_t blkno, int nblk)
      {
              struct ld_softc *sc = device_private(dev);
      
              if (sc->sc_dump == NULL)
                      return (ENODEV);
      
              return (*sc->sc_dump)(sc, va, blkno, nblk);
      }
      
      /*
       * Adjust the size of a transfer.
       */
      static void
      ldminphys(struct buf *bp)
      {
              int unit;
              struct ld_softc *sc;
      
              unit = DISKUNIT(bp->b_dev);
              sc = device_lookup_private(&ld_cd, unit);
      
              ld_iosize(sc->sc_dv, &bp->b_bcount);
              minphys(bp);
      }
      
      static void
      ld_iosize(device_t d, int *countp)
      {
              struct ld_softc *sc = device_private(d);
      
              if (*countp > sc->sc_maxxfer)
                      *countp = sc->sc_maxxfer;
      }
      
      static void
      ld_fake_geometry(struct ld_softc *sc)
      {
              uint64_t ncyl;
      
              if (sc->sc_secperunit <= 528 * 2048)                /* 528MB */
                      sc->sc_nheads = 16;
              else if (sc->sc_secperunit <= 1024 * 2048)        /* 1GB */
                      sc->sc_nheads = 32;
              else if (sc->sc_secperunit <= 21504 * 2048)        /* 21GB */
                      sc->sc_nheads = 64;
              else if (sc->sc_secperunit <= 43008 * 2048)        /* 42GB */
                      sc->sc_nheads = 128;
              else
                      sc->sc_nheads = 255;
      
              sc->sc_nsectors = 63;
              sc->sc_ncylinders = INT_MAX;
              ncyl = sc->sc_secperunit /
                  (sc->sc_nheads * sc->sc_nsectors);
              if (ncyl < INT_MAX)
                      sc->sc_ncylinders = (int)ncyl;
      }
      
      static void
      ld_set_geometry(struct ld_softc *sc)
      {
              struct dk_softc *dksc = &sc->sc_dksc;
              struct disk_geom *dg = &dksc->sc_dkdev.dk_geom;
              char tbuf[9];
      
              format_bytes(tbuf, sizeof(tbuf), sc->sc_secperunit *
                  sc->sc_secsize);
              aprint_normal_dev(dksc->sc_dev, "%s, %d cyl, %d head, %d sec, "
                  "%d bytes/sect x %"PRIu64" sectors\n",
                  tbuf, sc->sc_ncylinders, sc->sc_nheads,
                  sc->sc_nsectors, sc->sc_secsize, sc->sc_secperunit);
      
              memset(dg, 0, sizeof(*dg));
              dg->dg_secperunit = sc->sc_secperunit;
              dg->dg_secsize = sc->sc_secsize;
              dg->dg_nsectors = sc->sc_nsectors;
              dg->dg_ntracks = sc->sc_nheads;
              dg->dg_ncylinders = sc->sc_ncylinders;
      
              disk_set_info(dksc->sc_dev, &dksc->sc_dkdev, sc->sc_typename);
      }
      
      static void
      ld_config_interrupts(device_t d)
      {
              struct ld_softc *sc = device_private(d);
              struct dk_softc *dksc = &sc->sc_dksc;
      
              dkwedge_discover(&dksc->sc_dkdev);
      }
      
      static int
      ld_discard(device_t dev, off_t pos, off_t len)
      {
              struct ld_softc *sc = device_private(dev);
              struct buf dbuf, *bp = &dbuf;
              int error = 0;
      
              KASSERT(len <= INT_MAX);
      
              if (sc->sc_discard == NULL)
                      return (ENODEV);
      
              if ((sc->sc_flags & LDF_MPSAFE) == 0)
                      KERNEL_LOCK(1, curlwp);
      
              buf_init(bp);
              bp->b_vp = NULL;
              bp->b_data = NULL;
              bp->b_bufsize = 0;
              bp->b_rawblkno = pos / sc->sc_secsize;
              bp->b_bcount = len;
              bp->b_flags = B_WRITE;
              bp->b_cflags = BC_BUSY;
      
              error = (*sc->sc_discard)(sc, bp);
              if (error == 0)
                      error = biowait(bp);
      
              buf_destroy(bp);
      
              if ((sc->sc_flags & LDF_MPSAFE) == 0)
                      KERNEL_UNLOCK_ONE(curlwp);
      
              return error;
      }
      
      void
      lddiscardend(struct ld_softc *sc, struct buf *bp)
      {
      
              if (bp->b_error)
                      bp->b_resid = bp->b_bcount;
              biodone(bp);
      }
      
      static int
      lddiscard(dev_t dev, off_t pos, off_t len)
      {
              struct ld_softc *sc;
              struct dk_softc *dksc;
              int unit;
      
              unit = DISKUNIT(dev);
              sc = device_lookup_private(&ld_cd, unit);
              dksc = &sc->sc_dksc;
      
              return dk_discard(dksc, dev, pos, len);
      }
      
      MODULE(MODULE_CLASS_DRIVER, ld, "dk_subr");
      
      #ifdef _MODULE
      CFDRIVER_DECL(ld, DV_DISK, NULL);
      #endif
      
      static int
      ld_modcmd(modcmd_t cmd, void *opaque)
      {
      #ifdef _MODULE
              devmajor_t bmajor, cmajor;
      #endif
              int error = 0;
      
      #ifdef _MODULE
              switch (cmd) {
              case MODULE_CMD_INIT:
                      bmajor = cmajor = -1;
                      error = devsw_attach(ld_cd.cd_name, &ld_bdevsw, &bmajor,
                          &ld_cdevsw, &cmajor);
                      if (error)
                              break;
                      error = config_cfdriver_attach(&ld_cd);
                      break;
              case MODULE_CMD_FINI:
                      error = config_cfdriver_detach(&ld_cd);
                      if (error)
                              break;
                      devsw_detach(&ld_bdevsw, &ld_cdevsw);
                      break;
              default:
                      error = ENOTTY;
                      break;
              }
      #endif
      
              return error;
      }
      /*        $NetBSD: proc.h,v 1.367 2020/05/23 23:42:44 ad Exp $        */
      
      /*-
       * Copyright (c) 2006, 2007, 2008, 2020 The NetBSD Foundation, Inc.
       * All rights reserved.
       *
       * This code is derived from software contributed to The NetBSD Foundation
       * by Andrew Doran.
       *
       * Redistribution and use in source and binary forms, with or without
       * modification, are permitted provided that the following conditions
       * are met:
       * 1. Redistributions of source code must retain the above copyright
       *    notice, this list of conditions and the following disclaimer.
       * 2. Redistributions in binary form must reproduce the above copyright
       *    notice, this list of conditions and the following disclaimer in the
       *    documentation and/or other materials provided with the distribution.
       *
       * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
       * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
       * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
       * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
       * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
       * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
       * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
       * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
       * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
       * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
       * POSSIBILITY OF SUCH DAMAGE.
       */
      
      /*-
       * Copyright (c) 1986, 1989, 1991, 1993
       *        The Regents of the University of California.  All rights reserved.
       * (c) UNIX System Laboratories, Inc.
       * All or some portions of this file are derived from material licensed
       * to the University of California by American Telephone and Telegraph
       * Co. or Unix System Laboratories, Inc. and are reproduced herein with
       * the permission of UNIX System Laboratories, Inc.
       *
       * Redistribution and use in source and binary forms, with or without
       * modification, are permitted provided that the following conditions
       * are met:
       * 1. Redistributions of source code must retain the above copyright
       *    notice, this list of conditions and the following disclaimer.
       * 2. Redistributions in binary form must reproduce the above copyright
       *    notice, this list of conditions and the following disclaimer in the
       *    documentation and/or other materials provided with the distribution.
       * 3. Neither the name of the University nor the names of its contributors
       *    may be used to endorse or promote products derived from this software
       *    without specific prior written permission.
       *
       * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
       * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
       * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
       * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
       * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
       * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
       * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
       * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
       * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
       * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
       * SUCH DAMAGE.
       *
       *        @(#)proc.h        8.15 (Berkeley) 5/19/95
       */
      
      #ifndef _SYS_PROC_H_
      #define        _SYS_PROC_H_
      
      #include <sys/lwp.h>
      
      #if defined(_KMEMUSER) || defined(_KERNEL)
      
      #if defined(_KERNEL_OPT)
      #include "opt_multiprocessor.h"
      #include "opt_kstack.h"
      #include "opt_lockdebug.h"
      #endif
      
      #include <machine/proc.h>                /* Machine-dependent proc substruct */
      #include <machine/pcb.h>
      #include <sys/aio.h>
      #include <sys/idtype.h>
      #include <sys/rwlock.h>
      #include <sys/mqueue.h>
      #include <sys/mutex.h>
      #include <sys/condvar.h>
      #include <sys/queue.h>
      #include <sys/radixtree.h>
      #include <sys/signalvar.h>
      #include <sys/siginfo.h>
      #include <sys/event.h>
      #include <sys/specificdata.h>
      
      #ifndef _KERNEL
      #include <sys/time.h>
      #include <sys/resource.h>
      #endif
      
      /*
       * One structure allocated per session.
       */
      struct session {
              int                s_count;        /* Ref cnt; pgrps in session */
              u_int                s_flags;
      #define        S_LOGIN_SET        1                /* s_login set in this session */
              struct proc        *s_leader;        /* Session leader */
              struct vnode        *s_ttyvp;        /* Vnode of controlling terminal */
              struct tty        *s_ttyp;        /* Controlling terminal */
              char                s_login[MAXLOGNAME]; /* Setlogin() name */
              pid_t                s_sid;                /* Session ID (pid of leader) */
      };
      
      /*
       * One structure allocated per process group.
       */
      struct pgrp {
              LIST_HEAD(, proc) pg_members;        /* Pointer to pgrp members */
              struct session        *pg_session;        /* Pointer to session */
              pid_t                pg_id;                /* Pgrp id */
              int                pg_jobc;        /*
                                               * Number of processes qualifying
                                               * pgrp for job control
                                               */
      };
      
      /*
       * Autoloadable syscall definition
       */
      struct sc_autoload {
              u_int                al_code;
              const char        *al_module;
      };
      
      /*
       * One structure allocated per emulation.
       */
      struct exec_package;
      struct ras;
      struct kauth_cred;
      
      struct emul {
              const char        *e_name;        /* Symbolic name */
              const char        *e_path;        /* Extra emulation path (NULL if none)*/
      #ifndef __HAVE_MINIMAL_EMUL
              int                e_flags;        /* Miscellaneous flags, see above */
                                              /* Syscall handling function */
              const int        *e_errno;        /* Errno array */
              int                e_nosys;        /* Offset of the nosys() syscall */
              int                e_nsysent;        /* Number of system call entries */
      #endif
              struct sysent        *e_sysent;        /* System call array */
              const uint32_t        *e_nomodbits;        /* sys_nosys/sys_nomodule flags
                                               * for syscall_disestablish() */
              const char * const *e_syscallnames; /* System call name array */
              struct sc_autoload *e_sc_autoload; /* List of autoloadable syscalls */
                                              /* Signal sending function */
              void                (*e_sendsig)(const struct ksiginfo *,
                                                const sigset_t *);
              void                (*e_trapsignal)(struct lwp *, struct ksiginfo *);
              char                *e_sigcode;        /* Start of sigcode */
              char                *e_esigcode;        /* End of sigcode */
                                              /* Set registers before execution */
              struct uvm_object **e_sigobject;/* shared sigcode object */
              void                (*e_setregs)(struct lwp *, struct exec_package *,
                                                vaddr_t);
      
                                              /* Per-process hooks */
              void                (*e_proc_exec)(struct proc *, struct exec_package *);
              void                (*e_proc_fork)(struct proc *, struct lwp *, int);
              void                (*e_proc_exit)(struct proc *);
              void                (*e_lwp_fork)(struct lwp *, struct lwp *);
              void                (*e_lwp_exit)(struct lwp *);
      
      #ifdef __HAVE_SYSCALL_INTERN
              void                (*e_syscall_intern)(struct proc *);
      #else
              void                (*e_syscall)(void);
      #endif
                                              /* Emulation specific sysctl data */
              struct sysctlnode *e_sysctlovly;
      
              vaddr_t                (*e_vm_default_addr)(struct proc *, vaddr_t, vsize_t,
                                   int);
      
              /* Emulation-specific hook for userspace page faults */
              int                (*e_usertrap)(struct lwp *, vaddr_t, void *);
      
              size_t                e_ucsize;        /* size of ucontext_t */
              void                (*e_startlwp)(void *);
      
              /* Dtrace syscall probe */
              void                 (*e_dtrace_syscall)(uint32_t, register_t,
                                  const struct sysent *, const void *,
                                  const register_t *, int);
      
              /* Emulation specific support for ktracing signal posts */
              void                (*e_ktrpsig)(int, sig_t, const sigset_t *,
                                  const struct ksiginfo *);
      };
      
      /*
       * Emulation miscellaneous flags
       */
      #define        EMUL_HAS_SYS___syscall        0x001        /* Has SYS___syscall */
      
      /*
       * Description of a process.
       *
       * This structure contains the information needed to manage a thread of
       * control, known in UN*X as a process; it has references to substructures
       * containing descriptions of things that the process uses, but may share
       * with related processes.  The process structure and the substructures
       * are always addressible except for those marked "(PROC ONLY)" below,
       * which might be addressible only on a processor on which the process
       * is running.
       *
       * Field markings and the corresponding locks:
       *
       * a:        p_auxlock
       * k:        ktrace_mutex
       * l:        proc_lock
       * t:        p_stmutex
       * p:        p_lock
       * (:        updated atomically
       * ::        unlocked, stable
       */
      struct vmspace;
      
      struct proc {
              LIST_ENTRY(proc) p_list;        /* l: List of all processes */
              kmutex_t        *p_lock;        /* :: general mutex */
              kcondvar_t        p_waitcv;        /* p: wait, stop CV on children */
              kcondvar_t        p_lwpcv;        /* p: wait, stop CV on LWPs */
      
              /* Substructures: */
              struct kauth_cred *p_cred;        /* p: Master copy of credentials */
              struct filedesc        *p_fd;                /* :: Ptr to open files structure */
              struct cwdinfo        *p_cwdi;        /* :: cdir/rdir/cmask info */
              struct pstats        *p_stats;        /* :: Accounting/stats (PROC ONLY) */
              struct plimit        *p_limit;        /* :: Process limits */
              struct vmspace        *p_vmspace;        /* :: Address space */
              struct sigacts        *p_sigacts;        /* :: Process sigactions */
              struct aioproc        *p_aio;                /* p: Asynchronous I/O data */
              u_int                p_mqueue_cnt;        /* (: Count of open message queues */
              specificdata_reference
                              p_specdataref;        /*    subsystem proc-specific data */
      
              int                p_exitsig;        /* l: signal to send to parent on exit */
              int                p_flag;                /* p: PK_* flags */
              int                p_sflag;        /* p: PS_* flags */
              int                p_slflag;        /* p, l: PSL_* flags */
              int                p_lflag;        /* l: PL_* flags */
              int                p_stflag;        /* t: PST_* flags */
              char                p_stat;                /* p: S* process status. */
              char                p_trace_enabled;/* p: cached by syscall_intern() */
              char                p_pad1[2];        /*  unused */
      
              pid_t                p_pid;                /* :: Process identifier. */
              LIST_ENTRY(proc) p_pglist;        /* l: List of processes in pgrp. */
              struct proc         *p_pptr;        /* l: Pointer to parent process. */
              LIST_ENTRY(proc) p_sibling;        /* l: List of sibling processes. */
              LIST_HEAD(, proc) p_children;        /* l: List of children. */
              LIST_HEAD(, lwp) p_lwps;        /* p: List of LWPs. */
              struct ras        *p_raslist;        /* a: List of RAS entries */
      
      /* The following fields are all zeroed upon creation in fork. */
      #define        p_startzero        p_nlwps
      
              int                 p_nlwps;        /* p: Number of LWPs */
              int                 p_nzlwps;        /* p: Number of zombie LWPs */
              int                p_nrlwps;        /* p: Number running/sleeping LWPs */
              int                p_nlwpwait;        /* p: Number of LWPs in lwp_wait1() */
              int                p_ndlwps;        /* p: Number of detached LWPs */
              u_int                p_nstopchild;        /* l: Count of stopped/dead children */
              u_int                p_waited;        /* l: parent has waited on child */
              struct lwp        *p_zomblwp;        /* p: detached LWP to be reaped */
              struct lwp        *p_vforklwp;        /* p: parent LWP waiting at vfork() */
      
              /* scheduling */
              void                *p_sched_info;        /* p: Scheduler-specific structure */
              fixpt_t                p_estcpu;        /* p: Time avg. value of p_cpticks */
              fixpt_t                p_estcpu_inherited; /* p: cpu inherited from children */
              unsigned int        p_forktime;
              fixpt_t         p_pctcpu;       /* p: %cpu from dead LWPs */
      
              struct proc        *p_opptr;        /* l: save parent during ptrace. */
              struct ptimers        *p_timers;        /*    Timers: real, virtual, profiling */
              struct bintime         p_rtime;        /* p: real time */
              u_quad_t         p_uticks;        /* t: Statclock hits in user mode */
              u_quad_t         p_sticks;        /* t: Statclock hits in system mode */
              u_quad_t         p_iticks;        /* t: Statclock hits processing intr */
              uint64_t        p_xutime;        /* p: utime exposed to userspace */
              uint64_t        p_xstime;        /* p: stime exposed to userspace */
      
              int                p_traceflag;        /* k: Kernel trace points */
              void                *p_tracep;        /* k: Trace private data */
              struct vnode         *p_textvp;        /* :: Vnode of executable */
      
              struct emul        *p_emul;        /* :: emulation information */
              void                *p_emuldata;        /* :: per-proc emul data, or NULL */
              const struct execsw *p_execsw;        /* :: exec package information */
              struct klist        p_klist;        /* p: knotes attached to proc */
      
              LIST_HEAD(, lwp) p_sigwaiters;        /* p: LWPs waiting for signals */
              sigpend_t        p_sigpend;        /* p: pending signals */
              struct lcproc        *p_lwpctl;        /* p, a: _lwp_ctl() information */
              pid_t                p_ppid;                /* :: cached parent pid */
              pid_t                p_oppid;        /* :: cached original parent pid */
              char                *p_path;        /* :: full pathname of executable */
      
      /*
       * End area that is zeroed on creation
       */
      #define        p_endzero        p_startcopy
      
      /*
       * The following fields are all copied upon creation in fork.
       */
      #define        p_startcopy        p_sigctx
      
              struct sigctx         p_sigctx;        /* p: Shared signal state */
      
              u_char                p_nice;                /* p: Process "nice" value */
              char                p_comm[MAXCOMLEN+1];
                                              /* p: basename of last exec file */
              struct pgrp         *p_pgrp;        /* l: Pointer to process group */
      
              vaddr_t                p_psstrp;        /* :: address of process's ps_strings */
              u_int                p_pax;                /* :: PAX flags */
              int                p_xexit;        /* p: exit code */
      /*
       * End area that is copied on creation
       */
      #define        p_endcopy        p_xsig
              u_short                p_xsig;                /* p: stop signal */
              u_short                p_acflag;        /* p: Acc. flags; see struct lwp also */
              struct mdproc        p_md;                /* p: Any machine-dependent fields */
              vaddr_t                p_stackbase;        /* :: ASLR randomized stack base */
              struct kdtrace_proc *p_dtrace;        /* :: DTrace-specific data. */
      /*
       * Locks in their own cache line towards the end.
       */
              kmutex_t        p_auxlock        /* :: secondary, longer term lock */
                  __aligned(COHERENCY_UNIT);
              kmutex_t        p_stmutex;        /* :: mutex on profiling state */
              krwlock_t        p_reflock;        /* :: lock for debugger, procfs */
      };
      
      #define        p_rlimit        p_limit->pl_rlimit
      #define        p_session        p_pgrp->pg_session
      #define        p_pgid                p_pgrp->pg_id
      
      #endif        /* _KMEMUSER || _KERNEL */
      
      /*
       * Status values.
       */
      #define        SIDL                1                /* Process being created by fork */
      #define        SACTIVE                2                /* Process is not stopped */
      #define        SDYING                3                /* About to die */
      #define        SSTOP                4                /* Process debugging or suspension */
      #define        SZOMB                5                /* Awaiting collection by parent */
      #define        SDEAD                 6                /* Almost a zombie */
      
      #define        P_ZOMBIE(p)        \
          ((p)->p_stat == SZOMB || (p)->p_stat == SDYING || (p)->p_stat == SDEAD)
      
      /*
       * These flags are kept in p_flag and are protected by p_lock.  Access from
       * process context only.
       */
      #define        PK_ADVLOCK        0x00000001 /* Process may hold a POSIX advisory lock */
      #define        PK_SYSTEM        0x00000002 /* System process (kthread) */
      #define        PK_SYSVSEM        0x00000004 /* Used SysV semaphores */
      #define        PK_SUGID        0x00000100 /* Had set id privileges since last exec */
      #define        PK_KMEM                0x00000200 /* Has kmem access */
      #define        PK_EXEC                0x00004000 /* Process called exec */
      #define        PK_NOCLDWAIT        0x00020000 /* No zombies if child dies */
      #define        PK_32                0x00040000 /* 32-bit process (used on 64-bit kernels) */
      #define        PK_CLDSIGIGN        0x00080000 /* Process is ignoring SIGCHLD */
      #define        PK_MARKER        0x80000000 /* Is a dummy marker process */
      
      /*
       * These flags are kept in p_sflag and are protected by p_lock.  Access from
       * process context only.
       */
      #define        PS_NOCLDSTOP        0x00000008 /* No SIGCHLD when children stop */
      #define        PS_RUMP_LWPEXIT        0x00000400 /* LWPs in RUMP kernel should exit for GC */
      #define        PS_WCORE        0x00001000 /* Process needs to dump core */
      #define        PS_WEXIT        0x00002000 /* Working on exiting */
      #define        PS_STOPFORK        0x00800000 /* Child will be stopped on fork(2) */
      #define        PS_STOPEXEC        0x01000000 /* Will be stopped on exec(2) */
      #define        PS_STOPEXIT        0x02000000 /* Will be stopped at process exit */
      #define        PS_COREDUMP        0x20000000 /* Process core-dumped */
      #define        PS_CONTINUED        0x40000000 /* Process is continued */
      #define        PS_STOPPING        0x80000000 /* Transitioning SACTIVE -> SSTOP */
      
      /*
       * These flags are kept in p_slflag and are protected by the proc_lock
       * and p_lock.  Access from process context only.
       */
      #define        PSL_TRACEFORK        0x00000001 /* traced process wants fork events */
      #define        PSL_TRACEVFORK        0x00000002 /* traced process wants vfork events */
      #define        PSL_TRACEVFORK_DONE        \
                              0x00000004 /* traced process wants vfork done events */
      #define        PSL_TRACELWP_CREATE        \
                              0x00000008 /* traced process wants LWP create events */
      #define        PSL_TRACELWP_EXIT        \
                              0x00000010 /* traced process wants LWP exit events */
      #define        PSL_TRACEPOSIX_SPAWN        \
                              0x00000020 /* traced process wants posix_spawn events */
      
      #define        PSL_TRACED        0x00000800 /* Debugged process being traced */
      #define        PSL_TRACEDCHILD 0x00001000 /* Report process birth */
      #define        PSL_CHTRACED        0x00400000 /* Child has been traced & reparented */
      #define        PSL_SYSCALL        0x04000000 /* process has PT_SYSCALL enabled */
      #define        PSL_SYSCALLEMU        0x08000000 /* cancel in-progress syscall */
      
      /*
       * Kept in p_stflag and protected by p_stmutex.
       */
      #define        PST_PROFIL        0x00000020 /* Has started profiling */
      
      /*
       * Kept in p_lflag and protected by the proc_lock.  Access
       * from process context only.
       */
      #define        PL_CONTROLT        0x00000002 /* Has a controlling terminal */
      #define        PL_PPWAIT        0x00000010 /* Parent is waiting for child exec/exit */
      #define        PL_SIGCOMPAT        0x00000200 /* Has used compat signal trampoline */
      #define        PL_ORPHANPG        0x20000000 /* Member of an orphaned pgrp */
      
      #if defined(_KMEMUSER) || defined(_KERNEL)
      
      /*
       * Macro to compute the exit signal to be delivered.
       */
      #define        P_EXITSIG(p)        \
          (((p)->p_slflag & PSL_TRACED) ? SIGCHLD : p->p_exitsig)
      /*
       * Compute a wait(2) 16 bit exit status code
       */
      #define P_WAITSTATUS(p) W_EXITCODE((p)->p_xexit, ((p)->p_xsig | \
          (((p)->p_sflag & PS_COREDUMP) ? WCOREFLAG : 0)))
      
      LIST_HEAD(proclist, proc);                /* A list of processes */
      
      /*
       * This structure associates a proclist with its lock.
       */
      struct proclist_desc {
              struct proclist        *pd_list;        /* The list */
              /*
               * XXX Add a pointer to the proclist's lock eventually.
               */
      };
      
      #ifdef _KERNEL
      
      /*
       * We use process IDs <= PID_MAX until there are > 16k processes.
       * NO_PGID is used to represent "no process group" for a tty.
       */
      #define        PID_MAX                30000
      #define        NO_PGID                ((pid_t)-1)
      
      #define        SESS_LEADER(p)        ((p)->p_session->s_leader == (p))
      
      /*
       * Flags passed to fork1().
       */
      #define        FORK_PPWAIT        0x0001                /* Block parent until child exit */
      #define        FORK_SHAREVM        0x0002                /* Share vmspace with parent */
      #define        FORK_SHARECWD        0x0004                /* Share cdir/rdir/cmask */
      #define        FORK_SHAREFILES        0x0008                /* Share file descriptors */
      #define        FORK_SHARESIGS        0x0010                /* Share signal actions */
      #define        FORK_NOWAIT        0x0020                /* Make init the parent of the child */
      #define        FORK_CLEANFILES        0x0040                /* Start with a clean descriptor set */
      #define        FORK_SYSTEM        0x0080                /* Fork a kernel thread */
      
      extern struct proc        proc0;                /* Process slot for swapper */
      extern u_int                nprocs;                /* Current number of procs */
      extern int                maxproc;        /* Max number of procs */
      #define        vmspace_kernel()        (proc0.p_vmspace)
      
      extern kmutex_t                proc_lock;
      extern struct proclist        allproc;        /* List of all processes */
      extern struct proclist        zombproc;        /* List of zombie processes */
      
      extern struct proc        *initproc;        /* Process slots for init, pager */
      
      extern const struct proclist_desc proclists[];
      
      extern struct pool        ptimer_pool;        /* Memory pool for ptimers */
      
      int                proc_find_locked(struct lwp *, struct proc **, pid_t);
      proc_t *        proc_find_raw(pid_t);
      proc_t *        proc_find(pid_t);                /* Find process by ID */
      proc_t *        proc_find_lwpid(pid_t);                /* Find process by LWP ID */
      struct lwp *        proc_find_lwp(proc_t *, pid_t);        /* Find LWP in proc by ID */
      struct lwp *        proc_find_lwp_unlocked(proc_t *, pid_t);
                                                      /* Find LWP, acquire proc */
      struct lwp *        proc_find_lwp_acquire_proc(pid_t, proc_t **);
      struct pgrp *        pgrp_find(pid_t);                /* Find process group by ID */
      
      void        procinit(void);
      void        procinit_sysctl(void);
      int        proc_enterpgrp(struct proc *, pid_t, pid_t, bool);
      void        proc_leavepgrp(struct proc *);
      void        proc_sesshold(struct session *);
      void        proc_sessrele(struct session *);
      void        fixjobc(struct proc *, struct pgrp *, int);
      
      int        tsleep(wchan_t, pri_t, const char *, int);
      int        mtsleep(wchan_t, pri_t, const char *, int, kmutex_t *);
      void        wakeup(wchan_t);
      int        kpause(const char *, bool, int, kmutex_t *);
      void        exit1(struct lwp *, int, int) __dead;
      int        kill1(struct lwp *l, pid_t pid, ksiginfo_t *ksi, register_t *retval);
      int        do_sys_wait(int *, int *, int, struct rusage *);
      int        do_sys_waitid(idtype_t, id_t, int *, int *, int, struct wrusage *,
                  siginfo_t *);
      
      struct proc *proc_alloc(void);
      void        proc0_init(void);
      pid_t        proc_alloc_pid(struct proc *);
      void        proc_free_pid(pid_t);
      pid_t        proc_alloc_lwpid(struct proc *, struct lwp *);
      void        proc_free_lwpid(struct proc *, pid_t);
      void        proc_free_mem(struct proc *);
      void        exit_lwps(struct lwp *l);
      int        fork1(struct lwp *, int, int, void *, size_t,
                  void (*)(void *), void *, register_t *);
      int        pgid_in_session(struct proc *, pid_t);
      void        cpu_lwp_fork(struct lwp *, struct lwp *, void *, size_t,
                  void (*)(void *), void *);
      void        cpu_lwp_free(struct lwp *, int);
      void        cpu_lwp_free2(struct lwp *);
      void        cpu_spawn_return(struct lwp*);
      
      #ifdef __HAVE_SYSCALL_INTERN
      void        syscall_intern(struct proc *);
      #endif
      
      void        md_child_return(struct lwp *);
      void        child_return(void *);
      
      int        proc_isunder(struct proc *, struct lwp *);
      int        proc_uidmatch(kauth_cred_t, kauth_cred_t);
      
      int        proc_vmspace_getref(struct proc *, struct vmspace **);
      void        proc_crmod_leave(kauth_cred_t, kauth_cred_t, bool);
      void        proc_crmod_enter(void);
      int        proc_getauxv(struct proc *, void **, size_t *);
      
      int        proc_specific_key_create(specificdata_key_t *, specificdata_dtor_t);
      void        proc_specific_key_delete(specificdata_key_t);
      void        proc_initspecific(struct proc *);
      void        proc_finispecific(struct proc *);
      void *        proc_getspecific(struct proc *, specificdata_key_t);
      void        proc_setspecific(struct proc *, specificdata_key_t, void *);
      int        proc_compare(const struct proc *, const struct lwp *,
          const struct proc *, const struct lwp *);
      
      int        proclist_foreach_call(struct proclist *,
          int (*)(struct proc *, void *arg), void *);
      
      static __inline struct proc *
      _proclist_skipmarker(struct proc *p0)
      {
              struct proc *p = p0;
      
    5         while (p != NULL && p->p_flag & PK_MARKER)
                      p = LIST_NEXT(p, p_list);
      
              return p;
      }
      
      #define PROC_PTRSZ(p) (((p)->p_flag & PK_32) ? sizeof(int) : sizeof(void *))
      #define PROC_REGSZ(p) (((p)->p_flag & PK_32) ? \
          sizeof(process_reg32) : sizeof(struct reg))
      #define PROC_FPREGSZ(p) (((p)->p_flag & PK_32) ? \
          sizeof(process_fpreg32) : sizeof(struct fpreg))
      #define PROC_DBREGSZ(p) (((p)->p_flag & PK_32) ? \
          sizeof(process_dbreg32) : sizeof(struct dbreg))
      
      /*
       * PROCLIST_FOREACH: iterate on the given proclist, skipping PK_MARKER ones.
       */
      #define        PROCLIST_FOREACH(var, head)                                        \
              for ((var) = LIST_FIRST(head);                                        \
                      ((var) = _proclist_skipmarker(var)) != NULL;                \
                      (var) = LIST_NEXT(var, p_list))
      
      #ifdef KSTACK_CHECK_MAGIC
      void        kstack_setup_magic(const struct lwp *);
      void        kstack_check_magic(const struct lwp *);
      #else
      #define        kstack_setup_magic(x)
      #define        kstack_check_magic(x)
      #endif
      
      extern struct emul emul_netbsd;
      
      #endif        /* _KERNEL */
      
      /*
       * Kernel stack parameters.
       *
       * KSTACK_LOWEST_ADDR: return the lowest address of the LWP's kernel stack,
       * excluding red-zone.
       *
       * KSTACK_SIZE: the size kernel stack for a LWP, excluding red-zone.
       *
       * if <machine/proc.h> provides the MD definition, it will be used.
       */
      #ifndef KSTACK_LOWEST_ADDR
      #define        KSTACK_LOWEST_ADDR(l)        ((void *)ALIGN((struct pcb *)((l)->l_addr) + 1))
      #endif
      #ifndef KSTACK_SIZE
      #define        KSTACK_SIZE                (USPACE - ALIGN(sizeof(struct pcb)))
      #endif
      
      #endif        /* _KMEMUSER || _KERNEL */
      
      #endif        /* !_SYS_PROC_H_ */
      /*        $NetBSD: kern_acct.c,v 1.97 2020/05/23 23:42:43 ad Exp $        */
      
      /*-
       * Copyright (c) 1982, 1986, 1989, 1993
       *        The Regents of the University of California.  All rights reserved.
       * (c) UNIX System Laboratories, Inc.
       * All or some portions of this file are derived from material licensed
       * to the University of California by American Telephone and Telegraph
       * Co. or Unix System Laboratories, Inc. and are reproduced herein with
       * the permission of UNIX System Laboratories, Inc.
       *
       * Redistribution and use in source and binary forms, with or without
       * modification, are permitted provided that the following conditions
       * are met:
       * 1. Redistributions of source code must retain the above copyright
       *    notice, this list of conditions and the following disclaimer.
       * 2. Redistributions in binary form must reproduce the above copyright
       *    notice, this list of conditions and the following disclaimer in the
       *    documentation and/or other materials provided with the distribution.
       * 3. Neither the name of the University nor the names of its contributors
       *    may be used to endorse or promote products derived from this software
       *    without specific prior written permission.
       *
       * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
       * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
       * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
       * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
       * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
       * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
       * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
       * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
       * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
       * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
       * SUCH DAMAGE.
       *
       *        @(#)kern_acct.c        8.8 (Berkeley) 5/14/95
       */
      
      /*-
       * Copyright (c) 1994 Christopher G. Demetriou
       *
       * Redistribution and use in source and binary forms, with or without
       * modification, are permitted provided that the following conditions
       * are met:
       * 1. Redistributions of source code must retain the above copyright
       *    notice, this list of conditions and the following disclaimer.
       * 2. Redistributions in binary form must reproduce the above copyright
       *    notice, this list of conditions and the following disclaimer in the
       *    documentation and/or other materials provided with the distribution.
       * 3. All advertising materials mentioning features or use of this software
       *    must display the following acknowledgement:
       *        This product includes software developed by the University of
       *        California, Berkeley and its contributors.
       * 4. Neither the name of the University nor the names of its contributors
       *    may be used to endorse or promote products derived from this software
       *    without specific prior written permission.
       *
       * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
       * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
       * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
       * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
       * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
       * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
       * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
       * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
       * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
       * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
       * SUCH DAMAGE.
       *
       *        @(#)kern_acct.c        8.8 (Berkeley) 5/14/95
       */
      
      #include <sys/cdefs.h>
      __KERNEL_RCSID(0, "$NetBSD: kern_acct.c,v 1.97 2020/05/23 23:42:43 ad Exp $");
      
      #include <sys/param.h>
      #include <sys/systm.h>
      #include <sys/proc.h>
      #include <sys/mount.h>
      #include <sys/vnode.h>
      #include <sys/file.h>
      #include <sys/syslog.h>
      #include <sys/kernel.h>
      #include <sys/kthread.h>
      #include <sys/kmem.h>
      #include <sys/namei.h>
      #include <sys/errno.h>
      #include <sys/acct.h>
      #include <sys/resourcevar.h>
      #include <sys/ioctl.h>
      #include <sys/tty.h>
      #include <sys/kauth.h>
      
      #include <sys/syscallargs.h>
      
      /*
       * The routines implemented in this file are described in:
       *      Leffler, et al.: The Design and Implementation of the 4.3BSD
       *            UNIX Operating System (Addison Welley, 1989)
       * on pages 62-63.
       *
       * Arguably, to simplify accounting operations, this mechanism should
       * be replaced by one in which an accounting log file (similar to /dev/klog)
       * is read by a user process, etc.  However, that has its own problems.
       */
      
      /*
       * Lock to serialize system calls and kernel threads.
       */
      krwlock_t        acct_lock;
      
      /*
       * The global accounting state and related data.  Gain the mutex before
       * accessing these variables.
       */
      static enum {
              ACCT_STOP,
              ACCT_ACTIVE,
              ACCT_SUSPENDED
      } acct_state;                                /* The current accounting state. */
      static struct vnode *acct_vp;                /* Accounting vnode pointer. */
      static kauth_cred_t acct_cred;                /* Credential of accounting file
                                                 owner (i.e root).  Used when
                                                  accounting file i/o.  */
      static struct lwp *acct_dkwatcher;        /* Free disk space checker. */
      
      /*
       * Values associated with enabling and disabling accounting
       */
      int        acctsuspend = 2;        /* stop accounting when < 2% free space left */
      int        acctresume = 4;                /* resume when free space risen to > 4% */
      int        acctchkfreq = 15;        /* frequency (in seconds) to check space */
      
      /*
       * Encode_comp_t converts from ticks in seconds and microseconds
       * to ticks in 1/AHZ seconds.  The encoding is described in
       * Leffler, et al., on page 63.
       */
      
      #define        MANTSIZE        13                        /* 13 bit mantissa. */
      #define        EXPSIZE                3                        /* Base 8 (3 bit) exponent. */
      #define        MAXFRACT        ((1 << MANTSIZE) - 1)        /* Maximum fractional value. */
      
      static comp_t
      encode_comp_t(u_long s, u_long us)
      {
              int exp, rnd;
      
              exp = 0;
              rnd = 0;
              s *= AHZ;
              s += us / (1000000 / AHZ);        /* Maximize precision. */
      
              while (s > MAXFRACT) {
                      rnd = s & (1 << (EXPSIZE - 1));        /* Round up? */
                      s >>= EXPSIZE;                /* Base 8 exponent == 3 bit shift. */
                      exp++;
              }
      
              /* If we need to round up, do it (and handle overflow correctly). */
              if (rnd && (++s > MAXFRACT)) {
                      s >>= EXPSIZE;
                      exp++;
              }
      
              /* Clean it up and polish it off. */
              exp <<= MANTSIZE;                /* Shift the exponent into place */
              exp += s;                        /* and add on the mantissa. */
              return (exp);
      }
      
      static int
      acct_chkfree(void)
      {
              int error;
              struct statvfs *sb;
              fsblkcnt_t bavail;
      
    4         sb = kmem_alloc(sizeof(*sb), KM_SLEEP);
              error = VFS_STATVFS(acct_vp->v_mount, sb);
              if (error != 0) {
                      kmem_free(sb, sizeof(*sb));
                      return (error);
              }
      
    4         if (sb->f_bfree < sb->f_bresvd) {
                      bavail = 0;
              } else {
    4                 bavail = sb->f_bfree - sb->f_bresvd;
              }
      
              switch (acct_state) {
              case ACCT_SUSPENDED:
                      if (bavail > acctresume * sb->f_blocks / 100) {
                              acct_state = ACCT_ACTIVE;
                              log(LOG_NOTICE, "Accounting resumed\n");
                      }
                      break;
              case ACCT_ACTIVE:
    4                 if (bavail <= acctsuspend * sb->f_blocks / 100) {
                              acct_state = ACCT_SUSPENDED;
                              log(LOG_NOTICE, "Accounting suspended\n");
                      }
                      break;
              case ACCT_STOP:
                      break;
              }
    4         kmem_free(sb, sizeof(*sb));
    4         return (0);
      }
      
      static void
      acct_stop(void)
      {
              int error;
      
    5         KASSERT(rw_write_held(&acct_lock));
      
    5         if (acct_vp != NULLVP && acct_vp->v_type != VBAD) {
    4                 error = vn_close(acct_vp, FWRITE, acct_cred);
      #ifdef DIAGNOSTIC
                      if (error != 0)
                              printf("acct_stop: failed to close, errno = %d\n",
                                  error);
      #else
                      __USE(error);
      #endif
    4                 acct_vp = NULLVP;
              }
    5         if (acct_cred != NULL) {
    4                 kauth_cred_free(acct_cred);
                      acct_cred = NULL;
              }
    5         acct_state = ACCT_STOP;
      }
      
      /*
       * Periodically check the file system to see if accounting
       * should be turned on or off.  Beware the case where the vnode
       * has been vgone()'d out from underneath us, e.g. when the file
       * system containing the accounting file has been forcibly unmounted.
       */
      static void
      acctwatch(void *arg)
      {
              int error;
      
              log(LOG_NOTICE, "Accounting started\n");
              rw_enter(&acct_lock, RW_WRITER);
              while (acct_state != ACCT_STOP) {
                      if (acct_vp->v_type == VBAD) {
                              log(LOG_NOTICE, "Accounting terminated\n");
                              acct_stop();
                              continue;
                      }
      
                      error = acct_chkfree();
      #ifdef DIAGNOSTIC
                      if (error != 0)
                              printf("acctwatch: failed to statvfs, error = %d\n",
                                  error);
      #else
                      __USE(error);
      #endif
                      rw_exit(&acct_lock);
                      error = kpause("actwat", false, acctchkfreq * hz, NULL);
                      rw_enter(&acct_lock, RW_WRITER);
      #ifdef DIAGNOSTIC
                      if (error != 0 && error != EWOULDBLOCK)
                              printf("acctwatch: sleep error %d\n", error);
      #endif
              }
              acct_dkwatcher = NULL;
              rw_exit(&acct_lock);
      
              kthread_exit(0);
      }
      
      void
      acct_init(void)
      {
      
              acct_state = ACCT_STOP;
              acct_vp = NULLVP;
              acct_cred = NULL;
              rw_init(&acct_lock);
      }
      
      /*
       * Accounting system call.  Written based on the specification and
       * previous implementation done by Mark Tinguely.
       */
      int
      sys_acct(struct lwp *l, const struct sys_acct_args *uap, register_t *retval)
      {
              /* {
                      syscallarg(const char *) path;
              } */
   12         struct pathbuf *pb;
              struct nameidata nd;
              int error;
      
              /* Make sure that the caller is root. */
              if ((error = kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_ACCOUNTING,
                  0, NULL, NULL, NULL)))
                      return (error);
      
              /*
               * If accounting is to be started to a file, open that file for
               * writing and make sure it's a 'normal'.
               */
   11         if (SCARG(uap, path) != NULL) {
   10                 struct vattr va;
                      size_t pad;
      
                      error = pathbuf_copyin(SCARG(uap, path), &pb);
                      if (error) {
    3                         return error;
                      }
    9                 NDINIT(&nd, LOOKUP, FOLLOW | TRYEMULROOT, pb);
                      if ((error = vn_open(&nd, FWRITE|O_APPEND, 0)) != 0) {
    2                         pathbuf_destroy(pb);
                              return error;
                      }
    7                 if (nd.ni_vp->v_type != VREG) {
    3                         VOP_UNLOCK(nd.ni_vp);
                              error = EACCES;
    3                         goto bad;
                      }
    4                 if ((error = VOP_GETATTR(nd.ni_vp, &va, l->l_cred)) != 0) {
                              VOP_UNLOCK(nd.ni_vp);
                              goto bad;
                      }
      
    4                 if ((pad = (va.va_size % sizeof(struct acct))) != 0) {
    2                         u_quad_t size = va.va_size - pad;
      #ifdef DIAGNOSTIC
                              printf("Size of accounting file not a multiple of "
                                  "%lu - incomplete record truncated\n",
                                  (unsigned long)sizeof(struct acct));
      #endif
                              vattr_null(&va);
                              va.va_size = size;
                              error = VOP_SETATTR(nd.ni_vp, &va, l->l_cred);
                              if (error != 0) {
                                      VOP_UNLOCK(nd.ni_vp);
                                      goto bad;
                              }
                      }
    4                 VOP_UNLOCK(nd.ni_vp);
              }
      
    5         rw_enter(&acct_lock, RW_WRITER);
      
              /*
               * If accounting was previously enabled, kill the old space-watcher,
               * free credential for accounting file i/o,
               * ... (and, if no new file was specified, leave).
               */
              acct_stop();
              if (SCARG(uap, path) == NULL)
                      goto out;
      
              /*
               * Save the new accounting file vnode and credential,
               * and schedule the new free space watcher.
               */
    4         acct_state = ACCT_ACTIVE;
              acct_vp = nd.ni_vp;
              acct_cred = l->l_cred;
              kauth_cred_hold(acct_cred);
      
              pathbuf_destroy(pb);
      
              error = acct_chkfree();                /* Initial guess. */
              if (error != 0) {
                      acct_stop();
                      goto out;
              }
      
    4         if (acct_dkwatcher == NULL) {
    1                 error = kthread_create(PRI_NONE, KTHREAD_MPSAFE, NULL,
                          acctwatch, NULL, &acct_dkwatcher, "acctwatch");
                      if (error != 0)
                              acct_stop();
              }
      
    5  out:
              rw_exit(&acct_lock);
              return (error);
       bad:
              vn_close(nd.ni_vp, FWRITE, l->l_cred);
              pathbuf_destroy(pb);
   11         return error;
      }
      
      /*
       * Write out process accounting information, on process exit.
       * Data to be written out is specified in Leffler, et al.
       * and are enumerated below.  (They're also noted in the system
       * "acct.h" header file.)
       */
      int
      acct_process(struct lwp *l)
      {
              struct acct acct;
              struct timeval ut, st, tmp;
              struct rusage *r;
              int t, error = 0;
              struct rlimit orlim;
              struct proc *p = l->l_proc;
      
              if (acct_state != ACCT_ACTIVE)
                      return 0;
      
              memset(&acct, 0, sizeof(acct));        /* to zerofill padded data */
      
              rw_enter(&acct_lock, RW_READER);
      
              /* If accounting isn't enabled, don't bother */
              if (acct_state != ACCT_ACTIVE)
                      goto out;
      
              /*
               * Temporarily raise the file limit so that accounting can't
               * be stopped by the user.
               *
               * XXX We should think about the CPU limit, too.
               */
              lim_privatise(p);
              orlim = p->p_rlimit[RLIMIT_FSIZE];
              /* Set current and max to avoid illegal values */
              p->p_rlimit[RLIMIT_FSIZE].rlim_cur = RLIM_INFINITY;
              p->p_rlimit[RLIMIT_FSIZE].rlim_max = RLIM_INFINITY;
      
              /*
               * Get process accounting information.
               */
      
              /* (1) The name of the command that ran */
              strncpy(acct.ac_comm, p->p_comm, sizeof(acct.ac_comm));
      
              /* (2) The amount of user and system time that was used */
              mutex_enter(p->p_lock);
              calcru(p, &ut, &st, NULL, NULL);
              mutex_exit(p->p_lock);
              acct.ac_utime = encode_comp_t(ut.tv_sec, ut.tv_usec);
              acct.ac_stime = encode_comp_t(st.tv_sec, st.tv_usec);
      
              /* (3) The elapsed time the commmand ran (and its starting time) */
              acct.ac_btime = p->p_stats->p_start.tv_sec;
              getmicrotime(&tmp);
              timersub(&tmp, &p->p_stats->p_start, &tmp);
              acct.ac_etime = encode_comp_t(tmp.tv_sec, tmp.tv_usec);
      
              /* (4) The average amount of memory used */
              r = &p->p_stats->p_ru;
              timeradd(&ut, &st, &tmp);
              t = tmp.tv_sec * hz + tmp.tv_usec / tick;
              if (t)
                      acct.ac_mem = (r->ru_ixrss + r->ru_idrss + r->ru_isrss) / t;
              else
                      acct.ac_mem = 0;
      
              /* (5) The number of disk I/O operations done */
              acct.ac_io = encode_comp_t(r->ru_inblock + r->ru_oublock, 0);
      
              /* (6) The UID and GID of the process */
              acct.ac_uid = kauth_cred_getuid(l->l_cred);
              acct.ac_gid = kauth_cred_getgid(l->l_cred);
      
              /* (7) The terminal from which the process was started */
              mutex_enter(&proc_lock);
              if ((p->p_lflag & PL_CONTROLT) && p->p_pgrp->pg_session->s_ttyp)
                      acct.ac_tty = p->p_pgrp->pg_session->s_ttyp->t_dev;
              else
                      acct.ac_tty = NODEV;
              mutex_exit(&proc_lock);
      
              /* (8) The boolean flags that tell how the process terminated, etc. */
              acct.ac_flag = p->p_acflag;
      
              /*
               * Now, just write the accounting information to the file.
               */
              error = vn_rdwr(UIO_WRITE, acct_vp, (void *)&acct,
                  sizeof(acct), (off_t)0, UIO_SYSSPACE, IO_APPEND|IO_UNIT,
                  acct_cred, NULL, NULL);
              if (error != 0)
                      log(LOG_ERR, "Accounting: write failed %d\n", error);
      
              /* Restore limit - rather pointless since process is about to exit */
              p->p_rlimit[RLIMIT_FSIZE] = orlim;
      
       out:
              rw_exit(&acct_lock);
              return (error);
      }
      /*        $NetBSD: audio.c,v 1.75 2020/05/29 03:09:14 isaki Exp $        */
      
      /*-
       * Copyright (c) 2008 The NetBSD Foundation, Inc.
       * All rights reserved.
       *
       * This code is derived from software contributed to The NetBSD Foundation
       * by Andrew Doran.
       *
       * Redistribution and use in source and binary forms, with or without
       * modification, are permitted provided that the following conditions
       * are met:
       * 1. Redistributions of source code must retain the above copyright
       *    notice, this list of conditions and the following disclaimer.
       * 2. Redistributions in binary form must reproduce the above copyright
       *    notice, this list of conditions and the following disclaimer in the
       *    documentation and/or other materials provided with the distribution.
       *
       * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
       * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
       * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
       * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
       * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
       * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
       * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
       * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
       * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
       * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
       * POSSIBILITY OF SUCH DAMAGE.
       */
      
      /*
       * Copyright (c) 1991-1993 Regents of the University of California.
       * All rights reserved.
       *
       * Redistribution and use in source and binary forms, with or without
       * modification, are permitted provided that the following conditions
       * are met:
       * 1. Redistributions of source code must retain the above copyright
       *    notice, this list of conditions and the following disclaimer.
       * 2. Redistributions in binary form must reproduce the above copyright
       *    notice, this list of conditions and the following disclaimer in the
       *    documentation and/or other materials provided with the distribution.
       * 3. All advertising materials mentioning features or use of this software
       *    must display the following acknowledgement:
       *        This product includes software developed by the Computer Systems
       *        Engineering Group at Lawrence Berkeley Laboratory.
       * 4. Neither the name of the University nor of the Laboratory may be used
       *    to endorse or promote products derived from this software without
       *    specific prior written permission.
       *
       * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
       * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
       * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
       * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
       * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
       * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
       * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
       * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
       * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
       * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
       * SUCH DAMAGE.
       */
      
      /*
       * Locking: there are three locks per device.
       *
       * - sc_lock, provided by the underlying driver.  This is an adaptive lock,
       *   returned in the second parameter to hw_if->get_locks().  It is known
       *   as the "thread lock".
       *
       *   It serializes access to state in all places except the
       *   driver's interrupt service routine.  This lock is taken from process
       *   context (example: access to /dev/audio).  It is also taken from soft
       *   interrupt handlers in this module, primarily to serialize delivery of
       *   wakeups.  This lock may be used/provided by modules external to the
       *   audio subsystem, so take care not to introduce a lock order problem.
       *   LONG TERM SLEEPS MUST NOT OCCUR WITH THIS LOCK HELD.
       *
       * - sc_intr_lock, provided by the underlying driver.  This may be either a
       *   spinlock (at IPL_SCHED or IPL_VM) or an adaptive lock (IPL_NONE or
       *   IPL_SOFT*), returned in the first parameter to hw_if->get_locks().  It
       *   is known as the "interrupt lock".
       *
       *   It provides atomic access to the device's hardware state, and to audio
       *   channel data that may be accessed by the hardware driver's ISR.
       *   In all places outside the ISR, sc_lock must be held before taking
       *   sc_intr_lock.  This is to ensure that groups of hardware operations are
       *   made atomically.  SLEEPS CANNOT OCCUR WITH THIS LOCK HELD.
       *
       * - sc_exlock, private to this module.  This is a variable protected by
       *   sc_lock.  It is known as the "critical section".
       *   Some operations release sc_lock in order to allocate memory, to wait
       *   for in-flight I/O to complete, to copy to/from user context, etc.
       *   sc_exlock provides a critical section even under the circumstance.
       *   "+" in following list indicates the interfaces which necessary to be
       *   protected by sc_exlock.
       *
       * List of hardware interface methods, and which locks are held when each
       * is called by this module:
       *
       *        METHOD                        INTR        THREAD  NOTES
       *        ----------------------- ------- -------        -------------------------
       *        open                         x        x +
       *        close                         x        x +
       *        query_format                -        x
       *        set_format                -        x
       *        round_blocksize                -        x
       *        commit_settings                -        x
       *        init_output                 x        x
       *        init_input                 x        x
       *        start_output                 x        x +
       *        start_input                 x        x +
       *        halt_output                 x        x +
       *        halt_input                 x        x +
       *        speaker_ctl                 x        x
       *        getdev                         -        x
       *        set_port                 -        x +
       *        get_port                 -        x +
       *        query_devinfo                 -        x
       *        allocm                         -        - +
       *        freem                         -        - +
       *        round_buffersize         -        x
       *        get_props                 -        -        Called at attach time
       *        trigger_output                 x        x +
       *        trigger_input                 x        x +
       *        dev_ioctl                 -        x
       *        get_locks                 -        -        Called at attach time
       *
       * In addition, there is an additional lock.
       *
       * - track->lock.  This is an atomic variable and is similar to the
       *   "interrupt lock".  This is one for each track.  If any thread context
       *   (and software interrupt context) and hardware interrupt context who
       *   want to access some variables on this track, they must acquire this
       *   lock before.  It protects track's consistency between hardware
       *   interrupt context and others.
       */
      
      #include <sys/cdefs.h>
      __KERNEL_RCSID(0, "$NetBSD: audio.c,v 1.75 2020/05/29 03:09:14 isaki Exp $");
      
      #ifdef _KERNEL_OPT
      #include "audio.h"
      #include "midi.h"
      #endif
      
      #if NAUDIO > 0
      
      #include <sys/types.h>
      #include <sys/param.h>
      #include <sys/atomic.h>
      #include <sys/audioio.h>
      #include <sys/conf.h>
      #include <sys/cpu.h>
      #include <sys/device.h>
      #include <sys/fcntl.h>
      #include <sys/file.h>
      #include <sys/filedesc.h>
      #include <sys/intr.h>
      #include <sys/ioctl.h>
      #include <sys/kauth.h>
      #include <sys/kernel.h>
      #include <sys/kmem.h>
      #include <sys/malloc.h>
      #include <sys/mman.h>
      #include <sys/module.h>
      #include <sys/poll.h>
      #include <sys/proc.h>
      #include <sys/queue.h>
      #include <sys/select.h>
      #include <sys/signalvar.h>
      #include <sys/stat.h>
      #include <sys/sysctl.h>
      #include <sys/systm.h>
      #include <sys/syslog.h>
      #include <sys/vnode.h>
      
      #include <dev/audio/audio_if.h>
      #include <dev/audio/audiovar.h>
      #include <dev/audio/audiodef.h>
      #include <dev/audio/linear.h>
      #include <dev/audio/mulaw.h>
      
      #include <machine/endian.h>
      
      #include <uvm/uvm_extern.h>
      
      #include "ioconf.h"
      
      /*
       * 0: No debug logs
       * 1: action changes like open/close/set_format...
       * 2: + normal operations like read/write/ioctl...
       * 3: + TRACEs except interrupt
       * 4: + TRACEs including interrupt
       */
      //#define AUDIO_DEBUG 1
      
      #if defined(AUDIO_DEBUG)
      
      int audiodebug = AUDIO_DEBUG;
      static void audio_vtrace(struct audio_softc *sc, const char *, const char *,
              const char *, va_list);
      static void audio_trace(struct audio_softc *sc, const char *, const char *, ...)
              __printflike(3, 4);
      static void audio_tracet(const char *, audio_track_t *, const char *, ...)
              __printflike(3, 4);
      static void audio_tracef(const char *, audio_file_t *, const char *, ...)
              __printflike(3, 4);
      
      /* XXX sloppy memory logger */
      static void audio_mlog_init(void);
      static void audio_mlog_free(void);
      static void audio_mlog_softintr(void *);
      extern void audio_mlog_flush(void);
      extern void audio_mlog_printf(const char *, ...);
      
      static int mlog_refs;                /* reference counter */
      static char *mlog_buf[2];        /* double buffer */
      static int mlog_buflen;                /* buffer length */
      static int mlog_used;                /* used length */
      static int mlog_full;                /* number of dropped lines by buffer full */
      static int mlog_drop;                /* number of dropped lines by busy */
      static volatile uint32_t mlog_inuse;        /* in-use */
      static int mlog_wpage;                /* active page */
      static void *mlog_sih;                /* softint handle */
      
      static void
      audio_mlog_init(void)
      {
              mlog_refs++;
              if (mlog_refs > 1)
                      return;
              mlog_buflen = 4096;
              mlog_buf[0] = kmem_zalloc(mlog_buflen, KM_SLEEP);
              mlog_buf[1] = kmem_zalloc(mlog_buflen, KM_SLEEP);
              mlog_used = 0;
              mlog_full = 0;
              mlog_drop = 0;
              mlog_inuse = 0;
              mlog_wpage = 0;
              mlog_sih = softint_establish(SOFTINT_SERIAL, audio_mlog_softintr, NULL);
              if (mlog_sih == NULL)
                      printf("%s: softint_establish failed\n", __func__);
      }
      
      static void
      audio_mlog_free(void)
      {
              mlog_refs--;
              if (mlog_refs > 0)
                      return;
      
              audio_mlog_flush();
              if (mlog_sih)
                      softint_disestablish(mlog_sih);
              kmem_free(mlog_buf[0], mlog_buflen);
              kmem_free(mlog_buf[1], mlog_buflen);
      }
      
      /*
       * Flush memory buffer.
       * It must not be called from hardware interrupt context.
       */
      void
      audio_mlog_flush(void)
      {
              if (mlog_refs == 0)
                      return;
      
              /* Nothing to do if already in use ? */
              if (atomic_swap_32(&mlog_inuse, 1) == 1)
                      return;
      
              int rpage = mlog_wpage;
              mlog_wpage ^= 1;
              mlog_buf[mlog_wpage][0] = '\0';
              mlog_used = 0;
      
              atomic_swap_32(&mlog_inuse, 0);
      
              if (mlog_buf[rpage][0] != '\0') {
                      printf("%s", mlog_buf[rpage]);
                      if (mlog_drop > 0)
                              printf("mlog_drop %d\n", mlog_drop);
                      if (mlog_full > 0)
                              printf("mlog_full %d\n", mlog_full);
              }
              mlog_full = 0;
              mlog_drop = 0;
      }
      
      static void
      audio_mlog_softintr(void *cookie)
      {
              audio_mlog_flush();
      }
      
      void
      audio_mlog_printf(const char *fmt, ...)
      {
              int len;
              va_list ap;
      
              if (atomic_swap_32(&mlog_inuse, 1) == 1) {
                      /* already inuse */
                      mlog_drop++;
                      return;
              }
      
              va_start(ap, fmt);
              len = vsnprintf(
                  mlog_buf[mlog_wpage] + mlog_used,
                  mlog_buflen - mlog_used,
                  fmt, ap);
              va_end(ap);
      
              mlog_used += len;
              if (mlog_buflen - mlog_used <= 1) {
                      mlog_full++;
              }
      
              atomic_swap_32(&mlog_inuse, 0);
      
              if (mlog_sih)
                      softint_schedule(mlog_sih);
      }
      
      /* trace functions */
      static void
      audio_vtrace(struct audio_softc *sc, const char *funcname, const char *header,
              const char *fmt, va_list ap)
      {
              char buf[256];
              int n;
      
              n = 0;
              buf[0] = '\0';
              n += snprintf(buf + n, sizeof(buf) - n, "%s@%d %s",
                  funcname, device_unit(sc->sc_dev), header);
              n += vsnprintf(buf + n, sizeof(buf) - n, fmt, ap);
      
              if (cpu_intr_p()) {
                      audio_mlog_printf("%s\n", buf);
              } else {
                      audio_mlog_flush();
                      printf("%s\n", buf);
              }
      }
      
      static void
      audio_trace(struct audio_softc *sc, const char *funcname, const char *fmt, ...)
      {
              va_list ap;
      
              va_start(ap, fmt);
              audio_vtrace(sc, funcname, "", fmt, ap);
              va_end(ap);
      }
      
      static void
      audio_tracet(const char *funcname, audio_track_t *track, const char *fmt, ...)
      {
              char hdr[16];
              va_list ap;
      
              snprintf(hdr, sizeof(hdr), "#%d ", track->id);
              va_start(ap, fmt);
              audio_vtrace(track->mixer->sc, funcname, hdr, fmt, ap);
              va_end(ap);
      }
      
      static void
      audio_tracef(const char *funcname, audio_file_t *file, const char *fmt, ...)
      {
              char hdr[32];
              char phdr[16], rhdr[16];
              va_list ap;
      
              phdr[0] = '\0';
              rhdr[0] = '\0';
              if (file->ptrack)
                      snprintf(phdr, sizeof(phdr), "#%d", file->ptrack->id);
              if (file->rtrack)
                      snprintf(rhdr, sizeof(rhdr), "#%d", file->rtrack->id);
              snprintf(hdr, sizeof(hdr), "{%s,%s} ", phdr, rhdr);
      
              va_start(ap, fmt);
              audio_vtrace(file->sc, funcname, hdr, fmt, ap);
              va_end(ap);
      }
      
      #define DPRINTF(n, fmt...)        do {        \
              if (audiodebug >= (n)) {        \
                      audio_mlog_flush();        \
                      printf(fmt);                \
              }                                \
      } while (0)
      #define TRACE(n, fmt...)        do { \
              if (audiodebug >= (n)) audio_trace(sc, __func__, fmt); \
      } while (0)
      #define TRACET(n, t, fmt...)        do { \
              if (audiodebug >= (n)) audio_tracet(__func__, t, fmt); \
      } while (0)
      #define TRACEF(n, f, fmt...)        do { \
              if (audiodebug >= (n)) audio_tracef(__func__, f, fmt); \
      } while (0)
      
      struct audio_track_debugbuf {
              char usrbuf[32];
              char codec[32];
              char chvol[32];
              char chmix[32];
              char freq[32];
              char outbuf[32];
      };
      
      static void
      audio_track_bufstat(audio_track_t *track, struct audio_track_debugbuf *buf)
      {
      
              memset(buf, 0, sizeof(*buf));
      
              snprintf(buf->outbuf, sizeof(buf->outbuf), " out=%d/%d/%d",
                  track->outbuf.head, track->outbuf.used, track->outbuf.capacity);
              if (track->freq.filter)
                      snprintf(buf->freq, sizeof(buf->freq), " f=%d/%d/%d",
                          track->freq.srcbuf.head,
                          track->freq.srcbuf.used,
                          track->freq.srcbuf.capacity);
              if (track->chmix.filter)
                      snprintf(buf->chmix, sizeof(buf->chmix), " m=%d",
                          track->chmix.srcbuf.used);
              if (track->chvol.filter)
                      snprintf(buf->chvol, sizeof(buf->chvol), " v=%d",
                          track->chvol.srcbuf.used);
              if (track->codec.filter)
                      snprintf(buf->codec, sizeof(buf->codec), " e=%d",
                          track->codec.srcbuf.used);
              snprintf(buf->usrbuf, sizeof(buf->usrbuf), " usr=%d/%d/H%d",
                  track->usrbuf.head, track->usrbuf.used, track->usrbuf_usedhigh);
      }
      #else
      #define DPRINTF(n, fmt...)        do { } while (0)
      #define TRACE(n, fmt, ...)        do { } while (0)
      #define TRACET(n, t, fmt, ...)        do { } while (0)
      #define TRACEF(n, f, fmt, ...)        do { } while (0)
      #endif
      
      #define SPECIFIED(x)        ((x) != ~0)
      #define SPECIFIED_CH(x)        ((x) != (u_char)~0)
      
      /*
       * Default hardware blocksize in msec.
       *
       * We use 10 msec for most modern platforms.  This period is good enough to
       * play audio and video synchronizely.
       * In contrast, for very old platforms, this is usually too short and too
       * severe.  Also such platforms usually can not play video confortably, so
       * it's not so important to make the blocksize shorter.  If the platform
       * defines its own value as __AUDIO_BLK_MS in its <machine/param.h>, it
       * uses this instead.
       *
       * In either case, you can overwrite AUDIO_BLK_MS by your kernel
       * configuration file if you wish.
       */
      #if !defined(AUDIO_BLK_MS)
      # if defined(__AUDIO_BLK_MS)
      #  define AUDIO_BLK_MS __AUDIO_BLK_MS
      # else
      #  define AUDIO_BLK_MS (10)
      # endif
      #endif
      
      /* Device timeout in msec */
      #define AUDIO_TIMEOUT        (3000)
      
      /* #define AUDIO_PM_IDLE */
      #ifdef AUDIO_PM_IDLE
      int audio_idle_timeout = 30;
      #endif
      
      /* Number of elements of async mixer's pid */
      #define AM_CAPACITY        (4)
      
      struct portname {
              const char *name;
              int mask;
      };
      
      static int audiomatch(device_t, cfdata_t, void *);
      static void audioattach(device_t, device_t, void *);
      static int audiodetach(device_t, int);
      static int audioactivate(device_t, enum devact);
      static void audiochilddet(device_t, device_t);
      static int audiorescan(device_t, const char *, const int *);
      
      static int audio_modcmd(modcmd_t, void *);
      
      #ifdef AUDIO_PM_IDLE
      static void audio_idle(void *);
      static void audio_activity(device_t, devactive_t);
      #endif
      
      static bool audio_suspend(device_t dv, const pmf_qual_t *);
      static bool audio_resume(device_t dv, const pmf_qual_t *);
      static void audio_volume_down(device_t);
      static void audio_volume_up(device_t);
      static void audio_volume_toggle(device_t);
      
      static void audio_mixer_capture(struct audio_softc *);
      static void audio_mixer_restore(struct audio_softc *);
      
      static void audio_softintr_rd(void *);
      static void audio_softintr_wr(void *);
      
      static int audio_exlock_mutex_enter(struct audio_softc *);
      static void audio_exlock_mutex_exit(struct audio_softc *);
      static int audio_exlock_enter(struct audio_softc *);
      static void audio_exlock_exit(struct audio_softc *);
      static struct audio_softc *audio_file_enter(audio_file_t *, struct psref *);
      static void audio_file_exit(struct audio_softc *, struct psref *);
      static int audio_track_waitio(struct audio_softc *, audio_track_t *);
      
      static int audioclose(struct file *);
      static int audioread(struct file *, off_t *, struct uio *, kauth_cred_t, int);
      static int audiowrite(struct file *, off_t *, struct uio *, kauth_cred_t, int);
      static int audioioctl(struct file *, u_long, void *);
      static int audiopoll(struct file *, int);
      static int audiokqfilter(struct file *, struct knote *);
      static int audiommap(struct file *, off_t *, size_t, int, int *, int *,
              struct uvm_object **, int *);
      static int audiostat(struct file *, struct stat *);
      
      static void filt_audiowrite_detach(struct knote *);
      static int  filt_audiowrite_event(struct knote *, long);
      static void filt_audioread_detach(struct knote *);
      static int  filt_audioread_event(struct knote *, long);
      
      static int audio_open(dev_t, struct audio_softc *, int, int, struct lwp *,
              audio_file_t **);
      static int audio_close(struct audio_softc *, audio_file_t *);
      static int audio_unlink(struct audio_softc *, audio_file_t *);
      static int audio_read(struct audio_softc *, struct uio *, int, audio_file_t *);
      static int audio_write(struct audio_softc *, struct uio *, int, audio_file_t *);
      static void audio_file_clear(struct audio_softc *, audio_file_t *);
      static int audio_ioctl(dev_t, struct audio_softc *, u_long, void *, int,
              struct lwp *, audio_file_t *);
      static int audio_poll(struct audio_softc *, int, struct lwp *, audio_file_t *);
      static int audio_kqfilter(struct audio_softc *, audio_file_t *, struct knote *);
      static int audio_mmap(struct audio_softc *, off_t *, size_t, int, int *, int *,
              struct uvm_object **, int *, audio_file_t *);
      
      static int audioctl_open(dev_t, struct audio_softc *, int, int, struct lwp *);
      
      static void audio_pintr(void *);
      static void audio_rintr(void *);
      
      static int audio_query_devinfo(struct audio_softc *, mixer_devinfo_t *);
      
      static __inline int audio_track_readablebytes(const audio_track_t *);
      static int audio_file_setinfo(struct audio_softc *, audio_file_t *,
              const struct audio_info *);
      static int audio_track_setinfo_check(audio_track_t *,
              audio_format2_t *, const struct audio_prinfo *);
      static void audio_track_setinfo_water(audio_track_t *,
              const struct audio_info *);
      static int audio_hw_setinfo(struct audio_softc *, const struct audio_info *,
              struct audio_info *);
      static int audio_hw_set_format(struct audio_softc *, int,
              const audio_format2_t *, const audio_format2_t *,
              audio_filter_reg_t *, audio_filter_reg_t *);
      static int audiogetinfo(struct audio_softc *, struct audio_info *, int,
              audio_file_t *);
      static bool audio_can_playback(struct audio_softc *);
      static bool audio_can_capture(struct audio_softc *);
      static int audio_check_params(audio_format2_t *);
      static int audio_mixers_init(struct audio_softc *sc, int,
              const audio_format2_t *, const audio_format2_t *,
              const audio_filter_reg_t *, const audio_filter_reg_t *);
      static int audio_select_freq(const struct audio_format *);
      static int audio_hw_probe(struct audio_softc *, audio_format2_t *, int);
      static int audio_hw_validate_format(struct audio_softc *, int,
              const audio_format2_t *);
      static int audio_mixers_set_format(struct audio_softc *,
              const struct audio_info *);
      static void audio_mixers_get_format(struct audio_softc *, struct audio_info *);
      static int audio_sysctl_blk_ms(SYSCTLFN_PROTO);
      static int audio_sysctl_multiuser(SYSCTLFN_PROTO);
      #if defined(AUDIO_DEBUG)
      static int audio_sysctl_debug(SYSCTLFN_PROTO);
      static void audio_format2_tostr(char *, size_t, const audio_format2_t *);
      static void audio_print_format2(const char *, const audio_format2_t *) __unused;
      #endif
      
      static void *audio_realloc(void *, size_t);
      static int audio_realloc_usrbuf(audio_track_t *, int);
      static void audio_free_usrbuf(audio_track_t *);
      
      static audio_track_t *audio_track_create(struct audio_softc *,
              audio_trackmixer_t *);
      static void audio_track_destroy(audio_track_t *);
      static audio_filter_t audio_track_get_codec(audio_track_t *,
              const audio_format2_t *, const audio_format2_t *);
      static int audio_track_set_format(audio_track_t *, audio_format2_t *);
      static void audio_track_play(audio_track_t *);
      static int audio_track_drain(struct audio_softc *, audio_track_t *);
      static void audio_track_record(audio_track_t *);
      static void audio_track_clear(struct audio_softc *, audio_track_t *);
      
      static int audio_mixer_init(struct audio_softc *, int,
              const audio_format2_t *, const audio_filter_reg_t *);
      static void audio_mixer_destroy(struct audio_softc *, audio_trackmixer_t *);
      static void audio_pmixer_start(struct audio_softc *, bool);
      static void audio_pmixer_process(struct audio_softc *);
      static void audio_pmixer_agc(audio_trackmixer_t *, int);
      static int  audio_pmixer_mix_track(audio_trackmixer_t *, audio_track_t *, int);
      static void audio_pmixer_output(struct audio_softc *);
      static int  audio_pmixer_halt(struct audio_softc *);
      static void audio_rmixer_start(struct audio_softc *);
      static void audio_rmixer_process(struct audio_softc *);
      static void audio_rmixer_input(struct audio_softc *);
      static int  audio_rmixer_halt(struct audio_softc *);
      
      static void mixer_init(struct audio_softc *);
      static int mixer_open(dev_t, struct audio_softc *, int, int, struct lwp *);
      static int mixer_close(struct audio_softc *, audio_file_t *);
      static int mixer_ioctl(struct audio_softc *, u_long, void *, int, struct lwp *);
      static void mixer_async_add(struct audio_softc *, pid_t);
      static void mixer_async_remove(struct audio_softc *, pid_t);
      static void mixer_signal(struct audio_softc *);
      
      static int au_portof(struct audio_softc *, char *, int);
      
      static void au_setup_ports(struct audio_softc *, struct au_mixer_ports *,
              mixer_devinfo_t *, const struct portname *);
      static int au_set_lr_value(struct audio_softc *, mixer_ctrl_t *, int, int);
      static int au_get_lr_value(struct audio_softc *, mixer_ctrl_t *, int *, int *);
      static int au_set_gain(struct audio_softc *, struct au_mixer_ports *, int, int);
      static void au_get_gain(struct audio_softc *, struct au_mixer_ports *,
              u_int *, u_char *);
      static int au_set_port(struct audio_softc *, struct au_mixer_ports *, u_int);
      static int au_get_port(struct audio_softc *, struct au_mixer_ports *);
      static int au_set_monitor_gain(struct audio_softc *, int);
      static int au_get_monitor_gain(struct audio_softc *);
      static int audio_get_port(struct audio_softc *, mixer_ctrl_t *);
      static int audio_set_port(struct audio_softc *, mixer_ctrl_t *);
      
      static __inline struct audio_params
      format2_to_params(const audio_format2_t *f2)
      {
              audio_params_t p;
      
              /* validbits/precision <-> precision/stride */
              p.sample_rate = f2->sample_rate;
              p.channels    = f2->channels;
              p.encoding    = f2->encoding;
              p.validbits   = f2->precision;
              p.precision   = f2->stride;
              return p;
      }
      
      static __inline audio_format2_t
      params_to_format2(const struct audio_params *p)
      {
              audio_format2_t f2;
      
              /* precision/stride <-> validbits/precision */
              f2.sample_rate = p->sample_rate;
              f2.channels    = p->channels;
              f2.encoding    = p->encoding;
              f2.precision   = p->validbits;
              f2.stride      = p->precision;
              return f2;
      }
      
      /* Return true if this track is a playback track. */
      static __inline bool
      audio_track_is_playback(const audio_track_t *track)
      {
      
              return ((track->mode & AUMODE_PLAY) != 0);
      }
      
      /* Return true if this track is a recording track. */
      static __inline bool
      audio_track_is_record(const audio_track_t *track)
      {
      
              return ((track->mode & AUMODE_RECORD) != 0);
      }
      
      #if 0 /* XXX Not used yet */
      /*
       * Convert 0..255 volume used in userland to internal presentation 0..256.
       */
      static __inline u_int
      audio_volume_to_inner(u_int v)
      {
      
              return v < 127 ? v : v + 1;
      }
      
      /*
       * Convert 0..256 internal presentation to 0..255 volume used in userland.
       */
      static __inline u_int
      audio_volume_to_outer(u_int v)
      {
      
              return v < 127 ? v : v - 1;
      }
      #endif /* 0 */
      
      static dev_type_open(audioopen);
      /* XXXMRG use more dev_type_xxx */
      
      const struct cdevsw audio_cdevsw = {
              .d_open = audioopen,
              .d_close = noclose,
              .d_read = noread,
              .d_write = nowrite,
              .d_ioctl = noioctl,
              .d_stop = nostop,
              .d_tty = notty,
              .d_poll = nopoll,
              .d_mmap = nommap,
              .d_kqfilter = nokqfilter,
              .d_discard = nodiscard,
              .d_flag = D_OTHER | D_MPSAFE
      };
      
      const struct fileops audio_fileops = {
              .fo_name = "audio",
              .fo_read = audioread,
              .fo_write = audiowrite,
              .fo_ioctl = audioioctl,
              .fo_fcntl = fnullop_fcntl,
              .fo_stat = audiostat,
              .fo_poll = audiopoll,
              .fo_close = audioclose,
              .fo_mmap = audiommap,
              .fo_kqfilter = audiokqfilter,
              .fo_restart = fnullop_restart
      };
      
      /* The default audio mode: 8 kHz mono mu-law */
      static const struct audio_params audio_default = {
              .sample_rate = 8000,
              .encoding = AUDIO_ENCODING_ULAW,
              .precision = 8,
              .validbits = 8,
              .channels = 1,
      };
      
      static const char *encoding_names[] = {
              "none",
              AudioEmulaw,
              AudioEalaw,
              "pcm16",
              "pcm8",
              AudioEadpcm,
              AudioEslinear_le,
              AudioEslinear_be,
              AudioEulinear_le,
              AudioEulinear_be,
              AudioEslinear,
              AudioEulinear,
              AudioEmpeg_l1_stream,
              AudioEmpeg_l1_packets,
              AudioEmpeg_l1_system,
              AudioEmpeg_l2_stream,
              AudioEmpeg_l2_packets,
              AudioEmpeg_l2_system,
              AudioEac3,
      };
      
      /*
       * Returns encoding name corresponding to AUDIO_ENCODING_*.
       * Note that it may return a local buffer because it is mainly for debugging.
       */
      const char *
      audio_encoding_name(int encoding)
      {
              static char buf[16];
      
              if (0 <= encoding && encoding < __arraycount(encoding_names)) {
                      return encoding_names[encoding];
              } else {
                      snprintf(buf, sizeof(buf), "enc=%d", encoding);
                      return buf;
              }
      }
      
      /*
       * Supported encodings used by AUDIO_GETENC.
       * index and flags are set by code.
       * XXX is there any needs for SLINEAR_OE:>=16/ULINEAR_OE:>=16 ?
       */
      static const audio_encoding_t audio_encodings[] = {
              { 0, AudioEmulaw,        AUDIO_ENCODING_ULAW,                8,  0 },
              { 0, AudioEalaw,        AUDIO_ENCODING_ALAW,                8,  0 },
              { 0, AudioEslinear,        AUDIO_ENCODING_SLINEAR,                8,  0 },
              { 0, AudioEulinear,        AUDIO_ENCODING_ULINEAR,                8,  0 },
              { 0, AudioEslinear_le,        AUDIO_ENCODING_SLINEAR_LE,        16, 0 },
              { 0, AudioEulinear_le,        AUDIO_ENCODING_ULINEAR_LE,        16, 0 },
              { 0, AudioEslinear_be,        AUDIO_ENCODING_SLINEAR_BE,        16, 0 },
              { 0, AudioEulinear_be,        AUDIO_ENCODING_ULINEAR_BE,        16, 0 },
      #if defined(AUDIO_SUPPORT_LINEAR24)
              { 0, AudioEslinear_le,        AUDIO_ENCODING_SLINEAR_LE,        24, 0 },
              { 0, AudioEulinear_le,        AUDIO_ENCODING_ULINEAR_LE,        24, 0 },
              { 0, AudioEslinear_be,        AUDIO_ENCODING_SLINEAR_BE,        24, 0 },
              { 0, AudioEulinear_be,        AUDIO_ENCODING_ULINEAR_BE,        24, 0 },
      #endif
              { 0, AudioEslinear_le,        AUDIO_ENCODING_SLINEAR_LE,        32, 0 },
              { 0, AudioEulinear_le,        AUDIO_ENCODING_ULINEAR_LE,        32, 0 },
              { 0, AudioEslinear_be,        AUDIO_ENCODING_SLINEAR_BE,        32, 0 },
              { 0, AudioEulinear_be,        AUDIO_ENCODING_ULINEAR_BE,        32, 0 },
      };
      
      static const struct portname itable[] = {
              { AudioNmicrophone,        AUDIO_MICROPHONE },
              { AudioNline,                AUDIO_LINE_IN },
              { AudioNcd,                AUDIO_CD },
              { 0, 0 }
      };
      static const struct portname otable[] = {
              { AudioNspeaker,        AUDIO_SPEAKER },
              { AudioNheadphone,        AUDIO_HEADPHONE },
              { AudioNline,                AUDIO_LINE_OUT },
              { 0, 0 }
      };
      
      static struct psref_class *audio_psref_class __read_mostly;
      
      CFATTACH_DECL3_NEW(audio, sizeof(struct audio_softc),
          audiomatch, audioattach, audiodetach, audioactivate, audiorescan,
          audiochilddet, DVF_DETACH_SHUTDOWN);
      
      static int
      audiomatch(device_t parent, cfdata_t match, void *aux)
      {
              struct audio_attach_args *sa;
      
              sa = aux;
              DPRINTF(1, "%s: type=%d sa=%p hw=%p\n",
                   __func__, sa->type, sa, sa->hwif);
              return (sa->type == AUDIODEV_TYPE_AUDIO) ? 1 : 0;
      }
      
      static void
      audioattach(device_t parent, device_t self, void *aux)
      {
              struct audio_softc *sc;
              struct audio_attach_args *sa;
              const struct audio_hw_if *hw_if;
              audio_format2_t phwfmt;
              audio_format2_t rhwfmt;
              audio_filter_reg_t pfil;
              audio_filter_reg_t rfil;
              const struct sysctlnode *node;
              void *hdlp;
              bool has_playback;
              bool has_capture;
              bool has_indep;
              bool has_fulldup;
              int mode;
              int error;
      
              sc = device_private(self);
              sc->sc_dev = self;
              sa = (struct audio_attach_args *)aux;
              hw_if = sa->hwif;
              hdlp = sa->hdl;
      
              if (hw_if == NULL) {
                      panic("audioattach: missing hw_if method");
              }
              if (hw_if->get_locks == NULL || hw_if->get_props == NULL) {
                      aprint_error(": missing mandatory method\n");
                      return;
              }
      
              hw_if->get_locks(hdlp, &sc->sc_intr_lock, &sc->sc_lock);
              sc->sc_props = hw_if->get_props(hdlp);
      
              has_playback = (sc->sc_props & AUDIO_PROP_PLAYBACK);
              has_capture  = (sc->sc_props & AUDIO_PROP_CAPTURE);
              has_indep    = (sc->sc_props & AUDIO_PROP_INDEPENDENT);
              has_fulldup  = (sc->sc_props & AUDIO_PROP_FULLDUPLEX);
      
      #ifdef DIAGNOSTIC
              if (hw_if->query_format == NULL ||
                  hw_if->set_format == NULL ||
                  hw_if->getdev == NULL ||
                  hw_if->set_port == NULL ||
                  hw_if->get_port == NULL ||
                  hw_if->query_devinfo == NULL) {
                      aprint_error(": missing mandatory method\n");
                      return;
              }
              if (has_playback) {
                      if ((hw_if->start_output == NULL && hw_if->trigger_output == NULL) ||
                          hw_if->halt_output == NULL) {
                              aprint_error(": missing playback method\n");
                      }
              }
              if (has_capture) {
                      if ((hw_if->start_input == NULL && hw_if->trigger_input == NULL) ||
                          hw_if->halt_input == NULL) {
                              aprint_error(": missing capture method\n");
                      }
              }
      #endif
      
              sc->hw_if = hw_if;
              sc->hw_hdl = hdlp;
              sc->hw_dev = parent;
      
              sc->sc_exlock = 1;
              sc->sc_blk_ms = AUDIO_BLK_MS;
              SLIST_INIT(&sc->sc_files);
              cv_init(&sc->sc_exlockcv, "audiolk");
              sc->sc_am_capacity = 0;
              sc->sc_am_used = 0;
              sc->sc_am = NULL;
      
              /* MMAP is now supported by upper layer.  */
              sc->sc_props |= AUDIO_PROP_MMAP;
      
              KASSERT(has_playback || has_capture);
              /* Unidirectional device must have neither FULLDUP nor INDEPENDENT. */
              if (!has_playback || !has_capture) {
                      KASSERT(!has_indep);
                      KASSERT(!has_fulldup);
              }
      
              mode = 0;
              if (has_playback) {
                      aprint_normal(": playback");
                      mode |= AUMODE_PLAY;
              }
              if (has_capture) {
                      aprint_normal("%c capture", has_playback ? ',' : ':');
                      mode |= AUMODE_RECORD;
              }
              if (has_playback && has_capture) {
                      if (has_fulldup)
                              aprint_normal(", full duplex");
                      else
                              aprint_normal(", half duplex");
      
                      if (has_indep)
                              aprint_normal(", independent");
              }
      
              aprint_naive("\n");
              aprint_normal("\n");
      
              /* probe hw params */
              memset(&phwfmt, 0, sizeof(phwfmt));
              memset(&rhwfmt, 0, sizeof(rhwfmt));
              memset(&pfil, 0, sizeof(pfil));
              memset(&rfil, 0, sizeof(rfil));
              if (has_indep) {
                      int perror, rerror;
      
                      /* On independent devices, probe separately. */
                      perror = audio_hw_probe(sc, &phwfmt, AUMODE_PLAY);
                      rerror = audio_hw_probe(sc, &rhwfmt, AUMODE_RECORD);
                      if (perror && rerror) {
                              aprint_error_dev(self, "audio_hw_probe failed, "
                                  "perror = %d, rerror = %d\n", perror, rerror);
                              goto bad;
                      }
                      if (perror) {
                              mode &= ~AUMODE_PLAY;
                              aprint_error_dev(self, "audio_hw_probe failed with "
                                  "%d, playback disabled\n", perror);
                      }
                      if (rerror) {
                              mode &= ~AUMODE_RECORD;
                              aprint_error_dev(self, "audio_hw_probe failed with "
                                  "%d, capture disabled\n", rerror);
                      }
              } else {
                      /*
                       * On non independent devices or uni-directional devices,
                       * probe once (simultaneously).
                       */
                      audio_format2_t *fmt = has_playback ? &phwfmt : &rhwfmt;
                      error = audio_hw_probe(sc, fmt, mode);
                      if (error) {
                              aprint_error_dev(self, "audio_hw_probe failed, "
                                  "error = %d\n", error);
                              goto bad;
                      }
                      if (has_playback && has_capture)
                              rhwfmt = phwfmt;
              }
      
              /* Init hardware. */
              /* hw_probe() also validates [pr]hwfmt.  */
              error = audio_hw_set_format(sc, mode, &phwfmt, &rhwfmt, &pfil, &rfil);
              if (error) {
                      aprint_error_dev(self, "audio_hw_set_format failed, "
                          "error = %d\n", error);
                      goto bad;
              }
      
              /*
               * Init track mixers.  If at least one direction is available on
               * attach time, we assume a success.
               */
              error = audio_mixers_init(sc, mode, &phwfmt, &rhwfmt, &pfil, &rfil);
              if (sc->sc_pmixer == NULL && sc->sc_rmixer == NULL) {
                      aprint_error_dev(self, "audio_mixers_init failed, "
                          "error = %d\n", error);
                      goto bad;
              }
      
              sc->sc_psz = pserialize_create();
              psref_target_init(&sc->sc_psref, audio_psref_class);
      
              selinit(&sc->sc_wsel);
              selinit(&sc->sc_rsel);
      
              /* Initial parameter of /dev/sound */
              sc->sc_sound_pparams = params_to_format2(&audio_default);
              sc->sc_sound_rparams = params_to_format2(&audio_default);
              sc->sc_sound_ppause = false;
              sc->sc_sound_rpause = false;
      
              /* XXX TODO: consider about sc_ai */
      
              mixer_init(sc);
              TRACE(2, "inputs ports=0x%x, input master=%d, "
                  "output ports=0x%x, output master=%d",
                  sc->sc_inports.allports, sc->sc_inports.master,
                  sc->sc_outports.allports, sc->sc_outports.master);
      
              sysctl_createv(&sc->sc_log, 0, NULL, &node,
                  0,
                  CTLTYPE_NODE, device_xname(sc->sc_dev),
                  SYSCTL_DESCR("audio test"),
                  NULL, 0,
                  NULL, 0,
                  CTL_HW,
                  CTL_CREATE, CTL_EOL);
      
              if (node != NULL) {
                      sysctl_createv(&sc->sc_log, 0, NULL, NULL,
                          CTLFLAG_READWRITE,
                          CTLTYPE_INT, "blk_ms",
                          SYSCTL_DESCR("blocksize in msec"),
                          audio_sysctl_blk_ms, 0, (void *)sc, 0,
                          CTL_HW, node->sysctl_num, CTL_CREATE, CTL_EOL);
      
                      sysctl_createv(&sc->sc_log, 0, NULL, NULL,
                          CTLFLAG_READWRITE,
                          CTLTYPE_BOOL, "multiuser",
                          SYSCTL_DESCR("allow multiple user access"),
                          audio_sysctl_multiuser, 0, (void *)sc, 0,
                          CTL_HW, node->sysctl_num, CTL_CREATE, CTL_EOL);
      
      #if defined(AUDIO_DEBUG)
                      sysctl_createv(&sc->sc_log, 0, NULL, NULL,
                          CTLFLAG_READWRITE,
                          CTLTYPE_INT, "debug",
                          SYSCTL_DESCR("debug level (0..4)"),
                          audio_sysctl_debug, 0, (void *)sc, 0,
                          CTL_HW, node->sysctl_num, CTL_CREATE, CTL_EOL);
      #endif
              }
      
      #ifdef AUDIO_PM_IDLE
              callout_init(&sc->sc_idle_counter, 0);
              callout_setfunc(&sc->sc_idle_counter, audio_idle, self);
      #endif
      
              if (!pmf_device_register(self, audio_suspend, audio_resume))
                      aprint_error_dev(self, "couldn't establish power handler\n");
      #ifdef AUDIO_PM_IDLE
              if (!device_active_register(self, audio_activity))
                      aprint_error_dev(self, "couldn't register activity handler\n");
      #endif
      
              if (!pmf_event_register(self, PMFE_AUDIO_VOLUME_DOWN,
                  audio_volume_down, true))
                      aprint_error_dev(self, "couldn't add volume down handler\n");
              if (!pmf_event_register(self, PMFE_AUDIO_VOLUME_UP,
                  audio_volume_up, true))
                      aprint_error_dev(self, "couldn't add volume up handler\n");
              if (!pmf_event_register(self, PMFE_AUDIO_VOLUME_TOGGLE,
                  audio_volume_toggle, true))
                      aprint_error_dev(self, "couldn't add volume toggle handler\n");
      
      #ifdef AUDIO_PM_IDLE
              callout_schedule(&sc->sc_idle_counter, audio_idle_timeout * hz);
      #endif
      
      #if defined(AUDIO_DEBUG)
              audio_mlog_init();
      #endif
      
              audiorescan(self, "audio", NULL);
              sc->sc_exlock = 0;
              return;
      
      bad:
              /* Clearing hw_if means that device is attached but disabled. */
              sc->hw_if = NULL;
              sc->sc_exlock = 0;
              aprint_error_dev(sc->sc_dev, "disabled\n");
              return;
      }
      
      /*
       * Initialize hardware mixer.
       * This function is called from audioattach().
       */
      static void
      mixer_init(struct audio_softc *sc)
      {
              mixer_devinfo_t mi;
              int iclass, mclass, oclass, rclass;
              int record_master_found, record_source_found;
      
              iclass = mclass = oclass = rclass = -1;
              sc->sc_inports.index = -1;
              sc->sc_inports.master = -1;
              sc->sc_inports.nports = 0;
              sc->sc_inports.isenum = false;
              sc->sc_inports.allports = 0;
              sc->sc_inports.isdual = false;
              sc->sc_inports.mixerout = -1;
              sc->sc_inports.cur_port = -1;
              sc->sc_outports.index = -1;
              sc->sc_outports.master = -1;
              sc->sc_outports.nports = 0;
              sc->sc_outports.isenum = false;
              sc->sc_outports.allports = 0;
              sc->sc_outports.isdual = false;
              sc->sc_outports.mixerout = -1;
              sc->sc_outports.cur_port = -1;
              sc->sc_monitor_port = -1;
              /*
               * Read through the underlying driver's list, picking out the class
               * names from the mixer descriptions. We'll need them to decode the
               * mixer descriptions on the next pass through the loop.
               */
              mutex_enter(sc->sc_lock);
              for(mi.index = 0; ; mi.index++) {
                      if (audio_query_devinfo(sc, &mi) != 0)
                              break;
                       /*
                        * The type of AUDIO_MIXER_CLASS merely introduces a class.
                        * All the other types describe an actual mixer.
                        */
                      if (mi.type == AUDIO_MIXER_CLASS) {
                              if (strcmp(mi.label.name, AudioCinputs) == 0)
                                      iclass = mi.mixer_class;
                              if (strcmp(mi.label.name, AudioCmonitor) == 0)
                                      mclass = mi.mixer_class;
                              if (strcmp(mi.label.name, AudioCoutputs) == 0)
                                      oclass = mi.mixer_class;
                              if (strcmp(mi.label.name, AudioCrecord) == 0)
                                      rclass = mi.mixer_class;
                      }
              }
              mutex_exit(sc->sc_lock);
      
              /* Allocate save area.  Ensure non-zero allocation. */
              sc->sc_nmixer_states = mi.index;
              sc->sc_mixer_state = kmem_zalloc(sizeof(mixer_ctrl_t) *
                  (sc->sc_nmixer_states + 1), KM_SLEEP);
      
              /*
               * This is where we assign each control in the "audio" model, to the
               * underlying "mixer" control.  We walk through the whole list once,
               * assigning likely candidates as we come across them.
               */
              record_master_found = 0;
              record_source_found = 0;
              mutex_enter(sc->sc_lock);
              for(mi.index = 0; ; mi.index++) {
                      if (audio_query_devinfo(sc, &mi) != 0)
                              break;
                      KASSERT(mi.index < sc->sc_nmixer_states);
                      if (mi.type == AUDIO_MIXER_CLASS)
                              continue;
                      if (mi.mixer_class == iclass) {
                              /*
                               * AudioCinputs is only a fallback, when we don't
                               * find what we're looking for in AudioCrecord, so
                               * check the flags before accepting one of these.
                               */
                              if (strcmp(mi.label.name, AudioNmaster) == 0
                                  && record_master_found == 0)
                                      sc->sc_inports.master = mi.index;
                              if (strcmp(mi.label.name, AudioNsource) == 0
                                  && record_source_found == 0) {
                                      if (mi.type == AUDIO_MIXER_ENUM) {
                                          int i;
                                          for(i = 0; i < mi.un.e.num_mem; i++)
                                              if (strcmp(mi.un.e.member[i].label.name,
                                                          AudioNmixerout) == 0)
                                                      sc->sc_inports.mixerout =
                                                          mi.un.e.member[i].ord;
                                      }
                                      au_setup_ports(sc, &sc->sc_inports, &mi,
                                          itable);
                              }
                              if (strcmp(mi.label.name, AudioNdac) == 0 &&
                                  sc->sc_outports.master == -1)
                                      sc->sc_outports.master = mi.index;
                      } else if (mi.mixer_class == mclass) {
                              if (strcmp(mi.label.name, AudioNmonitor) == 0)
                                      sc->sc_monitor_port = mi.index;
                      } else if (mi.mixer_class == oclass) {
                              if (strcmp(mi.label.name, AudioNmaster) == 0)
                                      sc->sc_outports.master = mi.index;
                              if (strcmp(mi.label.name, AudioNselect) == 0)
                                      au_setup_ports(sc, &sc->sc_outports, &mi,
                                          otable);
                      } else if (mi.mixer_class == rclass) {
                              /*
                               * These are the preferred mixers for the audio record
                               * controls, so set the flags here, but don't check.
                               */
                              if (strcmp(mi.label.name, AudioNmaster) == 0) {
                                      sc->sc_inports.master = mi.index;
                                      record_master_found = 1;
                              }
      #if 1        /* Deprecated. Use AudioNmaster. */
                              if (strcmp(mi.label.name, AudioNrecord) == 0) {
                                      sc->sc_inports.master = mi.index;
                                      record_master_found = 1;
                              }
                              if (strcmp(mi.label.name, AudioNvolume) == 0) {
                                      sc->sc_inports.master = mi.index;
                                      record_master_found = 1;
                              }
      #endif
                              if (strcmp(mi.label.name, AudioNsource) == 0) {
                                      if (mi.type == AUDIO_MIXER_ENUM) {
                                          int i;
                                          for(i = 0; i < mi.un.e.num_mem; i++)
                                              if (strcmp(mi.un.e.member[i].label.name,
                                                          AudioNmixerout) == 0)
                                                      sc->sc_inports.mixerout =
                                                          mi.un.e.member[i].ord;
                                      }
                                      au_setup_ports(sc, &sc->sc_inports, &mi,
                                          itable);
                                      record_source_found = 1;
                              }
                      }
              }
              mutex_exit(sc->sc_lock);
      }
      
      static int
      audioactivate(device_t self, enum devact act)
      {
              struct audio_softc *sc = device_private(self);
      
              switch (act) {
              case DVACT_DEACTIVATE:
                      mutex_enter(sc->sc_lock);
                      sc->sc_dying = true;
                      cv_broadcast(&sc->sc_exlockcv);
                      mutex_exit(sc->sc_lock);
                      return 0;
              default:
                      return EOPNOTSUPP;
              }
      }
      
      static int
      audiodetach(device_t self, int flags)
      {
              struct audio_softc *sc;
              struct audio_file *file;
              int error;
      
              sc = device_private(self);
              TRACE(2, "flags=%d", flags);
      
              /* device is not initialized */
              if (sc->hw_if == NULL)
                      return 0;
      
              /* Start draining existing accessors of the device. */
              error = config_detach_children(self, flags);
              if (error)
                      return error;
      
              /* delete sysctl nodes */
              sysctl_teardown(&sc->sc_log);
      
              mutex_enter(sc->sc_lock);
              sc->sc_dying = true;
              cv_broadcast(&sc->sc_exlockcv);
              if (sc->sc_pmixer)
                      cv_broadcast(&sc->sc_pmixer->outcv);
              if (sc->sc_rmixer)
                      cv_broadcast(&sc->sc_rmixer->outcv);
      
              /* Prevent new users */
              SLIST_FOREACH(file, &sc->sc_files, entry) {
                      atomic_store_relaxed(&file->dying, true);
              }
      
              /*
               * Wait for existing users to drain.
               * - pserialize_perform waits for all pserialize_read sections on
               *   all CPUs; after this, no more new psref_acquire can happen.
               * - psref_target_destroy waits for all extant acquired psrefs to
               *   be psref_released.
               */
              pserialize_perform(sc->sc_psz);
              mutex_exit(sc->sc_lock);
              psref_target_destroy(&sc->sc_psref, audio_psref_class);
      
              /*
               * We are now guaranteed that there are no calls to audio fileops
               * that hold sc, and any new calls with files that were for sc will
               * fail.  Thus, we now have exclusive access to the softc.
               */
              sc->sc_exlock = 1;
      
              /*
               * Nuke all open instances.
               * Here, we no longer need any locks to traverse sc_files.
               */
              while ((file = SLIST_FIRST(&sc->sc_files)) != NULL) {
                      audio_unlink(sc, file);
              }
      
              pmf_event_deregister(self, PMFE_AUDIO_VOLUME_DOWN,
                  audio_volume_down, true);
              pmf_event_deregister(self, PMFE_AUDIO_VOLUME_UP,
                  audio_volume_up, true);
              pmf_event_deregister(self, PMFE_AUDIO_VOLUME_TOGGLE,
                  audio_volume_toggle, true);
      
      #ifdef AUDIO_PM_IDLE
              callout_halt(&sc->sc_idle_counter, sc->sc_lock);
      
              device_active_deregister(self, audio_activity);
      #endif
      
              pmf_device_deregister(self);
      
              /* Free resources */
              if (sc->sc_pmixer) {
                      audio_mixer_destroy(sc, sc->sc_pmixer);
                      kmem_free(sc->sc_pmixer, sizeof(*sc->sc_pmixer));
              }
              if (sc->sc_rmixer) {
                      audio_mixer_destroy(sc, sc->sc_rmixer);
                      kmem_free(sc->sc_rmixer, sizeof(*sc->sc_rmixer));
              }
              if (sc->sc_am)
                      kern_free(sc->sc_am);
      
              seldestroy(&sc->sc_wsel);
              seldestroy(&sc->sc_rsel);
      
      #ifdef AUDIO_PM_IDLE
              callout_destroy(&sc->sc_idle_counter);
      #endif
      
              cv_destroy(&sc->sc_exlockcv);
      
      #if defined(AUDIO_DEBUG)
              audio_mlog_free();
      #endif
      
              return 0;
      }
      
      static void
      audiochilddet(device_t self, device_t child)
      {
      
              /* we hold no child references, so do nothing */
      }
      
      static int
      audiosearch(device_t parent, cfdata_t cf, const int *locs, void *aux)
      {
      
              if (config_match(parent, cf, aux))
                      config_attach_loc(parent, cf, locs, aux, NULL);
      
              return 0;
      }
      
      static int
      audiorescan(device_t self, const char *ifattr, const int *flags)
      {
              struct audio_softc *sc = device_private(self);
      
              if (!ifattr_match(ifattr, "audio"))
                      return 0;
      
              config_search_loc(audiosearch, sc->sc_dev, "audio", NULL, NULL);
      
              return 0;
      }
      
      /*
       * Called from hardware driver.  This is where the MI audio driver gets
       * probed/attached to the hardware driver.
       */
      device_t
      audio_attach_mi(const struct audio_hw_if *ahwp, void *hdlp, device_t dev)
      {
              struct audio_attach_args arg;
      
      #ifdef DIAGNOSTIC
              if (ahwp == NULL) {
                      aprint_error("audio_attach_mi: NULL\n");
                      return 0;
              }
      #endif
              arg.type = AUDIODEV_TYPE_AUDIO;
              arg.hwif = ahwp;
              arg.hdl = hdlp;
              return config_found(dev, &arg, audioprint);
      }
      
      /*
       * Enter critical section and also keep sc_lock.
       * If successful, returns 0 with sc_lock held.  Otherwise returns errno.
       * Must be called without sc_lock held.
       */
      static int
      audio_exlock_mutex_enter(struct audio_softc *sc)
      {
              int error;
      
              mutex_enter(sc->sc_lock);
              if (sc->sc_dying) {
                      mutex_exit(sc->sc_lock);
                      return EIO;
              }
      
              while (__predict_false(sc->sc_exlock != 0)) {
                      error = cv_wait_sig(&sc->sc_exlockcv, sc->sc_lock);
                      if (sc->sc_dying)
                              error = EIO;
                      if (error) {
                              mutex_exit(sc->sc_lock);
                              return error;
                      }
              }
      
              /* Acquire */
              sc->sc_exlock = 1;
              return 0;
      }
      
      /*
       * Exit critical section and exit sc_lock.
       * Must be called with sc_lock held.
       */
      static void
      audio_exlock_mutex_exit(struct audio_softc *sc)
      {
      
              KASSERT(mutex_owned(sc->sc_lock));
      
              sc->sc_exlock = 0;
              cv_broadcast(&sc->sc_exlockcv);
              mutex_exit(sc->sc_lock);
      }
      
      /*
       * Enter critical section.
       * If successful, it returns 0.  Otherwise returns errno.
       * Must be called without sc_lock held.
       * This function returns without sc_lock held.
       */
      static int
      audio_exlock_enter(struct audio_softc *sc)
      {
              int error;
      
              error = audio_exlock_mutex_enter(sc);
              if (error)
                      return error;
              mutex_exit(sc->sc_lock);
              return 0;
      }
      
      /*
       * Exit critical section.
       * Must be called without sc_lock held.
       */
      static void
      audio_exlock_exit(struct audio_softc *sc)
      {
      
              mutex_enter(sc->sc_lock);
              audio_exlock_mutex_exit(sc);
      }
      
      /*
       * Acquire sc from file, and increment the psref count.
       * If successful, returns sc.  Otherwise returns NULL.
       */
      struct audio_softc *
      audio_file_enter(audio_file_t *file, struct psref *refp)
      {
              int s;
              bool dying;
      
              /* psref(9) forbids to migrate CPUs */
              curlwp_bind();
      
              /* Block audiodetach while we acquire a reference */
              s = pserialize_read_enter();
      
              /* If close or audiodetach already ran, tough -- no more audio */
              dying = atomic_load_relaxed(&file->dying);
              if (dying) {
                      pserialize_read_exit(s);
                      return NULL;
              }
      
              /* Acquire a reference */
              psref_acquire(refp, &file->sc->sc_psref, audio_psref_class);
      
              /* Now sc won't go away until we drop the reference count */
              pserialize_read_exit(s);
      
              return file->sc;
      }
      
      /*
       * Decrement the psref count.
       */
      void
      audio_file_exit(struct audio_softc *sc, struct psref *refp)
      {
      
              psref_release(refp, &sc->sc_psref, audio_psref_class);
      }
      
      /*
       * Wait for I/O to complete, releasing sc_lock.
       * Must be called with sc_lock held.
       */
      static int
      audio_track_waitio(struct audio_softc *sc, audio_track_t *track)
      {
              int error;
      
              KASSERT(track);
              KASSERT(mutex_owned(sc->sc_lock));
      
              /* Wait for pending I/O to complete. */
              error = cv_timedwait_sig(&track->mixer->outcv, sc->sc_lock,
                  mstohz(AUDIO_TIMEOUT));
              if (sc->sc_suspending) {
                      /* If it's about to suspend, ignore timeout error. */
                      if (error == EWOULDBLOCK) {
                              TRACET(2, track, "timeout (suspending)");
                              return 0;
                      }
              }
              if (sc->sc_dying) {
                      error = EIO;
              }
              if (error) {
                      TRACET(2, track, "cv_timedwait_sig failed %d", error);
                      if (error == EWOULDBLOCK)
                              device_printf(sc->sc_dev, "device timeout\n");
              } else {
                      TRACET(3, track, "wakeup");
              }
              return error;
      }
      
      /*
       * Try to acquire track lock.
       * It doesn't block if the track lock is already aquired.
       * Returns true if the track lock was acquired, or false if the track
       * lock was already acquired.
       */
      static __inline bool
      audio_track_lock_tryenter(audio_track_t *track)
      {
              return (atomic_cas_uint(&track->lock, 0, 1) == 0);
      }
      
      /*
       * Acquire track lock.
       */
      static __inline void
      audio_track_lock_enter(audio_track_t *track)
      {
              /* Don't sleep here. */
              while (audio_track_lock_tryenter(track) == false)
                      ;
      }
      
      /*
       * Release track lock.
       */
      static __inline void
      audio_track_lock_exit(audio_track_t *track)
      {
              atomic_swap_uint(&track->lock, 0);
      }
      
      
      static int
      audioopen(dev_t dev, int flags, int ifmt, struct lwp *l)
      {
              struct audio_softc *sc;
              int error;
      
              /* Find the device */
    1         sc = device_lookup_private(&audio_cd, AUDIOUNIT(dev));
              if (sc == NULL || sc->hw_if == NULL)
                      return ENXIO;
      
              error = audio_exlock_enter(sc);
    1         if (error)
                      return error;
      
              device_active(sc->sc_dev, DVA_SYSTEM);
              switch (AUDIODEV(dev)) {
              case SOUND_DEVICE:
              case AUDIO_DEVICE:
                      error = audio_open(dev, sc, flags, ifmt, l, NULL);
                      break;
              case AUDIOCTL_DEVICE:
                      error = audioctl_open(dev, sc, flags, ifmt, l);
                      break;
              case MIXER_DEVICE:
                      error = mixer_open(dev, sc, flags, ifmt, l);
                      break;
              default:
                      error = ENXIO;
                      break;
              }
              audio_exlock_exit(sc);
      
              return error;
      }
      
      static int
      audioclose(struct file *fp)
      {
              struct audio_softc *sc;
              struct psref sc_ref;
              audio_file_t *file;
              int error;
              dev_t dev;
      
              KASSERT(fp->f_audioctx);
              file = fp->f_audioctx;
              dev = file->dev;
              error = 0;
      
              /*
               * audioclose() must
               * - unplug track from the trackmixer (and unplug anything from softc),
               *   if sc exists.
               * - free all memory objects, regardless of sc.
               */
      
              sc = audio_file_enter(file, &sc_ref);
              if (sc) {
                      switch (AUDIODEV(dev)) {
                      case SOUND_DEVICE:
                      case AUDIO_DEVICE:
                              error = audio_close(sc, file);
                              break;
                      case AUDIOCTL_DEVICE:
                              error = 0;
                              break;
                      case MIXER_DEVICE:
                              error = mixer_close(sc, file);
                              break;
                      default:
                              error = ENXIO;
                              break;
                      }
      
                      audio_file_exit(sc, &sc_ref);
              }
      
              /* Free memory objects anyway */
              TRACEF(2, file, "free memory");
              if (file->ptrack)
                      audio_track_destroy(file->ptrack);
              if (file->rtrack)
                      audio_track_destroy(file->rtrack);
              kmem_free(file, sizeof(*file));
              fp->f_audioctx = NULL;
      
              return error;
      }
      
      static int
      audioread(struct file *fp, off_t *offp, struct uio *uio, kauth_cred_t cred,
              int ioflag)
      {
              struct audio_softc *sc;
              struct psref sc_ref;
              audio_file_t *file;
              int error;
              dev_t dev;
      
              KASSERT(fp->f_audioctx);
              file = fp->f_audioctx;
              dev = file->dev;
      
              sc = audio_file_enter(file, &sc_ref);
              if (sc == NULL)
                      return EIO;
      
              if (fp->f_flag & O_NONBLOCK)
                      ioflag |= IO_NDELAY;
      
              switch (AUDIODEV(dev)) {
              case SOUND_DEVICE:
              case AUDIO_DEVICE:
                      error = audio_read(sc, uio, ioflag, file);
                      break;
              case AUDIOCTL_DEVICE:
              case MIXER_DEVICE:
                      error = ENODEV;
                      break;
              default:
                      error = ENXIO;
                      break;
              }
      
              audio_file_exit(sc, &sc_ref);
              return error;
      }
      
      static int
      audiowrite(struct file *fp, off_t *offp, struct uio *uio, kauth_cred_t cred,
              int ioflag)
      {
              struct audio_softc *sc;
              struct psref sc_ref;
              audio_file_t *file;
              int error;
              dev_t dev;
      
              KASSERT(fp->f_audioctx);
              file = fp->f_audioctx;
              dev = file->dev;
      
              sc = audio_file_enter(file, &sc_ref);
              if (sc == NULL)
                      return EIO;
      
              if (fp->f_flag & O_NONBLOCK)
                      ioflag |= IO_NDELAY;
      
              switch (AUDIODEV(dev)) {
              case SOUND_DEVICE:
              case AUDIO_DEVICE:
                      error = audio_write(sc, uio, ioflag, file);
                      break;
              case AUDIOCTL_DEVICE:
              case MIXER_DEVICE:
                      error = ENODEV;
                      break;
              default:
                      error = ENXIO;
                      break;
              }
      
              audio_file_exit(sc, &sc_ref);
              return error;
      }
      
      static int
      audioioctl(struct file *fp, u_long cmd, void *addr)
      {
              struct audio_softc *sc;
              struct psref sc_ref;
              audio_file_t *file;
              struct lwp *l = curlwp;
              int error;
              dev_t dev;
      
              KASSERT(fp->f_audioctx);
              file = fp->f_audioctx;
              dev = file->dev;
      
              sc = audio_file_enter(file, &sc_ref);
              if (sc == NULL)
                      return EIO;
      
              switch (AUDIODEV(dev)) {
              case SOUND_DEVICE:
              case AUDIO_DEVICE:
              case AUDIOCTL_DEVICE:
                      mutex_enter(sc->sc_lock);
                      device_active(sc->sc_dev, DVA_SYSTEM);
                      mutex_exit(sc->sc_lock);
                      if (IOCGROUP(cmd) == IOCGROUP(AUDIO_MIXER_READ))
                              error = mixer_ioctl(sc, cmd, addr, fp->f_flag, l);
                      else
                              error = audio_ioctl(dev, sc, cmd, addr, fp->f_flag, l,
                                  file);
                      break;
              case MIXER_DEVICE:
                      error = mixer_ioctl(sc, cmd, addr, fp->f_flag, l);
                      break;
              default:
                      error = ENXIO;
                      break;
              }
      
              audio_file_exit(sc, &sc_ref);
              return error;
      }
      
      static int
      audiostat(struct file *fp, struct stat *st)
      {
              struct audio_softc *sc;
              struct psref sc_ref;
              audio_file_t *file;
      
              KASSERT(fp->f_audioctx);
              file = fp->f_audioctx;
      
              sc = audio_file_enter(file, &sc_ref);
              if (sc == NULL)
                      return EIO;
      
              memset(st, 0, sizeof(*st));
      
              st->st_dev = file->dev;
              st->st_uid = kauth_cred_geteuid(fp->f_cred);
              st->st_gid = kauth_cred_getegid(fp->f_cred);
              st->st_mode = S_IFCHR;
      
              audio_file_exit(sc, &sc_ref);
              return 0;
      }
      
      static int
      audiopoll(struct file *fp, int events)
      {
              struct audio_softc *sc;
              struct psref sc_ref;
              audio_file_t *file;
              struct lwp *l = curlwp;
              int revents;
              dev_t dev;
      
              KASSERT(fp->f_audioctx);
              file = fp->f_audioctx;
              dev = file->dev;
      
              sc = audio_file_enter(file, &sc_ref);
              if (sc == NULL)
                      return EIO;
      
              switch (AUDIODEV(dev)) {
              case SOUND_DEVICE:
              case AUDIO_DEVICE:
                      revents = audio_poll(sc, events, l, file);
                      break;
              case AUDIOCTL_DEVICE:
              case MIXER_DEVICE:
                      revents = 0;
                      break;
              default:
                      revents = POLLERR;
                      break;
              }
      
              audio_file_exit(sc, &sc_ref);
              return revents;
      }
      
      static int
      audiokqfilter(struct file *fp, struct knote *kn)
      {
              struct audio_softc *sc;
              struct psref sc_ref;
              audio_file_t *file;
              dev_t dev;
              int error;
      
              KASSERT(fp->f_audioctx);
              file = fp->f_audioctx;
              dev = file->dev;
      
              sc = audio_file_enter(file, &sc_ref);
              if (sc == NULL)
                      return EIO;
      
              switch (AUDIODEV(dev)) {
              case SOUND_DEVICE:
              case AUDIO_DEVICE:
                      error = audio_kqfilter(sc, file, kn);
                      break;
              case AUDIOCTL_DEVICE:
              case MIXER_DEVICE:
                      error = ENODEV;
                      break;
              default:
                      error = ENXIO;
                      break;
              }
      
              audio_file_exit(sc, &sc_ref);
              return error;
      }
      
      static int
      audiommap(struct file *fp, off_t *offp, size_t len, int prot, int *flagsp,
              int *advicep, struct uvm_object **uobjp, int *maxprotp)
      {
              struct audio_softc *sc;
              struct psref sc_ref;
              audio_file_t *file;
              dev_t dev;
              int error;
      
              KASSERT(fp->f_audioctx);
              file = fp->f_audioctx;
              dev = file->dev;
      
              sc = audio_file_enter(file, &sc_ref);
              if (sc == NULL)
                      return EIO;
      
              mutex_enter(sc->sc_lock);
              device_active(sc->sc_dev, DVA_SYSTEM); /* XXXJDM */
              mutex_exit(sc->sc_lock);
      
              switch (AUDIODEV(dev)) {
              case SOUND_DEVICE:
              case AUDIO_DEVICE:
                      error = audio_mmap(sc, offp, len, prot, flagsp, advicep,
                          uobjp, maxprotp, file);
                      break;
              case AUDIOCTL_DEVICE:
              case MIXER_DEVICE:
              default:
                      error = ENOTSUP;
                      break;
              }
      
              audio_file_exit(sc, &sc_ref);
              return error;
      }
      
      
      /* Exported interfaces for audiobell. */
      
      /*
       * Open for audiobell.
       * It stores allocated file to *filep.
       * If successful returns 0, otherwise errno.
       */
      int
      audiobellopen(dev_t dev, audio_file_t **filep)
      {
              struct audio_softc *sc;
              int error;
      
              /* Find the device */
              sc = device_lookup_private(&audio_cd, AUDIOUNIT(dev));
              if (sc == NULL || sc->hw_if == NULL)
                      return ENXIO;
      
              error = audio_exlock_enter(sc);
              if (error)
                      return error;
      
              device_active(sc->sc_dev, DVA_SYSTEM);
              error = audio_open(dev, sc, FWRITE, 0, curlwp, filep);
      
              audio_exlock_exit(sc);
              return error;
      }
      
      /* Close for audiobell */
      int
      audiobellclose(audio_file_t *file)
      {
              struct audio_softc *sc;
              struct psref sc_ref;
              int error;
      
              sc = audio_file_enter(file, &sc_ref);
              if (sc == NULL)
                      return EIO;
      
              error = audio_close(sc, file);
      
              audio_file_exit(sc, &sc_ref);
      
              KASSERT(file->ptrack);
              audio_track_destroy(file->ptrack);
              KASSERT(file->rtrack == NULL);
              kmem_free(file, sizeof(*file));
              return error;
      }
      
      /* Set sample rate for audiobell */
      int
      audiobellsetrate(audio_file_t *file, u_int sample_rate)
      {
              struct audio_softc *sc;
              struct psref sc_ref;
              struct audio_info ai;
              int error;
      
              sc = audio_file_enter(file, &sc_ref);
              if (sc == NULL)
                      return EIO;
      
              AUDIO_INITINFO(&ai);
              ai.play.sample_rate = sample_rate;
      
              error = audio_exlock_enter(sc);
              if (error)
                      goto done;
              error = audio_file_setinfo(sc, file, &ai);
              audio_exlock_exit(sc);
      
      done:
              audio_file_exit(sc, &sc_ref);
              return error;
      }
      
      /* Playback for audiobell */
      int
      audiobellwrite(audio_file_t *file, struct uio *uio)
      {
              struct audio_softc *sc;
              struct psref sc_ref;
              int error;
      
              sc = audio_file_enter(file, &sc_ref);
              if (sc == NULL)
                      return EIO;
      
              error = audio_write(sc, uio, 0, file);
      
              audio_file_exit(sc, &sc_ref);
              return error;
      }
      
      
      /*
       * Audio driver
       */
      
      /*
       * Must be called with sc_exlock held and without sc_lock held.
       */
      int
      audio_open(dev_t dev, struct audio_softc *sc, int flags, int ifmt,
              struct lwp *l, audio_file_t **bellfile)
      {
              struct audio_info ai;
              struct file *fp;
              audio_file_t *af;
              audio_ring_t *hwbuf;
              bool fullduplex;
              int fd;
              int error;
      
              KASSERT(sc->sc_exlock);
      
              TRACE(1, "%sdev=%s flags=0x%x po=%d ro=%d",
                  (audiodebug >= 3) ? "start " : "",
                  ISDEVSOUND(dev) ? "sound" : "audio",
                  flags, sc->sc_popens, sc->sc_ropens);
      
              af = kmem_zalloc(sizeof(audio_file_t), KM_SLEEP);
              af->sc = sc;
              af->dev = dev;
              if ((flags & FWRITE) != 0 && audio_can_playback(sc))
                      af->mode |= AUMODE_PLAY | AUMODE_PLAY_ALL;
              if ((flags & FREAD) != 0 && audio_can_capture(sc))
                      af->mode |= AUMODE_RECORD;
              if (af->mode == 0) {
                      error = ENXIO;
                      goto bad1;
              }
      
              fullduplex = (sc->sc_props & AUDIO_PROP_FULLDUPLEX);
      
              /*
               * On half duplex hardware,
               * 1. if mode is (PLAY | REC), let mode PLAY.
               * 2. if mode is PLAY, let mode PLAY if no rec tracks, otherwise error.
               * 3. if mode is REC, let mode REC if no play tracks, otherwise error.
               */
              if (fullduplex == false) {
                      if ((af->mode & AUMODE_PLAY)) {
                              if (sc->sc_ropens != 0) {
                                      TRACE(1, "record track already exists");
                                      error = ENODEV;
                                      goto bad1;
                              }
                              /* Play takes precedence */
                              af->mode &= ~AUMODE_RECORD;
                      }
                      if ((af->mode & AUMODE_RECORD)) {
                              if (sc->sc_popens != 0) {
                                      TRACE(1, "play track already exists");
                                      error = ENODEV;
                                      goto bad1;
                              }
                      }
              }
      
              /* Create tracks */
              if ((af->mode & AUMODE_PLAY))
                      af->ptrack = audio_track_create(sc, sc->sc_pmixer);
              if ((af->mode & AUMODE_RECORD))
                      af->rtrack = audio_track_create(sc, sc->sc_rmixer);
      
              /* Set parameters */
              AUDIO_INITINFO(&ai);
              if (bellfile) {
                      /* If audiobell, only sample_rate will be set later. */
                      ai.play.sample_rate   = audio_default.sample_rate;
                      ai.play.encoding      = AUDIO_ENCODING_SLINEAR_NE;
                      ai.play.channels      = 1;
                      ai.play.precision     = 16;
                      ai.play.pause         = 0;
              } else if (ISDEVAUDIO(dev)) {
                      /* If /dev/audio, initialize everytime. */
                      ai.play.sample_rate   = audio_default.sample_rate;
                      ai.play.encoding      = audio_default.encoding;
                      ai.play.channels      = audio_default.channels;
                      ai.play.precision     = audio_default.precision;
                      ai.play.pause         = 0;
                      ai.record.sample_rate = audio_default.sample_rate;
                      ai.record.encoding    = audio_default.encoding;
                      ai.record.channels    = audio_default.channels;
                      ai.record.precision   = audio_default.precision;
                      ai.record.pause       = 0;
              } else {
                      /* If /dev/sound, take over the previous parameters. */
                      ai.play.sample_rate   = sc->sc_sound_pparams.sample_rate;
                      ai.play.encoding      = sc->sc_sound_pparams.encoding;
                      ai.play.channels      = sc->sc_sound_pparams.channels;
                      ai.play.precision     = sc->sc_sound_pparams.precision;
                      ai.play.pause         = sc->sc_sound_ppause;
                      ai.record.sample_rate = sc->sc_sound_rparams.sample_rate;
                      ai.record.encoding    = sc->sc_sound_rparams.encoding;
                      ai.record.channels    = sc->sc_sound_rparams.channels;
                      ai.record.precision   = sc->sc_sound_rparams.precision;
                      ai.record.pause       = sc->sc_sound_rpause;
              }
              error = audio_file_setinfo(sc, af, &ai);
              if (error)
                      goto bad2;
      
              if (sc->sc_popens + sc->sc_ropens == 0) {
                      /* First open */
      
                      sc->sc_cred = kauth_cred_get();
                      kauth_cred_hold(sc->sc_cred);
      
                      if (sc->hw_if->open) {
                              int hwflags;
      
                              /*
                               * Call hw_if->open() only at first open of
                               * combination of playback and recording.
                               * On full duplex hardware, the flags passed to
                               * hw_if->open() is always (FREAD | FWRITE)
                               * regardless of this open()'s flags.
                               * see also dev/isa/aria.c
                               * On half duplex hardware, the flags passed to
                               * hw_if->open() is either FREAD or FWRITE.
                               * see also arch/evbarm/mini2440/audio_mini2440.c
                               */
                              if (fullduplex) {
                                      hwflags = FREAD | FWRITE;
                              } else {
                                      /* Construct hwflags from af->mode. */
                                      hwflags = 0;
                                      if ((af->mode & AUMODE_PLAY) != 0)
                                              hwflags |= FWRITE;
                                      if ((af->mode & AUMODE_RECORD) != 0)
                                              hwflags |= FREAD;
                              }
      
                              mutex_enter(sc->sc_lock);
                              mutex_enter(sc->sc_intr_lock);
                              error = sc->hw_if->open(sc->hw_hdl, hwflags);
                              mutex_exit(sc->sc_intr_lock);
                              mutex_exit(sc->sc_lock);
                              if (error)
                                      goto bad2;
                      }
      
                      /*
                       * Set speaker mode when a half duplex.
                       * XXX I'm not sure this is correct.
                       */
                      if (1/*XXX*/) {
                              if (sc->hw_if->speaker_ctl) {
                                      int on;
                                      if (af->ptrack) {
                                              on = 1;
                                      } else {
                                              on = 0;
                                      }
                                      mutex_enter(sc->sc_lock);
                                      mutex_enter(sc->sc_intr_lock);
                                      error = sc->hw_if->speaker_ctl(sc->hw_hdl, on);
                                      mutex_exit(sc->sc_intr_lock);
                                      mutex_exit(sc->sc_lock);
                                      if (error)
                                              goto bad3;
                              }
                      }
              } else if (sc->sc_multiuser == false) {
                      uid_t euid = kauth_cred_geteuid(kauth_cred_get());
                      if (euid != 0 && euid != kauth_cred_geteuid(sc->sc_cred)) {
                              error = EPERM;
                              goto bad2;
                      }
              }
      
              /* Call init_output if this is the first playback open. */
              if (af->ptrack && sc->sc_popens == 0) {
                      if (sc->hw_if->init_output) {
                              hwbuf = &sc->sc_pmixer->hwbuf;
                              mutex_enter(sc->sc_lock);
                              mutex_enter(sc->sc_intr_lock);
                              error = sc->hw_if->init_output(sc->hw_hdl,
                                  hwbuf->mem,
                                  hwbuf->capacity *
                                  hwbuf->fmt.channels * hwbuf->fmt.stride / NBBY);
                              mutex_exit(sc->sc_intr_lock);
                              mutex_exit(sc->sc_lock);
                              if (error)
                                      goto bad3;
                      }
              }
              /*
               * Call init_input and start rmixer, if this is the first recording
               * open.  See pause consideration notes.
               */
              if (af->rtrack && sc->sc_ropens == 0) {
                      if (sc->hw_if->init_input) {
                              hwbuf = &sc->sc_rmixer->hwbuf;
                              mutex_enter(sc->sc_lock);
                              mutex_enter(sc->sc_intr_lock);
                              error = sc->hw_if->init_input(sc->hw_hdl,
                                  hwbuf->mem,
                                  hwbuf->capacity *
                                  hwbuf->fmt.channels * hwbuf->fmt.stride / NBBY);
                              mutex_exit(sc->sc_intr_lock);
                              mutex_exit(sc->sc_lock);
                              if (error)
                                      goto bad3;
                      }
      
                      mutex_enter(sc->sc_lock);
                      audio_rmixer_start(sc);
                      mutex_exit(sc->sc_lock);
              }
      
              if (bellfile == NULL) {
                      error = fd_allocfile(&fp, &fd);
                      if (error)
                              goto bad3;
              }
      
              /*
               * Count up finally.
               * Don't fail from here.
               */
              mutex_enter(sc->sc_lock);
              if (af->ptrack)
                      sc->sc_popens++;
              if (af->rtrack)
                      sc->sc_ropens++;
              mutex_enter(sc->sc_intr_lock);
              SLIST_INSERT_HEAD(&sc->sc_files, af, entry);
              mutex_exit(sc->sc_intr_lock);
              mutex_exit(sc->sc_lock);
      
              if (bellfile) {
                      *bellfile = af;
              } else {
                      error = fd_clone(fp, fd, flags, &audio_fileops, af);
                      KASSERTMSG(error == EMOVEFD, "error=%d", error);
              }
      
              TRACEF(3, af, "done");
              return error;
      
              /*
               * Since track here is not yet linked to sc_files,
               * you can call track_destroy() without sc_intr_lock.
               */
      bad3:
              if (sc->sc_popens + sc->sc_ropens == 0) {
                      if (sc->hw_if->close) {
                              mutex_enter(sc->sc_lock);
                              mutex_enter(sc->sc_intr_lock);
                              sc->hw_if->close(sc->hw_hdl);
                              mutex_exit(sc->sc_intr_lock);
                              mutex_exit(sc->sc_lock);
                      }
              }
      bad2:
              if (af->rtrack) {
                      audio_track_destroy(af->rtrack);
                      af->rtrack = NULL;
              }
              if (af->ptrack) {
                      audio_track_destroy(af->ptrack);
                      af->ptrack = NULL;
              }
      bad1:
              kmem_free(af, sizeof(*af));
              return error;
      }
      
      /*
       * Must be called without sc_lock nor sc_exlock held.
       */
      int
      audio_close(struct audio_softc *sc, audio_file_t *file)
      {
      
              /* Protect entering new fileops to this file */
              atomic_store_relaxed(&file->dying, true);
      
              /*
               * Drain first.
               * It must be done before unlinking(acquiring exlock).
               */
              if (file->ptrack) {
                      mutex_enter(sc->sc_lock);
                      audio_track_drain(sc, file->ptrack);
                      mutex_exit(sc->sc_lock);
              }
      
              return audio_unlink(sc, file);
      }
      
      /*
       * Unlink this file, but not freeing memory here.
       * Must be called without sc_lock nor sc_exlock held.
       */
      int
      audio_unlink(struct audio_softc *sc, audio_file_t *file)
      {
              int error;
      
              mutex_enter(sc->sc_lock);
      
              TRACEF(1, file, "%spid=%d.%d po=%d ro=%d",
                  (audiodebug >= 3) ? "start " : "",
                  (int)curproc->p_pid, (int)curlwp->l_lid,
                  sc->sc_popens, sc->sc_ropens);
              KASSERTMSG(sc->sc_popens + sc->sc_ropens > 0,
                  "sc->sc_popens=%d, sc->sc_ropens=%d",
                  sc->sc_popens, sc->sc_ropens);
      
              /*
               * Acquire exlock to protect counters.
               * Does not use audio_exlock_enter() due to sc_dying.
               */
              while (__predict_false(sc->sc_exlock != 0)) {
                      error = cv_timedwait_sig(&sc->sc_exlockcv, sc->sc_lock,
                          mstohz(AUDIO_TIMEOUT));
                      /* XXX what should I do on error? */
                      if (error == EWOULDBLOCK) {
                              mutex_exit(sc->sc_lock);
                              device_printf(sc->sc_dev,
                                  "%s: cv_timedwait_sig failed %d", __func__, error);
                              return error;
                      }
              }
              sc->sc_exlock = 1;
      
              device_active(sc->sc_dev, DVA_SYSTEM);
      
              mutex_enter(sc->sc_intr_lock);
              SLIST_REMOVE(&sc->sc_files, file, audio_file, entry);
              mutex_exit(sc->sc_intr_lock);
      
              if (file->ptrack) {
                      TRACET(3, file->ptrack, "dropframes=%" PRIu64,
                          file->ptrack->dropframes);
      
                      KASSERT(sc->sc_popens > 0);
                      sc->sc_popens--;
      
                      /* Call hw halt_output if this is the last playback track. */
                      if (sc->sc_popens == 0 && sc->sc_pbusy) {
                              error = audio_pmixer_halt(sc);
                              if (error) {
                                      device_printf(sc->sc_dev,
                                          "halt_output failed with %d (ignored)\n",
                                          error);
                              }
                      }
      
                      /* Restore mixing volume if all tracks are gone. */
                      if (sc->sc_popens == 0) {
                              /* intr_lock is not necessary, but just manners. */
                              mutex_enter(sc->sc_intr_lock);
                              sc->sc_pmixer->volume = 256;
                              sc->sc_pmixer->voltimer = 0;
                              mutex_exit(sc->sc_intr_lock);
                      }
              }
              if (file->rtrack) {
                      TRACET(3, file->rtrack, "dropframes=%" PRIu64,
                          file->rtrack->dropframes);
      
                      KASSERT(sc->sc_ropens > 0);
                      sc->sc_ropens--;
      
                      /* Call hw halt_input if this is the last recording track. */
                      if (sc->sc_ropens == 0 && sc->sc_rbusy) {
                              error = audio_rmixer_halt(sc);
                              if (error) {
                                      device_printf(sc->sc_dev,
                                          "halt_input failed with %d (ignored)\n",
                                          error);
                              }
                      }
      
              }
      
              /* Call hw close if this is the last track. */
              if (sc->sc_popens + sc->sc_ropens == 0) {
                      if (sc->hw_if->close) {
                              TRACE(2, "hw_if close");
                              mutex_enter(sc->sc_intr_lock);
                              sc->hw_if->close(sc->hw_hdl);
                              mutex_exit(sc->sc_intr_lock);
                      }
              }
      
              mutex_exit(sc->sc_lock);
              if (sc->sc_popens + sc->sc_ropens == 0)
                      kauth_cred_free(sc->sc_cred);
      
              TRACE(3, "done");
              audio_exlock_exit(sc);
      
              return 0;
      }
      
      /*
       * Must be called without sc_lock nor sc_exlock held.
       */
      int
      audio_read(struct audio_softc *sc, struct uio *uio, int ioflag,
              audio_file_t *file)
      {
              audio_track_t *track;
              audio_ring_t *usrbuf;
              audio_ring_t *input;
              int error;
      
              /*
               * On half-duplex hardware, O_RDWR is treated as O_WRONLY.
               * However read() system call itself can be called because it's
               * opened with O_RDWR.  So in this case, deny this read().
               */
              track = file->rtrack;
              if (track == NULL) {
                      return EBADF;
              }
      
              /* I think it's better than EINVAL. */
              if (track->mmapped)
                      return EPERM;
      
              TRACET(2, track, "resid=%zd", uio->uio_resid);
      
      #ifdef AUDIO_PM_IDLE
              error = audio_exlock_mutex_enter(sc);
              if (error)
                      return error;
      
              if (device_is_active(&sc->sc_dev) || sc->sc_idle)
                      device_active(&sc->sc_dev, DVA_SYSTEM);
      
              /* In recording, unlike playback, read() never operates rmixer. */
      
              audio_exlock_mutex_exit(sc);
      #endif
      
              usrbuf = &track->usrbuf;
              input = track->input;
              error = 0;
      
              while (uio->uio_resid > 0 && error == 0) {
                      int bytes;
      
                      TRACET(3, track,
                          "while resid=%zd input=%d/%d/%d usrbuf=%d/%d/H%d",
                          uio->uio_resid,
                          input->head, input->used, input->capacity,
                          usrbuf->head, usrbuf->used, track->usrbuf_usedhigh);
      
                      /* Wait when buffers are empty. */
                      mutex_enter(sc->sc_lock);
                      for (;;) {
                              bool empty;
                              audio_track_lock_enter(track);
                              empty = (input->used == 0 && usrbuf->used == 0);
                              audio_track_lock_exit(track);
                              if (!empty)
                                      break;
      
                              if ((ioflag & IO_NDELAY)) {
                                      mutex_exit(sc->sc_lock);
                                      return EWOULDBLOCK;
                              }
      
                              TRACET(3, track, "sleep");
                              error = audio_track_waitio(sc, track);
                              if (error) {
                                      mutex_exit(sc->sc_lock);
                                      return error;
                              }
                      }
                      mutex_exit(sc->sc_lock);
      
                      audio_track_lock_enter(track);
                      audio_track_record(track);
      
                      /* uiomove from usrbuf as much as possible. */
                      bytes = uimin(usrbuf->used, uio->uio_resid);
                      while (bytes > 0) {
                              int head = usrbuf->head;
                              int len = uimin(bytes, usrbuf->capacity - head);
                              error = uiomove((uint8_t *)usrbuf->mem + head, len,
                                  uio);
                              if (error) {
                                      audio_track_lock_exit(track);
                                      device_printf(sc->sc_dev,
                                          "uiomove(len=%d) failed with %d\n",
                                          len, error);
                                      goto abort;
                              }
                              auring_take(usrbuf, len);
                              track->useriobytes += len;
                              TRACET(3, track, "uiomove(len=%d) usrbuf=%d/%d/C%d",
                                  len,
                                  usrbuf->head, usrbuf->used, usrbuf->capacity);
                              bytes -= len;
                      }
      
                      audio_track_lock_exit(track);
              }
      
      abort:
              return error;
      }
      
      
      /*
       * Clear file's playback and/or record track buffer immediately.
       */
      static void
      audio_file_clear(struct audio_softc *sc, audio_file_t *file)
      {
      
              if (file->ptrack)
                      audio_track_clear(sc, file->ptrack);
              if (file->rtrack)
                      audio_track_clear(sc, file->rtrack);
      }
      
      /*
       * Must be called without sc_lock nor sc_exlock held.
       */
      int
      audio_write(struct audio_softc *sc, struct uio *uio, int ioflag,
              audio_file_t *file)
      {
              audio_track_t *track;
              audio_ring_t *usrbuf;
              audio_ring_t *outbuf;
              int error;
      
              track = file->ptrack;
              KASSERT(track);
      
              /* I think it's better than EINVAL. */
              if (track->mmapped)
                      return EPERM;
      
              TRACET(2, track, "%sresid=%zd pid=%d.%d ioflag=0x%x",
                  audiodebug >= 3 ? "begin " : "",
                  uio->uio_resid, (int)curproc->p_pid, (int)curlwp->l_lid, ioflag);
      
              if (uio->uio_resid == 0) {
                      track->eofcounter++;
                      return 0;
              }
      
              error = audio_exlock_mutex_enter(sc);
              if (error)
                      return error;
      
      #ifdef AUDIO_PM_IDLE
              if (device_is_active(&sc->sc_dev) || sc->sc_idle)
                      device_active(&sc->sc_dev, DVA_SYSTEM);
      #endif
      
              /*
               * The first write starts pmixer.
               */
              if (sc->sc_pbusy == false)
                      audio_pmixer_start(sc, false);
              audio_exlock_mutex_exit(sc);
      
              usrbuf = &track->usrbuf;
              outbuf = &track->outbuf;
              track->pstate = AUDIO_STATE_RUNNING;
              error = 0;
      
              while (uio->uio_resid > 0 && error == 0) {
                      int bytes;
      
                      TRACET(3, track, "while resid=%zd usrbuf=%d/%d/H%d",
                          uio->uio_resid,
                          usrbuf->head, usrbuf->used, track->usrbuf_usedhigh);
      
                      /* Wait when buffers are full. */
                      mutex_enter(sc->sc_lock);
                      for (;;) {
                              bool full;
                              audio_track_lock_enter(track);
                              full = (usrbuf->used >= track->usrbuf_usedhigh &&
                                  outbuf->used >= outbuf->capacity);
                              audio_track_lock_exit(track);
                              if (!full)
                                      break;
      
                              if ((ioflag & IO_NDELAY)) {
                                      error = EWOULDBLOCK;
                                      mutex_exit(sc->sc_lock);
                                      goto abort;
                              }
      
                              TRACET(3, track, "sleep usrbuf=%d/H%d",
                                  usrbuf->used, track->usrbuf_usedhigh);
                              error = audio_track_waitio(sc, track);
                              if (error) {
                                      mutex_exit(sc->sc_lock);
                                      goto abort;
                              }
                      }
                      mutex_exit(sc->sc_lock);
      
                      audio_track_lock_enter(track);
      
                      /* uiomove to usrbuf as much as possible. */
                      bytes = uimin(track->usrbuf_usedhigh - usrbuf->used,
                          uio->uio_resid);
                      while (bytes > 0) {
                              int tail = auring_tail(usrbuf);
                              int len = uimin(bytes, usrbuf->capacity - tail);
                              error = uiomove((uint8_t *)usrbuf->mem + tail, len,
                                  uio);
                              if (error) {
                                      audio_track_lock_exit(track);
                                      device_printf(sc->sc_dev,
                                          "uiomove(len=%d) failed with %d\n",
                                          len, error);
                                      goto abort;
                              }
                              auring_push(usrbuf, len);
                              track->useriobytes += len;
                              TRACET(3, track, "uiomove(len=%d) usrbuf=%d/%d/C%d",
                                  len,
                                  usrbuf->head, usrbuf->used, usrbuf->capacity);
                              bytes -= len;
                      }
      
                      /* Convert them as much as possible. */
                      while (usrbuf->used >= track->usrbuf_blksize &&
                          outbuf->used < outbuf->capacity) {
                              audio_track_play(track);
                      }
      
                      audio_track_lock_exit(track);
              }
      
      abort:
              TRACET(3, track, "done error=%d", error);
              return error;
      }
      
      /*
       * Must be called without sc_lock nor sc_exlock held.
       */
      int
      audio_ioctl(dev_t dev, struct audio_softc *sc, u_long cmd, void *addr, int flag,
              struct lwp *l, audio_file_t *file)
      {
              struct audio_offset *ao;
              struct audio_info ai;
              audio_track_t *track;
              audio_encoding_t *ae;
              audio_format_query_t *query;
              u_int stamp;
              u_int offs;
              int fd;
              int index;
              int error;
      
      #if defined(AUDIO_DEBUG)
              const char *ioctlnames[] = {
                      " AUDIO_GETINFO",        /* 21 */
                      " AUDIO_SETINFO",        /* 22 */
                      " AUDIO_DRAIN",                /* 23 */
                      " AUDIO_FLUSH",                /* 24 */
                      " AUDIO_WSEEK",                /* 25 */
                      " AUDIO_RERROR",        /* 26 */
                      " AUDIO_GETDEV",        /* 27 */
                      " AUDIO_GETENC",        /* 28 */
                      " AUDIO_GETFD",                /* 29 */
                      " AUDIO_SETFD",                /* 30 */
                      " AUDIO_PERROR",        /* 31 */
                      " AUDIO_GETIOFFS",        /* 32 */
                      " AUDIO_GETOOFFS",        /* 33 */
                      " AUDIO_GETPROPS",        /* 34 */
                      " AUDIO_GETBUFINFO",        /* 35 */
                      " AUDIO_SETCHAN",        /* 36 */
                      " AUDIO_GETCHAN",        /* 37 */
                      " AUDIO_QUERYFORMAT",        /* 38 */
                      " AUDIO_GETFORMAT",        /* 39 */
                      " AUDIO_SETFORMAT",        /* 40 */
              };
              int nameidx = (cmd & 0xff);
              const char *ioctlname = "";
              if (21 <= nameidx && nameidx <= 21 + __arraycount(ioctlnames))
                      ioctlname = ioctlnames[nameidx - 21];
              TRACEF(2, file, "(%lu,'%c',%lu)%s pid=%d.%d",
                  IOCPARM_LEN(cmd), (char)IOCGROUP(cmd), cmd&0xff, ioctlname,
                  (int)curproc->p_pid, (int)l->l_lid);
      #endif
      
              error = 0;
              switch (cmd) {
              case FIONBIO:
                      /* All handled in the upper FS layer. */
                      break;
      
              case FIONREAD:
                      /* Get the number of bytes that can be read. */
                      if (file->rtrack) {
                              *(int *)addr = audio_track_readablebytes(file->rtrack);
                      } else {
                              *(int *)addr = 0;
                      }
                      break;
      
              case FIOASYNC:
                      /* Set/Clear ASYNC I/O. */
                      if (*(int *)addr) {
                              file->async_audio = curproc->p_pid;
                              TRACEF(2, file, "FIOASYNC pid %d", file->async_audio);
                      } else {
                              file->async_audio = 0;
                              TRACEF(2, file, "FIOASYNC off");
                      }
                      break;
      
              case AUDIO_FLUSH:
                      /* XXX TODO: clear errors and restart? */
                      audio_file_clear(sc, file);
                      break;
      
              case AUDIO_RERROR:
                      /*
                       * Number of read bytes dropped.  We don't know where
                       * or when they were dropped (including conversion stage).
                       * Therefore, the number of accurate bytes or samples is
                       * also unknown.
                       */
                      track = file->rtrack;
                      if (track) {
                              *(int *)addr = frametobyte(&track->usrbuf.fmt,
                                  track->dropframes);
                      }
                      break;
      
              case AUDIO_PERROR:
                      /*
                       * Number of write bytes dropped.  We don't know where
                       * or when they were dropped (including conversion stage).
                       * Therefore, the number of accurate bytes or samples is
                       * also unknown.
                       */
                      track = file->ptrack;
                      if (track) {
                              *(int *)addr = frametobyte(&track->usrbuf.fmt,
                                  track->dropframes);
                      }
                      break;
      
              case AUDIO_GETIOFFS:
                      /* XXX TODO */
                      ao = (struct audio_offset *)addr;
                      ao->samples = 0;
                      ao->deltablks = 0;
                      ao->offset = 0;
                      break;
      
              case AUDIO_GETOOFFS:
                      ao = (struct audio_offset *)addr;
                      track = file->ptrack;
                      if (track == NULL) {
                              ao->samples = 0;
                              ao->deltablks = 0;
                              ao->offset = 0;
                              break;
                      }
                      mutex_enter(sc->sc_lock);
                      mutex_enter(sc->sc_intr_lock);
                      /* figure out where next DMA will start */
                      stamp = track->usrbuf_stamp;
                      offs = track->usrbuf.head;
                      mutex_exit(sc->sc_intr_lock);
                      mutex_exit(sc->sc_lock);
      
                      ao->samples = stamp;
                      ao->deltablks = (stamp / track->usrbuf_blksize) -
                          (track->usrbuf_stamp_last / track->usrbuf_blksize);
                      track->usrbuf_stamp_last = stamp;
                      offs = rounddown(offs, track->usrbuf_blksize)
                          + track->usrbuf_blksize;
                      if (offs >= track->usrbuf.capacity)
                              offs -= track->usrbuf.capacity;
                      ao->offset = offs;
      
                      TRACET(3, track, "GETOOFFS: samples=%u deltablks=%u offset=%u",
                          ao->samples, ao->deltablks, ao->offset);
                      break;
      
              case AUDIO_WSEEK:
                      /* XXX return value does not include outbuf one. */
                      if (file->ptrack)
                              *(u_long *)addr = file->ptrack->usrbuf.used;
                      break;
      
              case AUDIO_SETINFO:
                      error = audio_exlock_enter(sc);
                      if (error)
                              break;
                      error = audio_file_setinfo(sc, file, (struct audio_info *)addr);
                      if (error) {
                              audio_exlock_exit(sc);
                              break;
                      }
                      /* XXX TODO: update last_ai if /dev/sound ? */
                      if (ISDEVSOUND(dev))
                              error = audiogetinfo(sc, &sc->sc_ai, 0, file);
                      audio_exlock_exit(sc);
                      break;
      
              case AUDIO_GETINFO:
                      error = audio_exlock_enter(sc);
                      if (error)
                              break;
                      error = audiogetinfo(sc, (struct audio_info *)addr, 1, file);
                      audio_exlock_exit(sc);
                      break;
      
              case AUDIO_GETBUFINFO:
                      error = audio_exlock_enter(sc);
                      if (error)
                              break;
                      error = audiogetinfo(sc, (struct audio_info *)addr, 0, file);
                      audio_exlock_exit(sc);
                      break;
      
              case AUDIO_DRAIN:
                      if (file->ptrack) {
                              mutex_enter(sc->sc_lock);
                              error = audio_track_drain(sc, file->ptrack);
                              mutex_exit(sc->sc_lock);
                      }
                      break;
      
              case AUDIO_GETDEV:
                      mutex_enter(sc->sc_lock);
                      error = sc->hw_if->getdev(sc->hw_hdl, (audio_device_t *)addr);
                      mutex_exit(sc->sc_lock);
                      break;
      
              case AUDIO_GETENC:
                      ae = (audio_encoding_t *)addr;
                      index = ae->index;
                      if (index < 0 || index >= __arraycount(audio_encodings)) {
                              error = EINVAL;
                              break;
                      }
                      *ae = audio_encodings[index];
                      ae->index = index;
                      /*
                       * EMULATED always.
                       * EMULATED flag at that time used to mean that it could
                       * not be passed directly to the hardware as-is.  But
                       * currently, all formats including hardware native is not
                       * passed directly to the hardware.  So I set EMULATED
                       * flag for all formats.
                       */
                      ae->flags = AUDIO_ENCODINGFLAG_EMULATED;
                      break;
      
              case AUDIO_GETFD:
                      /*
                       * Returns the current setting of full duplex mode.
                       * If HW has full duplex mode and there are two mixers,
                       * it is full duplex.  Otherwise half duplex.
                       */
                      error = audio_exlock_enter(sc);
                      if (error)
                              break;
                      fd = (sc->sc_props & AUDIO_PROP_FULLDUPLEX)
                          && (sc->sc_pmixer && sc->sc_rmixer);
                      audio_exlock_exit(sc);
                      *(int *)addr = fd;
                      break;
      
              case AUDIO_GETPROPS:
                      *(int *)addr = sc->sc_props;
                      break;
      
              case AUDIO_QUERYFORMAT:
                      query = (audio_format_query_t *)addr;
                      mutex_enter(sc->sc_lock);
                      error = sc->hw_if->query_format(sc->hw_hdl, query);
                      mutex_exit(sc->sc_lock);
                      /* Hide internal infomations */
                      query->fmt.driver_data = NULL;
                      break;
      
              case AUDIO_GETFORMAT:
                      error = audio_exlock_enter(sc);
                      if (error)
                              break;
                      audio_mixers_get_format(sc, (struct audio_info *)addr);
                      audio_exlock_exit(sc);
                      break;
      
              case AUDIO_SETFORMAT:
                      error = audio_exlock_enter(sc);
                      audio_mixers_get_format(sc, &ai);
                      error = audio_mixers_set_format(sc, (struct audio_info *)addr);
                      if (error) {
                              /* Rollback */
                              audio_mixers_set_format(sc, &ai);
                      }
                      audio_exlock_exit(sc);
                      break;
      
              case AUDIO_SETFD:
              case AUDIO_SETCHAN:
              case AUDIO_GETCHAN:
                      /* Obsoleted */
                      break;
      
              default:
                      if (sc->hw_if->dev_ioctl) {
                              mutex_enter(sc->sc_lock);
                              error = sc->hw_if->dev_ioctl(sc->hw_hdl,
                                  cmd, addr, flag, l);
                              mutex_exit(sc->sc_lock);
                      } else {
                              TRACEF(2, file, "unknown ioctl");
                              error = EINVAL;
                      }
                      break;
              }
              TRACEF(2, file, "(%lu,'%c',%lu)%s result %d",
                  IOCPARM_LEN(cmd), (char)IOCGROUP(cmd), cmd&0xff, ioctlname,
                  error);
              return error;
      }
      
      /*
       * Returns the number of bytes that can be read on recording buffer.
       */
      static __inline int
      audio_track_readablebytes(const audio_track_t *track)
      {
              int bytes;
      
              KASSERT(track);
              KASSERT(track->mode == AUMODE_RECORD);
      
              /*
               * Although usrbuf is primarily readable data, recorded data
               * also stays in track->input until reading.  So it is necessary
               * to add it.  track->input is in frame, usrbuf is in byte.
               */
              bytes = track->usrbuf.used +
                  track->input->used * frametobyte(&track->usrbuf.fmt, 1);
              return bytes;
      }
      
      /*
       * Must be called without sc_lock nor sc_exlock held.
       */
      int
      audio_poll(struct audio_softc *sc, int events, struct lwp *l,
              audio_file_t *file)
      {
              audio_track_t *track;
              int revents;
              bool in_is_valid;
              bool out_is_valid;
      
      #if defined(AUDIO_DEBUG)
      #define POLLEV_BITMAP "\177\020" \
                  "b\10WRBAND\0" \
                  "b\7RDBAND\0" "b\6RDNORM\0" "b\5NVAL\0" "b\4HUP\0" \
                  "b\3ERR\0" "b\2OUT\0" "b\1PRI\0" "b\0IN\0"
              char evbuf[64];
              snprintb(evbuf, sizeof(evbuf), POLLEV_BITMAP, events);
              TRACEF(2, file, "pid=%d.%d events=%s",
                  (int)curproc->p_pid, (int)l->l_lid, evbuf);
      #endif
      
              revents = 0;
              in_is_valid = false;
              out_is_valid = false;
              if (events & (POLLIN | POLLRDNORM)) {
                      track = file->rtrack;
                      if (track) {
                              int used;
                              in_is_valid = true;
                              used = audio_track_readablebytes(track);
                              if (used > 0)
                                      revents |= events & (POLLIN | POLLRDNORM);
                      }
              }
              if (events & (POLLOUT | POLLWRNORM)) {
                      track = file->ptrack;
                      if (track) {
                              out_is_valid = true;
                              if (track->usrbuf.used <= track->usrbuf_usedlow)
                                      revents |= events & (POLLOUT | POLLWRNORM);
                      }
              }
      
              if (revents == 0) {
                      mutex_enter(sc->sc_lock);
                      if (in_is_valid) {
                              TRACEF(3, file, "selrecord rsel");
                              selrecord(l, &sc->sc_rsel);
                      }
                      if (out_is_valid) {
                              TRACEF(3, file, "selrecord wsel");
                              selrecord(l, &sc->sc_wsel);
                      }
                      mutex_exit(sc->sc_lock);
              }
      
      #if defined(AUDIO_DEBUG)
              snprintb(evbuf, sizeof(evbuf), POLLEV_BITMAP, revents);
              TRACEF(2, file, "revents=%s", evbuf);
      #endif
              return revents;
      }
      
      static const struct filterops audioread_filtops = {
              .f_isfd = 1,
              .f_attach = NULL,
              .f_detach = filt_audioread_detach,
              .f_event = filt_audioread_event,
      };
      
      static void
      filt_audioread_detach(struct knote *kn)
      {
              struct audio_softc *sc;
              audio_file_t *file;
      
              file = kn->kn_hook;
              sc = file->sc;
              TRACEF(3, file, "");
      
              mutex_enter(sc->sc_lock);
              SLIST_REMOVE(&sc->sc_rsel.sel_klist, kn, knote, kn_selnext);
              mutex_exit(sc->sc_lock);
      }
      
      static int
      filt_audioread_event(struct knote *kn, long hint)
      {
              audio_file_t *file;
              audio_track_t *track;
      
              file = kn->kn_hook;
              track = file->rtrack;
      
              /*
               * kn_data must contain the number of bytes can be read.
               * The return value indicates whether the event occurs or not.
               */
      
              if (track == NULL) {
                      /* can not read with this descriptor. */
                      kn->kn_data = 0;
                      return 0;
              }
      
              kn->kn_data = audio_track_readablebytes(track);
              TRACEF(3, file, "data=%" PRId64, kn->kn_data);
              return kn->kn_data > 0;
      }
      
      static const struct filterops audiowrite_filtops = {
              .f_isfd = 1,
              .f_attach = NULL,
              .f_detach = filt_audiowrite_detach,
              .f_event = filt_audiowrite_event,
      };
      
      static void
      filt_audiowrite_detach(struct knote *kn)
      {
              struct audio_softc *sc;
              audio_file_t *file;
      
              file = kn->kn_hook;
              sc = file->sc;
              TRACEF(3, file, "");
      
              mutex_enter(sc->sc_lock);
              SLIST_REMOVE(&sc->sc_wsel.sel_klist, kn, knote, kn_selnext);
              mutex_exit(sc->sc_lock);
      }
      
      static int
      filt_audiowrite_event(struct knote *kn, long hint)
      {
              audio_file_t *file;
              audio_track_t *track;
      
              file = kn->kn_hook;
              track = file->ptrack;
      
              /*
               * kn_data must contain the number of bytes can be write.
               * The return value indicates whether the event occurs or not.
               */
      
              if (track == NULL) {
                      /* can not write with this descriptor. */
                      kn->kn_data = 0;
                      return 0;
              }
      
              kn->kn_data = track->usrbuf_usedhigh - track->usrbuf.used;
              TRACEF(3, file, "data=%" PRId64, kn->kn_data);
              return (track->usrbuf.used < track->usrbuf_usedlow);
      }
      
      /*
       * Must be called without sc_lock nor sc_exlock held.
       */
      int
      audio_kqfilter(struct audio_softc *sc, audio_file_t *file, struct knote *kn)
      {
              struct klist *klist;
      
              TRACEF(3, file, "kn=%p kn_filter=%x", kn, (int)kn->kn_filter);
      
              mutex_enter(sc->sc_lock);
              switch (kn->kn_filter) {
              case EVFILT_READ:
                      klist = &sc->sc_rsel.sel_klist;
                      kn->kn_fop = &audioread_filtops;
                      break;
      
              case EVFILT_WRITE:
                      klist = &sc->sc_wsel.sel_klist;
                      kn->kn_fop = &audiowrite_filtops;
                      break;
      
              default:
                      mutex_exit(sc->sc_lock);
                      return EINVAL;
              }
      
              kn->kn_hook = file;
      
              SLIST_INSERT_HEAD(klist, kn, kn_selnext);
              mutex_exit(sc->sc_lock);
      
              return 0;
      }
      
      /*
       * Must be called without sc_lock nor sc_exlock held.
       */
      int
      audio_mmap(struct audio_softc *sc, off_t *offp, size_t len, int prot,
              int *flagsp, int *advicep, struct uvm_object **uobjp, int *maxprotp,
              audio_file_t *file)
      {
              audio_track_t *track;
              vsize_t vsize;
              int error;
      
              TRACEF(2, file, "off=%lld, prot=%d", (long long)(*offp), prot);
      
              if (*offp < 0)
                      return EINVAL;
      
      #if 0
              /* XXX
               * The idea here was to use the protection to determine if
               * we are mapping the read or write buffer, but it fails.
               * The VM system is broken in (at least) two ways.
               * 1) If you map memory VM_PROT_WRITE you SIGSEGV
               *    when writing to it, so VM_PROT_READ|VM_PROT_WRITE
               *    has to be used for mmapping the play buffer.
               * 2) Even if calling mmap() with VM_PROT_READ|VM_PROT_WRITE
               *    audio_mmap will get called at some point with VM_PROT_READ
               *    only.
               * So, alas, we always map the play buffer for now.
               */
              if (prot == (VM_PROT_READ|VM_PROT_WRITE) ||
                  prot == VM_PROT_WRITE)
                      track = file->ptrack;
              else if (prot == VM_PROT_READ)
                      track = file->rtrack;
              else
                      return EINVAL;
      #else
              track = file->ptrack;
      #endif
              if (track == NULL)
                      return EACCES;
      
              vsize = roundup2(MAX(track->usrbuf.capacity, PAGE_SIZE), PAGE_SIZE);
              if (len > vsize)
                      return EOVERFLOW;
              if (*offp > (uint)(vsize - len))
                      return EOVERFLOW;
      
              /* XXX TODO: what happens when mmap twice. */
              if (!track->mmapped) {
                      track->mmapped = true;
      
                      if (!track->is_pause) {
                              error = audio_exlock_mutex_enter(sc);
                              if (error)
                                      return error;
                              if (sc->sc_pbusy == false)
                                      audio_pmixer_start(sc, true);
                              audio_exlock_mutex_exit(sc);
                      }
                      /* XXX mmapping record buffer is not supported */
              }
      
              /* get ringbuffer */
              *uobjp = track->uobj;
      
              /* Acquire a reference for the mmap.  munmap will release. */
              uao_reference(*uobjp);
              *maxprotp = prot;
              *advicep = UVM_ADV_RANDOM;
              *flagsp = MAP_SHARED;
              return 0;
      }
      
      /*
       * /dev/audioctl has to be able to open at any time without interference
       * with any /dev/audio or /dev/sound.
       * Must be called with sc_exlock held and without sc_lock held.
       */
      static int
      audioctl_open(dev_t dev, struct audio_softc *sc, int flags, int ifmt,
              struct lwp *l)
      {
              struct file *fp;
              audio_file_t *af;
              int fd;
              int error;
      
              KASSERT(sc->sc_exlock);
      
              TRACE(1, "");
      
              error = fd_allocfile(&fp, &fd);
              if (error)
                      return error;
      
              af = kmem_zalloc(sizeof(audio_file_t), KM_SLEEP);
              af->sc = sc;
              af->dev = dev;
      
              /* Not necessary to insert sc_files. */
      
              error = fd_clone(fp, fd, flags, &audio_fileops, af);
              KASSERTMSG(error == EMOVEFD, "error=%d", error);
      
              return error;
      }
      
      /*
       * Free 'mem' if available, and initialize the pointer.
       * For this reason, this is implemented as macro.
       */
      #define audio_free(mem)        do {        \
              if (mem != NULL) {        \
                      kern_free(mem);        \
                      mem = NULL;        \
              }        \
      } while (0)
      
      /*
       * (Re)allocate 'memblock' with specified 'bytes'.
       * bytes must not be 0.
       * This function never returns NULL.
       */
      static void *
      audio_realloc(void *memblock, size_t bytes)
      {
      
              KASSERT(bytes != 0);
              audio_free(memblock);
              return kern_malloc(bytes, M_WAITOK);
      }
      
      /*
       * (Re)allocate usrbuf with 'newbufsize' bytes.
       * Use this function for usrbuf because only usrbuf can be mmapped.
       * If successful, it updates track->usrbuf.mem, track->usrbuf.capacity and
       * returns 0.  Otherwise, it clears track->usrbuf.mem, track->usrbuf.capacity
       * and returns errno.
       * It must be called before updating usrbuf.capacity.
       */
      static int
      audio_realloc_usrbuf(audio_track_t *track, int newbufsize)
      {
              struct audio_softc *sc;
              vaddr_t vstart;
              vsize_t oldvsize;
              vsize_t newvsize;
              int error;
      
              KASSERT(newbufsize > 0);
              sc = track->mixer->sc;
      
              /* Get a nonzero multiple of PAGE_SIZE */
              newvsize = roundup2(MAX(newbufsize, PAGE_SIZE), PAGE_SIZE);
      
              if (track->usrbuf.mem != NULL) {
                      oldvsize = roundup2(MAX(track->usrbuf.capacity, PAGE_SIZE),
                          PAGE_SIZE);
                      if (oldvsize == newvsize) {
                              track->usrbuf.capacity = newbufsize;
                              return 0;
                      }
                      vstart = (vaddr_t)track->usrbuf.mem;
                      uvm_unmap(kernel_map, vstart, vstart + oldvsize);
                      /* uvm_unmap also detach uobj */
                      track->uobj = NULL;                /* paranoia */
                      track->usrbuf.mem = NULL;
              }
      
              /* Create a uvm anonymous object */
              track->uobj = uao_create(newvsize, 0);
      
              /* Map it into the kernel virtual address space */
              vstart = 0;
              error = uvm_map(kernel_map, &vstart, newvsize, track->uobj, 0, 0,
                  UVM_MAPFLAG(UVM_PROT_RW, UVM_PROT_RW, UVM_INH_NONE,
                  UVM_ADV_RANDOM, 0));
              if (error) {
                      device_printf(sc->sc_dev, "uvm_map failed with %d\n", error);
                      uao_detach(track->uobj);        /* release reference */
                      goto abort;
              }
      
              error = uvm_map_pageable(kernel_map, vstart, vstart + newvsize,
                  false, 0);
              if (error) {
                      device_printf(sc->sc_dev, "uvm_map_pageable failed with %d\n",
                          error);
                      uvm_unmap(kernel_map, vstart, vstart + newvsize);
                      /* uvm_unmap also detach uobj */
                      goto abort;
              }
      
              track->usrbuf.mem = (void *)vstart;
              track->usrbuf.capacity = newbufsize;
              memset(track->usrbuf.mem, 0, newvsize);
              return 0;
      
              /* failure */
      abort:
              track->uobj = NULL;                /* paranoia */
              track->usrbuf.mem = NULL;
              track->usrbuf.capacity = 0;
              return error;
      }
      
      /*
       * Free usrbuf (if available).
       */
      static void
      audio_free_usrbuf(audio_track_t *track)
      {
              vaddr_t vstart;
              vsize_t vsize;
      
              vstart = (vaddr_t)track->usrbuf.mem;
              vsize = roundup2(MAX(track->usrbuf.capacity, PAGE_SIZE), PAGE_SIZE);
              if (track->usrbuf.mem != NULL) {
                      /*
                       * Unmap the kernel mapping.  uvm_unmap releases the
                       * reference to the uvm object, and this should be the
                       * last virtual mapping of the uvm object, so no need
                       * to explicitly release (`detach') the object.
                       */
                      uvm_unmap(kernel_map, vstart, vstart + vsize);
      
                      track->uobj = NULL;
                      track->usrbuf.mem = NULL;
                      track->usrbuf.capacity = 0;
              }
      }
      
      /*
       * This filter changes the volume for each channel.
       * arg->context points track->ch_volume[].
       */
      static void
      audio_track_chvol(audio_filter_arg_t *arg)
      {
              int16_t *ch_volume;
              const aint_t *s;
              aint_t *d;
              u_int i;
              u_int ch;
              u_int channels;
      
              DIAGNOSTIC_filter_arg(arg);
              KASSERTMSG(arg->srcfmt->channels == arg->dstfmt->channels,
                  "arg->srcfmt->channels=%d, arg->dstfmt->channels=%d",
                  arg->srcfmt->channels, arg->dstfmt->channels);
              KASSERT(arg->context != NULL);
              KASSERTMSG(arg->srcfmt->channels <= AUDIO_MAX_CHANNELS,
                  "arg->srcfmt->channels=%d", arg->srcfmt->channels);
      
              s = arg->src;
              d = arg->dst;
              ch_volume = arg->context;
      
              channels = arg->srcfmt->channels;
              for (i = 0; i < arg->count; i++) {
                      for (ch = 0; ch < channels; ch++) {
                              aint2_t val;
                              val = *s++;
                              val = AUDIO_SCALEDOWN(val * ch_volume[ch], 8);
                              *d++ = (aint_t)val;
                      }
              }
      }
      
      /*
       * This filter performs conversion from stereo (or more channels) to mono.
       */
      static void
      audio_track_chmix_mixLR(audio_filter_arg_t *arg)
      {
              const aint_t *s;
              aint_t *d;
              u_int i;
      
              DIAGNOSTIC_filter_arg(arg);
      
              s = arg->src;
              d = arg->dst;
      
              for (i = 0; i < arg->count; i++) {
                      *d++ = AUDIO_SCALEDOWN(s[0], 1) + AUDIO_SCALEDOWN(s[1], 1);
                      s += arg->srcfmt->channels;
              }
      }
      
      /*
       * This filter performs conversion from mono to stereo (or more channels).
       */
      static void
      audio_track_chmix_dupLR(audio_filter_arg_t *arg)
      {
              const aint_t *s;
              aint_t *d;
              u_int i;
              u_int ch;
              u_int dstchannels;
      
              DIAGNOSTIC_filter_arg(arg);
      
              s = arg->src;
              d = arg->dst;
              dstchannels = arg->dstfmt->channels;
      
              for (i = 0; i < arg->count; i++) {
                      d[0] = s[0];
                      d[1] = s[0];
                      s++;
                      d += dstchannels;
              }
              if (dstchannels > 2) {
                      d = arg->dst;
                      for (i = 0; i < arg->count; i++) {
                              for (ch = 2; ch < dstchannels; ch++) {
                                      d[ch] = 0;
                              }
                              d += dstchannels;
                      }
              }
      }
      
      /*
       * This filter shrinks M channels into N channels.
       * Extra channels are discarded.
       */
      static void
      audio_track_chmix_shrink(audio_filter_arg_t *arg)
      {
              const aint_t *s;
              aint_t *d;
              u_int i;
              u_int ch;
      
              DIAGNOSTIC_filter_arg(arg);
      
              s = arg->src;
              d = arg->dst;
      
              for (i = 0; i < arg->count; i++) {
                      for (ch = 0; ch < arg->dstfmt->channels; ch++) {
                              *d++ = s[ch];
                      }
                      s += arg->srcfmt->channels;
              }
      }
      
      /*
       * This filter expands M channels into N channels.
       * Silence is inserted for missing channels.
       */
      static void
      audio_track_chmix_expand(audio_filter_arg_t *arg)
      {
              const aint_t *s;
              aint_t *d;
              u_int i;
              u_int ch;
              u_int srcchannels;
              u_int dstchannels;
      
              DIAGNOSTIC_filter_arg(arg);
      
              s = arg->src;
              d = arg->dst;
      
              srcchannels = arg->srcfmt->channels;
              dstchannels = arg->dstfmt->channels;
              for (i = 0; i < arg->count; i++) {
                      for (ch = 0; ch < srcchannels; ch++) {
                              *d++ = *s++;
                      }
                      for (; ch < dstchannels; ch++) {
                              *d++ = 0;
                      }
              }
      }
      
      /*
       * This filter performs frequency conversion (up sampling).
       * It uses linear interpolation.
       */
      static void
      audio_track_freq_up(audio_filter_arg_t *arg)
      {
              audio_track_t *track;
              audio_ring_t *src;
              audio_ring_t *dst;
              const aint_t *s;
              aint_t *d;
              aint_t prev[AUDIO_MAX_CHANNELS];
              aint_t curr[AUDIO_MAX_CHANNELS];
              aint_t grad[AUDIO_MAX_CHANNELS];
              u_int i;
              u_int t;
              u_int step;
              u_int channels;
              u_int ch;
              int srcused;
      
              track = arg->context;
              KASSERT(track);
              src = &track->freq.srcbuf;
              dst = track->freq.dst;
              DIAGNOSTIC_ring(dst);
              DIAGNOSTIC_ring(src);
              KASSERT(src->used > 0);
              KASSERTMSG(src->fmt.channels == dst->fmt.channels,
                  "src->fmt.channels=%d dst->fmt.channels=%d",
                  src->fmt.channels, dst->fmt.channels);
              KASSERTMSG(src->head % track->mixer->frames_per_block == 0,
                  "src->head=%d track->mixer->frames_per_block=%d",
                  src->head, track->mixer->frames_per_block);
      
              s = arg->src;
              d = arg->dst;
      
              /*
               * In order to faciliate interpolation for each block, slide (delay)
               * input by one sample.  As a result, strictly speaking, the output
               * phase is delayed by 1/dstfreq.  However, I believe there is no
               * observable impact.
               *
               * Example)
               * srcfreq:dstfreq = 1:3
               *
               *  A - -
               *  |
               *  |
               *  |     B - -
               *  +-----+-----> input timeframe
               *  0     1
               *
               *  0     1
               *  +-----+-----> input timeframe
               *  |     A
               *  |   x   x
               *  | x       x
               *  x          (B)
               *  +-+-+-+-+-+-> output timeframe
               *  0 1 2 3 4 5
               */
      
              /* Last samples in previous block */
              channels = src->fmt.channels;
              for (ch = 0; ch < channels; ch++) {
                      prev[ch] = track->freq_prev[ch];
                      curr[ch] = track->freq_curr[ch];
                      grad[ch] = curr[ch] - prev[ch];
              }
      
              step = track->freq_step;
              t = track->freq_current;
      //#define FREQ_DEBUG
      #if defined(FREQ_DEBUG)
      #define PRINTF(fmt...)        printf(fmt)
      #else
      #define PRINTF(fmt...)        do { } while (0)
      #endif
              srcused = src->used;
              PRINTF("upstart step=%d leap=%d", step, track->freq_leap);
              PRINTF(" srcused=%d arg->count=%u", src->used, arg->count);
              PRINTF(" prev=%d curr=%d grad=%d", prev[0], curr[0], grad[0]);
              PRINTF(" t=%d\n", t);
      
              for (i = 0; i < arg->count; i++) {
                      PRINTF("i=%d t=%5d", i, t);
                      if (t >= 65536) {
                              for (ch = 0; ch < channels; ch++) {
                                      prev[ch] = curr[ch];
                                      curr[ch] = *s++;
                                      grad[ch] = curr[ch] - prev[ch];
                              }
                              PRINTF(" prev=%d s[%d]=%d",
                                  prev[0], src->used - srcused, curr[0]);
      
                              /* Update */
                              t -= 65536;
                              srcused--;
                              if (srcused < 0) {
                                      PRINTF(" break\n");
                                      break;
                              }
                      }
      
                      for (ch = 0; ch < channels; ch++) {
                              *d++ = prev[ch] + (aint2_t)grad[ch] * t / 65536;
      #if defined(FREQ_DEBUG)
                              if (ch == 0)
                                      printf(" t=%5d *d=%d", t, d[-1]);
      #endif
                      }
                      t += step;
      
                      PRINTF("\n");
              }
              PRINTF("end prev=%d curr=%d\n", prev[0], curr[0]);
      
              auring_take(src, src->used);
              auring_push(dst, i);
      
              /* Adjust */
              t += track->freq_leap;
      
              track->freq_current = t;
              for (ch = 0; ch < channels; ch++) {
                      track->freq_prev[ch] = prev[ch];
                      track->freq_curr[ch] = curr[ch];
              }
      }
      
      /*
       * This filter performs frequency conversion (down sampling).
       * It uses simple thinning.
       */
      static void
      audio_track_freq_down(audio_filter_arg_t *arg)
      {
              audio_track_t *track;
              audio_ring_t *src;
              audio_ring_t *dst;
              const aint_t *s0;
              aint_t *d;
              u_int i;
              u_int t;
              u_int step;
              u_int ch;
              u_int channels;
      
              track = arg->context;
              KASSERT(track);
              src = &track->freq.srcbuf;
              dst = track->freq.dst;
      
              DIAGNOSTIC_ring(dst);
              DIAGNOSTIC_ring(src);
              KASSERT(src->used > 0);
              KASSERTMSG(src->fmt.channels == dst->fmt.channels,
                  "src->fmt.channels=%d dst->fmt.channels=%d",
                  src->fmt.channels, dst->fmt.channels);
              KASSERTMSG(src->head % track->mixer->frames_per_block == 0,
                  "src->head=%d track->mixer->frames_per_block=%d",
                  src->head, track->mixer->frames_per_block);
      
              s0 = arg->src;
              d = arg->dst;
              t = track->freq_current;
              step = track->freq_step;
              channels = dst->fmt.channels;
              PRINTF("downstart step=%d leap=%d", step, track->freq_leap);
              PRINTF(" srcused=%d arg->count=%u", src->used, arg->count);
              PRINTF(" t=%d\n", t);
      
              for (i = 0; i < arg->count && t / 65536 < src->used; i++) {
                      const aint_t *s;
                      PRINTF("i=%4d t=%10d", i, t);
                      s = s0 + (t / 65536) * channels;
                      PRINTF(" s=%5ld", (s - s0) / channels);
                      for (ch = 0; ch < channels; ch++) {
                              if (ch == 0) PRINTF(" *s=%d", s[ch]);
                              *d++ = s[ch];
                      }
                      PRINTF("\n");
                      t += step;
              }
              t += track->freq_leap;
              PRINTF("end t=%d\n", t);
              auring_take(src, src->used);
              auring_push(dst, i);
              track->freq_current = t % 65536;
      }
      
      /*
       * Creates track and returns it.
       * Must be called without sc_lock held.
       */
      audio_track_t *
      audio_track_create(struct audio_softc *sc, audio_trackmixer_t *mixer)
      {
              audio_track_t *track;
              static int newid = 0;
      
              track = kmem_zalloc(sizeof(*track), KM_SLEEP);
      
              track->id = newid++;
              track->mixer = mixer;
              track->mode = mixer->mode;
      
              /* Do TRACE after id is assigned. */
              TRACET(3, track, "for %s",
                  mixer->mode == AUMODE_PLAY ? "playback" : "recording");
      
      #if defined(AUDIO_SUPPORT_TRACK_VOLUME)
              track->volume = 256;
      #endif
              for (int i = 0; i < AUDIO_MAX_CHANNELS; i++) {
                      track->ch_volume[i] = 256;
              }
      
              return track;
      }
      
      /*
       * Release all resources of the track and track itself.
       * track must not be NULL.  Don't specify the track within the file
       * structure linked from sc->sc_files.
       */
      static void
      audio_track_destroy(audio_track_t *track)
      {
      
              KASSERT(track);
      
              audio_free_usrbuf(track);
              audio_free(track->codec.srcbuf.mem);
              audio_free(track->chvol.srcbuf.mem);
              audio_free(track->chmix.srcbuf.mem);
              audio_free(track->freq.srcbuf.mem);
              audio_free(track->outbuf.mem);
      
              kmem_free(track, sizeof(*track));
      }
      
      /*
       * It returns encoding conversion filter according to src and dst format.
       * If it is not a convertible pair, it returns NULL.  Either src or dst
       * must be internal format.
       */
      static audio_filter_t
      audio_track_get_codec(audio_track_t *track, const audio_format2_t *src,
              const audio_format2_t *dst)
      {
      
              if (audio_format2_is_internal(src)) {
                      if (dst->encoding == AUDIO_ENCODING_ULAW) {
                              return audio_internal_to_mulaw;
                      } else if (dst->encoding == AUDIO_ENCODING_ALAW) {
                              return audio_internal_to_alaw;
                      } else if (audio_format2_is_linear(dst)) {
                              switch (dst->stride) {
                              case 8:
                                      return audio_internal_to_linear8;
                              case 16:
                                      return audio_internal_to_linear16;
      #if defined(AUDIO_SUPPORT_LINEAR24)
                              case 24:
                                      return audio_internal_to_linear24;
      #endif
                              case 32:
                                      return audio_internal_to_linear32;
                              default:
                                      TRACET(1, track, "unsupported %s stride %d",
                                          "dst", dst->stride);
                                      goto abort;
                              }
                      }
              } else if (audio_format2_is_internal(dst)) {
                      if (src->encoding == AUDIO_ENCODING_ULAW) {
                              return audio_mulaw_to_internal;
                      } else if (src->encoding == AUDIO_ENCODING_ALAW) {
                              return audio_alaw_to_internal;
                      } else if (audio_format2_is_linear(src)) {
                              switch (src->stride) {
                              case 8:
                                      return audio_linear8_to_internal;
                              case 16:
                                      return audio_linear16_to_internal;
      #if defined(AUDIO_SUPPORT_LINEAR24)
                              case 24:
                                      return audio_linear24_to_internal;
      #endif
                              case 32:
                                      return audio_linear32_to_internal;
                              default:
                                      TRACET(1, track, "unsupported %s stride %d",
                                          "src", src->stride);
                                      goto abort;
                              }
                      }
              }
      
              TRACET(1, track, "unsupported encoding");
      abort:
      #if defined(AUDIO_DEBUG)
              if (audiodebug >= 2) {
                      char buf[100];
                      audio_format2_tostr(buf, sizeof(buf), src);
                      TRACET(2, track, "src %s", buf);
                      audio_format2_tostr(buf, sizeof(buf), dst);
                      TRACET(2, track, "dst %s", buf);
              }
      #endif
              return NULL;
      }
      
      /*
       * Initialize the codec stage of this track as necessary.
       * If successful, it initializes the codec stage as necessary, stores updated
       * last_dst in *last_dstp in any case, and returns 0.
       * Otherwise, it returns errno without modifying *last_dstp.
       */
      static int
      audio_track_init_codec(audio_track_t *track, audio_ring_t **last_dstp)
      {
              audio_ring_t *last_dst;
              audio_ring_t *srcbuf;
              audio_format2_t *srcfmt;
              audio_format2_t *dstfmt;
              audio_filter_arg_t *arg;
              u_int len;
              int error;
      
              KASSERT(track);
      
              last_dst = *last_dstp;
              dstfmt = &last_dst->fmt;
              srcfmt = &track->inputfmt;
              srcbuf = &track->codec.srcbuf;
              error = 0;
      
              if (srcfmt->encoding != dstfmt->encoding
               || srcfmt->precision != dstfmt->precision
               || srcfmt->stride != dstfmt->stride) {
                      track->codec.dst = last_dst;
      
                      srcbuf->fmt = *dstfmt;
                      srcbuf->fmt.encoding = srcfmt->encoding;
                      srcbuf->fmt.precision = srcfmt->precision;
                      srcbuf->fmt.stride = srcfmt->stride;
      
                      track->codec.filter = audio_track_get_codec(track,
                          &srcbuf->fmt, dstfmt);
                      if (track->codec.filter == NULL) {
                              error = EINVAL;
                              goto abort;
                      }
      
                      srcbuf->head = 0;
                      srcbuf->used = 0;
                      srcbuf->capacity = frame_per_block(track->mixer, &srcbuf->fmt);
                      len = auring_bytelen(srcbuf);
                      srcbuf->mem = audio_realloc(srcbuf->mem, len);
      
                      arg = &track->codec.arg;
                      arg->srcfmt = &srcbuf->fmt;
                      arg->dstfmt = dstfmt;
                      arg->context = NULL;
      
                      *last_dstp = srcbuf;
                      return 0;
              }
      
      abort:
              track->codec.filter = NULL;
              audio_free(srcbuf->mem);
              return error;
      }
      
      /*
       * Initialize the chvol stage of this track as necessary.
       * If successful, it initializes the chvol stage as necessary, stores updated
       * last_dst in *last_dstp in any case, and returns 0.
       * Otherwise, it returns errno without modifying *last_dstp.
       */
      static int
      audio_track_init_chvol(audio_track_t *track, audio_ring_t **last_dstp)
      {
              audio_ring_t *last_dst;
              audio_ring_t *srcbuf;
              audio_format2_t *srcfmt;
              audio_format2_t *dstfmt;
              audio_filter_arg_t *arg;
              u_int len;
              int error;
      
              KASSERT(track);
      
              last_dst = *last_dstp;
              dstfmt = &last_dst->fmt;
              srcfmt = &track->inputfmt;
              srcbuf = &track->chvol.srcbuf;
              error = 0;
      
              /* Check whether channel volume conversion is necessary. */
              bool use_chvol = false;
              for (int ch = 0; ch < srcfmt->channels; ch++) {
                      if (track->ch_volume[ch] != 256) {
                              use_chvol = true;
                              break;
                      }
              }
      
              if (use_chvol == true) {
                      track->chvol.dst = last_dst;
                      track->chvol.filter = audio_track_chvol;
      
                      srcbuf->fmt = *dstfmt;
                      /* no format conversion occurs */
      
                      srcbuf->head = 0;
                      srcbuf->used = 0;
                      srcbuf->capacity = frame_per_block(track->mixer, &srcbuf->fmt);
                      len = auring_bytelen(srcbuf);
                      srcbuf->mem = audio_realloc(srcbuf->mem, len);
      
                      arg = &track->chvol.arg;
                      arg->srcfmt = &srcbuf->fmt;
                      arg->dstfmt = dstfmt;
                      arg->context = track->ch_volume;
      
                      *last_dstp = srcbuf;
                      return 0;
              }
      
              track->chvol.filter = NULL;
              audio_free(srcbuf->mem);
              return error;
      }
      
      /*
       * Initialize the chmix stage of this track as necessary.
       * If successful, it initializes the chmix stage as necessary, stores updated
       * last_dst in *last_dstp in any case, and returns 0.
       * Otherwise, it returns errno without modifying *last_dstp.
       */
      static int
      audio_track_init_chmix(audio_track_t *track, audio_ring_t **last_dstp)
      {
              audio_ring_t *last_dst;
              audio_ring_t *srcbuf;
              audio_format2_t *srcfmt;
              audio_format2_t *dstfmt;
              audio_filter_arg_t *arg;
              u_int srcch;
              u_int dstch;
              u_int len;
              int error;
      
              KASSERT(track);
      
              last_dst = *last_dstp;
              dstfmt = &last_dst->fmt;
              srcfmt = &track->inputfmt;
              srcbuf = &track->chmix.srcbuf;
              error = 0;
      
              srcch = srcfmt->channels;
              dstch = dstfmt->channels;
              if (srcch != dstch) {
                      track->chmix.dst = last_dst;
      
                      if (srcch >= 2 && dstch == 1) {
                              track->chmix.filter = audio_track_chmix_mixLR;
                      } else if (srcch == 1 && dstch >= 2) {
                              track->chmix.filter = audio_track_chmix_dupLR;
                      } else if (srcch > dstch) {
                              track->chmix.filter = audio_track_chmix_shrink;
                      } else {
                              track->chmix.filter = audio_track_chmix_expand;
                      }
      
                      srcbuf->fmt = *dstfmt;
                      srcbuf->fmt.channels = srcch;
      
                      srcbuf->head = 0;
                      srcbuf->used = 0;
                      /* XXX The buffer size should be able to calculate. */
                      srcbuf->capacity = frame_per_block(track->mixer, &srcbuf->fmt);
                      len = auring_bytelen(srcbuf);
                      srcbuf->mem = audio_realloc(srcbuf->mem, len);
      
                      arg = &track->chmix.arg;
                      arg->srcfmt = &srcbuf->fmt;
                      arg->dstfmt = dstfmt;
                      arg->context = NULL;
      
                      *last_dstp = srcbuf;
                      return 0;
              }
      
              track->chmix.filter = NULL;
              audio_free(srcbuf->mem);
              return error;
      }
      
      /*
       * Initialize the freq stage of this track as necessary.
       * If successful, it initializes the freq stage as necessary, stores updated
       * last_dst in *last_dstp in any case, and returns 0.
       * Otherwise, it returns errno without modifying *last_dstp.
       */
      static int
      audio_track_init_freq(audio_track_t *track, audio_ring_t **last_dstp)
      {
              audio_ring_t *last_dst;
              audio_ring_t *srcbuf;
              audio_format2_t *srcfmt;
              audio_format2_t *dstfmt;
              audio_filter_arg_t *arg;
              uint32_t srcfreq;
              uint32_t dstfreq;
              u_int dst_capacity;
              u_int mod;
              u_int len;
              int error;
      
              KASSERT(track);
      
              last_dst = *last_dstp;
              dstfmt = &last_dst->fmt;
              srcfmt = &track->inputfmt;
              srcbuf = &track->freq.srcbuf;
              error = 0;
      
              srcfreq = srcfmt->sample_rate;
              dstfreq = dstfmt->sample_rate;
              if (srcfreq != dstfreq) {
                      track->freq.dst = last_dst;
      
                      memset(track->freq_prev, 0, sizeof(track->freq_prev));
                      memset(track->freq_curr, 0, sizeof(track->freq_curr));
      
                      /* freq_step is the ratio of src/dst when let dst 65536. */
                      track->freq_step = (uint64_t)srcfreq * 65536 / dstfreq;
      
                      dst_capacity = frame_per_block(track->mixer, dstfmt);
                      mod = (uint64_t)srcfreq * 65536 % dstfreq;
                      track->freq_leap = (mod * dst_capacity + dstfreq / 2) / dstfreq;
      
                      if (track->freq_step < 65536) {
                              track->freq.filter = audio_track_freq_up;
                              /* In order to carry at the first time. */
                              track->freq_current = 65536;
                      } else {
                              track->freq.filter = audio_track_freq_down;
                              track->freq_current = 0;
                      }
      
                      srcbuf->fmt = *dstfmt;
                      srcbuf->fmt.sample_rate = srcfreq;
      
                      srcbuf->head = 0;
                      srcbuf->used = 0;
                      srcbuf->capacity = frame_per_block(track->mixer, &srcbuf->fmt);
                      len = auring_bytelen(srcbuf);
                      srcbuf->mem = audio_realloc(srcbuf->mem, len);
      
                      arg = &track->freq.arg;
                      arg->srcfmt = &srcbuf->fmt;
                      arg->dstfmt = dstfmt;/*&last_dst->fmt;*/
                      arg->context = track;
      
                      *last_dstp = srcbuf;
                      return 0;
              }
      
              track->freq.filter = NULL;
              audio_free(srcbuf->mem);
              return error;
      }
      
      /*
       * When playing back: (e.g. if codec and freq stage are valid)
       *
       *               write
       *                | uiomove
       *                v
       *  usrbuf      [...............]  byte ring buffer (mmap-able)
       *                | memcpy
       *                v
       *  codec.srcbuf[....]             1 block (ring) buffer   <-- stage input
       *       .dst ----+
       *                | convert
       *                v
       *  freq.srcbuf [....]             1 block (ring) buffer
       *      .dst  ----+
       *                | convert
       *                v
       *  outbuf      [...............]  NBLKOUT blocks ring buffer
       *
       *
       * When recording:
       *
       *  freq.srcbuf [...............]  NBLKOUT blocks ring buffer <-- stage input
       *      .dst  ----+
       *                | convert
       *                v
       *  codec.srcbuf[.....]            1 block (ring) buffer
       *       .dst ----+
       *                | convert
       *                v
       *  outbuf      [.....]            1 block (ring) buffer
       *                | memcpy
       *                v
       *  usrbuf      [...............]  byte ring buffer (mmap-able *)
       *                | uiomove
       *                v
       *               read
       *
       *    *: usrbuf for recording is also mmap-able due to symmetry with
       *       playback buffer, but for now mmap will never happen for recording.
       */
      
      /*
       * Set the userland format of this track.
       * usrfmt argument should be parameter verified with audio_check_params().
       * It will release and reallocate all internal conversion buffers.
       * It returns 0 if successful.  Otherwise it returns errno with clearing all
       * internal buffers.
       * It must be called without sc_intr_lock since uvm_* routines require non
       * intr_lock state.
       * It must be called with track lock held since it may release and reallocate
       * outbuf.
       */
      static int
      audio_track_set_format(audio_track_t *track, audio_format2_t *usrfmt)
      {
              struct audio_softc *sc;
              u_int newbufsize;
              u_int oldblksize;
              u_int len;
              int error;
      
              KASSERT(track);
              sc = track->mixer->sc;
      
              /* usrbuf is the closest buffer to the userland. */
              track->usrbuf.fmt = *usrfmt;
      
              /*
               * For references, one block size (in 40msec) is:
               *  320 bytes    = 204 blocks/64KB for mulaw/8kHz/1ch
               *  7680 bytes   = 8 blocks/64KB for s16/48kHz/2ch
               *  30720 bytes  = 90 KB/3blocks for s16/48kHz/8ch
               *  61440 bytes  = 180 KB/3blocks for s16/96kHz/8ch
               *  245760 bytes = 720 KB/3blocks for s32/192kHz/8ch
               *
               * For example,
               * 1) If usrbuf_blksize = 7056 (s16/44.1k/2ch) and PAGE_SIZE = 8192,
               *     newbufsize = rounddown(65536 / 7056) = 63504
               *     newvsize = roundup2(63504, PAGE_SIZE) = 65536
               *    Therefore it maps 8 * 8K pages and usrbuf->capacity = 63504.
               *
               * 2) If usrbuf_blksize = 7680 (s16/48k/2ch) and PAGE_SIZE = 4096,
               *     newbufsize = rounddown(65536 / 7680) = 61440
               *     newvsize = roundup2(61440, PAGE_SIZE) = 61440 (= 15 pages)
               *    Therefore it maps 15 * 4K pages and usrbuf->capacity = 61440.
               */
              oldblksize = track->usrbuf_blksize;
              track->usrbuf_blksize = frametobyte(&track->usrbuf.fmt,
                  frame_per_block(track->mixer, &track->usrbuf.fmt));
              track->usrbuf.head = 0;
              track->usrbuf.used = 0;
              newbufsize = MAX(track->usrbuf_blksize * AUMINNOBLK, 65536);
              newbufsize = rounddown(newbufsize, track->usrbuf_blksize);
              error = audio_realloc_usrbuf(track, newbufsize);
              if (error) {
                      device_printf(sc->sc_dev, "malloc usrbuf(%d) failed\n",
                          newbufsize);
                      goto error;
              }
      
              /* Recalc water mark. */
              if (track->usrbuf_blksize != oldblksize) {
                      if (audio_track_is_playback(track)) {
                              /* Set high at 100%, low at 75%.  */
                              track->usrbuf_usedhigh = track->usrbuf.capacity;
                              track->usrbuf_usedlow = track->usrbuf.capacity * 3 / 4;
                      } else {
                              /* Set high at 100% minus 1block(?), low at 0% */
                              track->usrbuf_usedhigh = track->usrbuf.capacity -
                                  track->usrbuf_blksize;
                              track->usrbuf_usedlow = 0;
                      }
              }
      
              /* Stage buffer */
              audio_ring_t *last_dst = &track->outbuf;
              if (audio_track_is_playback(track)) {
                      /* On playback, initialize from the mixer side in order. */
                      track->inputfmt = *usrfmt;
                      track->outbuf.fmt =  track->mixer->track_fmt;
      
                      if ((error = audio_track_init_freq(track, &last_dst)) != 0)
                              goto error;
                      if ((error = audio_track_init_chmix(track, &last_dst)) != 0)
                              goto error;
                      if ((error = audio_track_init_chvol(track, &last_dst)) != 0)
                              goto error;
                      if ((error = audio_track_init_codec(track, &last_dst)) != 0)
                              goto error;
              } else {
                      /* On recording, initialize from userland side in order. */
                      track->inputfmt = track->mixer->track_fmt;
                      track->outbuf.fmt = *usrfmt;
      
                      if ((error = audio_track_init_codec(track, &last_dst)) != 0)
                              goto error;
                      if ((error = audio_track_init_chvol(track, &last_dst)) != 0)
                              goto error;
                      if ((error = audio_track_init_chmix(track, &last_dst)) != 0)
                              goto error;
                      if ((error = audio_track_init_freq(track, &last_dst)) != 0)
                              goto error;
              }
      #if 0
              /* debug */
              if (track->freq.filter) {
                      audio_print_format2("freq src", &track->freq.srcbuf.fmt);
                      audio_print_format2("freq dst", &track->freq.dst->fmt);
              }
              if (track->chmix.filter) {
                      audio_print_format2("chmix src", &track->chmix.srcbuf.fmt);
                      audio_print_format2("chmix dst", &track->chmix.dst->fmt);
              }
              if (track->chvol.filter) {
                      audio_print_format2("chvol src", &track->chvol.srcbuf.fmt);
                      audio_print_format2("chvol dst", &track->chvol.dst->fmt);
              }
              if (track->codec.filter) {
                      audio_print_format2("codec src", &track->codec.srcbuf.fmt);
                      audio_print_format2("codec dst", &track->codec.dst->fmt);
              }
      #endif
      
              /* Stage input buffer */
              track->input = last_dst;
      
              /*
               * On the recording track, make the first stage a ring buffer.
               * XXX is there a better way?
               */
              if (audio_track_is_record(track)) {
                      track->input->capacity = NBLKOUT *
                          frame_per_block(track->mixer, &track->input->fmt);
                      len = auring_bytelen(track->input);
                      track->input->mem = audio_realloc(track->input->mem, len);
              }
      
              /*
               * Output buffer.
               * On the playback track, its capacity is NBLKOUT blocks.
               * On the recording track, its capacity is 1 block.
               */
              track->outbuf.head = 0;
              track->outbuf.used = 0;
              track->outbuf.capacity = frame_per_block(track->mixer,
                  &track->outbuf.fmt);
              if (audio_track_is_playback(track))
                      track->outbuf.capacity *= NBLKOUT;
              len = auring_bytelen(&track->outbuf);
              track->outbuf.mem = audio_realloc(track->outbuf.mem, len);
              if (track->outbuf.mem == NULL) {
                      device_printf(sc->sc_dev, "malloc outbuf(%d) failed\n", len);
                      error = ENOMEM;
                      goto error;
              }
      
      #if defined(AUDIO_DEBUG)
              if (audiodebug >= 3) {
                      struct audio_track_debugbuf m;
      
                      memset(&m, 0, sizeof(m));
                      snprintf(m.outbuf, sizeof(m.outbuf), " out=%d",
                          track->outbuf.capacity * frametobyte(&track->outbuf.fmt,1));
                      if (track->freq.filter)
                              snprintf(m.freq, sizeof(m.freq), " freq=%d",
                                  track->freq.srcbuf.capacity *
                                  frametobyte(&track->freq.srcbuf.fmt, 1));
                      if (track->chmix.filter)
                              snprintf(m.chmix, sizeof(m.chmix), " chmix=%d",
                                  track->chmix.srcbuf.capacity *
                                  frametobyte(&track->chmix.srcbuf.fmt, 1));
                      if (track->chvol.filter)
                              snprintf(m.chvol, sizeof(m.chvol), " chvol=%d",
                                  track->chvol.srcbuf.capacity *
                                  frametobyte(&track->chvol.srcbuf.fmt, 1));
                      if (track->codec.filter)
                              snprintf(m.codec, sizeof(m.codec), " codec=%d",
                                  track->codec.srcbuf.capacity *
                                  frametobyte(&track->codec.srcbuf.fmt, 1));
                      snprintf(m.usrbuf, sizeof(m.usrbuf),
                          " usr=%d", track->usrbuf.capacity);
      
                      if (audio_track_is_playback(track)) {
                              TRACET(0, track, "bufsize%s%s%s%s%s%s",
                                  m.outbuf, m.freq, m.chmix,
                                  m.chvol, m.codec, m.usrbuf);
                      } else {
                              TRACET(0, track, "bufsize%s%s%s%s%s%s",
                                  m.freq, m.chmix, m.chvol,
                                  m.codec, m.outbuf, m.usrbuf);
                      }
              }
      #endif
              return 0;
      
      error:
              audio_free_usrbuf(track);
              audio_free(track->codec.srcbuf.mem);
              audio_free(track->chvol.srcbuf.mem);
              audio_free(track->chmix.srcbuf.mem);
              audio_free(track->freq.srcbuf.mem);
              audio_free(track->outbuf.mem);
              return error;
      }
      
      /*
       * Fill silence frames (as the internal format) up to 1 block
       * if the ring is not empty and less than 1 block.
       * It returns the number of appended frames.
       */
      static int
      audio_append_silence(audio_track_t *track, audio_ring_t *ring)
      {
              int fpb;
              int n;
      
              KASSERT(track);
              KASSERT(audio_format2_is_internal(&ring->fmt));
      
              /* XXX is n correct? */
              /* XXX memset uses frametobyte()? */
      
              if (ring->used == 0)
                      return 0;
      
              fpb = frame_per_block(track->mixer, &ring->fmt);
              if (ring->used >= fpb)
                      return 0;
      
              n = (ring->capacity - ring->used) % fpb;
      
              KASSERTMSG(auring_get_contig_free(ring) >= n,
                  "auring_get_contig_free(ring)=%d n=%d",
                  auring_get_contig_free(ring), n);
      
              memset(auring_tailptr_aint(ring), 0,
                  n * ring->fmt.channels * sizeof(aint_t));
              auring_push(ring, n);
              return n;
      }
      
      /*
       * Execute the conversion stage.
       * It prepares arg from this stage and executes stage->filter.
       * It must be called only if stage->filter is not NULL.
       *
       * For stages other than frequency conversion, the function increments
       * src and dst counters here.  For frequency conversion stage, on the
       * other hand, the function does not touch src and dst counters and
       * filter side has to increment them.
       */
      static void
      audio_apply_stage(audio_track_t *track, audio_stage_t *stage, bool isfreq)
      {
              audio_filter_arg_t *arg;
              int srccount;
              int dstcount;
              int count;
      
              KASSERT(track);
              KASSERT(stage->filter);
      
              srccount = auring_get_contig_used(&stage->srcbuf);
              dstcount = auring_get_contig_free(stage->dst);
      
              if (isfreq) {
                      KASSERTMSG(srccount > 0, "freq but srccount=%d", srccount);
                      count = uimin(dstcount, track->mixer->frames_per_block);
              } else {
                      count = uimin(srccount, dstcount);
              }
      
              if (count > 0) {
                      arg = &stage->arg;
                      arg->src = auring_headptr(&stage->srcbuf);
                      arg->dst = auring_tailptr(stage->dst);
                      arg->count = count;
      
                      stage->filter(arg);
      
                      if (!isfreq) {
                              auring_take(&stage->srcbuf, count);
                              auring_push(stage->dst, count);
                      }
              }
      }
      
      /*
       * Produce output buffer for playback from user input buffer.
       * It must be called only if usrbuf is not empty and outbuf is
       * available at least one free block.
       */
      static void
      audio_track_play(audio_track_t *track)
      {
              audio_ring_t *usrbuf;
              audio_ring_t *input;
              int count;
              int framesize;
              int bytes;
      
              KASSERT(track);
              KASSERT(track->lock);
              TRACET(4, track, "start pstate=%d", track->pstate);
      
              /* At this point usrbuf must not be empty. */
              KASSERT(track->usrbuf.used > 0);
              /* Also, outbuf must be available at least one block. */
              count = auring_get_contig_free(&track->outbuf);
              KASSERTMSG(count >= frame_per_block(track->mixer, &track->outbuf.fmt),
                  "count=%d fpb=%d",
                  count, frame_per_block(track->mixer, &track->outbuf.fmt));
      
              /* XXX TODO: is this necessary for now? */
              int track_count_0 = track->outbuf.used;
      
              usrbuf = &track->usrbuf;
              input = track->input;
      
              /*
               * framesize is always 1 byte or more since all formats supported as
               * usrfmt(=input) have 8bit or more stride.
               */
              framesize = frametobyte(&input->fmt, 1);
              KASSERT(framesize >= 1);
      
              /* The next stage of usrbuf (=input) must be available. */
              KASSERT(auring_get_contig_free(input) > 0);
      
              /*
               * Copy usrbuf up to 1block to input buffer.
               * count is the number of frames to copy from usrbuf.
               * bytes is the number of bytes to copy from usrbuf.  However it is
               * not copied less than one frame.
               */
              count = uimin(usrbuf->used, track->usrbuf_blksize) / framesize;
              bytes = count * framesize;
      
              track->usrbuf_stamp += bytes;
      
              if (usrbuf->head + bytes < usrbuf->capacity) {
                      memcpy((uint8_t *)input->mem + auring_tail(input) * framesize,
                          (uint8_t *)usrbuf->mem + usrbuf->head,
                          bytes);
                      auring_push(input, count);
                      auring_take(usrbuf, bytes);
              } else {
                      int bytes1;
                      int bytes2;
      
                      bytes1 = auring_get_contig_used(usrbuf);
                      KASSERTMSG(bytes1 % framesize == 0,
                          "bytes1=%d framesize=%d", bytes1, framesize);
                      memcpy((uint8_t *)input->mem + auring_tail(input) * framesize,
                          (uint8_t *)usrbuf->mem + usrbuf->head,
                          bytes1);
                      auring_push(input, bytes1 / framesize);
                      auring_take(usrbuf, bytes1);
      
                      bytes2 = bytes - bytes1;
                      memcpy((uint8_t *)input->mem + auring_tail(input) * framesize,
                          (uint8_t *)usrbuf->mem + usrbuf->head,
                          bytes2);
                      auring_push(input, bytes2 / framesize);
                      auring_take(usrbuf, bytes2);
              }
      
              /* Encoding conversion */
              if (track->codec.filter)
                      audio_apply_stage(track, &track->codec, false);
      
              /* Channel volume */
              if (track->chvol.filter)
                      audio_apply_stage(track, &track->chvol, false);
      
              /* Channel mix */
              if (track->chmix.filter)
                      audio_apply_stage(track, &track->chmix, false);
      
              /* Frequency conversion */
              /*
               * Since the frequency conversion needs correction for each block,
               * it rounds up to 1 block.
               */
              if (track->freq.filter) {
                      int n;
                      n = audio_append_silence(track, &track->freq.srcbuf);
                      if (n > 0) {
                              TRACET(4, track,
                                  "freq.srcbuf add silence %d -> %d/%d/%d",
                                  n,
                                  track->freq.srcbuf.head,
                                  track->freq.srcbuf.used,
                                  track->freq.srcbuf.capacity);
                      }
                      if (track->freq.srcbuf.used > 0) {
                              audio_apply_stage(track, &track->freq, true);
                      }
              }
      
              if (bytes < track->usrbuf_blksize) {
                      /*
                       * Clear all conversion buffer pointer if the conversion was
                       * not exactly one block.  These conversion stage buffers are
                       * certainly circular buffers because of symmetry with the
                       * previous and next stage buffer.  However, since they are
                       * treated as simple contiguous buffers in operation, so head
                       * always should point 0.  This may happen during drain-age.
                       */
                      TRACET(4, track, "reset stage");
                      if (track->codec.filter) {
                              KASSERT(track->codec.srcbuf.used == 0);
                              track->codec.srcbuf.head = 0;
                      }
                      if (track->chvol.filter) {
                              KASSERT(track->chvol.srcbuf.used == 0);
                              track->chvol.srcbuf.head = 0;
                      }
                      if (track->chmix.filter) {
                              KASSERT(track->chmix.srcbuf.used == 0);
                              track->chmix.srcbuf.head = 0;
                      }
                      if (track->freq.filter) {
                              KASSERT(track->freq.srcbuf.used == 0);
                              track->freq.srcbuf.head = 0;
                      }
              }
      
              if (track->input == &track->outbuf) {
                      track->outputcounter = track->inputcounter;
              } else {
                      track->outputcounter += track->outbuf.used - track_count_0;
              }
      
      #if defined(AUDIO_DEBUG)
              if (audiodebug >= 3) {
                      struct audio_track_debugbuf m;
                      audio_track_bufstat(track, &m);
                      TRACET(0, track, "end%s%s%s%s%s%s",
                          m.outbuf, m.freq, m.chvol, m.chmix, m.codec, m.usrbuf);
              }
      #endif
      }
      
      /*
       * Produce user output buffer for recording from input buffer.
       */
      static void
      audio_track_record(audio_track_t *track)
      {
              audio_ring_t *outbuf;
              audio_ring_t *usrbuf;
              int count;
              int bytes;
              int framesize;
      
              KASSERT(track);
              KASSERT(track->lock);
      
              /* Number of frames to process */
              count = auring_get_contig_used(track->input);
              count = uimin(count, track->mixer->frames_per_block);
              if (count == 0) {
                      TRACET(4, track, "count == 0");
                      return;
              }
      
              /* Frequency conversion */
              if (track->freq.filter) {
                      if (track->freq.srcbuf.used > 0) {
                              audio_apply_stage(track, &track->freq, true);
                              /* XXX should input of freq be from beginning of buf? */
                      }
              }
      
              /* Channel mix */
              if (track->chmix.filter)
                      audio_apply_stage(track, &track->chmix, false);
      
              /* Channel volume */
              if (track->chvol.filter)
                      audio_apply_stage(track, &track->chvol, false);
      
              /* Encoding conversion */
              if (track->codec.filter)
                      audio_apply_stage(track, &track->codec, false);
      
              /* Copy outbuf to usrbuf */
              outbuf = &track->outbuf;
              usrbuf = &track->usrbuf;
              /*
               * framesize is always 1 byte or more since all formats supported
               * as usrfmt(=output) have 8bit or more stride.
               */
              framesize = frametobyte(&outbuf->fmt, 1);
              KASSERT(framesize >= 1);
              /*
               * count is the number of frames to copy to usrbuf.
               * bytes is the number of bytes to copy to usrbuf.
               */
              count = outbuf->used;
              count = uimin(count,
                  (track->usrbuf_usedhigh - usrbuf->used) / framesize);
              bytes = count * framesize;
              if (auring_tail(usrbuf) + bytes < usrbuf->capacity) {
                      memcpy((uint8_t *)usrbuf->mem + auring_tail(usrbuf),
                          (uint8_t *)outbuf->mem + outbuf->head * framesize,
                          bytes);
                      auring_push(usrbuf, bytes);
                      auring_take(outbuf, count);
              } else {
                      int bytes1;
                      int bytes2;
      
                      bytes1 = auring_get_contig_free(usrbuf);
                      KASSERTMSG(bytes1 % framesize == 0,
                          "bytes1=%d framesize=%d", bytes1, framesize);
                      memcpy((uint8_t *)usrbuf->mem + auring_tail(usrbuf),
                          (uint8_t *)outbuf->mem + outbuf->head * framesize,
                          bytes1);
                      auring_push(usrbuf, bytes1);
                      auring_take(outbuf, bytes1 / framesize);
      
                      bytes2 = bytes - bytes1;
                      memcpy((uint8_t *)usrbuf->mem + auring_tail(usrbuf),
                          (uint8_t *)outbuf->mem + outbuf->head * framesize,
                          bytes2);
                      auring_push(usrbuf, bytes2);
                      auring_take(outbuf, bytes2 / framesize);
              }
      
              /* XXX TODO: any counters here? */
      
      #if defined(AUDIO_DEBUG)
              if (audiodebug >= 3) {
                      struct audio_track_debugbuf m;
                      audio_track_bufstat(track, &m);
                      TRACET(0, track, "end%s%s%s%s%s%s",
                          m.freq, m.chvol, m.chmix, m.codec, m.outbuf, m.usrbuf);
              }
      #endif
      }
      
      /*
       * Calcurate blktime [msec] from mixer(.hwbuf.fmt).
       * Must be called with sc_exlock held.
       */
      static u_int
      audio_mixer_calc_blktime(struct audio_softc *sc, audio_trackmixer_t *mixer)
      {
              audio_format2_t *fmt;
              u_int blktime;
              u_int frames_per_block;
      
              KASSERT(sc->sc_exlock);
      
              fmt = &mixer->hwbuf.fmt;
              blktime = sc->sc_blk_ms;
      
              /*
               * If stride is not multiples of 8, special treatment is necessary.
               * For now, it is only x68k's vs(4), 4 bit/sample ADPCM.
               */
              if (fmt->stride == 4) {
                      frames_per_block = fmt->sample_rate * blktime / 1000;
                      if ((frames_per_block & 1) != 0)
                              blktime *= 2;
              }
      #ifdef DIAGNOSTIC
              else if (fmt->stride % NBBY != 0) {
                      panic("unsupported HW stride %d", fmt->stride);
              }
      #endif
      
              return blktime;
      }
      
      /*
       * Initialize the mixer corresponding to the mode.
       * Set AUMODE_PLAY to the 'mode' for playback or AUMODE_RECORD for recording.
       * sc->sc_[pr]mixer (corresponding to the 'mode') must be zero-filled.
       * This function returns 0 on successful.  Otherwise returns errno.
       * Must be called with sc_exlock held and without sc_lock held.
       */
      static int
      audio_mixer_init(struct audio_softc *sc, int mode,
              const audio_format2_t *hwfmt, const audio_filter_reg_t *reg)
      {
              char codecbuf[64];
              char blkdmsbuf[8];
              audio_trackmixer_t *mixer;
              void (*softint_handler)(void *);
              int len;
              int blksize;
              int capacity;
              size_t bufsize;
              int hwblks;
              int blkms;
              int blkdms;
              int error;
      
              KASSERT(hwfmt != NULL);
              KASSERT(reg != NULL);
              KASSERT(sc->sc_exlock);
      
              error = 0;
              if (mode == AUMODE_PLAY)
                      mixer = sc->sc_pmixer;
              else
                      mixer = sc->sc_rmixer;
      
              mixer->sc = sc;
              mixer->mode = mode;
      
              mixer->hwbuf.fmt = *hwfmt;
              mixer->volume = 256;
              mixer->blktime_d = 1000;
              mixer->blktime_n = audio_mixer_calc_blktime(sc, mixer);
              sc->sc_blk_ms = mixer->blktime_n;
              hwblks = NBLKHW;
      
              mixer->frames_per_block = frame_per_block(mixer, &mixer->hwbuf.fmt);
              blksize = frametobyte(&mixer->hwbuf.fmt, mixer->frames_per_block);
              if (sc->hw_if->round_blocksize) {
                      int rounded;
                      audio_params_t p = format2_to_params(&mixer->hwbuf.fmt);
                      mutex_enter(sc->sc_lock);
                      rounded = sc->hw_if->round_blocksize(sc->hw_hdl, blksize,
                          mode, &p);
                      mutex_exit(sc->sc_lock);
                      TRACE(1, "round_blocksize %d -> %d", blksize, rounded);
                      if (rounded != blksize) {
                              if ((rounded * NBBY) % (mixer->hwbuf.fmt.stride *
                                  mixer->hwbuf.fmt.channels) != 0) {
                                      device_printf(sc->sc_dev,
                                          "round_blocksize must return blocksize "
                                          "divisible by framesize: "
                                          "blksize=%d rounded=%d "
                                          "stride=%ubit channels=%u\n",
                                          blksize, rounded,
                                          mixer->hwbuf.fmt.stride,
                                          mixer->hwbuf.fmt.channels);
                                      return EINVAL;
                              }
                              /* Recalculation */
                              blksize = rounded;
                              mixer->frames_per_block = blksize * NBBY /
                                  (mixer->hwbuf.fmt.stride *
                                   mixer->hwbuf.fmt.channels);
                      }
              }
              mixer->blktime_n = mixer->frames_per_block;
              mixer->blktime_d = mixer->hwbuf.fmt.sample_rate;
      
              capacity = mixer->frames_per_block * hwblks;
              bufsize = frametobyte(&mixer->hwbuf.fmt, capacity);
              if (sc->hw_if->round_buffersize) {
                      size_t rounded;
                      mutex_enter(sc->sc_lock);
                      rounded = sc->hw_if->round_buffersize(sc->hw_hdl, mode,
                          bufsize);
                      mutex_exit(sc->sc_lock);
                      TRACE(1, "round_buffersize %zd -> %zd", bufsize, rounded);
                      if (rounded < bufsize) {
                              /* buffersize needs NBLKHW blocks at least. */
                              device_printf(sc->sc_dev,
                                  "buffersize too small: buffersize=%zd blksize=%d\n",
                                  rounded, blksize);
                              return EINVAL;
                      }
                      if (rounded % blksize != 0) {
                              /* buffersize/blksize constraint mismatch? */
                              device_printf(sc->sc_dev,
                                  "buffersize must be multiple of blksize: "
                                  "buffersize=%zu blksize=%d\n",
                                  rounded, blksize);
                              return EINVAL;
                      }
                      if (rounded != bufsize) {
                              /* Recalcuration */
                              bufsize = rounded;
                              hwblks = bufsize / blksize;
                              capacity = mixer->frames_per_block * hwblks;
                      }
              }
              TRACE(1, "buffersize for %s = %zu",
                  (mode == AUMODE_PLAY) ? "playback" : "recording",
                  bufsize);
              mixer->hwbuf.capacity = capacity;
      
              if (sc->hw_if->allocm) {
                      /* sc_lock is not necessary for allocm */
                      mixer->hwbuf.mem = sc->hw_if->allocm(sc->hw_hdl, mode, bufsize);
                      if (mixer->hwbuf.mem == NULL) {
                              device_printf(sc->sc_dev, "%s: allocm(%zu) failed\n",
                                  __func__, bufsize);
                              return ENOMEM;
                      }
              } else {
                      mixer->hwbuf.mem = kmem_alloc(bufsize, KM_SLEEP);
              }
      
              /* From here, audio_mixer_destroy is necessary to exit. */
              if (mode == AUMODE_PLAY) {
                      cv_init(&mixer->outcv, "audiowr");
              } else {
                      cv_init(&mixer->outcv, "audiord");
              }
      
              if (mode == AUMODE_PLAY) {
                      softint_handler = audio_softintr_wr;
              } else {
                      softint_handler = audio_softintr_rd;
              }
              mixer->sih = softint_establish(SOFTINT_SERIAL | SOFTINT_MPSAFE,
                  softint_handler, sc);
              if (mixer->sih == NULL) {
                      device_printf(sc->sc_dev, "softint_establish failed\n");
                      goto abort;
              }
      
              mixer->track_fmt.encoding = AUDIO_ENCODING_SLINEAR_NE;
              mixer->track_fmt.precision = AUDIO_INTERNAL_BITS;
              mixer->track_fmt.stride = AUDIO_INTERNAL_BITS;
              mixer->track_fmt.channels = mixer->hwbuf.fmt.channels;
              mixer->track_fmt.sample_rate = mixer->hwbuf.fmt.sample_rate;
      
              if (mixer->hwbuf.fmt.encoding == AUDIO_ENCODING_SLINEAR_OE &&
                  mixer->hwbuf.fmt.precision == AUDIO_INTERNAL_BITS) {
                      mixer->swap_endian = true;
                      TRACE(1, "swap_endian");
              }
      
              if (mode == AUMODE_PLAY) {
                      /* Mixing buffer */
                      mixer->mixfmt = mixer->track_fmt;
                      mixer->mixfmt.precision *= 2;
                      mixer->mixfmt.stride *= 2;
                      /* XXX TODO: use some macros? */
                      len = mixer->frames_per_block * mixer->mixfmt.channels *
                          mixer->mixfmt.stride / NBBY;
                      mixer->mixsample = audio_realloc(mixer->mixsample, len);
              } else {
                      /* No mixing buffer for recording */
              }
      
              if (reg->codec) {
                      mixer->codec = reg->codec;
                      mixer->codecarg.context = reg->context;
                      if (mode == AUMODE_PLAY) {
                              mixer->codecarg.srcfmt = &mixer->track_fmt;
                              mixer->codecarg.dstfmt = &mixer->hwbuf.fmt;
                      } else {
                              mixer->codecarg.srcfmt = &mixer->hwbuf.fmt;
                              mixer->codecarg.dstfmt = &mixer->track_fmt;
                      }
                      mixer->codecbuf.fmt = mixer->track_fmt;
                      mixer->codecbuf.capacity = mixer->frames_per_block;
                      len = auring_bytelen(&mixer->codecbuf);
                      mixer->codecbuf.mem = audio_realloc(mixer->codecbuf.mem, len);
                      if (mixer->codecbuf.mem == NULL) {
                              device_printf(sc->sc_dev,
                                  "%s: malloc codecbuf(%d) failed\n",
                                  __func__, len);
                              error = ENOMEM;
                              goto abort;
                      }
              }
      
              /* Succeeded so display it. */
              codecbuf[0] = '\0';
              if (mixer->codec || mixer->swap_endian) {
                      snprintf(codecbuf, sizeof(codecbuf), " %s %s:%d",
                          (mode == AUMODE_PLAY) ? "->" : "<-",
                          audio_encoding_name(mixer->hwbuf.fmt.encoding),
                          mixer->hwbuf.fmt.precision);
              }
              blkms = mixer->blktime_n * 1000 / mixer->blktime_d;
              blkdms = (mixer->blktime_n * 10000 / mixer->blktime_d) % 10;
              blkdmsbuf[0] = '\0';
              if (blkdms != 0) {
                      snprintf(blkdmsbuf, sizeof(blkdmsbuf), ".%1d", blkdms);
              }
              aprint_normal_dev(sc->sc_dev,
                  "%s:%d%s %dch %dHz, blk %d bytes (%d%sms) for %s\n",
                  audio_encoding_name(mixer->track_fmt.encoding),
                  mixer->track_fmt.precision,
                  codecbuf,
                  mixer->track_fmt.channels,
                  mixer->track_fmt.sample_rate,
                  blksize,
                  blkms, blkdmsbuf,
                  (mode == AUMODE_PLAY) ? "playback" : "recording");
      
              return 0;
      
      abort:
              audio_mixer_destroy(sc, mixer);
              return error;
      }
      
      /*
       * Releases all resources of 'mixer'.
       * Note that it does not release the memory area of 'mixer' itself.
       * Must be called with sc_exlock held and without sc_lock held.
       */
      static void
      audio_mixer_destroy(struct audio_softc *sc, audio_trackmixer_t *mixer)
      {
              int bufsize;
      
              KASSERT(sc->sc_exlock == 1);
      
              bufsize = frametobyte(&mixer->hwbuf.fmt, mixer->hwbuf.capacity);
      
              if (mixer->hwbuf.mem != NULL) {
                      if (sc->hw_if->freem) {
                              /* sc_lock is not necessary for freem */
                              sc->hw_if->freem(sc->hw_hdl, mixer->hwbuf.mem, bufsize);
                      } else {
                              kmem_free(mixer->hwbuf.mem, bufsize);
                      }
                      mixer->hwbuf.mem = NULL;
              }
      
              audio_free(mixer->codecbuf.mem);
              audio_free(mixer->mixsample);
      
              cv_destroy(&mixer->outcv);
      
              if (mixer->sih) {
                      softint_disestablish(mixer->sih);
                      mixer->sih = NULL;
              }
      }
      
      /*
       * Starts playback mixer.
       * Must be called only if sc_pbusy is false.
       * Must be called with sc_lock && sc_exlock held.
       * Must not be called from the interrupt context.
       */
      static void
      audio_pmixer_start(struct audio_softc *sc, bool force)
      {
              audio_trackmixer_t *mixer;
              int minimum;
      
              KASSERT(mutex_owned(sc->sc_lock));
              KASSERT(sc->sc_exlock);
              KASSERT(sc->sc_pbusy == false);
      
              mutex_enter(sc->sc_intr_lock);
      
              mixer = sc->sc_pmixer;
              TRACE(2, "%smixseq=%d hwseq=%d hwbuf=%d/%d/%d%s",
                  (audiodebug >= 3) ? "begin " : "",
                  (int)mixer->mixseq, (int)mixer->hwseq,
                  mixer->hwbuf.head, mixer->hwbuf.used, mixer->hwbuf.capacity,
                  force ? " force" : "");
      
              /* Need two blocks to start normally. */
              minimum = (force) ? 1 : 2;
              while (mixer->hwbuf.used < mixer->frames_per_block * minimum) {
                      audio_pmixer_process(sc);
              }
      
              /* Start output */
              audio_pmixer_output(sc);
              sc->sc_pbusy = true;
      
              TRACE(3, "end   mixseq=%d hwseq=%d hwbuf=%d/%d/%d",
                  (int)mixer->mixseq, (int)mixer->hwseq,
                  mixer->hwbuf.head, mixer->hwbuf.used, mixer->hwbuf.capacity);
      
              mutex_exit(sc->sc_intr_lock);
      }
      
      /*
       * When playing back with MD filter:
       *
       *           track track ...
       *               v v
       *                +  mix (with aint2_t)
       *                |  master volume (with aint2_t)
       *                v
       *    mixsample [::::]                  wide-int 1 block (ring) buffer
       *                |
       *                |  convert aint2_t -> aint_t
       *                v
       *    codecbuf  [....]                  1 block (ring) buffer
       *                |
       *                |  convert to hw format
       *                v
       *    hwbuf     [............]          NBLKHW blocks ring buffer
       *
       * When playing back without MD filter:
       *
       *    mixsample [::::]                  wide-int 1 block (ring) buffer
       *                |
       *                |  convert aint2_t -> aint_t
       *                |  (with byte swap if necessary)
       *                v
       *    hwbuf     [............]          NBLKHW blocks ring buffer
       *
       * mixsample: slinear_NE, wide internal precision, HW ch, HW freq.
       * codecbuf:  slinear_NE, internal precision,      HW ch, HW freq.
       * hwbuf:     HW encoding, HW precision,           HW ch, HW freq.
       */
      
      /*
       * Performs track mixing and converts it to hwbuf.
       * Note that this function doesn't transfer hwbuf to hardware.
       * Must be called with sc_intr_lock held.
       */
      static void
      audio_pmixer_process(struct audio_softc *sc)
      {
              audio_trackmixer_t *mixer;
              audio_file_t *f;
              int frame_count;
              int sample_count;
              int mixed;
              int i;
              aint2_t *m;
              aint_t *h;
      
              mixer = sc->sc_pmixer;
      
              frame_count = mixer->frames_per_block;
              KASSERTMSG(auring_get_contig_free(&mixer->hwbuf) >= frame_count,
                  "auring_get_contig_free()=%d frame_count=%d",
                  auring_get_contig_free(&mixer->hwbuf), frame_count);
              sample_count = frame_count * mixer->mixfmt.channels;
      
              mixer->mixseq++;
      
              /* Mix all tracks */
              mixed = 0;
              SLIST_FOREACH(f, &sc->sc_files, entry) {
                      audio_track_t *track = f->ptrack;
      
                      if (track == NULL)
                              continue;
      
                      if (track->is_pause) {
                              TRACET(4, track, "skip; paused");
                              continue;
                      }
      
                      /* Skip if the track is used by process context. */
                      if (audio_track_lock_tryenter(track) == false) {
                              TRACET(4, track, "skip; in use");
                              continue;
                      }
      
                      /* Emulate mmap'ped track */
                      if (track->mmapped) {
                              auring_push(&track->usrbuf, track->usrbuf_blksize);
                              TRACET(4, track, "mmap; usr=%d/%d/C%d",
                                  track->usrbuf.head,
                                  track->usrbuf.used,
                                  track->usrbuf.capacity);
                      }
      
                      if (track->outbuf.used < mixer->frames_per_block &&
                          track->usrbuf.used > 0) {
                              TRACET(4, track, "process");
                              audio_track_play(track);
                      }
      
                      if (track->outbuf.used > 0) {
                              mixed = audio_pmixer_mix_track(mixer, track, mixed);
                      } else {
                              TRACET(4, track, "skip; empty");
                      }
      
                      audio_track_lock_exit(track);
              }
      
              if (mixed == 0) {
                      /* Silence */
                      memset(mixer->mixsample, 0,
                          frametobyte(&mixer->mixfmt, frame_count));
              } else {
                      if (mixed > 1) {
                              /* If there are multiple tracks, do auto gain control */
                              audio_pmixer_agc(mixer, sample_count);
                      }
      
                      /* Apply master volume */
                      if (mixer->volume < 256) {
                              m = mixer->mixsample;
                              for (i = 0; i < sample_count; i++) {
                                      *m = AUDIO_SCALEDOWN(*m * mixer->volume, 8);
                                      m++;
                              }
      
                              /*
                               * Recover the volume gradually at the pace of
                               * several times per second.  If it's too fast, you
                               * can recognize that the volume changes up and down
                               * quickly and it's not so comfortable.
                               */
                              mixer->voltimer += mixer->blktime_n;
                              if (mixer->voltimer * 4 >= mixer->blktime_d) {
                                      mixer->volume++;
                                      mixer->voltimer = 0;
      #if defined(AUDIO_DEBUG_AGC)
                                      TRACE(1, "volume recover: %d", mixer->volume);
      #endif
                              }
                      }
              }
      
              /*
               * The rest is the hardware part.
               */
      
              if (mixer->codec) {
                      h = auring_tailptr_aint(&mixer->codecbuf);
              } else {
                      h = auring_tailptr_aint(&mixer->hwbuf);
              }
      
              m = mixer->mixsample;
              if (mixer->swap_endian) {
                      for (i = 0; i < sample_count; i++) {
                              *h++ = bswap16(*m++);
                      }
              } else {
                      for (i = 0; i < sample_count; i++) {
                              *h++ = *m++;
                      }
              }
      
              /* Hardware driver's codec */
              if (mixer->codec) {
                      auring_push(&mixer->codecbuf, frame_count);
                      mixer->codecarg.src = auring_headptr(&mixer->codecbuf);
                      mixer->codecarg.dst = auring_tailptr(&mixer->hwbuf);
                      mixer->codecarg.count = frame_count;
                      mixer->codec(&mixer->codecarg);
                      auring_take(&mixer->codecbuf, mixer->codecarg.count);
              }
      
              auring_push(&mixer->hwbuf, frame_count);
      
              TRACE(4, "done mixseq=%d hwbuf=%d/%d/%d%s",
                  (int)mixer->mixseq,
                  mixer->hwbuf.head, mixer->hwbuf.used, mixer->hwbuf.capacity,
                  (mixed == 0) ? " silent" : "");
      }
      
      /*
       * Do auto gain control.
       * Must be called sc_intr_lock held.
       */
      static void
      audio_pmixer_agc(audio_trackmixer_t *mixer, int sample_count)
      {
              struct audio_softc *sc __unused;
              aint2_t val;
              aint2_t maxval;
              aint2_t minval;
              aint2_t over_plus;
              aint2_t over_minus;
              aint2_t *m;
              int newvol;
              int i;
      
              sc = mixer->sc;
      
              /* Overflow detection */
              maxval = AINT_T_MAX;
              minval = AINT_T_MIN;
              m = mixer->mixsample;
              for (i = 0; i < sample_count; i++) {
                      val = *m++;
                      if (val > maxval)
                              maxval = val;
                      else if (val < minval)
                              minval = val;
              }
      
              /* Absolute value of overflowed amount */
              over_plus = maxval - AINT_T_MAX;
              over_minus = AINT_T_MIN - minval;
      
              if (over_plus > 0 || over_minus > 0) {
                      if (over_plus > over_minus) {
                              newvol = (int)((aint2_t)AINT_T_MAX * 256 / maxval);
                      } else {
                              newvol = (int)((aint2_t)AINT_T_MIN * 256 / minval);
                      }
      
                      /*
                       * Change the volume only if new one is smaller.
                       * Reset the timer even if the volume isn't changed.
                       */
                      if (newvol <= mixer->volume) {
                              mixer->volume = newvol;
                              mixer->voltimer = 0;
      #if defined(AUDIO_DEBUG_AGC)
                              TRACE(1, "auto volume adjust: %d", mixer->volume);
      #endif
                      }
              }
      }
      
      /*
       * Mix one track.
       * 'mixed' specifies the number of tracks mixed so far.
       * It returns the number of tracks mixed.  In other words, it returns
       * mixed + 1 if this track is mixed.
       */
      static int
      audio_pmixer_mix_track(audio_trackmixer_t *mixer, audio_track_t *track,
              int mixed)
      {
              int count;
              int sample_count;
              int remain;
              int i;
              const aint_t *s;
              aint2_t *d;
      
              /* XXX TODO: Is this necessary for now? */
              if (mixer->mixseq < track->seq)
                      return mixed;
      
              count = auring_get_contig_used(&track->outbuf);
              count = uimin(count, mixer->frames_per_block);
      
              s = auring_headptr_aint(&track->outbuf);
              d = mixer->mixsample;
      
              /*
               * Apply track volume with double-sized integer and perform
               * additive synthesis.
               *
               * XXX If you limit the track volume to 1.0 or less (<= 256),
               *     it would be better to do this in the track conversion stage
               *     rather than here.  However, if you accept the volume to
               *     be greater than 1.0 (> 256), it's better to do it here.
               *     Because the operation here is done by double-sized integer.
               */
              sample_count = count * mixer->mixfmt.channels;
              if (mixed == 0) {
                      /* If this is the first track, assignment can be used. */
      #if defined(AUDIO_SUPPORT_TRACK_VOLUME)
                      if (track->volume != 256) {
                              for (i = 0; i < sample_count; i++) {
                                      aint2_t v;
                                      v = *s++;
                                      *d++ = AUDIO_SCALEDOWN(v * track->volume, 8)
                              }
                      } else
      #endif
                      {
                              for (i = 0; i < sample_count; i++) {
                                      *d++ = ((aint2_t)*s++);
                              }
                      }
                      /* Fill silence if the first track is not filled. */
                      for (; i < mixer->frames_per_block * mixer->mixfmt.channels; i++)
                              *d++ = 0;
              } else {
                      /* If this is the second or later, add it. */
      #if defined(AUDIO_SUPPORT_TRACK_VOLUME)
                      if (track->volume != 256) {
                              for (i = 0; i < sample_count; i++) {
                                      aint2_t v;
                                      v = *s++;
                                      *d++ += AUDIO_SCALEDOWN(v * track->volume, 8);
                              }
                      } else
      #endif
                      {
                              for (i = 0; i < sample_count; i++) {
                                      *d++ += ((aint2_t)*s++);
                              }
                      }
              }
      
              auring_take(&track->outbuf, count);
              /*
               * The counters have to align block even if outbuf is less than
               * one block. XXX Is this still necessary?
               */
              remain = mixer->frames_per_block - count;
              if (__predict_false(remain != 0)) {
                      auring_push(&track->outbuf, remain);
                      auring_take(&track->outbuf, remain);
              }
      
              /*
               * Update track sequence.
               * mixseq has previous value yet at this point.
               */
              track->seq = mixer->mixseq + 1;
      
              return mixed + 1;
      }
      
      /*
       * Output one block from hwbuf to HW.
       * Must be called with sc_intr_lock held.
       */
      static void
      audio_pmixer_output(struct audio_softc *sc)
      {
              audio_trackmixer_t *mixer;
              audio_params_t params;
              void *start;
              void *end;
              int blksize;
              int error;
      
              mixer = sc->sc_pmixer;
              TRACE(4, "pbusy=%d hwbuf=%d/%d/%d",
                  sc->sc_pbusy,
                  mixer->hwbuf.head, mixer->hwbuf.used, mixer->hwbuf.capacity);
              KASSERTMSG(mixer->hwbuf.used >= mixer->frames_per_block,
                  "mixer->hwbuf.used=%d mixer->frames_per_block=%d",
                  mixer->hwbuf.used, mixer->frames_per_block);
      
              blksize = frametobyte(&mixer->hwbuf.fmt, mixer->frames_per_block);
      
              if (sc->hw_if->trigger_output) {
                      /* trigger (at once) */
                      if (!sc->sc_pbusy) {
                              start = mixer->hwbuf.mem;
                              end = (uint8_t *)start + auring_bytelen(&mixer->hwbuf);
                              params = format2_to_params(&mixer->hwbuf.fmt);
      
                              error = sc->hw_if->trigger_output(sc->hw_hdl,
                                  start, end, blksize, audio_pintr, sc, &params);
                              if (error) {
                                      device_printf(sc->sc_dev,
                                          "trigger_output failed with %d\n", error);
                                      return;
                              }
                      }
              } else {
                      /* start (everytime) */
                      start = auring_headptr(&mixer->hwbuf);
      
                      error = sc->hw_if->start_output(sc->hw_hdl,
                          start, blksize, audio_pintr, sc);
                      if (error) {
                              device_printf(sc->sc_dev,
                                  "start_output failed with %d\n", error);
                              return;
                      }
              }
      }
      
      /*
       * This is an interrupt handler for playback.
       * It is called with sc_intr_lock held.
       *
       * It is usually called from hardware interrupt.  However, note that
       * for some drivers (e.g. uaudio) it is called from software interrupt.
       */
      static void
      audio_pintr(void *arg)
      {
              struct audio_softc *sc;
              audio_trackmixer_t *mixer;
      
              sc = arg;
              KASSERT(mutex_owned(sc->sc_intr_lock));
      
              if (sc->sc_dying)
                      return;
              if (sc->sc_pbusy == false) {
      #if defined(DIAGNOSTIC)
                      device_printf(sc->sc_dev,
                          "DIAGNOSTIC: %s raised stray interrupt\n",
                          device_xname(sc->hw_dev));
      #endif
                      return;
              }
      
              mixer = sc->sc_pmixer;
              mixer->hw_complete_counter += mixer->frames_per_block;
              mixer->hwseq++;
      
              auring_take(&mixer->hwbuf, mixer->frames_per_block);
      
              TRACE(4,
                  "HW_INT ++hwseq=%" PRIu64 " cmplcnt=%" PRIu64 " hwbuf=%d/%d/%d",
                  mixer->hwseq, mixer->hw_complete_counter,
                  mixer->hwbuf.head, mixer->hwbuf.used, mixer->hwbuf.capacity);
      
      #if defined(AUDIO_HW_SINGLE_BUFFER)
              /*
               * Create a new block here and output it immediately.
               * It makes a latency lower but needs machine power.
               */
              audio_pmixer_process(sc);
              audio_pmixer_output(sc);
      #else
              /*
               * It is called when block N output is done.
               * Output immediately block N+1 created by the last interrupt.
               * And then create block N+2 for the next interrupt.
               * This method makes playback robust even on slower machines.
               * Instead the latency is increased by one block.
               */
      
              /* At first, output ready block. */
              if (mixer->hwbuf.used >= mixer->frames_per_block) {
                      audio_pmixer_output(sc);
              }
      
              bool later = false;
      
              if (mixer->hwbuf.used < mixer->frames_per_block) {
                      later = true;
              }
      
              /* Then, process next block. */
              audio_pmixer_process(sc);
      
              if (later) {
                      audio_pmixer_output(sc);
              }
      #endif
      
              /*
               * When this interrupt is the real hardware interrupt, disabling
               * preemption here is not necessary.  But some drivers (e.g. uaudio)
               * emulate it by software interrupt, so kpreempt_disable is necessary.
               */
              kpreempt_disable();
              softint_schedule(mixer->sih);
              kpreempt_enable();
      }
      
      /*
       * Starts record mixer.
       * Must be called only if sc_rbusy is false.
       * Must be called with sc_lock && sc_exlock held.
       * Must not be called from the interrupt context.
       */
      static void
      audio_rmixer_start(struct audio_softc *sc)
      {
      
              KASSERT(mutex_owned(sc->sc_lock));
              KASSERT(sc->sc_exlock);
              KASSERT(sc->sc_rbusy == false);
      
              mutex_enter(sc->sc_intr_lock);
      
              TRACE(2, "%s", (audiodebug >= 3) ? "begin" : "");
              audio_rmixer_input(sc);
              sc->sc_rbusy = true;
              TRACE(3, "end");
      
              mutex_exit(sc->sc_intr_lock);
      }
      
      /*
       * When recording with MD filter:
       *
       *    hwbuf     [............]          NBLKHW blocks ring buffer
       *                |
       *                | convert from hw format
       *                v
       *    codecbuf  [....]                  1 block (ring) buffer
       *               |  |
       *               v  v
       *            track track ...
       *
       * When recording without MD filter:
       *
       *    hwbuf     [............]          NBLKHW blocks ring buffer
       *               |  |
       *               v  v
       *            track track ...
       *
       * hwbuf:     HW encoding, HW precision, HW ch, HW freq.
       * codecbuf:  slinear_NE, internal precision, HW ch, HW freq.
       */
      
      /*
       * Distribute a recorded block to all recording tracks.
       */
      static void
      audio_rmixer_process(struct audio_softc *sc)
      {
              audio_trackmixer_t *mixer;
              audio_ring_t *mixersrc;
              audio_file_t *f;
              aint_t *p;
              int count;
              int bytes;
              int i;
      
              mixer = sc->sc_rmixer;
      
              /*
               * count is the number of frames to be retrieved this time.
               * count should be one block.
               */
              count = auring_get_contig_used(&mixer->hwbuf);
              count = uimin(count, mixer->frames_per_block);
              if (count <= 0) {
                      TRACE(4, "count %d: too short", count);
                      return;
              }
              bytes = frametobyte(&mixer->track_fmt, count);
      
              /* Hardware driver's codec */
              if (mixer->codec) {
                      mixer->codecarg.src = auring_headptr(&mixer->hwbuf);
                      mixer->codecarg.dst = auring_tailptr(&mixer->codecbuf);
                      mixer->codecarg.count = count;
                      mixer->codec(&mixer->codecarg);
                      auring_take(&mixer->hwbuf, mixer->codecarg.count);
                      auring_push(&mixer->codecbuf, mixer->codecarg.count);
                      mixersrc = &mixer->codecbuf;
              } else {
                      mixersrc = &mixer->hwbuf;
              }
      
              if (mixer->swap_endian) {
                      /* inplace conversion */
                      p = auring_headptr_aint(mixersrc);
                      for (i = 0; i < count * mixer->track_fmt.channels; i++, p++) {
                              *p = bswap16(*p);
                      }
              }
      
              /* Distribute to all tracks. */
              SLIST_FOREACH(f, &sc->sc_files, entry) {
                      audio_track_t *track = f->rtrack;
                      audio_ring_t *input;
      
                      if (track == NULL)
                              continue;
      
                      if (track->is_pause) {
                              TRACET(4, track, "skip; paused");
                              continue;
                      }
      
                      if (audio_track_lock_tryenter(track) == false) {
                              TRACET(4, track, "skip; in use");
                              continue;
                      }
      
                      /* If the track buffer is full, discard the oldest one? */
                      input = track->input;
                      if (input->capacity - input->used < mixer->frames_per_block) {
                              int drops = mixer->frames_per_block -
                                  (input->capacity - input->used);
                              track->dropframes += drops;
                              TRACET(4, track, "drop %d frames: inp=%d/%d/%d",
                                  drops,
                                  input->head, input->used, input->capacity);
                              auring_take(input, drops);
                      }
                      KASSERTMSG(input->used % mixer->frames_per_block == 0,
                          "input->used=%d mixer->frames_per_block=%d",
                          input->used, mixer->frames_per_block);
      
                      memcpy(auring_tailptr_aint(input),
                          auring_headptr_aint(mixersrc),
                          bytes);
                      auring_push(input, count);
      
                      /* XXX sequence counter? */
      
                      audio_track_lock_exit(track);
              }
      
              auring_take(mixersrc, count);
      }
      
      /*
       * Input one block from HW to hwbuf.
       * Must be called with sc_intr_lock held.
       */
      static void
      audio_rmixer_input(struct audio_softc *sc)
      {
              audio_trackmixer_t *mixer;
              audio_params_t params;
              void *start;
              void *end;
              int blksize;
              int error;
      
              mixer = sc->sc_rmixer;
              blksize = frametobyte(&mixer->hwbuf.fmt, mixer->frames_per_block);
      
              if (sc->hw_if->trigger_input) {
                      /* trigger (at once) */
                      if (!sc->sc_rbusy) {
                              start = mixer->hwbuf.mem;
                              end = (uint8_t *)start + auring_bytelen(&mixer->hwbuf);
                              params = format2_to_params(&mixer->hwbuf.fmt);
      
                              error = sc->hw_if->trigger_input(sc->hw_hdl,
                                  start, end, blksize, audio_rintr, sc, &params);
                              if (error) {
                                      device_printf(sc->sc_dev,
                                          "trigger_input failed with %d\n", error);
                                      return;
                              }
                      }
              } else {
                      /* start (everytime) */
                      start = auring_tailptr(&mixer->hwbuf);
      
                      error = sc->hw_if->start_input(sc->hw_hdl,
                          start, blksize, audio_rintr, sc);
                      if (error) {
                              device_printf(sc->sc_dev,
                                  "start_input failed with %d\n", error);
                              return;
                      }
              }
      }
      
      /*
       * This is an interrupt handler for recording.
       * It is called with sc_intr_lock.
       *
       * It is usually called from hardware interrupt.  However, note that
       * for some drivers (e.g. uaudio) it is called from software interrupt.
       */
      static void
      audio_rintr(void *arg)
      {
              struct audio_softc *sc;
              audio_trackmixer_t *mixer;
      
              sc = arg;
              KASSERT(mutex_owned(sc->sc_intr_lock));
      
              if (sc->sc_dying)
                      return;
              if (sc->sc_rbusy == false) {
      #if defined(DIAGNOSTIC)
                      device_printf(sc->sc_dev,
                          "DIAGNOSTIC: %s raised stray interrupt\n",
                          device_xname(sc->hw_dev));
      #endif
                      return;
              }
      
              mixer = sc->sc_rmixer;
              mixer->hw_complete_counter += mixer->frames_per_block;
              mixer->hwseq++;
      
              auring_push(&mixer->hwbuf, mixer->frames_per_block);
      
              TRACE(4,
                  "HW_INT ++hwseq=%" PRIu64 " cmplcnt=%" PRIu64 " hwbuf=%d/%d/%d",
                  mixer->hwseq, mixer->hw_complete_counter,
                  mixer->hwbuf.head, mixer->hwbuf.used, mixer->hwbuf.capacity);
      
              /* Distrubute recorded block */
              audio_rmixer_process(sc);
      
              /* Request next block */
              audio_rmixer_input(sc);
      
              /*
               * When this interrupt is the real hardware interrupt, disabling
               * preemption here is not necessary.  But some drivers (e.g. uaudio)
               * emulate it by software interrupt, so kpreempt_disable is necessary.
               */
              kpreempt_disable();
              softint_schedule(mixer->sih);
              kpreempt_enable();
      }
      
      /*
       * Halts playback mixer.
       * This function also clears related parameters, so call this function
       * instead of calling halt_output directly.
       * Must be called only if sc_pbusy is true.
       * Must be called with sc_lock && sc_exlock held.
       */
      static int
      audio_pmixer_halt(struct audio_softc *sc)
      {
              int error;
      
              TRACE(2, "");
              KASSERT(mutex_owned(sc->sc_lock));
              KASSERT(sc->sc_exlock);
      
              mutex_enter(sc->sc_intr_lock);
              error = sc->hw_if->halt_output(sc->hw_hdl);
      
              /* Halts anyway even if some error has occurred. */
              sc->sc_pbusy = false;
              sc->sc_pmixer->hwbuf.head = 0;
              sc->sc_pmixer->hwbuf.used = 0;
              sc->sc_pmixer->mixseq = 0;
              sc->sc_pmixer->hwseq = 0;
              mutex_exit(sc->sc_intr_lock);
      
              return error;
      }
      
      /*
       * Halts recording mixer.
       * This function also clears related parameters, so call this function
       * instead of calling halt_input directly.
       * Must be called only if sc_rbusy is true.
       * Must be called with sc_lock && sc_exlock held.
       */
      static int
      audio_rmixer_halt(struct audio_softc *sc)
      {
              int error;
      
              TRACE(2, "");
              KASSERT(mutex_owned(sc->sc_lock));
              KASSERT(sc->sc_exlock);
      
              mutex_enter(sc->sc_intr_lock);
              error = sc->hw_if->halt_input(sc->hw_hdl);
      
              /* Halts anyway even if some error has occurred. */
              sc->sc_rbusy = false;
              sc->sc_rmixer->hwbuf.head = 0;
              sc->sc_rmixer->hwbuf.used = 0;
              sc->sc_rmixer->mixseq = 0;
              sc->sc_rmixer->hwseq = 0;
              mutex_exit(sc->sc_intr_lock);
      
              return error;
      }
      
      /*
       * Flush this track.
       * Halts all operations, clears all buffers, reset error counters.
       * XXX I'm not sure...
       */
      static void
      audio_track_clear(struct audio_softc *sc, audio_track_t *track)
      {
      
              KASSERT(track);
              TRACET(3, track, "clear");
      
              audio_track_lock_enter(track);
      
              track->usrbuf.used = 0;
              /* Clear all internal parameters. */
              if (track->codec.filter) {
                      track->codec.srcbuf.used = 0;
                      track->codec.srcbuf.head = 0;
              }
              if (track->chvol.filter) {
                      track->chvol.srcbuf.used = 0;
                      track->chvol.srcbuf.head = 0;
              }
              if (track->chmix.filter) {
                      track->chmix.srcbuf.used = 0;
                      track->chmix.srcbuf.head = 0;
              }
              if (track->freq.filter) {
                      track->freq.srcbuf.used = 0;
                      track->freq.srcbuf.head = 0;
                      if (track->freq_step < 65536)
                              track->freq_current = 65536;
                      else
                              track->freq_current = 0;
                      memset(track->freq_prev, 0, sizeof(track->freq_prev));
                      memset(track->freq_curr, 0, sizeof(track->freq_curr));
              }
              /* Clear buffer, then operation halts naturally. */
              track->outbuf.used = 0;
      
              /* Clear counters. */
              track->dropframes = 0;
      
              audio_track_lock_exit(track);
      }
      
      /*
       * Drain the track.
       * track must be present and for playback.
       * If successful, it returns 0.  Otherwise returns errno.
       * Must be called with sc_lock held.
       */
      static int
      audio_track_drain(struct audio_softc *sc, audio_track_t *track)
      {
              audio_trackmixer_t *mixer;
              int done;
              int error;
      
              KASSERT(track);
              TRACET(3, track, "start");
              mixer = track->mixer;
              KASSERT(mutex_owned(sc->sc_lock));
      
              /* Ignore them if pause. */
              if (track->is_pause) {
                      TRACET(3, track, "pause -> clear");
                      track->pstate = AUDIO_STATE_CLEAR;
              }
              /* Terminate early here if there is no data in the track. */
              if (track->pstate == AUDIO_STATE_CLEAR) {
                      TRACET(3, track, "no need to drain");
                      return 0;
              }
              track->pstate = AUDIO_STATE_DRAINING;
      
              for (;;) {
                      /* I want to display it before condition evaluation. */
                      TRACET(3, track, "pid=%d.%d trkseq=%d hwseq=%d out=%d/%d/%d",
                          (int)curproc->p_pid, (int)curlwp->l_lid,
                          (int)track->seq, (int)mixer->hwseq,
                          track->outbuf.head, track->outbuf.used,
                          track->outbuf.capacity);
      
                      /* Condition to terminate */
                      audio_track_lock_enter(track);
                      done = (track->usrbuf.used < frametobyte(&track->inputfmt, 1) &&
                          track->outbuf.used == 0 &&
                          track->seq <= mixer->hwseq);
                      audio_track_lock_exit(track);
                      if (done)
                              break;
      
                      TRACET(3, track, "sleep");
                      error = audio_track_waitio(sc, track);
                      if (error)
                              return error;
      
                      /* XXX call audio_track_play here ? */
              }
      
              track->pstate = AUDIO_STATE_CLEAR;
              TRACET(3, track, "done trk_inp=%d trk_out=%d",
                      (int)track->inputcounter, (int)track->outputcounter);
              return 0;
      }
      
      /*
       * Send signal to process.
       * This is intended to be called only from audio_softintr_{rd,wr}.
       * Must be called without sc_intr_lock held.
       */
      static inline void
      audio_psignal(struct audio_softc *sc, pid_t pid, int signum)
      {
              proc_t *p;
      
              KASSERT(pid != 0);
      
              /*
               * psignal() must be called without spin lock held.
               */
      
              mutex_enter(&proc_lock);
              p = proc_find(pid);
              if (p)
                      psignal(p, signum);
              mutex_exit(&proc_lock);
      }
      
      /*
       * This is software interrupt handler for record.
       * It is called from recording hardware interrupt everytime.
       * It does:
       * - Deliver SIGIO for all async processes.
       * - Notify to audio_read() that data has arrived.
       * - selnotify() for select/poll-ing processes.
       */
      /*
       * XXX If a process issues FIOASYNC between hardware interrupt and
       *     software interrupt, (stray) SIGIO will be sent to the process
       *     despite the fact that it has not receive recorded data yet.
       */
      static void
      audio_softintr_rd(void *cookie)
      {
              struct audio_softc *sc = cookie;
              audio_file_t *f;
              pid_t pid;
      
              mutex_enter(sc->sc_lock);
      
              SLIST_FOREACH(f, &sc->sc_files, entry) {
                      audio_track_t *track = f->rtrack;
      
                      if (track == NULL)
                              continue;
      
                      TRACET(4, track, "broadcast; inp=%d/%d/%d",
                          track->input->head,
                          track->input->used,
                          track->input->capacity);
      
                      pid = f->async_audio;
                      if (pid != 0) {
                              TRACEF(4, f, "sending SIGIO %d", pid);
                              audio_psignal(sc, pid, SIGIO);
                      }
              }
      
              /* Notify that data has arrived. */
              selnotify(&sc->sc_rsel, 0, NOTE_SUBMIT);
              KNOTE(&sc->sc_rsel.sel_klist, 0);
              cv_broadcast(&sc->sc_rmixer->outcv);
      
              mutex_exit(sc->sc_lock);
      }
      
      /*
       * This is software interrupt handler for playback.
       * It is called from playback hardware interrupt everytime.
       * It does:
       * - Deliver SIGIO for all async and writable (used < lowat) processes.
       * - Notify to audio_write() that outbuf block available.
       * - selnotify() for select/poll-ing processes if there are any writable
       *   (used < lowat) processes.  Checking each descriptor will be done by
       *   filt_audiowrite_event().
       */
      static void
      audio_softintr_wr(void *cookie)
      {
              struct audio_softc *sc = cookie;
              audio_file_t *f;
              bool found;
              pid_t pid;
      
              TRACE(4, "called");
              found = false;
      
              mutex_enter(sc->sc_lock);
      
              SLIST_FOREACH(f, &sc->sc_files, entry) {
                      audio_track_t *track = f->ptrack;
      
                      if (track == NULL)
                              continue;
      
                      TRACET(4, track, "broadcast; trseq=%d out=%d/%d/%d",
                          (int)track->seq,
                          track->outbuf.head,
                          track->outbuf.used,
                          track->outbuf.capacity);
      
                      /*
                       * Send a signal if the process is async mode and
                       * used is lower than lowat.
                       */
                      if (track->usrbuf.used <= track->usrbuf_usedlow &&
                          !track->is_pause) {
                              /* For selnotify */
                              found = true;
                              /* For SIGIO */
                              pid = f->async_audio;
                              if (pid != 0) {
                                      TRACEF(4, f, "sending SIGIO %d", pid);
                                      audio_psignal(sc, pid, SIGIO);
                              }
                      }
              }
      
              /*
               * Notify for select/poll when someone become writable.
               * It needs sc_lock (and not sc_intr_lock).
               */
              if (found) {
                      TRACE(4, "selnotify");
                      selnotify(&sc->sc_wsel, 0, NOTE_SUBMIT);
                      KNOTE(&sc->sc_wsel.sel_klist, 0);
              }
      
              /* Notify to audio_write() that outbuf available. */
              cv_broadcast(&sc->sc_pmixer->outcv);
      
              mutex_exit(sc->sc_lock);
      }
      
      /*
       * Check (and convert) the format *p came from userland.
       * If successful, it writes back the converted format to *p if necessary
       * and returns 0.  Otherwise returns errno (*p may change even this case).
       */
      static int
      audio_check_params(audio_format2_t *p)
      {
      
              /*
               * Convert obsolete AUDIO_ENCODING_PCM encodings.
               * 
               * AUDIO_ENCODING_PCM16 == AUDIO_ENCODING_LINEAR
               * So, it's always signed, as in SunOS.
               *
               * AUDIO_ENCODING_PCM8 == AUDIO_ENCODING_LINEAR8
               * So, it's always unsigned, as in SunOS.
               */
              if (p->encoding == AUDIO_ENCODING_PCM16) {
                      p->encoding = AUDIO_ENCODING_SLINEAR;
              } else if (p->encoding == AUDIO_ENCODING_PCM8) {
                      if (p->precision == 8)
                              p->encoding = AUDIO_ENCODING_ULINEAR;
                      else
                              return EINVAL;
              }
      
              /*
               * Convert obsoleted AUDIO_ENCODING_[SU]LINEAR without endianness
               * suffix.
               */
              if (p->encoding == AUDIO_ENCODING_SLINEAR)
                      p->encoding = AUDIO_ENCODING_SLINEAR_NE;
              if (p->encoding == AUDIO_ENCODING_ULINEAR)
                      p->encoding = AUDIO_ENCODING_ULINEAR_NE;
      
              switch (p->encoding) {
              case AUDIO_ENCODING_ULAW:
              case AUDIO_ENCODING_ALAW:
                      if (p->precision != 8)
                              return EINVAL;
                      break;
              case AUDIO_ENCODING_ADPCM:
                      if (p->precision != 4 && p->precision != 8)
                              return EINVAL;
                      break;
              case AUDIO_ENCODING_SLINEAR_LE:
              case AUDIO_ENCODING_SLINEAR_BE:
              case AUDIO_ENCODING_ULINEAR_LE:
              case AUDIO_ENCODING_ULINEAR_BE:
                      if (p->precision !=  8 && p->precision != 16 &&
                          p->precision != 24 && p->precision != 32)
                              return EINVAL;
      
                      /* 8bit format does not have endianness. */
                      if (p->precision == 8) {
                              if (p->encoding == AUDIO_ENCODING_SLINEAR_OE)
                                      p->encoding = AUDIO_ENCODING_SLINEAR_NE;
                              if (p->encoding == AUDIO_ENCODING_ULINEAR_OE)
                                      p->encoding = AUDIO_ENCODING_ULINEAR_NE;
                      }
      
                      if (p->precision > p->stride)
                              return EINVAL;
                      break;
              case AUDIO_ENCODING_MPEG_L1_STREAM:
              case AUDIO_ENCODING_MPEG_L1_PACKETS:
              case AUDIO_ENCODING_MPEG_L1_SYSTEM:
              case AUDIO_ENCODING_MPEG_L2_STREAM:
              case AUDIO_ENCODING_MPEG_L2_PACKETS:
              case AUDIO_ENCODING_MPEG_L2_SYSTEM:
              case AUDIO_ENCODING_AC3:
                      break;
              default:
                      return EINVAL;
              }
      
              /* sanity check # of channels*/
              if (p->channels < 1 || p->channels > AUDIO_MAX_CHANNELS)
                      return EINVAL;
      
              return 0;
      }
      
      /*
       * Initialize playback and record mixers.
       * mode (AUMODE_{PLAY,RECORD}) indicates the mixer to be initialized.
       * phwfmt and rhwfmt indicate the hardware format.  pfil and rfil indicate
       * the filter registration information.  These four must not be NULL.
       * If successful returns 0.  Otherwise returns errno.
       * Must be called with sc_exlock held and without sc_lock held.
       * Must not be called if there are any tracks.
       * Caller should check that the initialization succeed by whether
       * sc_[pr]mixer is not NULL.
       */
      static int
      audio_mixers_init(struct audio_softc *sc, int mode,
              const audio_format2_t *phwfmt, const audio_format2_t *rhwfmt,
              const audio_filter_reg_t *pfil, const audio_filter_reg_t *rfil)
      {
              int error;
      
              KASSERT(phwfmt != NULL);
              KASSERT(rhwfmt != NULL);
              KASSERT(pfil != NULL);
              KASSERT(rfil != NULL);
              KASSERT(sc->sc_exlock);
      
              if ((mode & AUMODE_PLAY)) {
                      if (sc->sc_pmixer == NULL) {
                              sc->sc_pmixer = kmem_zalloc(sizeof(*sc->sc_pmixer),
                                  KM_SLEEP);
                      } else {
                              /* destroy() doesn't free memory. */
                              audio_mixer_destroy(sc, sc->sc_pmixer);
                              memset(sc->sc_pmixer, 0, sizeof(*sc->sc_pmixer));
                      }
                      error = audio_mixer_init(sc, AUMODE_PLAY, phwfmt, pfil);
                      if (error) {
                              device_printf(sc->sc_dev,
                                  "configuring playback mode failed with %d\n",
                                  error);
                              kmem_free(sc->sc_pmixer, sizeof(*sc->sc_pmixer));
                              sc->sc_pmixer = NULL;
                              return error;
                      }
              }
              if ((mode & AUMODE_RECORD)) {
                      if (sc->sc_rmixer == NULL) {
                              sc->sc_rmixer = kmem_zalloc(sizeof(*sc->sc_rmixer),
                                  KM_SLEEP);
                      } else {
                              /* destroy() doesn't free memory. */
                              audio_mixer_destroy(sc, sc->sc_rmixer);
                              memset(sc->sc_rmixer, 0, sizeof(*sc->sc_rmixer));
                      }
                      error = audio_mixer_init(sc, AUMODE_RECORD, rhwfmt, rfil);
                      if (error) {
                              device_printf(sc->sc_dev,
                                  "configuring record mode failed with %d\n",
                                  error);
                              kmem_free(sc->sc_rmixer, sizeof(*sc->sc_rmixer));
                              sc->sc_rmixer = NULL;
                              return error;
                      }
              }
      
              return 0;
      }
      
      /*
       * Select a frequency.
       * Prioritize 48kHz and 44.1kHz.  Otherwise choose the highest one.
       * XXX Better algorithm?
       */
      static int
      audio_select_freq(const struct audio_format *fmt)
      {
              int freq;
              int high;
              int low;
              int j;
      
              if (fmt->frequency_type == 0) {
                      low = fmt->frequency[0];
                      high = fmt->frequency[1];
                      freq = 48000;
                      if (low <= freq && freq <= high) {
                              return freq;
                      }
                      freq = 44100;
                      if (low <= freq && freq <= high) {
                              return freq;
                      }
                      return high;
              } else {
                      for (j = 0; j < fmt->frequency_type; j++) {
                              if (fmt->frequency[j] == 48000) {
                                      return fmt->frequency[j];
                              }
                      }
                      high = 0;
                      for (j = 0; j < fmt->frequency_type; j++) {
                              if (fmt->frequency[j] == 44100) {
                                      return fmt->frequency[j];
                              }
                              if (fmt->frequency[j] > high) {
                                      high = fmt->frequency[j];
                              }
                      }
                      return high;
              }
      }
      
      /*
       * Choose the most preferred hardware format.
       * If successful, it will store the chosen format into *cand and return 0.
       * Otherwise, return errno.
       * Must be called without sc_lock held.
       */
      static int
      audio_hw_probe(struct audio_softc *sc, audio_format2_t *cand, int mode)
      {
              audio_format_query_t query;
              int cand_score;
              int score;
              int i;
              int error;
      
              /*
               * Score each formats and choose the highest one.
               *
               *                 +---- priority(0-3)
               *                 |+--- encoding/precision
               *                 ||+-- channels
               * score = 0x000000PEC
               */
      
              cand_score = 0;
              for (i = 0; ; i++) {
                      memset(&query, 0, sizeof(query));
                      query.index = i;
      
                      mutex_enter(sc->sc_lock);
                      error = sc->hw_if->query_format(sc->hw_hdl, &query);
                      mutex_exit(sc->sc_lock);
                      if (error == EINVAL)
                              break;
                      if (error)
                              return error;
      
      #if defined(AUDIO_DEBUG)
                      DPRINTF(1, "fmt[%d] %c%c pri=%d %s,%d/%dbit,%dch,", i,
                          (query.fmt.mode & AUMODE_PLAY)   ? 'P' : '-',
                          (query.fmt.mode & AUMODE_RECORD) ? 'R' : '-',
                          query.fmt.priority,
                          audio_encoding_name(query.fmt.encoding),
                          query.fmt.validbits,
                          query.fmt.precision,
                          query.fmt.channels);
                      if (query.fmt.frequency_type == 0) {
                              DPRINTF(1, "{%d-%d",
                                  query.fmt.frequency[0], query.fmt.frequency[1]);
                      } else {
                              int j;
                              for (j = 0; j < query.fmt.frequency_type; j++) {
                                      DPRINTF(1, "%c%d",
                                          (j == 0) ? '{' : ',',
                                          query.fmt.frequency[j]);
                              }
                      }
                      DPRINTF(1, "}\n");
      #endif
      
                      if ((query.fmt.mode & mode) == 0) {
                              DPRINTF(1, "fmt[%d] skip; mode not match %d\n", i,
                                  mode);
                              continue;
                      }
      
                      if (query.fmt.priority < 0) {
                              DPRINTF(1, "fmt[%d] skip; unsupported encoding\n", i);
                              continue;
                      }
      
                      /* Score */
                      score = (query.fmt.priority & 3) * 0x100;
                      if (query.fmt.encoding == AUDIO_ENCODING_SLINEAR_NE &&