From 59cf8412055c31299b8d3d1d15662320656d7ea2 Mon Sep 17 00:00:00 2001
From: Ed Maste <emaste@FreeBSD.org>
Date: Tue, 7 Apr 2009 01:13:44 +0000
Subject: [PATCH 001/279] Flatten the dist tree.

---
 contrib/gdb/COPYING => COPYING                          | 0
 contrib/gdb/COPYING.LIB => COPYING.LIB                  | 0
 contrib/gdb/FREEBSD-diffs => FREEBSD-diffs              | 0
 contrib/gdb/README => README                            | 0
 contrib/gdb/config-ml.in => config-ml.in                | 0
 contrib/gdb/djunpack.bat => djunpack.bat                | 0
 {contrib/gdb/gdb => gdb}/CONTRIBUTE                     | 0
 {contrib/gdb/gdb => gdb}/COPYING                        | 0
 {contrib/gdb/gdb => gdb}/MAINTAINERS                    | 0
 {contrib/gdb/gdb => gdb}/NEWS                           | 0
 {contrib/gdb/gdb => gdb}/PROBLEMS                       | 0
 {contrib/gdb/gdb => gdb}/README                         | 0
 {contrib/gdb/gdb => gdb}/TODO                           | 0
 {contrib/gdb/gdb => gdb}/abug-rom.c                     | 0
 {contrib/gdb/gdb => gdb}/acinclude.m4                   | 0
 {contrib/gdb/gdb => gdb}/aclocal.m4                     | 0
 {contrib/gdb/gdb => gdb}/ada-exp.c                      | 0
 {contrib/gdb/gdb => gdb}/ada-exp.y                      | 0
 {contrib/gdb/gdb => gdb}/ada-lang.c                     | 0
 {contrib/gdb/gdb => gdb}/ada-lang.h                     | 0
 {contrib/gdb/gdb => gdb}/ada-lex.l                      | 0
 {contrib/gdb/gdb => gdb}/ada-tasks.c                    | 0
 {contrib/gdb/gdb => gdb}/ada-typeprint.c                | 0
 {contrib/gdb/gdb => gdb}/ada-valprint.c                 | 0
 {contrib/gdb/gdb => gdb}/alpha-mdebug-tdep.c            | 0
 {contrib/gdb/gdb => gdb}/alpha-nat.c                    | 0
 {contrib/gdb/gdb => gdb}/alpha-tdep.c                   | 0
 {contrib/gdb/gdb => gdb}/alpha-tdep.h                   | 0
 {contrib/gdb/gdb => gdb}/alphabsd-nat.c                 | 0
 {contrib/gdb/gdb => gdb}/alphabsd-tdep.c                | 0
 {contrib/gdb/gdb => gdb}/alphabsd-tdep.h                | 0
 {contrib/gdb/gdb => gdb}/alphafbsd-tdep.c               | 0
 {contrib/gdb/gdb => gdb}/alphanbsd-tdep.c               | 0
 {contrib/gdb/gdb => gdb}/amd64-nat.c                    | 0
 {contrib/gdb/gdb => gdb}/amd64-nat.h                    | 0
 {contrib/gdb/gdb => gdb}/amd64-tdep.c                   | 0
 {contrib/gdb/gdb => gdb}/amd64-tdep.h                   | 0
 {contrib/gdb/gdb => gdb}/amd64bsd-nat.c                 | 0
 {contrib/gdb/gdb => gdb}/amd64fbsd-nat.c                | 0
 {contrib/gdb/gdb => gdb}/amd64fbsd-tdep.c               | 0
 {contrib/gdb/gdb => gdb}/amd64nbsd-nat.c                | 0
 {contrib/gdb/gdb => gdb}/amd64nbsd-tdep.c               | 0
 {contrib/gdb/gdb => gdb}/amd64obsd-nat.c                | 0
 {contrib/gdb/gdb => gdb}/amd64obsd-tdep.c               | 0
 {contrib/gdb/gdb => gdb}/annotate.c                     | 0
 {contrib/gdb/gdb => gdb}/annotate.h                     | 0
 {contrib/gdb/gdb => gdb}/arch-utils.c                   | 0
 {contrib/gdb/gdb => gdb}/arch-utils.h                   | 0
 {contrib/gdb/gdb => gdb}/arm-tdep.c                     | 0
 {contrib/gdb/gdb => gdb}/arm-tdep.h                     | 0
 {contrib/gdb/gdb => gdb}/armnbsd-nat.c                  | 0
 {contrib/gdb/gdb => gdb}/armnbsd-tdep.c                 | 0
 {contrib/gdb/gdb => gdb}/auxv.c                         | 0
 {contrib/gdb/gdb => gdb}/auxv.h                         | 0
 {contrib/gdb/gdb => gdb}/ax-gdb.c                       | 0
 {contrib/gdb/gdb => gdb}/ax-gdb.h                       | 0
 {contrib/gdb/gdb => gdb}/ax-general.c                   | 0
 {contrib/gdb/gdb => gdb}/ax.h                           | 0
 {contrib/gdb/gdb => gdb}/bcache.c                       | 0
 {contrib/gdb/gdb => gdb}/bcache.h                       | 0
 {contrib/gdb/gdb => gdb}/bfd-target.c                   | 0
 {contrib/gdb/gdb => gdb}/bfd-target.h                   | 0
 {contrib/gdb/gdb => gdb}/block.c                        | 0
 {contrib/gdb/gdb => gdb}/block.h                        | 0
 {contrib/gdb/gdb => gdb}/blockframe.c                   | 0
 {contrib/gdb/gdb => gdb}/breakpoint.c                   | 0
 {contrib/gdb/gdb => gdb}/breakpoint.h                   | 0
 {contrib/gdb/gdb => gdb}/buildsym.c                     | 0
 {contrib/gdb/gdb => gdb}/buildsym.h                     | 0
 {contrib/gdb/gdb => gdb}/c-exp.c                        | 0
 {contrib/gdb/gdb => gdb}/c-exp.y                        | 0
 {contrib/gdb/gdb => gdb}/c-lang.c                       | 0
 {contrib/gdb/gdb => gdb}/c-lang.h                       | 0
 {contrib/gdb/gdb => gdb}/c-typeprint.c                  | 0
 {contrib/gdb/gdb => gdb}/c-valprint.c                   | 0
 {contrib/gdb/gdb => gdb}/call-cmds.h                    | 0
 {contrib/gdb/gdb => gdb}/charset.c                      | 0
 {contrib/gdb/gdb => gdb}/charset.h                      | 0
 {contrib/gdb/gdb => gdb}/cli-out.c                      | 0
 {contrib/gdb/gdb => gdb}/cli-out.h                      | 0
 {contrib/gdb/gdb => gdb}/cli/cli-cmds.c                 | 0
 {contrib/gdb/gdb => gdb}/cli/cli-cmds.h                 | 0
 {contrib/gdb/gdb => gdb}/cli/cli-decode.c               | 0
 {contrib/gdb/gdb => gdb}/cli/cli-decode.h               | 0
 {contrib/gdb/gdb => gdb}/cli/cli-dump.c                 | 0
 {contrib/gdb/gdb => gdb}/cli/cli-dump.h                 | 0
 {contrib/gdb/gdb => gdb}/cli/cli-interp.c               | 0
 {contrib/gdb/gdb => gdb}/cli/cli-logging.c              | 0
 {contrib/gdb/gdb => gdb}/cli/cli-script.c               | 0
 {contrib/gdb/gdb => gdb}/cli/cli-script.h               | 0
 {contrib/gdb/gdb => gdb}/cli/cli-setshow.c              | 0
 {contrib/gdb/gdb => gdb}/cli/cli-setshow.h              | 0
 {contrib/gdb/gdb => gdb}/cli/cli-utils.c                | 0
 {contrib/gdb/gdb => gdb}/cli/cli-utils.h                | 0
 {contrib/gdb/gdb => gdb}/coff-pe-read.c                 | 0
 {contrib/gdb/gdb => gdb}/coff-pe-read.h                 | 0
 {contrib/gdb/gdb => gdb}/coff-solib.c                   | 0
 {contrib/gdb/gdb => gdb}/coff-solib.h                   | 0
 {contrib/gdb/gdb => gdb}/coffread.c                     | 0
 {contrib/gdb/gdb => gdb}/command.h                      | 0
 {contrib/gdb/gdb => gdb}/complaints.c                   | 0
 {contrib/gdb/gdb => gdb}/complaints.h                   | 0
 {contrib/gdb/gdb => gdb}/completer.c                    | 0
 {contrib/gdb/gdb => gdb}/completer.h                    | 0
 {contrib/gdb/gdb => gdb}/config/alpha/alpha-osf2.mh     | 0
 {contrib/gdb/gdb => gdb}/config/alpha/alpha-osf3.mh     | 0
 {contrib/gdb/gdb => gdb}/config/alpha/alpha.mt          | 0
 {contrib/gdb/gdb => gdb}/config/alpha/fbsd.mh           | 0
 {contrib/gdb/gdb => gdb}/config/alpha/fbsd.mt           | 0
 {contrib/gdb/gdb => gdb}/config/alpha/nbsd.mh           | 0
 {contrib/gdb/gdb => gdb}/config/alpha/nbsd.mt           | 0
 {contrib/gdb/gdb => gdb}/config/alpha/nm-fbsd.h         | 0
 {contrib/gdb/gdb => gdb}/config/alpha/nm-nbsd.h         | 0
 {contrib/gdb/gdb => gdb}/config/alpha/nm-osf.h          | 0
 {contrib/gdb/gdb => gdb}/config/alpha/nm-osf2.h         | 0
 {contrib/gdb/gdb => gdb}/config/alpha/nm-osf3.h         | 0
 {contrib/gdb/gdb => gdb}/config/alpha/tm-alpha.h        | 0
 {contrib/gdb/gdb => gdb}/config/alpha/tm-fbsd.h         | 0
 {contrib/gdb/gdb => gdb}/config/alpha/tm-nbsd.h         | 0
 {contrib/gdb/gdb => gdb}/config/alpha/xm-alphaosf.h     | 0
 {contrib/gdb/gdb => gdb}/config/arm/embed.mt            | 0
 {contrib/gdb/gdb => gdb}/config/arm/nbsd.mt             | 0
 {contrib/gdb/gdb => gdb}/config/arm/nbsdaout.mh         | 0
 {contrib/gdb/gdb => gdb}/config/arm/nbsdelf.mh          | 0
 {contrib/gdb/gdb => gdb}/config/arm/nm-nbsd.h           | 0
 {contrib/gdb/gdb => gdb}/config/arm/nm-nbsdaout.h       | 0
 {contrib/gdb/gdb => gdb}/config/arm/tm-arm.h            | 0
 {contrib/gdb/gdb => gdb}/config/arm/tm-embed.h          | 0
 {contrib/gdb/gdb => gdb}/config/arm/tm-nbsd.h           | 0
 {contrib/gdb/gdb => gdb}/config/arm/tm-wince.h          | 0
 {contrib/gdb/gdb => gdb}/config/arm/wince.mt            | 0
 {contrib/gdb/gdb => gdb}/config/arm/xm-nbsd.h           | 0
 {contrib/gdb/gdb => gdb}/config/i386/embed.mt           | 0
 {contrib/gdb/gdb => gdb}/config/i386/fbsd.mh            | 0
 {contrib/gdb/gdb => gdb}/config/i386/fbsd.mt            | 0
 {contrib/gdb/gdb => gdb}/config/i386/fbsd64.mh          | 0
 {contrib/gdb/gdb => gdb}/config/i386/fbsd64.mt          | 0
 {contrib/gdb/gdb => gdb}/config/i386/go32.mh            | 0
 {contrib/gdb/gdb => gdb}/config/i386/go32.mt            | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386aout.mt        | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386gnu.mh         | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386gnu.mt         | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386lynx.mh        | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386lynx.mt        | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386nw.mt          | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386sco.mh         | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386sco4.mh        | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386sco5.mh        | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386sol2.mh        | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386sol2.mt        | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386v.mh           | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386v.mt           | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386v4.mh          | 0
 {contrib/gdb/gdb => gdb}/config/i386/i386v42mp.mh       | 0
 {contrib/gdb/gdb => gdb}/config/i386/nbsd.mt            | 0
 {contrib/gdb/gdb => gdb}/config/i386/nbsd64.mh          | 0
 {contrib/gdb/gdb => gdb}/config/i386/nbsd64.mt          | 0
 {contrib/gdb/gdb => gdb}/config/i386/nbsdaout.mh        | 0
 {contrib/gdb/gdb => gdb}/config/i386/nbsdelf.mh         | 0
 {contrib/gdb/gdb => gdb}/config/i386/ncr3000.mh         | 0
 {contrib/gdb/gdb => gdb}/config/i386/ncr3000.mt         | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-fbsd.h          | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-fbsd64.h        | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-go32.h          | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-i386.h          | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-i386gnu.h       | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-i386lynx.h      | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-i386sco.h       | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-i386sco4.h      | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-i386sco5.h      | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-i386sol2.h      | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-i386v.h         | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-i386v4.h        | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-i386v42mp.h     | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-nbsd.h          | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-nbsdaout.h      | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-nto.h           | 0
 {contrib/gdb/gdb => gdb}/config/i386/nm-obsd.h          | 0
 {contrib/gdb/gdb => gdb}/config/i386/nto.mh             | 0
 {contrib/gdb/gdb => gdb}/config/i386/nto.mt             | 0
 {contrib/gdb/gdb => gdb}/config/i386/obsd.mh            | 0
 {contrib/gdb/gdb => gdb}/config/i386/obsd.mt            | 0
 {contrib/gdb/gdb => gdb}/config/i386/obsd64.mh          | 0
 {contrib/gdb/gdb => gdb}/config/i386/obsd64.mt          | 0
 {contrib/gdb/gdb => gdb}/config/i386/obsdaout.mh        | 0
 {contrib/gdb/gdb => gdb}/config/i386/tm-fbsd.h          | 0
 {contrib/gdb/gdb => gdb}/config/i386/tm-go32.h          | 0
 {contrib/gdb/gdb => gdb}/config/i386/tm-i386.h          | 0
 {contrib/gdb/gdb => gdb}/config/i386/tm-i386lynx.h      | 0
 {contrib/gdb/gdb => gdb}/config/i386/tm-i386sol2.h      | 0
 {contrib/gdb/gdb => gdb}/config/i386/tm-nbsd.h          | 0
 {contrib/gdb/gdb => gdb}/config/i386/tm-nto.h           | 0
 {contrib/gdb/gdb => gdb}/config/i386/tm-vxworks.h       | 0
 {contrib/gdb/gdb => gdb}/config/i386/vxworks.mt         | 0
 {contrib/gdb/gdb => gdb}/config/i386/xm-go32.h          | 0
 {contrib/gdb/gdb => gdb}/config/i386/xm-i386.h          | 0
 {contrib/gdb/gdb => gdb}/config/i386/xm-i386sco.h       | 0
 {contrib/gdb/gdb => gdb}/config/i386/xm-i386v.h         | 0
 {contrib/gdb/gdb => gdb}/config/i386/xm-i386v4.h        | 0
 {contrib/gdb/gdb => gdb}/config/i386/xm-nbsd.h          | 0
 {contrib/gdb/gdb => gdb}/config/ia64/fbsd.mh            | 0
 {contrib/gdb/gdb => gdb}/config/ia64/fbsd.mt            | 0
 {contrib/gdb/gdb => gdb}/config/ia64/ia64.mt            | 0
 {contrib/gdb/gdb => gdb}/config/ia64/nm-fbsd.h          | 0
 {contrib/gdb/gdb => gdb}/config/ia64/tm-fbsd.h          | 0
 {contrib/gdb/gdb => gdb}/config/ia64/tm-ia64.h          | 0
 {contrib/gdb/gdb => gdb}/config/mips/decstation.mh      | 0
 {contrib/gdb/gdb => gdb}/config/mips/embed.mt           | 0
 {contrib/gdb/gdb => gdb}/config/mips/littlemips.mh      | 0
 {contrib/gdb/gdb => gdb}/config/mips/mipsv4.mh          | 0
 {contrib/gdb/gdb => gdb}/config/mips/mipsv4.mt          | 0
 {contrib/gdb/gdb => gdb}/config/mips/nbsd.mh            | 0
 {contrib/gdb/gdb => gdb}/config/mips/nbsd.mt            | 0
 {contrib/gdb/gdb => gdb}/config/mips/news-mips.mh       | 0
 {contrib/gdb/gdb => gdb}/config/mips/nm-mips.h          | 0
 {contrib/gdb/gdb => gdb}/config/mips/nm-nbsd.h          | 0
 {contrib/gdb/gdb => gdb}/config/mips/nm-news-mips.h     | 0
 {contrib/gdb/gdb => gdb}/config/mips/nm-riscos.h        | 0
 {contrib/gdb/gdb => gdb}/config/mips/riscos.mh          | 0
 {contrib/gdb/gdb => gdb}/config/mips/tm-mips.h          | 0
 {contrib/gdb/gdb => gdb}/config/mips/tm-mipsv4.h        | 0
 {contrib/gdb/gdb => gdb}/config/mips/tm-nbsd.h          | 0
 {contrib/gdb/gdb => gdb}/config/mips/tm-vxmips.h        | 0
 {contrib/gdb/gdb => gdb}/config/mips/tm-wince.h         | 0
 {contrib/gdb/gdb => gdb}/config/mips/vxmips.mt          | 0
 {contrib/gdb/gdb => gdb}/config/mips/wince.mt           | 0
 {contrib/gdb/gdb => gdb}/config/mips/xm-mips.h          | 0
 {contrib/gdb/gdb => gdb}/config/mips/xm-mipsv4.h        | 0
 {contrib/gdb/gdb => gdb}/config/mips/xm-riscos.h        | 0
 {contrib/gdb/gdb => gdb}/config/nm-gnu.h                | 0
 {contrib/gdb/gdb => gdb}/config/nm-lynx.h               | 0
 {contrib/gdb/gdb => gdb}/config/nm-nbsd.h               | 0
 {contrib/gdb/gdb => gdb}/config/nm-nbsdaout.h           | 0
 {contrib/gdb/gdb => gdb}/config/nm-sysv4.h              | 0
 {contrib/gdb/gdb => gdb}/config/powerpc/fbsd.mh         | 0
 {contrib/gdb/gdb => gdb}/config/powerpc/fbsd.mt         | 0
 {contrib/gdb/gdb => gdb}/config/powerpc/nbsd.mh         | 0
 {contrib/gdb/gdb => gdb}/config/powerpc/nbsd.mt         | 0
 {contrib/gdb/gdb => gdb}/config/powerpc/nm-fbsd.h       | 0
 {contrib/gdb/gdb => gdb}/config/powerpc/nm-nbsd.h       | 0
 {contrib/gdb/gdb => gdb}/config/powerpc/ppc-eabi.mt     | 0
 {contrib/gdb/gdb => gdb}/config/powerpc/ppc-sim.mt      | 0
 {contrib/gdb/gdb => gdb}/config/powerpc/tm-nbsd.h       | 0
 {contrib/gdb/gdb => gdb}/config/powerpc/tm-ppc-eabi.h   | 0
 {contrib/gdb/gdb => gdb}/config/powerpc/tm-ppcle-eabi.h | 0
 {contrib/gdb/gdb => gdb}/config/powerpc/tm-ppcle-sim.h  | 0
 {contrib/gdb/gdb => gdb}/config/powerpc/tm-vxworks.h    | 0
 {contrib/gdb/gdb => gdb}/config/powerpc/vxworks.mt      | 0
 {contrib/gdb/gdb => gdb}/config/rs6000/nm-rs6000.h      | 0
 {contrib/gdb/gdb => gdb}/config/rs6000/nm-rs6000ly.h    | 0
 {contrib/gdb/gdb => gdb}/config/rs6000/rs6000.mh        | 0
 {contrib/gdb/gdb => gdb}/config/rs6000/rs6000.mt        | 0
 {contrib/gdb/gdb => gdb}/config/rs6000/rs6000lynx.mh    | 0
 {contrib/gdb/gdb => gdb}/config/rs6000/rs6000lynx.mt    | 0
 {contrib/gdb/gdb => gdb}/config/rs6000/tm-rs6000.h      | 0
 {contrib/gdb/gdb => gdb}/config/rs6000/tm-rs6000ly.h    | 0
 {contrib/gdb/gdb => gdb}/config/rs6000/xm-rs6000.h      | 0
 {contrib/gdb/gdb => gdb}/config/s390/s390.mh            | 0
 {contrib/gdb/gdb => gdb}/config/s390/s390.mt            | 0
 {contrib/gdb/gdb => gdb}/config/sparc/fbsd.mh           | 0
 {contrib/gdb/gdb => gdb}/config/sparc/fbsd.mt           | 0
 {contrib/gdb/gdb => gdb}/config/sparc/nbsd.mt           | 0
 {contrib/gdb/gdb => gdb}/config/sparc/nbsd64.mh         | 0
 {contrib/gdb/gdb => gdb}/config/sparc/nbsd64.mt         | 0
 {contrib/gdb/gdb => gdb}/config/sparc/nbsdaout.mh       | 0
 {contrib/gdb/gdb => gdb}/config/sparc/nbsdelf.mh        | 0
 {contrib/gdb/gdb => gdb}/config/sparc/nm-fbsd.h         | 0
 {contrib/gdb/gdb => gdb}/config/sparc/nm-nbsd.h         | 0
 {contrib/gdb/gdb => gdb}/config/sparc/nm-nbsdaout.h     | 0
 {contrib/gdb/gdb => gdb}/config/sparc/nm-sol2.h         | 0
 {contrib/gdb/gdb => gdb}/config/sparc/obsd.mt           | 0
 {contrib/gdb/gdb => gdb}/config/sparc/obsd64.mt         | 0
 {contrib/gdb/gdb => gdb}/config/sparc/sol2-64.mt        | 0
 {contrib/gdb/gdb => gdb}/config/sparc/sol2.mh           | 0
 {contrib/gdb/gdb => gdb}/config/sparc/sol2.mt           | 0
 {contrib/gdb/gdb => gdb}/config/sparc/sparc.mt          | 0
 {contrib/gdb/gdb => gdb}/config/sparc/sparc64.mt        | 0
 {contrib/gdb/gdb => gdb}/config/sparc/tm-fbsd.h         | 0
 {contrib/gdb/gdb => gdb}/config/sparc/tm-nbsd.h         | 0
 {contrib/gdb/gdb => gdb}/config/sparc/tm-nbsd64.h       | 0
 {contrib/gdb/gdb => gdb}/config/sparc/tm-sol2.h         | 0
 {contrib/gdb/gdb => gdb}/config/sparc/tm-vxworks.h      | 0
 {contrib/gdb/gdb => gdb}/config/sparc/vxworks.mt        | 0
 {contrib/gdb/gdb => gdb}/config/tm-lynx.h               | 0
 {contrib/gdb/gdb => gdb}/config/tm-nto.h                | 0
 {contrib/gdb/gdb => gdb}/config/tm-sunos.h              | 0
 {contrib/gdb/gdb => gdb}/config/tm-sysv4.h              | 0
 {contrib/gdb/gdb => gdb}/config/tm-vxworks.h            | 0
 {contrib/gdb/gdb => gdb}/config/xm-nbsd.h               | 0
 {contrib/gdb/gdb => gdb}/config/xm-sysv4.h              | 0
 {contrib/gdb/gdb => gdb}/copying.awk                    | 0
 {contrib/gdb/gdb => gdb}/copying.c                      | 0
 {contrib/gdb/gdb => gdb}/core-aout.c                    | 0
 {contrib/gdb/gdb => gdb}/core-regset.c                  | 0
 {contrib/gdb/gdb => gdb}/corefile.c                     | 0
 {contrib/gdb/gdb => gdb}/corelow.c                      | 0
 {contrib/gdb/gdb => gdb}/cp-abi.c                       | 0
 {contrib/gdb/gdb => gdb}/cp-abi.h                       | 0
 {contrib/gdb/gdb => gdb}/cp-namespace.c                 | 0
 {contrib/gdb/gdb => gdb}/cp-support.c                   | 0
 {contrib/gdb/gdb => gdb}/cp-support.h                   | 0
 {contrib/gdb/gdb => gdb}/cp-valprint.c                  | 0
 {contrib/gdb/gdb => gdb}/cpu32bug-rom.c                 | 0
 {contrib/gdb/gdb => gdb}/dbug-rom.c                     | 0
 {contrib/gdb/gdb => gdb}/dbxread.c                      | 0
 {contrib/gdb/gdb => gdb}/dcache.c                       | 0
 {contrib/gdb/gdb => gdb}/dcache.h                       | 0
 {contrib/gdb/gdb => gdb}/defs.h                         | 0
 {contrib/gdb/gdb => gdb}/delta68-nat.c                  | 0
 {contrib/gdb/gdb => gdb}/demangle.c                     | 0
 {contrib/gdb/gdb => gdb}/dictionary.c                   | 0
 {contrib/gdb/gdb => gdb}/dictionary.h                   | 0
 {contrib/gdb/gdb => gdb}/dink32-rom.c                   | 0
 {contrib/gdb/gdb => gdb}/disasm.c                       | 0
 {contrib/gdb/gdb => gdb}/disasm.h                       | 0
 {contrib/gdb/gdb => gdb}/doc/GDBvn.texi                 | 0
 {contrib/gdb/gdb => gdb}/doc/LRS                        | 0
 {contrib/gdb/gdb => gdb}/doc/a4rc.sed                   | 0
 {contrib/gdb/gdb => gdb}/doc/agentexpr.texi             | 0
 {contrib/gdb/gdb => gdb}/doc/all-cfg.texi               | 0
 {contrib/gdb/gdb => gdb}/doc/annotate.texinfo           | 0
 {contrib/gdb/gdb => gdb}/doc/fdl.texi                   | 0
 {contrib/gdb/gdb => gdb}/doc/gdb.info-1                 | 0
 {contrib/gdb/gdb => gdb}/doc/gdb.info-2                 | 0
 {contrib/gdb/gdb => gdb}/doc/gdb.info-3                 | 0
 {contrib/gdb/gdb => gdb}/doc/gdb.texinfo                | 0
 {contrib/gdb/gdb => gdb}/doc/gdbint.texinfo             | 0
 {contrib/gdb/gdb => gdb}/doc/gpl.texi                   | 0
 {contrib/gdb/gdb => gdb}/doc/lpsrc.sed                  | 0
 {contrib/gdb/gdb => gdb}/doc/observer.texi              | 0
 {contrib/gdb/gdb => gdb}/doc/psrc.sed                   | 0
 {contrib/gdb/gdb => gdb}/doc/refcard.tex                | 0
 {contrib/gdb/gdb => gdb}/doc/stabs.texinfo              | 0
 {contrib/gdb/gdb => gdb}/doublest.c                     | 0
 {contrib/gdb/gdb => gdb}/doublest.h                     | 0
 {contrib/gdb/gdb => gdb}/dpx2-nat.c                     | 0
 {contrib/gdb/gdb => gdb}/dsrec.c                        | 0
 {contrib/gdb/gdb => gdb}/dummy-frame.c                  | 0
 {contrib/gdb/gdb => gdb}/dummy-frame.h                  | 0
 {contrib/gdb/gdb => gdb}/dve3900-rom.c                  | 0
 {contrib/gdb/gdb => gdb}/dwarf2-frame.c                 | 0
 {contrib/gdb/gdb => gdb}/dwarf2-frame.h                 | 0
 {contrib/gdb/gdb => gdb}/dwarf2expr.c                   | 0
 {contrib/gdb/gdb => gdb}/dwarf2expr.h                   | 0
 {contrib/gdb/gdb => gdb}/dwarf2loc.c                    | 0
 {contrib/gdb/gdb => gdb}/dwarf2loc.h                    | 0
 {contrib/gdb/gdb => gdb}/dwarf2read.c                   | 0
 {contrib/gdb/gdb => gdb}/dwarfread.c                    | 0
 {contrib/gdb/gdb => gdb}/elfread.c                      | 0
 {contrib/gdb/gdb => gdb}/environ.c                      | 0
 {contrib/gdb/gdb => gdb}/environ.h                      | 0
 {contrib/gdb/gdb => gdb}/eval.c                         | 0
 {contrib/gdb/gdb => gdb}/event-loop.c                   | 0
 {contrib/gdb/gdb => gdb}/event-loop.h                   | 0
 {contrib/gdb/gdb => gdb}/event-top.c                    | 0
 {contrib/gdb/gdb => gdb}/event-top.h                    | 0
 {contrib/gdb/gdb => gdb}/exc_request.defs               | 0
 {contrib/gdb/gdb => gdb}/exec.c                         | 0
 {contrib/gdb/gdb => gdb}/exec.h                         | 0
 {contrib/gdb/gdb => gdb}/expprint.c                     | 0
 {contrib/gdb/gdb => gdb}/expression.h                   | 0
 {contrib/gdb/gdb => gdb}/f-exp.c                        | 0
 {contrib/gdb/gdb => gdb}/f-exp.y                        | 0
 {contrib/gdb/gdb => gdb}/f-lang.c                       | 0
 {contrib/gdb/gdb => gdb}/f-lang.h                       | 0
 {contrib/gdb/gdb => gdb}/f-typeprint.c                  | 0
 {contrib/gdb/gdb => gdb}/f-valprint.c                   | 0
 {contrib/gdb/gdb => gdb}/fbsd-proc.c                    | 0
 {contrib/gdb/gdb => gdb}/findvar.c                      | 0
 {contrib/gdb/gdb => gdb}/fork-child.c                   | 0
 {contrib/gdb/gdb => gdb}/frame-base.c                   | 0
 {contrib/gdb/gdb => gdb}/frame-base.h                   | 0
 {contrib/gdb/gdb => gdb}/frame-unwind.c                 | 0
 {contrib/gdb/gdb => gdb}/frame-unwind.h                 | 0
 {contrib/gdb/gdb => gdb}/frame.c                        | 0
 {contrib/gdb/gdb => gdb}/frame.h                        | 0
 {contrib/gdb/gdb => gdb}/gcore.c                        | 0
 {contrib/gdb/gdb => gdb}/gdb-events.c                   | 0
 {contrib/gdb/gdb => gdb}/gdb-events.h                   | 0
 {contrib/gdb/gdb => gdb}/gdb-events.sh                  | 0
 {contrib/gdb/gdb => gdb}/gdb-stabs.h                    | 0
 {contrib/gdb/gdb => gdb}/gdb.1                          | 0
 {contrib/gdb/gdb => gdb}/gdb.c                          | 0
 {contrib/gdb/gdb => gdb}/gdb.gdb                        | 0
 {contrib/gdb/gdb => gdb}/gdb.h                          | 0
 {contrib/gdb/gdb => gdb}/gdb_assert.h                   | 0
 {contrib/gdb/gdb => gdb}/gdb_curses.h                   | 0
 {contrib/gdb/gdb => gdb}/gdb_dirent.h                   | 0
 {contrib/gdb/gdb => gdb}/gdb_gcore.sh                   | 0
 {contrib/gdb/gdb => gdb}/gdb_indent.sh                  | 0
 {contrib/gdb/gdb => gdb}/gdb_locale.h                   | 0
 {contrib/gdb/gdb => gdb}/gdb_mbuild.sh                  | 0
 {contrib/gdb/gdb => gdb}/gdb_obstack.h                  | 0
 {contrib/gdb/gdb => gdb}/gdb_proc_service.h             | 0
 {contrib/gdb/gdb => gdb}/gdb_regex.h                    | 0
 {contrib/gdb/gdb => gdb}/gdb_stat.h                     | 0
 {contrib/gdb/gdb => gdb}/gdb_string.h                   | 0
 {contrib/gdb/gdb => gdb}/gdb_thread_db.h                | 0
 {contrib/gdb/gdb => gdb}/gdb_vfork.h                    | 0
 {contrib/gdb/gdb => gdb}/gdb_wait.h                     | 0
 {contrib/gdb/gdb => gdb}/gdbarch.c                      | 0
 {contrib/gdb/gdb => gdb}/gdbarch.h                      | 0
 {contrib/gdb/gdb => gdb}/gdbarch.sh                     | 0
 {contrib/gdb/gdb => gdb}/gdbcmd.h                       | 0
 {contrib/gdb/gdb => gdb}/gdbcore.h                      | 0
 {contrib/gdb/gdb => gdb}/gdbinit.in                     | 0
 {contrib/gdb/gdb => gdb}/gdbserver/README               | 0
 {contrib/gdb/gdb => gdb}/gdbserver/acinclude.m4         | 0
 {contrib/gdb/gdb => gdb}/gdbserver/aclocal.m4           | 0
 {contrib/gdb/gdb => gdb}/gdbserver/gdbreplay.c          | 0
 {contrib/gdb/gdb => gdb}/gdbserver/gdbserver.1          | 0
 {contrib/gdb/gdb => gdb}/gdbserver/i387-fp.c            | 0
 {contrib/gdb/gdb => gdb}/gdbserver/i387-fp.h            | 0
 {contrib/gdb/gdb => gdb}/gdbserver/inferiors.c          | 0
 {contrib/gdb/gdb => gdb}/gdbserver/mem-break.c          | 0
 {contrib/gdb/gdb => gdb}/gdbserver/mem-break.h          | 0
 {contrib/gdb/gdb => gdb}/gdbserver/proc-service.c       | 0
 {contrib/gdb/gdb => gdb}/gdbserver/regcache.c           | 0
 {contrib/gdb/gdb => gdb}/gdbserver/regcache.h           | 0
 {contrib/gdb/gdb => gdb}/gdbserver/remote-utils.c       | 0
 {contrib/gdb/gdb => gdb}/gdbserver/server.c             | 0
 {contrib/gdb/gdb => gdb}/gdbserver/server.h             | 0
 {contrib/gdb/gdb => gdb}/gdbserver/target.c             | 0
 {contrib/gdb/gdb => gdb}/gdbserver/target.h             | 0
 {contrib/gdb/gdb => gdb}/gdbserver/terminal.h           | 0
 {contrib/gdb/gdb => gdb}/gdbserver/thread-db.c          | 0
 {contrib/gdb/gdb => gdb}/gdbserver/utils.c              | 0
 {contrib/gdb/gdb => gdb}/gdbthread.h                    | 0
 {contrib/gdb/gdb => gdb}/gdbtypes.c                     | 0
 {contrib/gdb/gdb => gdb}/gdbtypes.h                     | 0
 {contrib/gdb/gdb => gdb}/glibc-tdep.c                   | 0
 {contrib/gdb/gdb => gdb}/glibc-tdep.h                   | 0
 {contrib/gdb/gdb => gdb}/gnu-nat.c                      | 0
 {contrib/gdb/gdb => gdb}/gnu-nat.h                      | 0
 {contrib/gdb/gdb => gdb}/gnu-v2-abi.c                   | 0
 {contrib/gdb/gdb => gdb}/gnu-v3-abi.c                   | 0
 {contrib/gdb/gdb => gdb}/go32-nat.c                     | 0
 {contrib/gdb/gdb => gdb}/gregset.h                      | 0
 {contrib/gdb/gdb => gdb}/hpacc-abi.c                    | 0
 {contrib/gdb/gdb => gdb}/hpread.c                       | 0
 {contrib/gdb/gdb => gdb}/i386-nat.c                     | 0
 {contrib/gdb/gdb => gdb}/i386-nto-tdep.c                | 0
 {contrib/gdb/gdb => gdb}/i386-sol2-tdep.c               | 0
 {contrib/gdb/gdb => gdb}/i386-stub.c                    | 0
 {contrib/gdb/gdb => gdb}/i386-tdep.c                    | 0
 {contrib/gdb/gdb => gdb}/i386-tdep.h                    | 0
 {contrib/gdb/gdb => gdb}/i386bsd-nat.c                  | 0
 {contrib/gdb/gdb => gdb}/i386bsd-tdep.c                 | 0
 {contrib/gdb/gdb => gdb}/i386fbsd-nat.c                 | 0
 {contrib/gdb/gdb => gdb}/i386fbsd-tdep.c                | 0
 {contrib/gdb/gdb => gdb}/i386gnu-nat.c                  | 0
 {contrib/gdb/gdb => gdb}/i386gnu-tdep.c                 | 0
 {contrib/gdb/gdb => gdb}/i386ly-tdep.c                  | 0
 {contrib/gdb/gdb => gdb}/i386nbsd-tdep.c                | 0
 {contrib/gdb/gdb => gdb}/i386obsd-nat.c                 | 0
 {contrib/gdb/gdb => gdb}/i386obsd-tdep.c                | 0
 {contrib/gdb/gdb => gdb}/i386v-nat.c                    | 0
 {contrib/gdb/gdb => gdb}/i386v4-nat.c                   | 0
 {contrib/gdb/gdb => gdb}/i387-tdep.c                    | 0
 {contrib/gdb/gdb => gdb}/i387-tdep.h                    | 0
 {contrib/gdb/gdb => gdb}/ia64-fbsd-nat.c                | 0
 {contrib/gdb/gdb => gdb}/ia64-fbsd-tdep.c               | 0
 {contrib/gdb/gdb => gdb}/ia64-tdep.c                    | 0
 {contrib/gdb/gdb => gdb}/ia64-tdep.h                    | 0
 {contrib/gdb/gdb => gdb}/inf-loop.c                     | 0
 {contrib/gdb/gdb => gdb}/inf-loop.h                     | 0
 {contrib/gdb/gdb => gdb}/infcall.c                      | 0
 {contrib/gdb/gdb => gdb}/infcall.h                      | 0
 {contrib/gdb/gdb => gdb}/infcmd.c                       | 0
 {contrib/gdb/gdb => gdb}/inferior.h                     | 0
 {contrib/gdb/gdb => gdb}/inflow.c                       | 0
 {contrib/gdb/gdb => gdb}/inflow.h                       | 0
 {contrib/gdb/gdb => gdb}/infptrace.c                    | 0
 {contrib/gdb/gdb => gdb}/infrun.c                       | 0
 {contrib/gdb/gdb => gdb}/inftarg.c                      | 0
 {contrib/gdb/gdb => gdb}/infttrace.c                    | 0
 {contrib/gdb/gdb => gdb}/infttrace.h                    | 0
 {contrib/gdb/gdb => gdb}/interps.c                      | 0
 {contrib/gdb/gdb => gdb}/interps.h                      | 0
 {contrib/gdb/gdb => gdb}/jv-exp.c                       | 0
 {contrib/gdb/gdb => gdb}/jv-exp.y                       | 0
 {contrib/gdb/gdb => gdb}/jv-lang.c                      | 0
 {contrib/gdb/gdb => gdb}/jv-lang.h                      | 0
 {contrib/gdb/gdb => gdb}/jv-typeprint.c                 | 0
 {contrib/gdb/gdb => gdb}/jv-valprint.c                  | 0
 {contrib/gdb/gdb => gdb}/kod-cisco.c                    | 0
 {contrib/gdb/gdb => gdb}/kod.c                          | 0
 {contrib/gdb/gdb => gdb}/kod.h                          | 0
 {contrib/gdb/gdb => gdb}/language.c                     | 0
 {contrib/gdb/gdb => gdb}/language.h                     | 0
 {contrib/gdb/gdb => gdb}/libunwind-frame.c              | 0
 {contrib/gdb/gdb => gdb}/libunwind-frame.h              | 0
 {contrib/gdb/gdb => gdb}/lin-lwp.c                      | 0
 {contrib/gdb/gdb => gdb}/linespec.c                     | 0
 {contrib/gdb/gdb => gdb}/linespec.h                     | 0
 {contrib/gdb/gdb => gdb}/lynx-nat.c                     | 0
 {contrib/gdb/gdb => gdb}/m2-exp.c                       | 0
 {contrib/gdb/gdb => gdb}/m2-exp.y                       | 0
 {contrib/gdb/gdb => gdb}/m2-lang.c                      | 0
 {contrib/gdb/gdb => gdb}/m2-lang.h                      | 0
 {contrib/gdb/gdb => gdb}/m2-typeprint.c                 | 0
 {contrib/gdb/gdb => gdb}/m2-valprint.c                  | 0
 {contrib/gdb/gdb => gdb}/macrocmd.c                     | 0
 {contrib/gdb/gdb => gdb}/macroexp.c                     | 0
 {contrib/gdb/gdb => gdb}/macroexp.h                     | 0
 {contrib/gdb/gdb => gdb}/macroscope.c                   | 0
 {contrib/gdb/gdb => gdb}/macroscope.h                   | 0
 {contrib/gdb/gdb => gdb}/macrotab.c                     | 0
 {contrib/gdb/gdb => gdb}/macrotab.h                     | 0
 {contrib/gdb/gdb => gdb}/main.c                         | 0
 {contrib/gdb/gdb => gdb}/main.h                         | 0
 {contrib/gdb/gdb => gdb}/maint.c                        | 0
 {contrib/gdb/gdb => gdb}/mdebugread.c                   | 0
 {contrib/gdb/gdb => gdb}/mem-break.c                    | 0
 {contrib/gdb/gdb => gdb}/memattr.c                      | 0
 {contrib/gdb/gdb => gdb}/memattr.h                      | 0
 {contrib/gdb/gdb => gdb}/mi/mi-cmd-break.c              | 0
 {contrib/gdb/gdb => gdb}/mi/mi-cmd-disas.c              | 0
 {contrib/gdb/gdb => gdb}/mi/mi-cmd-env.c                | 0
 {contrib/gdb/gdb => gdb}/mi/mi-cmd-file.c               | 0
 {contrib/gdb/gdb => gdb}/mi/mi-cmd-stack.c              | 0
 {contrib/gdb/gdb => gdb}/mi/mi-cmd-var.c                | 0
 {contrib/gdb/gdb => gdb}/mi/mi-cmds.c                   | 0
 {contrib/gdb/gdb => gdb}/mi/mi-cmds.h                   | 0
 {contrib/gdb/gdb => gdb}/mi/mi-console.c                | 0
 {contrib/gdb/gdb => gdb}/mi/mi-console.h                | 0
 {contrib/gdb/gdb => gdb}/mi/mi-getopt.c                 | 0
 {contrib/gdb/gdb => gdb}/mi/mi-getopt.h                 | 0
 {contrib/gdb/gdb => gdb}/mi/mi-interp.c                 | 0
 {contrib/gdb/gdb => gdb}/mi/mi-main.c                   | 0
 {contrib/gdb/gdb => gdb}/mi/mi-main.h                   | 0
 {contrib/gdb/gdb => gdb}/mi/mi-out.c                    | 0
 {contrib/gdb/gdb => gdb}/mi/mi-out.h                    | 0
 {contrib/gdb/gdb => gdb}/mi/mi-parse.c                  | 0
 {contrib/gdb/gdb => gdb}/mi/mi-parse.h                  | 0
 {contrib/gdb/gdb => gdb}/mi/mi-symbol-cmds.c            | 0
 {contrib/gdb/gdb => gdb}/minimon.h                      | 0
 {contrib/gdb/gdb => gdb}/minsyms.c                      | 0
 {contrib/gdb/gdb => gdb}/mips-nat.c                     | 0
 {contrib/gdb/gdb => gdb}/mips-tdep.c                    | 0
 {contrib/gdb/gdb => gdb}/mips-tdep.h                    | 0
 {contrib/gdb/gdb => gdb}/mipsnbsd-nat.c                 | 0
 {contrib/gdb/gdb => gdb}/mipsnbsd-tdep.c                | 0
 {contrib/gdb/gdb => gdb}/mipsnbsd-tdep.h                | 0
 {contrib/gdb/gdb => gdb}/mipsread.c                     | 0
 {contrib/gdb/gdb => gdb}/mipsv4-nat.c                   | 0
 {contrib/gdb/gdb => gdb}/monitor.c                      | 0
 {contrib/gdb/gdb => gdb}/monitor.h                      | 0
 {contrib/gdb/gdb => gdb}/msg.defs                       | 0
 {contrib/gdb/gdb => gdb}/msg_reply.defs                 | 0
 {contrib/gdb/gdb => gdb}/nbsd-tdep.c                    | 0
 {contrib/gdb/gdb => gdb}/nbsd-tdep.h                    | 0
 {contrib/gdb/gdb => gdb}/nlmread.c                      | 0
 {contrib/gdb/gdb => gdb}/notify.defs                    | 0
 {contrib/gdb/gdb => gdb}/nto-procfs.c                   | 0
 {contrib/gdb/gdb => gdb}/nto-tdep.c                     | 0
 {contrib/gdb/gdb => gdb}/nto-tdep.h                     | 0
 {contrib/gdb/gdb => gdb}/objc-exp.c                     | 0
 {contrib/gdb/gdb => gdb}/objc-exp.y                     | 0
 {contrib/gdb/gdb => gdb}/objc-lang.c                    | 0
 {contrib/gdb/gdb => gdb}/objc-lang.h                    | 0
 {contrib/gdb/gdb => gdb}/objfiles.c                     | 0
 {contrib/gdb/gdb => gdb}/objfiles.h                     | 0
 {contrib/gdb/gdb => gdb}/observer.c                     | 0
 {contrib/gdb/gdb => gdb}/observer.h                     | 0
 {contrib/gdb/gdb => gdb}/ocd.c                          | 0
 {contrib/gdb/gdb => gdb}/ocd.h                          | 0
 {contrib/gdb/gdb => gdb}/osabi.c                        | 0
 {contrib/gdb/gdb => gdb}/osabi.h                        | 0
 {contrib/gdb/gdb => gdb}/p-exp.c                        | 0
 {contrib/gdb/gdb => gdb}/p-exp.y                        | 0
 {contrib/gdb/gdb => gdb}/p-lang.c                       | 0
 {contrib/gdb/gdb => gdb}/p-lang.h                       | 0
 {contrib/gdb/gdb => gdb}/p-typeprint.c                  | 0
 {contrib/gdb/gdb => gdb}/p-valprint.c                   | 0
 {contrib/gdb/gdb => gdb}/pa64solib.c                    | 0
 {contrib/gdb/gdb => gdb}/pa64solib.h                    | 0
 {contrib/gdb/gdb => gdb}/parse.c                        | 0
 {contrib/gdb/gdb => gdb}/parser-defs.h                  | 0
 {contrib/gdb/gdb => gdb}/ppc-bdm.c                      | 0
 {contrib/gdb/gdb => gdb}/ppc-sysv-tdep.c                | 0
 {contrib/gdb/gdb => gdb}/ppc-tdep.h                     | 0
 {contrib/gdb/gdb => gdb}/ppcbug-rom.c                   | 0
 {contrib/gdb/gdb => gdb}/ppcfbsd-nat.c                  | 0
 {contrib/gdb/gdb => gdb}/ppcfbsd-tdep.c                 | 0
 {contrib/gdb/gdb => gdb}/ppcfbsd-tdep.h                 | 0
 {contrib/gdb/gdb => gdb}/ppcnbsd-nat.c                  | 0
 {contrib/gdb/gdb => gdb}/ppcnbsd-tdep.c                 | 0
 {contrib/gdb/gdb => gdb}/ppcnbsd-tdep.h                 | 0
 {contrib/gdb/gdb => gdb}/printcmd.c                     | 0
 {contrib/gdb/gdb => gdb}/proc-api.c                     | 0
 {contrib/gdb/gdb => gdb}/proc-events.c                  | 0
 {contrib/gdb/gdb => gdb}/proc-flags.c                   | 0
 {contrib/gdb/gdb => gdb}/proc-service.c                 | 0
 {contrib/gdb/gdb => gdb}/proc-utils.h                   | 0
 {contrib/gdb/gdb => gdb}/proc-why.c                     | 0
 {contrib/gdb/gdb => gdb}/process_reply.defs             | 0
 {contrib/gdb/gdb => gdb}/procfs.c                       | 0
 {contrib/gdb/gdb => gdb}/regcache.c                     | 0
 {contrib/gdb/gdb => gdb}/regcache.h                     | 0
 {contrib/gdb/gdb => gdb}/regformats/reg-arm.dat         | 0
 {contrib/gdb/gdb => gdb}/regformats/reg-i386.dat        | 0
 {contrib/gdb/gdb => gdb}/regformats/reg-ia64.dat        | 0
 {contrib/gdb/gdb => gdb}/regformats/reg-m68k.dat        | 0
 {contrib/gdb/gdb => gdb}/regformats/reg-mips.dat        | 0
 {contrib/gdb/gdb => gdb}/regformats/reg-ppc.dat         | 0
 {contrib/gdb/gdb => gdb}/regformats/reg-s390.dat        | 0
 {contrib/gdb/gdb => gdb}/regformats/reg-s390x.dat       | 0
 {contrib/gdb/gdb => gdb}/regformats/reg-sh.dat          | 0
 {contrib/gdb/gdb => gdb}/regformats/reg-x86-64.dat      | 0
 {contrib/gdb/gdb => gdb}/regformats/regdat.sh           | 0
 {contrib/gdb/gdb => gdb}/regformats/regdef.h            | 0
 {contrib/gdb/gdb => gdb}/reggroups.c                    | 0
 {contrib/gdb/gdb => gdb}/reggroups.h                    | 0
 {contrib/gdb/gdb => gdb}/regset.h                       | 0
 {contrib/gdb/gdb => gdb}/remote-e7000.c                 | 0
 {contrib/gdb/gdb => gdb}/remote-est.c                   | 0
 {contrib/gdb/gdb => gdb}/remote-fileio.c                | 0
 {contrib/gdb/gdb => gdb}/remote-fileio.h                | 0
 {contrib/gdb/gdb => gdb}/remote-hms.c                   | 0
 {contrib/gdb/gdb => gdb}/remote-mips.c                  | 0
 {contrib/gdb/gdb => gdb}/remote-rdi.c                   | 0
 {contrib/gdb/gdb => gdb}/remote-rdp.c                   | 0
 {contrib/gdb/gdb => gdb}/remote-sds.c                   | 0
 {contrib/gdb/gdb => gdb}/remote-sim.c                   | 0
 {contrib/gdb/gdb => gdb}/remote-st.c                    | 0
 {contrib/gdb/gdb => gdb}/remote-utils.c                 | 0
 {contrib/gdb/gdb => gdb}/remote-utils.h                 | 0
 {contrib/gdb/gdb => gdb}/remote-vx.c                    | 0
 {contrib/gdb/gdb => gdb}/remote-vx68.c                  | 0
 {contrib/gdb/gdb => gdb}/remote-vxmips.c                | 0
 {contrib/gdb/gdb => gdb}/remote-vxsparc.c               | 0
 {contrib/gdb/gdb => gdb}/remote.c                       | 0
 {contrib/gdb/gdb => gdb}/remote.h                       | 0
 {contrib/gdb/gdb => gdb}/reply_mig_hack.awk             | 0
 {contrib/gdb/gdb => gdb}/rom68k-rom.c                   | 0
 {contrib/gdb/gdb => gdb}/rs6000-nat.c                   | 0
 {contrib/gdb/gdb => gdb}/rs6000-tdep.c                  | 0
 {contrib/gdb/gdb => gdb}/s390-nat.c                     | 0
 {contrib/gdb/gdb => gdb}/s390-tdep.c                    | 0
 {contrib/gdb/gdb => gdb}/s390-tdep.h                    | 0
 {contrib/gdb/gdb => gdb}/scm-exp.c                      | 0
 {contrib/gdb/gdb => gdb}/scm-lang.c                     | 0
 {contrib/gdb/gdb => gdb}/scm-lang.h                     | 0
 {contrib/gdb/gdb => gdb}/scm-tags.h                     | 0
 {contrib/gdb/gdb => gdb}/scm-valprint.c                 | 0
 {contrib/gdb/gdb => gdb}/sentinel-frame.c               | 0
 {contrib/gdb/gdb => gdb}/sentinel-frame.h               | 0
 {contrib/gdb/gdb => gdb}/ser-e7kpc.c                    | 0
 {contrib/gdb/gdb => gdb}/ser-go32.c                     | 0
 {contrib/gdb/gdb => gdb}/ser-pipe.c                     | 0
 {contrib/gdb/gdb => gdb}/ser-tcp.c                      | 0
 {contrib/gdb/gdb => gdb}/ser-unix.c                     | 0
 {contrib/gdb/gdb => gdb}/ser-unix.h                     | 0
 {contrib/gdb/gdb => gdb}/serial.c                       | 0
 {contrib/gdb/gdb => gdb}/serial.h                       | 0
 {contrib/gdb/gdb => gdb}/signals/signals.c              | 0
 {contrib/gdb/gdb => gdb}/sim-regno.h                    | 0
 {contrib/gdb/gdb => gdb}/sol-thread.c                   | 0
 {contrib/gdb/gdb => gdb}/solib-legacy.c                 | 0
 {contrib/gdb/gdb => gdb}/solib-osf.c                    | 0
 {contrib/gdb/gdb => gdb}/solib-sunos.c                  | 0
 {contrib/gdb/gdb => gdb}/solib-svr4.c                   | 0
 {contrib/gdb/gdb => gdb}/solib-svr4.h                   | 0
 {contrib/gdb/gdb => gdb}/solib.c                        | 0
 {contrib/gdb/gdb => gdb}/solib.h                        | 0
 {contrib/gdb/gdb => gdb}/solist.h                       | 0
 {contrib/gdb/gdb => gdb}/somread.c                      | 0
 {contrib/gdb/gdb => gdb}/somsolib.c                     | 0
 {contrib/gdb/gdb => gdb}/somsolib.h                     | 0
 {contrib/gdb/gdb => gdb}/source.c                       | 0
 {contrib/gdb/gdb => gdb}/source.h                       | 0
 {contrib/gdb/gdb => gdb}/sparc-nat.c                    | 0
 {contrib/gdb/gdb => gdb}/sparc-nat.h                    | 0
 {contrib/gdb/gdb => gdb}/sparc-sol2-nat.c               | 0
 {contrib/gdb/gdb => gdb}/sparc-sol2-tdep.c              | 0
 {contrib/gdb/gdb => gdb}/sparc-stub.c                   | 0
 {contrib/gdb/gdb => gdb}/sparc-tdep.c                   | 0
 {contrib/gdb/gdb => gdb}/sparc-tdep.h                   | 0
 {contrib/gdb/gdb => gdb}/sparc64-nat.c                  | 0
 {contrib/gdb/gdb => gdb}/sparc64-sol2-tdep.c            | 0
 {contrib/gdb/gdb => gdb}/sparc64-tdep.c                 | 0
 {contrib/gdb/gdb => gdb}/sparc64-tdep.h                 | 0
 {contrib/gdb/gdb => gdb}/sparc64fbsd-nat.c              | 0
 {contrib/gdb/gdb => gdb}/sparc64fbsd-tdep.c             | 0
 {contrib/gdb/gdb => gdb}/sparc64nbsd-nat.c              | 0
 {contrib/gdb/gdb => gdb}/sparc64nbsd-tdep.c             | 0
 {contrib/gdb/gdb => gdb}/sparc64obsd-tdep.c             | 0
 {contrib/gdb/gdb => gdb}/sparcnbsd-nat.c                | 0
 {contrib/gdb/gdb => gdb}/sparcnbsd-tdep.c               | 0
 {contrib/gdb/gdb => gdb}/sparcobsd-tdep.c               | 0
 {contrib/gdb/gdb => gdb}/srec.h                         | 0
 {contrib/gdb/gdb => gdb}/stabsread.c                    | 0
 {contrib/gdb/gdb => gdb}/stabsread.h                    | 0
 {contrib/gdb/gdb => gdb}/stack.c                        | 0
 {contrib/gdb/gdb => gdb}/stack.h                        | 0
 {contrib/gdb/gdb => gdb}/standalone.c                   | 0
 {contrib/gdb/gdb => gdb}/std-regs.c                     | 0
 {contrib/gdb/gdb => gdb}/stop-gdb.c                     | 0
 {contrib/gdb/gdb => gdb}/sun3-nat.c                     | 0
 {contrib/gdb/gdb => gdb}/symfile.c                      | 0
 {contrib/gdb/gdb => gdb}/symfile.h                      | 0
 {contrib/gdb/gdb => gdb}/symmisc.c                      | 0
 {contrib/gdb/gdb => gdb}/symtab.c                       | 0
 {contrib/gdb/gdb => gdb}/symtab.h                       | 0
 {contrib/gdb/gdb => gdb}/target.c                       | 0
 {contrib/gdb/gdb => gdb}/target.h                       | 0
 {contrib/gdb/gdb => gdb}/terminal.h                     | 0
 {contrib/gdb/gdb => gdb}/thread-db.c                    | 0
 {contrib/gdb/gdb => gdb}/thread.c                       | 0
 {contrib/gdb/gdb => gdb}/top.c                          | 0
 {contrib/gdb/gdb => gdb}/top.h                          | 0
 {contrib/gdb/gdb => gdb}/tracepoint.c                   | 0
 {contrib/gdb/gdb => gdb}/tracepoint.h                   | 0
 {contrib/gdb/gdb => gdb}/trad-frame.c                   | 0
 {contrib/gdb/gdb => gdb}/trad-frame.h                   | 0
 {contrib/gdb/gdb => gdb}/tui/tui-command.c              | 0
 {contrib/gdb/gdb => gdb}/tui/tui-command.h              | 0
 {contrib/gdb/gdb => gdb}/tui/tui-data.c                 | 0
 {contrib/gdb/gdb => gdb}/tui/tui-data.h                 | 0
 {contrib/gdb/gdb => gdb}/tui/tui-disasm.c               | 0
 {contrib/gdb/gdb => gdb}/tui/tui-disasm.h               | 0
 {contrib/gdb/gdb => gdb}/tui/tui-file.c                 | 0
 {contrib/gdb/gdb => gdb}/tui/tui-file.h                 | 0
 {contrib/gdb/gdb => gdb}/tui/tui-hooks.c                | 0
 {contrib/gdb/gdb => gdb}/tui/tui-hooks.h                | 0
 {contrib/gdb/gdb => gdb}/tui/tui-interp.c               | 0
 {contrib/gdb/gdb => gdb}/tui/tui-io.c                   | 0
 {contrib/gdb/gdb => gdb}/tui/tui-io.h                   | 0
 {contrib/gdb/gdb => gdb}/tui/tui-layout.c               | 0
 {contrib/gdb/gdb => gdb}/tui/tui-layout.h               | 0
 {contrib/gdb/gdb => gdb}/tui/tui-main.c                 | 0
 {contrib/gdb/gdb => gdb}/tui/tui-out.c                  | 0
 {contrib/gdb/gdb => gdb}/tui/tui-regs.c                 | 0
 {contrib/gdb/gdb => gdb}/tui/tui-regs.h                 | 0
 {contrib/gdb/gdb => gdb}/tui/tui-source.c               | 0
 {contrib/gdb/gdb => gdb}/tui/tui-source.h               | 0
 {contrib/gdb/gdb => gdb}/tui/tui-stack.c                | 0
 {contrib/gdb/gdb => gdb}/tui/tui-stack.h                | 0
 {contrib/gdb/gdb => gdb}/tui/tui-win.c                  | 0
 {contrib/gdb/gdb => gdb}/tui/tui-win.h                  | 0
 {contrib/gdb/gdb => gdb}/tui/tui-windata.c              | 0
 {contrib/gdb/gdb => gdb}/tui/tui-windata.h              | 0
 {contrib/gdb/gdb => gdb}/tui/tui-wingeneral.c           | 0
 {contrib/gdb/gdb => gdb}/tui/tui-wingeneral.h           | 0
 {contrib/gdb/gdb => gdb}/tui/tui-winsource.c            | 0
 {contrib/gdb/gdb => gdb}/tui/tui-winsource.h            | 0
 {contrib/gdb/gdb => gdb}/tui/tui.c                      | 0
 {contrib/gdb/gdb => gdb}/tui/tui.h                      | 0
 {contrib/gdb/gdb => gdb}/typeprint.c                    | 0
 {contrib/gdb/gdb => gdb}/typeprint.h                    | 0
 {contrib/gdb/gdb => gdb}/ui-file.c                      | 0
 {contrib/gdb/gdb => gdb}/ui-file.h                      | 0
 {contrib/gdb/gdb => gdb}/ui-out.c                       | 0
 {contrib/gdb/gdb => gdb}/ui-out.h                       | 0
 {contrib/gdb/gdb => gdb}/user-regs.c                    | 0
 {contrib/gdb/gdb => gdb}/user-regs.h                    | 0
 {contrib/gdb/gdb => gdb}/utils.c                        | 0
 {contrib/gdb/gdb => gdb}/uw-thread.c                    | 0
 {contrib/gdb/gdb => gdb}/valarith.c                     | 0
 {contrib/gdb/gdb => gdb}/valops.c                       | 0
 {contrib/gdb/gdb => gdb}/valprint.c                     | 0
 {contrib/gdb/gdb => gdb}/valprint.h                     | 0
 {contrib/gdb/gdb => gdb}/value.h                        | 0
 {contrib/gdb/gdb => gdb}/values.c                       | 0
 {contrib/gdb/gdb => gdb}/varobj.c                       | 0
 {contrib/gdb/gdb => gdb}/varobj.h                       | 0
 {contrib/gdb/gdb => gdb}/version.h                      | 0
 {contrib/gdb/gdb => gdb}/version.in                     | 0
 {contrib/gdb/gdb => gdb}/win32-nat.c                    | 0
 {contrib/gdb/gdb => gdb}/wince-stub.c                   | 0
 {contrib/gdb/gdb => gdb}/wince-stub.h                   | 0
 {contrib/gdb/gdb => gdb}/wince.c                        | 0
 {contrib/gdb/gdb => gdb}/wrapper.c                      | 0
 {contrib/gdb/gdb => gdb}/wrapper.h                      | 0
 {contrib/gdb/gdb => gdb}/xcoffread.c                    | 0
 {contrib/gdb/gdb => gdb}/xcoffsolib.c                   | 0
 {contrib/gdb/gdb => gdb}/xcoffsolib.h                   | 0
 {contrib/gdb/gdb => gdb}/xmodem.c                       | 0
 {contrib/gdb/gdb => gdb}/xmodem.h                       | 0
 contrib/gdb/gettext.m4 => gettext.m4                    | 0
 {contrib/gdb/include => include}/COPYING                | 0
 {contrib/gdb/include => include}/MAINTAINERS            | 0
 {contrib/gdb/include => include}/alloca-conf.h          | 0
 {contrib/gdb/include => include}/ansidecl.h             | 0
 {contrib/gdb/include => include}/bfdlink.h              | 0
 {contrib/gdb/include => include}/bin-bugs.h             | 0
 {contrib/gdb/include => include}/bout.h                 | 0
 {contrib/gdb/include => include}/demangle.h             | 0
 {contrib/gdb/include => include}/dis-asm.h              | 0
 {contrib/gdb/include => include}/dyn-string.h           | 0
 {contrib/gdb/include => include}/fibheap.h              | 0
 {contrib/gdb/include => include}/filenames.h            | 0
 {contrib/gdb/include => include}/floatformat.h          | 0
 {contrib/gdb/include => include}/fnmatch.h              | 0
 {contrib/gdb/include => include}/fopen-bin.h            | 0
 {contrib/gdb/include => include}/fopen-same.h           | 0
 {contrib/gdb/include => include}/fopen-vms.h            | 0
 {contrib/gdb/include => include}/gdb/callback.h         | 0
 {contrib/gdb/include => include}/gdb/fileio.h           | 0
 {contrib/gdb/include => include}/gdb/remote-sim.h       | 0
 {contrib/gdb/include => include}/gdb/signals.h          | 0
 {contrib/gdb/include => include}/gdb/sim-arm.h          | 0
 {contrib/gdb/include => include}/gdb/sim-d10v.h         | 0
 {contrib/gdb/include => include}/gdb/sim-frv.h          | 0
 {contrib/gdb/include => include}/gdb/sim-h8300.h        | 0
 {contrib/gdb/include => include}/gdb/sim-sh.h           | 0
 {contrib/gdb/include => include}/gdbm.h                 | 0
 {contrib/gdb/include => include}/getopt.h               | 0
 {contrib/gdb/include => include}/hashtab.h              | 0
 {contrib/gdb/include => include}/hp-symtab.h            | 0
 {contrib/gdb/include => include}/ieee.h                 | 0
 {contrib/gdb/include => include}/libiberty.h            | 0
 {contrib/gdb/include => include}/md5.h                  | 0
 {contrib/gdb/include => include}/oasys.h                | 0
 {contrib/gdb/include => include}/objalloc.h             | 0
 {contrib/gdb/include => include}/obstack.h              | 0
 {contrib/gdb/include => include}/os9k.h                 | 0
 {contrib/gdb/include => include}/partition.h            | 0
 {contrib/gdb/include => include}/progress.h             | 0
 {contrib/gdb/include => include}/safe-ctype.h           | 0
 {contrib/gdb/include => include}/sort.h                 | 0
 {contrib/gdb/include => include}/splay-tree.h           | 0
 {contrib/gdb/include => include}/symcat.h               | 0
 {contrib/gdb/include => include}/ternary.h              | 0
 {contrib/gdb/include => include}/xregex.h               | 0
 {contrib/gdb/include => include}/xregex2.h              | 0
 {contrib/gdb/include => include}/xtensa-isa-internal.h  | 0
 {contrib/gdb/include => include}/xtensa-isa.h           | 0
 contrib/gdb/install-sh => install-sh                    | 0
 contrib/gdb/libtool.m4 => libtool.m4                    | 0
 contrib/gdb/ltcf-c.sh => ltcf-c.sh                      | 0
 contrib/gdb/ltcf-cxx.sh => ltcf-cxx.sh                  | 0
 contrib/gdb/ltcf-gcj.sh => ltcf-gcj.sh                  | 0
 contrib/gdb/ltconfig => ltconfig                        | 0
 contrib/gdb/ltmain.sh => ltmain.sh                      | 0
 contrib/gdb/md5.sum => md5.sum                          | 0
 contrib/gdb/missing => missing                          | 0
 contrib/gdb/mkinstalldirs => mkinstalldirs              | 0
 contrib/gdb/move-if-change => move-if-change            | 0
 contrib/gdb/src-release => src-release                  | 0
 contrib/gdb/symlink-tree => symlink-tree                | 0
 contrib/gdb/ylwrap => ylwrap                            | 0
 843 files changed, 0 insertions(+), 0 deletions(-)
 rename contrib/gdb/COPYING => COPYING (100%)
 rename contrib/gdb/COPYING.LIB => COPYING.LIB (100%)
 rename contrib/gdb/FREEBSD-diffs => FREEBSD-diffs (100%)
 rename contrib/gdb/README => README (100%)
 rename contrib/gdb/config-ml.in => config-ml.in (100%)
 rename contrib/gdb/djunpack.bat => djunpack.bat (100%)
 rename {contrib/gdb/gdb => gdb}/CONTRIBUTE (100%)
 rename {contrib/gdb/gdb => gdb}/COPYING (100%)
 rename {contrib/gdb/gdb => gdb}/MAINTAINERS (100%)
 rename {contrib/gdb/gdb => gdb}/NEWS (100%)
 rename {contrib/gdb/gdb => gdb}/PROBLEMS (100%)
 rename {contrib/gdb/gdb => gdb}/README (100%)
 rename {contrib/gdb/gdb => gdb}/TODO (100%)
 rename {contrib/gdb/gdb => gdb}/abug-rom.c (100%)
 rename {contrib/gdb/gdb => gdb}/acinclude.m4 (100%)
 rename {contrib/gdb/gdb => gdb}/aclocal.m4 (100%)
 rename {contrib/gdb/gdb => gdb}/ada-exp.c (100%)
 rename {contrib/gdb/gdb => gdb}/ada-exp.y (100%)
 rename {contrib/gdb/gdb => gdb}/ada-lang.c (100%)
 rename {contrib/gdb/gdb => gdb}/ada-lang.h (100%)
 rename {contrib/gdb/gdb => gdb}/ada-lex.l (100%)
 rename {contrib/gdb/gdb => gdb}/ada-tasks.c (100%)
 rename {contrib/gdb/gdb => gdb}/ada-typeprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/ada-valprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/alpha-mdebug-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/alpha-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/alpha-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/alpha-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/alphabsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/alphabsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/alphabsd-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/alphafbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/alphanbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/amd64-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/amd64-nat.h (100%)
 rename {contrib/gdb/gdb => gdb}/amd64-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/amd64-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/amd64bsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/amd64fbsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/amd64fbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/amd64nbsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/amd64nbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/amd64obsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/amd64obsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/annotate.c (100%)
 rename {contrib/gdb/gdb => gdb}/annotate.h (100%)
 rename {contrib/gdb/gdb => gdb}/arch-utils.c (100%)
 rename {contrib/gdb/gdb => gdb}/arch-utils.h (100%)
 rename {contrib/gdb/gdb => gdb}/arm-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/arm-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/armnbsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/armnbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/auxv.c (100%)
 rename {contrib/gdb/gdb => gdb}/auxv.h (100%)
 rename {contrib/gdb/gdb => gdb}/ax-gdb.c (100%)
 rename {contrib/gdb/gdb => gdb}/ax-gdb.h (100%)
 rename {contrib/gdb/gdb => gdb}/ax-general.c (100%)
 rename {contrib/gdb/gdb => gdb}/ax.h (100%)
 rename {contrib/gdb/gdb => gdb}/bcache.c (100%)
 rename {contrib/gdb/gdb => gdb}/bcache.h (100%)
 rename {contrib/gdb/gdb => gdb}/bfd-target.c (100%)
 rename {contrib/gdb/gdb => gdb}/bfd-target.h (100%)
 rename {contrib/gdb/gdb => gdb}/block.c (100%)
 rename {contrib/gdb/gdb => gdb}/block.h (100%)
 rename {contrib/gdb/gdb => gdb}/blockframe.c (100%)
 rename {contrib/gdb/gdb => gdb}/breakpoint.c (100%)
 rename {contrib/gdb/gdb => gdb}/breakpoint.h (100%)
 rename {contrib/gdb/gdb => gdb}/buildsym.c (100%)
 rename {contrib/gdb/gdb => gdb}/buildsym.h (100%)
 rename {contrib/gdb/gdb => gdb}/c-exp.c (100%)
 rename {contrib/gdb/gdb => gdb}/c-exp.y (100%)
 rename {contrib/gdb/gdb => gdb}/c-lang.c (100%)
 rename {contrib/gdb/gdb => gdb}/c-lang.h (100%)
 rename {contrib/gdb/gdb => gdb}/c-typeprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/c-valprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/call-cmds.h (100%)
 rename {contrib/gdb/gdb => gdb}/charset.c (100%)
 rename {contrib/gdb/gdb => gdb}/charset.h (100%)
 rename {contrib/gdb/gdb => gdb}/cli-out.c (100%)
 rename {contrib/gdb/gdb => gdb}/cli-out.h (100%)
 rename {contrib/gdb/gdb => gdb}/cli/cli-cmds.c (100%)
 rename {contrib/gdb/gdb => gdb}/cli/cli-cmds.h (100%)
 rename {contrib/gdb/gdb => gdb}/cli/cli-decode.c (100%)
 rename {contrib/gdb/gdb => gdb}/cli/cli-decode.h (100%)
 rename {contrib/gdb/gdb => gdb}/cli/cli-dump.c (100%)
 rename {contrib/gdb/gdb => gdb}/cli/cli-dump.h (100%)
 rename {contrib/gdb/gdb => gdb}/cli/cli-interp.c (100%)
 rename {contrib/gdb/gdb => gdb}/cli/cli-logging.c (100%)
 rename {contrib/gdb/gdb => gdb}/cli/cli-script.c (100%)
 rename {contrib/gdb/gdb => gdb}/cli/cli-script.h (100%)
 rename {contrib/gdb/gdb => gdb}/cli/cli-setshow.c (100%)
 rename {contrib/gdb/gdb => gdb}/cli/cli-setshow.h (100%)
 rename {contrib/gdb/gdb => gdb}/cli/cli-utils.c (100%)
 rename {contrib/gdb/gdb => gdb}/cli/cli-utils.h (100%)
 rename {contrib/gdb/gdb => gdb}/coff-pe-read.c (100%)
 rename {contrib/gdb/gdb => gdb}/coff-pe-read.h (100%)
 rename {contrib/gdb/gdb => gdb}/coff-solib.c (100%)
 rename {contrib/gdb/gdb => gdb}/coff-solib.h (100%)
 rename {contrib/gdb/gdb => gdb}/coffread.c (100%)
 rename {contrib/gdb/gdb => gdb}/command.h (100%)
 rename {contrib/gdb/gdb => gdb}/complaints.c (100%)
 rename {contrib/gdb/gdb => gdb}/complaints.h (100%)
 rename {contrib/gdb/gdb => gdb}/completer.c (100%)
 rename {contrib/gdb/gdb => gdb}/completer.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/alpha-osf2.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/alpha-osf3.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/alpha.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/fbsd.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/fbsd.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/nbsd.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/nbsd.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/nm-fbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/nm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/nm-osf.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/nm-osf2.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/nm-osf3.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/tm-alpha.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/tm-fbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/tm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/alpha/xm-alphaosf.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/arm/embed.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/arm/nbsd.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/arm/nbsdaout.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/arm/nbsdelf.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/arm/nm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/arm/nm-nbsdaout.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/arm/tm-arm.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/arm/tm-embed.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/arm/tm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/arm/tm-wince.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/arm/wince.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/arm/xm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/embed.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/fbsd.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/fbsd.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/fbsd64.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/fbsd64.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/go32.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/go32.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386aout.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386gnu.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386gnu.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386lynx.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386lynx.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386nw.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386sco.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386sco4.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386sco5.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386sol2.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386sol2.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386v.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386v.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386v4.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/i386v42mp.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nbsd.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nbsd64.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nbsd64.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nbsdaout.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nbsdelf.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/ncr3000.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/ncr3000.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-fbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-fbsd64.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-go32.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-i386.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-i386gnu.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-i386lynx.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-i386sco.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-i386sco4.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-i386sco5.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-i386sol2.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-i386v.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-i386v4.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-i386v42mp.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-nbsdaout.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-nto.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nm-obsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nto.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/nto.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/obsd.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/obsd.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/obsd64.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/obsd64.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/obsdaout.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/tm-fbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/tm-go32.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/tm-i386.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/tm-i386lynx.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/tm-i386sol2.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/tm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/tm-nto.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/tm-vxworks.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/vxworks.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/xm-go32.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/xm-i386.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/xm-i386sco.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/xm-i386v.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/xm-i386v4.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/i386/xm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/ia64/fbsd.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/ia64/fbsd.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/ia64/ia64.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/ia64/nm-fbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/ia64/tm-fbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/ia64/tm-ia64.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/decstation.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/embed.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/littlemips.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/mipsv4.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/mipsv4.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/nbsd.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/nbsd.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/news-mips.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/nm-mips.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/nm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/nm-news-mips.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/nm-riscos.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/riscos.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/tm-mips.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/tm-mipsv4.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/tm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/tm-vxmips.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/tm-wince.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/vxmips.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/wince.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/xm-mips.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/xm-mipsv4.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/mips/xm-riscos.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/nm-gnu.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/nm-lynx.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/nm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/nm-nbsdaout.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/nm-sysv4.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/powerpc/fbsd.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/powerpc/fbsd.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/powerpc/nbsd.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/powerpc/nbsd.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/powerpc/nm-fbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/powerpc/nm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/powerpc/ppc-eabi.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/powerpc/ppc-sim.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/powerpc/tm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/powerpc/tm-ppc-eabi.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/powerpc/tm-ppcle-eabi.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/powerpc/tm-ppcle-sim.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/powerpc/tm-vxworks.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/powerpc/vxworks.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/rs6000/nm-rs6000.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/rs6000/nm-rs6000ly.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/rs6000/rs6000.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/rs6000/rs6000.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/rs6000/rs6000lynx.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/rs6000/rs6000lynx.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/rs6000/tm-rs6000.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/rs6000/tm-rs6000ly.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/rs6000/xm-rs6000.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/s390/s390.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/s390/s390.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/fbsd.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/fbsd.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/nbsd.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/nbsd64.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/nbsd64.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/nbsdaout.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/nbsdelf.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/nm-fbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/nm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/nm-nbsdaout.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/nm-sol2.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/obsd.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/obsd64.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/sol2-64.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/sol2.mh (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/sol2.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/sparc.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/sparc64.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/tm-fbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/tm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/tm-nbsd64.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/tm-sol2.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/tm-vxworks.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/sparc/vxworks.mt (100%)
 rename {contrib/gdb/gdb => gdb}/config/tm-lynx.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/tm-nto.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/tm-sunos.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/tm-sysv4.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/tm-vxworks.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/xm-nbsd.h (100%)
 rename {contrib/gdb/gdb => gdb}/config/xm-sysv4.h (100%)
 rename {contrib/gdb/gdb => gdb}/copying.awk (100%)
 rename {contrib/gdb/gdb => gdb}/copying.c (100%)
 rename {contrib/gdb/gdb => gdb}/core-aout.c (100%)
 rename {contrib/gdb/gdb => gdb}/core-regset.c (100%)
 rename {contrib/gdb/gdb => gdb}/corefile.c (100%)
 rename {contrib/gdb/gdb => gdb}/corelow.c (100%)
 rename {contrib/gdb/gdb => gdb}/cp-abi.c (100%)
 rename {contrib/gdb/gdb => gdb}/cp-abi.h (100%)
 rename {contrib/gdb/gdb => gdb}/cp-namespace.c (100%)
 rename {contrib/gdb/gdb => gdb}/cp-support.c (100%)
 rename {contrib/gdb/gdb => gdb}/cp-support.h (100%)
 rename {contrib/gdb/gdb => gdb}/cp-valprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/cpu32bug-rom.c (100%)
 rename {contrib/gdb/gdb => gdb}/dbug-rom.c (100%)
 rename {contrib/gdb/gdb => gdb}/dbxread.c (100%)
 rename {contrib/gdb/gdb => gdb}/dcache.c (100%)
 rename {contrib/gdb/gdb => gdb}/dcache.h (100%)
 rename {contrib/gdb/gdb => gdb}/defs.h (100%)
 rename {contrib/gdb/gdb => gdb}/delta68-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/demangle.c (100%)
 rename {contrib/gdb/gdb => gdb}/dictionary.c (100%)
 rename {contrib/gdb/gdb => gdb}/dictionary.h (100%)
 rename {contrib/gdb/gdb => gdb}/dink32-rom.c (100%)
 rename {contrib/gdb/gdb => gdb}/disasm.c (100%)
 rename {contrib/gdb/gdb => gdb}/disasm.h (100%)
 rename {contrib/gdb/gdb => gdb}/doc/GDBvn.texi (100%)
 rename {contrib/gdb/gdb => gdb}/doc/LRS (100%)
 rename {contrib/gdb/gdb => gdb}/doc/a4rc.sed (100%)
 rename {contrib/gdb/gdb => gdb}/doc/agentexpr.texi (100%)
 rename {contrib/gdb/gdb => gdb}/doc/all-cfg.texi (100%)
 rename {contrib/gdb/gdb => gdb}/doc/annotate.texinfo (100%)
 rename {contrib/gdb/gdb => gdb}/doc/fdl.texi (100%)
 rename {contrib/gdb/gdb => gdb}/doc/gdb.info-1 (100%)
 rename {contrib/gdb/gdb => gdb}/doc/gdb.info-2 (100%)
 rename {contrib/gdb/gdb => gdb}/doc/gdb.info-3 (100%)
 rename {contrib/gdb/gdb => gdb}/doc/gdb.texinfo (100%)
 rename {contrib/gdb/gdb => gdb}/doc/gdbint.texinfo (100%)
 rename {contrib/gdb/gdb => gdb}/doc/gpl.texi (100%)
 rename {contrib/gdb/gdb => gdb}/doc/lpsrc.sed (100%)
 rename {contrib/gdb/gdb => gdb}/doc/observer.texi (100%)
 rename {contrib/gdb/gdb => gdb}/doc/psrc.sed (100%)
 rename {contrib/gdb/gdb => gdb}/doc/refcard.tex (100%)
 rename {contrib/gdb/gdb => gdb}/doc/stabs.texinfo (100%)
 rename {contrib/gdb/gdb => gdb}/doublest.c (100%)
 rename {contrib/gdb/gdb => gdb}/doublest.h (100%)
 rename {contrib/gdb/gdb => gdb}/dpx2-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/dsrec.c (100%)
 rename {contrib/gdb/gdb => gdb}/dummy-frame.c (100%)
 rename {contrib/gdb/gdb => gdb}/dummy-frame.h (100%)
 rename {contrib/gdb/gdb => gdb}/dve3900-rom.c (100%)
 rename {contrib/gdb/gdb => gdb}/dwarf2-frame.c (100%)
 rename {contrib/gdb/gdb => gdb}/dwarf2-frame.h (100%)
 rename {contrib/gdb/gdb => gdb}/dwarf2expr.c (100%)
 rename {contrib/gdb/gdb => gdb}/dwarf2expr.h (100%)
 rename {contrib/gdb/gdb => gdb}/dwarf2loc.c (100%)
 rename {contrib/gdb/gdb => gdb}/dwarf2loc.h (100%)
 rename {contrib/gdb/gdb => gdb}/dwarf2read.c (100%)
 rename {contrib/gdb/gdb => gdb}/dwarfread.c (100%)
 rename {contrib/gdb/gdb => gdb}/elfread.c (100%)
 rename {contrib/gdb/gdb => gdb}/environ.c (100%)
 rename {contrib/gdb/gdb => gdb}/environ.h (100%)
 rename {contrib/gdb/gdb => gdb}/eval.c (100%)
 rename {contrib/gdb/gdb => gdb}/event-loop.c (100%)
 rename {contrib/gdb/gdb => gdb}/event-loop.h (100%)
 rename {contrib/gdb/gdb => gdb}/event-top.c (100%)
 rename {contrib/gdb/gdb => gdb}/event-top.h (100%)
 rename {contrib/gdb/gdb => gdb}/exc_request.defs (100%)
 rename {contrib/gdb/gdb => gdb}/exec.c (100%)
 rename {contrib/gdb/gdb => gdb}/exec.h (100%)
 rename {contrib/gdb/gdb => gdb}/expprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/expression.h (100%)
 rename {contrib/gdb/gdb => gdb}/f-exp.c (100%)
 rename {contrib/gdb/gdb => gdb}/f-exp.y (100%)
 rename {contrib/gdb/gdb => gdb}/f-lang.c (100%)
 rename {contrib/gdb/gdb => gdb}/f-lang.h (100%)
 rename {contrib/gdb/gdb => gdb}/f-typeprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/f-valprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/fbsd-proc.c (100%)
 rename {contrib/gdb/gdb => gdb}/findvar.c (100%)
 rename {contrib/gdb/gdb => gdb}/fork-child.c (100%)
 rename {contrib/gdb/gdb => gdb}/frame-base.c (100%)
 rename {contrib/gdb/gdb => gdb}/frame-base.h (100%)
 rename {contrib/gdb/gdb => gdb}/frame-unwind.c (100%)
 rename {contrib/gdb/gdb => gdb}/frame-unwind.h (100%)
 rename {contrib/gdb/gdb => gdb}/frame.c (100%)
 rename {contrib/gdb/gdb => gdb}/frame.h (100%)
 rename {contrib/gdb/gdb => gdb}/gcore.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdb-events.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdb-events.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdb-events.sh (100%)
 rename {contrib/gdb/gdb => gdb}/gdb-stabs.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdb.1 (100%)
 rename {contrib/gdb/gdb => gdb}/gdb.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdb.gdb (100%)
 rename {contrib/gdb/gdb => gdb}/gdb.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_assert.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_curses.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_dirent.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_gcore.sh (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_indent.sh (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_locale.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_mbuild.sh (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_obstack.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_proc_service.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_regex.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_stat.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_string.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_thread_db.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_vfork.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdb_wait.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdbarch.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdbarch.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdbarch.sh (100%)
 rename {contrib/gdb/gdb => gdb}/gdbcmd.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdbcore.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdbinit.in (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/README (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/acinclude.m4 (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/aclocal.m4 (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/gdbreplay.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/gdbserver.1 (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/i387-fp.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/i387-fp.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/inferiors.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/mem-break.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/mem-break.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/proc-service.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/regcache.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/regcache.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/remote-utils.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/server.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/server.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/target.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/target.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/terminal.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/thread-db.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdbserver/utils.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdbthread.h (100%)
 rename {contrib/gdb/gdb => gdb}/gdbtypes.c (100%)
 rename {contrib/gdb/gdb => gdb}/gdbtypes.h (100%)
 rename {contrib/gdb/gdb => gdb}/glibc-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/glibc-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/gnu-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/gnu-nat.h (100%)
 rename {contrib/gdb/gdb => gdb}/gnu-v2-abi.c (100%)
 rename {contrib/gdb/gdb => gdb}/gnu-v3-abi.c (100%)
 rename {contrib/gdb/gdb => gdb}/go32-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/gregset.h (100%)
 rename {contrib/gdb/gdb => gdb}/hpacc-abi.c (100%)
 rename {contrib/gdb/gdb => gdb}/hpread.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386-nto-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386-sol2-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386-stub.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/i386bsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386bsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386fbsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386fbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386gnu-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386gnu-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386ly-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386nbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386obsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386obsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386v-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/i386v4-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/i387-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/i387-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/ia64-fbsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/ia64-fbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/ia64-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/ia64-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/inf-loop.c (100%)
 rename {contrib/gdb/gdb => gdb}/inf-loop.h (100%)
 rename {contrib/gdb/gdb => gdb}/infcall.c (100%)
 rename {contrib/gdb/gdb => gdb}/infcall.h (100%)
 rename {contrib/gdb/gdb => gdb}/infcmd.c (100%)
 rename {contrib/gdb/gdb => gdb}/inferior.h (100%)
 rename {contrib/gdb/gdb => gdb}/inflow.c (100%)
 rename {contrib/gdb/gdb => gdb}/inflow.h (100%)
 rename {contrib/gdb/gdb => gdb}/infptrace.c (100%)
 rename {contrib/gdb/gdb => gdb}/infrun.c (100%)
 rename {contrib/gdb/gdb => gdb}/inftarg.c (100%)
 rename {contrib/gdb/gdb => gdb}/infttrace.c (100%)
 rename {contrib/gdb/gdb => gdb}/infttrace.h (100%)
 rename {contrib/gdb/gdb => gdb}/interps.c (100%)
 rename {contrib/gdb/gdb => gdb}/interps.h (100%)
 rename {contrib/gdb/gdb => gdb}/jv-exp.c (100%)
 rename {contrib/gdb/gdb => gdb}/jv-exp.y (100%)
 rename {contrib/gdb/gdb => gdb}/jv-lang.c (100%)
 rename {contrib/gdb/gdb => gdb}/jv-lang.h (100%)
 rename {contrib/gdb/gdb => gdb}/jv-typeprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/jv-valprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/kod-cisco.c (100%)
 rename {contrib/gdb/gdb => gdb}/kod.c (100%)
 rename {contrib/gdb/gdb => gdb}/kod.h (100%)
 rename {contrib/gdb/gdb => gdb}/language.c (100%)
 rename {contrib/gdb/gdb => gdb}/language.h (100%)
 rename {contrib/gdb/gdb => gdb}/libunwind-frame.c (100%)
 rename {contrib/gdb/gdb => gdb}/libunwind-frame.h (100%)
 rename {contrib/gdb/gdb => gdb}/lin-lwp.c (100%)
 rename {contrib/gdb/gdb => gdb}/linespec.c (100%)
 rename {contrib/gdb/gdb => gdb}/linespec.h (100%)
 rename {contrib/gdb/gdb => gdb}/lynx-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/m2-exp.c (100%)
 rename {contrib/gdb/gdb => gdb}/m2-exp.y (100%)
 rename {contrib/gdb/gdb => gdb}/m2-lang.c (100%)
 rename {contrib/gdb/gdb => gdb}/m2-lang.h (100%)
 rename {contrib/gdb/gdb => gdb}/m2-typeprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/m2-valprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/macrocmd.c (100%)
 rename {contrib/gdb/gdb => gdb}/macroexp.c (100%)
 rename {contrib/gdb/gdb => gdb}/macroexp.h (100%)
 rename {contrib/gdb/gdb => gdb}/macroscope.c (100%)
 rename {contrib/gdb/gdb => gdb}/macroscope.h (100%)
 rename {contrib/gdb/gdb => gdb}/macrotab.c (100%)
 rename {contrib/gdb/gdb => gdb}/macrotab.h (100%)
 rename {contrib/gdb/gdb => gdb}/main.c (100%)
 rename {contrib/gdb/gdb => gdb}/main.h (100%)
 rename {contrib/gdb/gdb => gdb}/maint.c (100%)
 rename {contrib/gdb/gdb => gdb}/mdebugread.c (100%)
 rename {contrib/gdb/gdb => gdb}/mem-break.c (100%)
 rename {contrib/gdb/gdb => gdb}/memattr.c (100%)
 rename {contrib/gdb/gdb => gdb}/memattr.h (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-cmd-break.c (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-cmd-disas.c (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-cmd-env.c (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-cmd-file.c (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-cmd-stack.c (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-cmd-var.c (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-cmds.c (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-cmds.h (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-console.c (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-console.h (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-getopt.c (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-getopt.h (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-interp.c (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-main.c (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-main.h (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-out.c (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-out.h (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-parse.c (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-parse.h (100%)
 rename {contrib/gdb/gdb => gdb}/mi/mi-symbol-cmds.c (100%)
 rename {contrib/gdb/gdb => gdb}/minimon.h (100%)
 rename {contrib/gdb/gdb => gdb}/minsyms.c (100%)
 rename {contrib/gdb/gdb => gdb}/mips-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/mips-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/mips-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/mipsnbsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/mipsnbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/mipsnbsd-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/mipsread.c (100%)
 rename {contrib/gdb/gdb => gdb}/mipsv4-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/monitor.c (100%)
 rename {contrib/gdb/gdb => gdb}/monitor.h (100%)
 rename {contrib/gdb/gdb => gdb}/msg.defs (100%)
 rename {contrib/gdb/gdb => gdb}/msg_reply.defs (100%)
 rename {contrib/gdb/gdb => gdb}/nbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/nbsd-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/nlmread.c (100%)
 rename {contrib/gdb/gdb => gdb}/notify.defs (100%)
 rename {contrib/gdb/gdb => gdb}/nto-procfs.c (100%)
 rename {contrib/gdb/gdb => gdb}/nto-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/nto-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/objc-exp.c (100%)
 rename {contrib/gdb/gdb => gdb}/objc-exp.y (100%)
 rename {contrib/gdb/gdb => gdb}/objc-lang.c (100%)
 rename {contrib/gdb/gdb => gdb}/objc-lang.h (100%)
 rename {contrib/gdb/gdb => gdb}/objfiles.c (100%)
 rename {contrib/gdb/gdb => gdb}/objfiles.h (100%)
 rename {contrib/gdb/gdb => gdb}/observer.c (100%)
 rename {contrib/gdb/gdb => gdb}/observer.h (100%)
 rename {contrib/gdb/gdb => gdb}/ocd.c (100%)
 rename {contrib/gdb/gdb => gdb}/ocd.h (100%)
 rename {contrib/gdb/gdb => gdb}/osabi.c (100%)
 rename {contrib/gdb/gdb => gdb}/osabi.h (100%)
 rename {contrib/gdb/gdb => gdb}/p-exp.c (100%)
 rename {contrib/gdb/gdb => gdb}/p-exp.y (100%)
 rename {contrib/gdb/gdb => gdb}/p-lang.c (100%)
 rename {contrib/gdb/gdb => gdb}/p-lang.h (100%)
 rename {contrib/gdb/gdb => gdb}/p-typeprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/p-valprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/pa64solib.c (100%)
 rename {contrib/gdb/gdb => gdb}/pa64solib.h (100%)
 rename {contrib/gdb/gdb => gdb}/parse.c (100%)
 rename {contrib/gdb/gdb => gdb}/parser-defs.h (100%)
 rename {contrib/gdb/gdb => gdb}/ppc-bdm.c (100%)
 rename {contrib/gdb/gdb => gdb}/ppc-sysv-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/ppc-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/ppcbug-rom.c (100%)
 rename {contrib/gdb/gdb => gdb}/ppcfbsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/ppcfbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/ppcfbsd-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/ppcnbsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/ppcnbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/ppcnbsd-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/printcmd.c (100%)
 rename {contrib/gdb/gdb => gdb}/proc-api.c (100%)
 rename {contrib/gdb/gdb => gdb}/proc-events.c (100%)
 rename {contrib/gdb/gdb => gdb}/proc-flags.c (100%)
 rename {contrib/gdb/gdb => gdb}/proc-service.c (100%)
 rename {contrib/gdb/gdb => gdb}/proc-utils.h (100%)
 rename {contrib/gdb/gdb => gdb}/proc-why.c (100%)
 rename {contrib/gdb/gdb => gdb}/process_reply.defs (100%)
 rename {contrib/gdb/gdb => gdb}/procfs.c (100%)
 rename {contrib/gdb/gdb => gdb}/regcache.c (100%)
 rename {contrib/gdb/gdb => gdb}/regcache.h (100%)
 rename {contrib/gdb/gdb => gdb}/regformats/reg-arm.dat (100%)
 rename {contrib/gdb/gdb => gdb}/regformats/reg-i386.dat (100%)
 rename {contrib/gdb/gdb => gdb}/regformats/reg-ia64.dat (100%)
 rename {contrib/gdb/gdb => gdb}/regformats/reg-m68k.dat (100%)
 rename {contrib/gdb/gdb => gdb}/regformats/reg-mips.dat (100%)
 rename {contrib/gdb/gdb => gdb}/regformats/reg-ppc.dat (100%)
 rename {contrib/gdb/gdb => gdb}/regformats/reg-s390.dat (100%)
 rename {contrib/gdb/gdb => gdb}/regformats/reg-s390x.dat (100%)
 rename {contrib/gdb/gdb => gdb}/regformats/reg-sh.dat (100%)
 rename {contrib/gdb/gdb => gdb}/regformats/reg-x86-64.dat (100%)
 rename {contrib/gdb/gdb => gdb}/regformats/regdat.sh (100%)
 rename {contrib/gdb/gdb => gdb}/regformats/regdef.h (100%)
 rename {contrib/gdb/gdb => gdb}/reggroups.c (100%)
 rename {contrib/gdb/gdb => gdb}/reggroups.h (100%)
 rename {contrib/gdb/gdb => gdb}/regset.h (100%)
 rename {contrib/gdb/gdb => gdb}/remote-e7000.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote-est.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote-fileio.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote-fileio.h (100%)
 rename {contrib/gdb/gdb => gdb}/remote-hms.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote-mips.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote-rdi.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote-rdp.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote-sds.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote-sim.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote-st.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote-utils.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote-utils.h (100%)
 rename {contrib/gdb/gdb => gdb}/remote-vx.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote-vx68.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote-vxmips.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote-vxsparc.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote.c (100%)
 rename {contrib/gdb/gdb => gdb}/remote.h (100%)
 rename {contrib/gdb/gdb => gdb}/reply_mig_hack.awk (100%)
 rename {contrib/gdb/gdb => gdb}/rom68k-rom.c (100%)
 rename {contrib/gdb/gdb => gdb}/rs6000-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/rs6000-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/s390-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/s390-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/s390-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/scm-exp.c (100%)
 rename {contrib/gdb/gdb => gdb}/scm-lang.c (100%)
 rename {contrib/gdb/gdb => gdb}/scm-lang.h (100%)
 rename {contrib/gdb/gdb => gdb}/scm-tags.h (100%)
 rename {contrib/gdb/gdb => gdb}/scm-valprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/sentinel-frame.c (100%)
 rename {contrib/gdb/gdb => gdb}/sentinel-frame.h (100%)
 rename {contrib/gdb/gdb => gdb}/ser-e7kpc.c (100%)
 rename {contrib/gdb/gdb => gdb}/ser-go32.c (100%)
 rename {contrib/gdb/gdb => gdb}/ser-pipe.c (100%)
 rename {contrib/gdb/gdb => gdb}/ser-tcp.c (100%)
 rename {contrib/gdb/gdb => gdb}/ser-unix.c (100%)
 rename {contrib/gdb/gdb => gdb}/ser-unix.h (100%)
 rename {contrib/gdb/gdb => gdb}/serial.c (100%)
 rename {contrib/gdb/gdb => gdb}/serial.h (100%)
 rename {contrib/gdb/gdb => gdb}/signals/signals.c (100%)
 rename {contrib/gdb/gdb => gdb}/sim-regno.h (100%)
 rename {contrib/gdb/gdb => gdb}/sol-thread.c (100%)
 rename {contrib/gdb/gdb => gdb}/solib-legacy.c (100%)
 rename {contrib/gdb/gdb => gdb}/solib-osf.c (100%)
 rename {contrib/gdb/gdb => gdb}/solib-sunos.c (100%)
 rename {contrib/gdb/gdb => gdb}/solib-svr4.c (100%)
 rename {contrib/gdb/gdb => gdb}/solib-svr4.h (100%)
 rename {contrib/gdb/gdb => gdb}/solib.c (100%)
 rename {contrib/gdb/gdb => gdb}/solib.h (100%)
 rename {contrib/gdb/gdb => gdb}/solist.h (100%)
 rename {contrib/gdb/gdb => gdb}/somread.c (100%)
 rename {contrib/gdb/gdb => gdb}/somsolib.c (100%)
 rename {contrib/gdb/gdb => gdb}/somsolib.h (100%)
 rename {contrib/gdb/gdb => gdb}/source.c (100%)
 rename {contrib/gdb/gdb => gdb}/source.h (100%)
 rename {contrib/gdb/gdb => gdb}/sparc-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparc-nat.h (100%)
 rename {contrib/gdb/gdb => gdb}/sparc-sol2-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparc-sol2-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparc-stub.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparc-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparc-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/sparc64-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparc64-sol2-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparc64-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparc64-tdep.h (100%)
 rename {contrib/gdb/gdb => gdb}/sparc64fbsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparc64fbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparc64nbsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparc64nbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparc64obsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparcnbsd-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparcnbsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/sparcobsd-tdep.c (100%)
 rename {contrib/gdb/gdb => gdb}/srec.h (100%)
 rename {contrib/gdb/gdb => gdb}/stabsread.c (100%)
 rename {contrib/gdb/gdb => gdb}/stabsread.h (100%)
 rename {contrib/gdb/gdb => gdb}/stack.c (100%)
 rename {contrib/gdb/gdb => gdb}/stack.h (100%)
 rename {contrib/gdb/gdb => gdb}/standalone.c (100%)
 rename {contrib/gdb/gdb => gdb}/std-regs.c (100%)
 rename {contrib/gdb/gdb => gdb}/stop-gdb.c (100%)
 rename {contrib/gdb/gdb => gdb}/sun3-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/symfile.c (100%)
 rename {contrib/gdb/gdb => gdb}/symfile.h (100%)
 rename {contrib/gdb/gdb => gdb}/symmisc.c (100%)
 rename {contrib/gdb/gdb => gdb}/symtab.c (100%)
 rename {contrib/gdb/gdb => gdb}/symtab.h (100%)
 rename {contrib/gdb/gdb => gdb}/target.c (100%)
 rename {contrib/gdb/gdb => gdb}/target.h (100%)
 rename {contrib/gdb/gdb => gdb}/terminal.h (100%)
 rename {contrib/gdb/gdb => gdb}/thread-db.c (100%)
 rename {contrib/gdb/gdb => gdb}/thread.c (100%)
 rename {contrib/gdb/gdb => gdb}/top.c (100%)
 rename {contrib/gdb/gdb => gdb}/top.h (100%)
 rename {contrib/gdb/gdb => gdb}/tracepoint.c (100%)
 rename {contrib/gdb/gdb => gdb}/tracepoint.h (100%)
 rename {contrib/gdb/gdb => gdb}/trad-frame.c (100%)
 rename {contrib/gdb/gdb => gdb}/trad-frame.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-command.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-command.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-data.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-data.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-disasm.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-disasm.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-file.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-file.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-hooks.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-hooks.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-interp.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-io.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-io.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-layout.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-layout.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-main.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-out.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-regs.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-regs.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-source.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-source.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-stack.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-stack.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-win.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-win.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-windata.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-windata.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-wingeneral.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-wingeneral.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-winsource.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui-winsource.h (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui.c (100%)
 rename {contrib/gdb/gdb => gdb}/tui/tui.h (100%)
 rename {contrib/gdb/gdb => gdb}/typeprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/typeprint.h (100%)
 rename {contrib/gdb/gdb => gdb}/ui-file.c (100%)
 rename {contrib/gdb/gdb => gdb}/ui-file.h (100%)
 rename {contrib/gdb/gdb => gdb}/ui-out.c (100%)
 rename {contrib/gdb/gdb => gdb}/ui-out.h (100%)
 rename {contrib/gdb/gdb => gdb}/user-regs.c (100%)
 rename {contrib/gdb/gdb => gdb}/user-regs.h (100%)
 rename {contrib/gdb/gdb => gdb}/utils.c (100%)
 rename {contrib/gdb/gdb => gdb}/uw-thread.c (100%)
 rename {contrib/gdb/gdb => gdb}/valarith.c (100%)
 rename {contrib/gdb/gdb => gdb}/valops.c (100%)
 rename {contrib/gdb/gdb => gdb}/valprint.c (100%)
 rename {contrib/gdb/gdb => gdb}/valprint.h (100%)
 rename {contrib/gdb/gdb => gdb}/value.h (100%)
 rename {contrib/gdb/gdb => gdb}/values.c (100%)
 rename {contrib/gdb/gdb => gdb}/varobj.c (100%)
 rename {contrib/gdb/gdb => gdb}/varobj.h (100%)
 rename {contrib/gdb/gdb => gdb}/version.h (100%)
 rename {contrib/gdb/gdb => gdb}/version.in (100%)
 rename {contrib/gdb/gdb => gdb}/win32-nat.c (100%)
 rename {contrib/gdb/gdb => gdb}/wince-stub.c (100%)
 rename {contrib/gdb/gdb => gdb}/wince-stub.h (100%)
 rename {contrib/gdb/gdb => gdb}/wince.c (100%)
 rename {contrib/gdb/gdb => gdb}/wrapper.c (100%)
 rename {contrib/gdb/gdb => gdb}/wrapper.h (100%)
 rename {contrib/gdb/gdb => gdb}/xcoffread.c (100%)
 rename {contrib/gdb/gdb => gdb}/xcoffsolib.c (100%)
 rename {contrib/gdb/gdb => gdb}/xcoffsolib.h (100%)
 rename {contrib/gdb/gdb => gdb}/xmodem.c (100%)
 rename {contrib/gdb/gdb => gdb}/xmodem.h (100%)
 rename contrib/gdb/gettext.m4 => gettext.m4 (100%)
 rename {contrib/gdb/include => include}/COPYING (100%)
 rename {contrib/gdb/include => include}/MAINTAINERS (100%)
 rename {contrib/gdb/include => include}/alloca-conf.h (100%)
 rename {contrib/gdb/include => include}/ansidecl.h (100%)
 rename {contrib/gdb/include => include}/bfdlink.h (100%)
 rename {contrib/gdb/include => include}/bin-bugs.h (100%)
 rename {contrib/gdb/include => include}/bout.h (100%)
 rename {contrib/gdb/include => include}/demangle.h (100%)
 rename {contrib/gdb/include => include}/dis-asm.h (100%)
 rename {contrib/gdb/include => include}/dyn-string.h (100%)
 rename {contrib/gdb/include => include}/fibheap.h (100%)
 rename {contrib/gdb/include => include}/filenames.h (100%)
 rename {contrib/gdb/include => include}/floatformat.h (100%)
 rename {contrib/gdb/include => include}/fnmatch.h (100%)
 rename {contrib/gdb/include => include}/fopen-bin.h (100%)
 rename {contrib/gdb/include => include}/fopen-same.h (100%)
 rename {contrib/gdb/include => include}/fopen-vms.h (100%)
 rename {contrib/gdb/include => include}/gdb/callback.h (100%)
 rename {contrib/gdb/include => include}/gdb/fileio.h (100%)
 rename {contrib/gdb/include => include}/gdb/remote-sim.h (100%)
 rename {contrib/gdb/include => include}/gdb/signals.h (100%)
 rename {contrib/gdb/include => include}/gdb/sim-arm.h (100%)
 rename {contrib/gdb/include => include}/gdb/sim-d10v.h (100%)
 rename {contrib/gdb/include => include}/gdb/sim-frv.h (100%)
 rename {contrib/gdb/include => include}/gdb/sim-h8300.h (100%)
 rename {contrib/gdb/include => include}/gdb/sim-sh.h (100%)
 rename {contrib/gdb/include => include}/gdbm.h (100%)
 rename {contrib/gdb/include => include}/getopt.h (100%)
 rename {contrib/gdb/include => include}/hashtab.h (100%)
 rename {contrib/gdb/include => include}/hp-symtab.h (100%)
 rename {contrib/gdb/include => include}/ieee.h (100%)
 rename {contrib/gdb/include => include}/libiberty.h (100%)
 rename {contrib/gdb/include => include}/md5.h (100%)
 rename {contrib/gdb/include => include}/oasys.h (100%)
 rename {contrib/gdb/include => include}/objalloc.h (100%)
 rename {contrib/gdb/include => include}/obstack.h (100%)
 rename {contrib/gdb/include => include}/os9k.h (100%)
 rename {contrib/gdb/include => include}/partition.h (100%)
 rename {contrib/gdb/include => include}/progress.h (100%)
 rename {contrib/gdb/include => include}/safe-ctype.h (100%)
 rename {contrib/gdb/include => include}/sort.h (100%)
 rename {contrib/gdb/include => include}/splay-tree.h (100%)
 rename {contrib/gdb/include => include}/symcat.h (100%)
 rename {contrib/gdb/include => include}/ternary.h (100%)
 rename {contrib/gdb/include => include}/xregex.h (100%)
 rename {contrib/gdb/include => include}/xregex2.h (100%)
 rename {contrib/gdb/include => include}/xtensa-isa-internal.h (100%)
 rename {contrib/gdb/include => include}/xtensa-isa.h (100%)
 rename contrib/gdb/install-sh => install-sh (100%)
 rename contrib/gdb/libtool.m4 => libtool.m4 (100%)
 rename contrib/gdb/ltcf-c.sh => ltcf-c.sh (100%)
 rename contrib/gdb/ltcf-cxx.sh => ltcf-cxx.sh (100%)
 rename contrib/gdb/ltcf-gcj.sh => ltcf-gcj.sh (100%)
 rename contrib/gdb/ltconfig => ltconfig (100%)
 rename contrib/gdb/ltmain.sh => ltmain.sh (100%)
 rename contrib/gdb/md5.sum => md5.sum (100%)
 rename contrib/gdb/missing => missing (100%)
 rename contrib/gdb/mkinstalldirs => mkinstalldirs (100%)
 rename contrib/gdb/move-if-change => move-if-change (100%)
 rename contrib/gdb/src-release => src-release (100%)
 rename contrib/gdb/symlink-tree => symlink-tree (100%)
 rename contrib/gdb/ylwrap => ylwrap (100%)

diff --git a/contrib/gdb/COPYING b/COPYING
similarity index 100%
rename from contrib/gdb/COPYING
rename to COPYING
diff --git a/contrib/gdb/COPYING.LIB b/COPYING.LIB
similarity index 100%
rename from contrib/gdb/COPYING.LIB
rename to COPYING.LIB
diff --git a/contrib/gdb/FREEBSD-diffs b/FREEBSD-diffs
similarity index 100%
rename from contrib/gdb/FREEBSD-diffs
rename to FREEBSD-diffs
diff --git a/contrib/gdb/README b/README
similarity index 100%
rename from contrib/gdb/README
rename to README
diff --git a/contrib/gdb/config-ml.in b/config-ml.in
similarity index 100%
rename from contrib/gdb/config-ml.in
rename to config-ml.in
diff --git a/contrib/gdb/djunpack.bat b/djunpack.bat
similarity index 100%
rename from contrib/gdb/djunpack.bat
rename to djunpack.bat
diff --git a/contrib/gdb/gdb/CONTRIBUTE b/gdb/CONTRIBUTE
similarity index 100%
rename from contrib/gdb/gdb/CONTRIBUTE
rename to gdb/CONTRIBUTE
diff --git a/contrib/gdb/gdb/COPYING b/gdb/COPYING
similarity index 100%
rename from contrib/gdb/gdb/COPYING
rename to gdb/COPYING
diff --git a/contrib/gdb/gdb/MAINTAINERS b/gdb/MAINTAINERS
similarity index 100%
rename from contrib/gdb/gdb/MAINTAINERS
rename to gdb/MAINTAINERS
diff --git a/contrib/gdb/gdb/NEWS b/gdb/NEWS
similarity index 100%
rename from contrib/gdb/gdb/NEWS
rename to gdb/NEWS
diff --git a/contrib/gdb/gdb/PROBLEMS b/gdb/PROBLEMS
similarity index 100%
rename from contrib/gdb/gdb/PROBLEMS
rename to gdb/PROBLEMS
diff --git a/contrib/gdb/gdb/README b/gdb/README
similarity index 100%
rename from contrib/gdb/gdb/README
rename to gdb/README
diff --git a/contrib/gdb/gdb/TODO b/gdb/TODO
similarity index 100%
rename from contrib/gdb/gdb/TODO
rename to gdb/TODO
diff --git a/contrib/gdb/gdb/abug-rom.c b/gdb/abug-rom.c
similarity index 100%
rename from contrib/gdb/gdb/abug-rom.c
rename to gdb/abug-rom.c
diff --git a/contrib/gdb/gdb/acinclude.m4 b/gdb/acinclude.m4
similarity index 100%
rename from contrib/gdb/gdb/acinclude.m4
rename to gdb/acinclude.m4
diff --git a/contrib/gdb/gdb/aclocal.m4 b/gdb/aclocal.m4
similarity index 100%
rename from contrib/gdb/gdb/aclocal.m4
rename to gdb/aclocal.m4
diff --git a/contrib/gdb/gdb/ada-exp.c b/gdb/ada-exp.c
similarity index 100%
rename from contrib/gdb/gdb/ada-exp.c
rename to gdb/ada-exp.c
diff --git a/contrib/gdb/gdb/ada-exp.y b/gdb/ada-exp.y
similarity index 100%
rename from contrib/gdb/gdb/ada-exp.y
rename to gdb/ada-exp.y
diff --git a/contrib/gdb/gdb/ada-lang.c b/gdb/ada-lang.c
similarity index 100%
rename from contrib/gdb/gdb/ada-lang.c
rename to gdb/ada-lang.c
diff --git a/contrib/gdb/gdb/ada-lang.h b/gdb/ada-lang.h
similarity index 100%
rename from contrib/gdb/gdb/ada-lang.h
rename to gdb/ada-lang.h
diff --git a/contrib/gdb/gdb/ada-lex.l b/gdb/ada-lex.l
similarity index 100%
rename from contrib/gdb/gdb/ada-lex.l
rename to gdb/ada-lex.l
diff --git a/contrib/gdb/gdb/ada-tasks.c b/gdb/ada-tasks.c
similarity index 100%
rename from contrib/gdb/gdb/ada-tasks.c
rename to gdb/ada-tasks.c
diff --git a/contrib/gdb/gdb/ada-typeprint.c b/gdb/ada-typeprint.c
similarity index 100%
rename from contrib/gdb/gdb/ada-typeprint.c
rename to gdb/ada-typeprint.c
diff --git a/contrib/gdb/gdb/ada-valprint.c b/gdb/ada-valprint.c
similarity index 100%
rename from contrib/gdb/gdb/ada-valprint.c
rename to gdb/ada-valprint.c
diff --git a/contrib/gdb/gdb/alpha-mdebug-tdep.c b/gdb/alpha-mdebug-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/alpha-mdebug-tdep.c
rename to gdb/alpha-mdebug-tdep.c
diff --git a/contrib/gdb/gdb/alpha-nat.c b/gdb/alpha-nat.c
similarity index 100%
rename from contrib/gdb/gdb/alpha-nat.c
rename to gdb/alpha-nat.c
diff --git a/contrib/gdb/gdb/alpha-tdep.c b/gdb/alpha-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/alpha-tdep.c
rename to gdb/alpha-tdep.c
diff --git a/contrib/gdb/gdb/alpha-tdep.h b/gdb/alpha-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/alpha-tdep.h
rename to gdb/alpha-tdep.h
diff --git a/contrib/gdb/gdb/alphabsd-nat.c b/gdb/alphabsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/alphabsd-nat.c
rename to gdb/alphabsd-nat.c
diff --git a/contrib/gdb/gdb/alphabsd-tdep.c b/gdb/alphabsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/alphabsd-tdep.c
rename to gdb/alphabsd-tdep.c
diff --git a/contrib/gdb/gdb/alphabsd-tdep.h b/gdb/alphabsd-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/alphabsd-tdep.h
rename to gdb/alphabsd-tdep.h
diff --git a/contrib/gdb/gdb/alphafbsd-tdep.c b/gdb/alphafbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/alphafbsd-tdep.c
rename to gdb/alphafbsd-tdep.c
diff --git a/contrib/gdb/gdb/alphanbsd-tdep.c b/gdb/alphanbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/alphanbsd-tdep.c
rename to gdb/alphanbsd-tdep.c
diff --git a/contrib/gdb/gdb/amd64-nat.c b/gdb/amd64-nat.c
similarity index 100%
rename from contrib/gdb/gdb/amd64-nat.c
rename to gdb/amd64-nat.c
diff --git a/contrib/gdb/gdb/amd64-nat.h b/gdb/amd64-nat.h
similarity index 100%
rename from contrib/gdb/gdb/amd64-nat.h
rename to gdb/amd64-nat.h
diff --git a/contrib/gdb/gdb/amd64-tdep.c b/gdb/amd64-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/amd64-tdep.c
rename to gdb/amd64-tdep.c
diff --git a/contrib/gdb/gdb/amd64-tdep.h b/gdb/amd64-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/amd64-tdep.h
rename to gdb/amd64-tdep.h
diff --git a/contrib/gdb/gdb/amd64bsd-nat.c b/gdb/amd64bsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/amd64bsd-nat.c
rename to gdb/amd64bsd-nat.c
diff --git a/contrib/gdb/gdb/amd64fbsd-nat.c b/gdb/amd64fbsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/amd64fbsd-nat.c
rename to gdb/amd64fbsd-nat.c
diff --git a/contrib/gdb/gdb/amd64fbsd-tdep.c b/gdb/amd64fbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/amd64fbsd-tdep.c
rename to gdb/amd64fbsd-tdep.c
diff --git a/contrib/gdb/gdb/amd64nbsd-nat.c b/gdb/amd64nbsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/amd64nbsd-nat.c
rename to gdb/amd64nbsd-nat.c
diff --git a/contrib/gdb/gdb/amd64nbsd-tdep.c b/gdb/amd64nbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/amd64nbsd-tdep.c
rename to gdb/amd64nbsd-tdep.c
diff --git a/contrib/gdb/gdb/amd64obsd-nat.c b/gdb/amd64obsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/amd64obsd-nat.c
rename to gdb/amd64obsd-nat.c
diff --git a/contrib/gdb/gdb/amd64obsd-tdep.c b/gdb/amd64obsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/amd64obsd-tdep.c
rename to gdb/amd64obsd-tdep.c
diff --git a/contrib/gdb/gdb/annotate.c b/gdb/annotate.c
similarity index 100%
rename from contrib/gdb/gdb/annotate.c
rename to gdb/annotate.c
diff --git a/contrib/gdb/gdb/annotate.h b/gdb/annotate.h
similarity index 100%
rename from contrib/gdb/gdb/annotate.h
rename to gdb/annotate.h
diff --git a/contrib/gdb/gdb/arch-utils.c b/gdb/arch-utils.c
similarity index 100%
rename from contrib/gdb/gdb/arch-utils.c
rename to gdb/arch-utils.c
diff --git a/contrib/gdb/gdb/arch-utils.h b/gdb/arch-utils.h
similarity index 100%
rename from contrib/gdb/gdb/arch-utils.h
rename to gdb/arch-utils.h
diff --git a/contrib/gdb/gdb/arm-tdep.c b/gdb/arm-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/arm-tdep.c
rename to gdb/arm-tdep.c
diff --git a/contrib/gdb/gdb/arm-tdep.h b/gdb/arm-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/arm-tdep.h
rename to gdb/arm-tdep.h
diff --git a/contrib/gdb/gdb/armnbsd-nat.c b/gdb/armnbsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/armnbsd-nat.c
rename to gdb/armnbsd-nat.c
diff --git a/contrib/gdb/gdb/armnbsd-tdep.c b/gdb/armnbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/armnbsd-tdep.c
rename to gdb/armnbsd-tdep.c
diff --git a/contrib/gdb/gdb/auxv.c b/gdb/auxv.c
similarity index 100%
rename from contrib/gdb/gdb/auxv.c
rename to gdb/auxv.c
diff --git a/contrib/gdb/gdb/auxv.h b/gdb/auxv.h
similarity index 100%
rename from contrib/gdb/gdb/auxv.h
rename to gdb/auxv.h
diff --git a/contrib/gdb/gdb/ax-gdb.c b/gdb/ax-gdb.c
similarity index 100%
rename from contrib/gdb/gdb/ax-gdb.c
rename to gdb/ax-gdb.c
diff --git a/contrib/gdb/gdb/ax-gdb.h b/gdb/ax-gdb.h
similarity index 100%
rename from contrib/gdb/gdb/ax-gdb.h
rename to gdb/ax-gdb.h
diff --git a/contrib/gdb/gdb/ax-general.c b/gdb/ax-general.c
similarity index 100%
rename from contrib/gdb/gdb/ax-general.c
rename to gdb/ax-general.c
diff --git a/contrib/gdb/gdb/ax.h b/gdb/ax.h
similarity index 100%
rename from contrib/gdb/gdb/ax.h
rename to gdb/ax.h
diff --git a/contrib/gdb/gdb/bcache.c b/gdb/bcache.c
similarity index 100%
rename from contrib/gdb/gdb/bcache.c
rename to gdb/bcache.c
diff --git a/contrib/gdb/gdb/bcache.h b/gdb/bcache.h
similarity index 100%
rename from contrib/gdb/gdb/bcache.h
rename to gdb/bcache.h
diff --git a/contrib/gdb/gdb/bfd-target.c b/gdb/bfd-target.c
similarity index 100%
rename from contrib/gdb/gdb/bfd-target.c
rename to gdb/bfd-target.c
diff --git a/contrib/gdb/gdb/bfd-target.h b/gdb/bfd-target.h
similarity index 100%
rename from contrib/gdb/gdb/bfd-target.h
rename to gdb/bfd-target.h
diff --git a/contrib/gdb/gdb/block.c b/gdb/block.c
similarity index 100%
rename from contrib/gdb/gdb/block.c
rename to gdb/block.c
diff --git a/contrib/gdb/gdb/block.h b/gdb/block.h
similarity index 100%
rename from contrib/gdb/gdb/block.h
rename to gdb/block.h
diff --git a/contrib/gdb/gdb/blockframe.c b/gdb/blockframe.c
similarity index 100%
rename from contrib/gdb/gdb/blockframe.c
rename to gdb/blockframe.c
diff --git a/contrib/gdb/gdb/breakpoint.c b/gdb/breakpoint.c
similarity index 100%
rename from contrib/gdb/gdb/breakpoint.c
rename to gdb/breakpoint.c
diff --git a/contrib/gdb/gdb/breakpoint.h b/gdb/breakpoint.h
similarity index 100%
rename from contrib/gdb/gdb/breakpoint.h
rename to gdb/breakpoint.h
diff --git a/contrib/gdb/gdb/buildsym.c b/gdb/buildsym.c
similarity index 100%
rename from contrib/gdb/gdb/buildsym.c
rename to gdb/buildsym.c
diff --git a/contrib/gdb/gdb/buildsym.h b/gdb/buildsym.h
similarity index 100%
rename from contrib/gdb/gdb/buildsym.h
rename to gdb/buildsym.h
diff --git a/contrib/gdb/gdb/c-exp.c b/gdb/c-exp.c
similarity index 100%
rename from contrib/gdb/gdb/c-exp.c
rename to gdb/c-exp.c
diff --git a/contrib/gdb/gdb/c-exp.y b/gdb/c-exp.y
similarity index 100%
rename from contrib/gdb/gdb/c-exp.y
rename to gdb/c-exp.y
diff --git a/contrib/gdb/gdb/c-lang.c b/gdb/c-lang.c
similarity index 100%
rename from contrib/gdb/gdb/c-lang.c
rename to gdb/c-lang.c
diff --git a/contrib/gdb/gdb/c-lang.h b/gdb/c-lang.h
similarity index 100%
rename from contrib/gdb/gdb/c-lang.h
rename to gdb/c-lang.h
diff --git a/contrib/gdb/gdb/c-typeprint.c b/gdb/c-typeprint.c
similarity index 100%
rename from contrib/gdb/gdb/c-typeprint.c
rename to gdb/c-typeprint.c
diff --git a/contrib/gdb/gdb/c-valprint.c b/gdb/c-valprint.c
similarity index 100%
rename from contrib/gdb/gdb/c-valprint.c
rename to gdb/c-valprint.c
diff --git a/contrib/gdb/gdb/call-cmds.h b/gdb/call-cmds.h
similarity index 100%
rename from contrib/gdb/gdb/call-cmds.h
rename to gdb/call-cmds.h
diff --git a/contrib/gdb/gdb/charset.c b/gdb/charset.c
similarity index 100%
rename from contrib/gdb/gdb/charset.c
rename to gdb/charset.c
diff --git a/contrib/gdb/gdb/charset.h b/gdb/charset.h
similarity index 100%
rename from contrib/gdb/gdb/charset.h
rename to gdb/charset.h
diff --git a/contrib/gdb/gdb/cli-out.c b/gdb/cli-out.c
similarity index 100%
rename from contrib/gdb/gdb/cli-out.c
rename to gdb/cli-out.c
diff --git a/contrib/gdb/gdb/cli-out.h b/gdb/cli-out.h
similarity index 100%
rename from contrib/gdb/gdb/cli-out.h
rename to gdb/cli-out.h
diff --git a/contrib/gdb/gdb/cli/cli-cmds.c b/gdb/cli/cli-cmds.c
similarity index 100%
rename from contrib/gdb/gdb/cli/cli-cmds.c
rename to gdb/cli/cli-cmds.c
diff --git a/contrib/gdb/gdb/cli/cli-cmds.h b/gdb/cli/cli-cmds.h
similarity index 100%
rename from contrib/gdb/gdb/cli/cli-cmds.h
rename to gdb/cli/cli-cmds.h
diff --git a/contrib/gdb/gdb/cli/cli-decode.c b/gdb/cli/cli-decode.c
similarity index 100%
rename from contrib/gdb/gdb/cli/cli-decode.c
rename to gdb/cli/cli-decode.c
diff --git a/contrib/gdb/gdb/cli/cli-decode.h b/gdb/cli/cli-decode.h
similarity index 100%
rename from contrib/gdb/gdb/cli/cli-decode.h
rename to gdb/cli/cli-decode.h
diff --git a/contrib/gdb/gdb/cli/cli-dump.c b/gdb/cli/cli-dump.c
similarity index 100%
rename from contrib/gdb/gdb/cli/cli-dump.c
rename to gdb/cli/cli-dump.c
diff --git a/contrib/gdb/gdb/cli/cli-dump.h b/gdb/cli/cli-dump.h
similarity index 100%
rename from contrib/gdb/gdb/cli/cli-dump.h
rename to gdb/cli/cli-dump.h
diff --git a/contrib/gdb/gdb/cli/cli-interp.c b/gdb/cli/cli-interp.c
similarity index 100%
rename from contrib/gdb/gdb/cli/cli-interp.c
rename to gdb/cli/cli-interp.c
diff --git a/contrib/gdb/gdb/cli/cli-logging.c b/gdb/cli/cli-logging.c
similarity index 100%
rename from contrib/gdb/gdb/cli/cli-logging.c
rename to gdb/cli/cli-logging.c
diff --git a/contrib/gdb/gdb/cli/cli-script.c b/gdb/cli/cli-script.c
similarity index 100%
rename from contrib/gdb/gdb/cli/cli-script.c
rename to gdb/cli/cli-script.c
diff --git a/contrib/gdb/gdb/cli/cli-script.h b/gdb/cli/cli-script.h
similarity index 100%
rename from contrib/gdb/gdb/cli/cli-script.h
rename to gdb/cli/cli-script.h
diff --git a/contrib/gdb/gdb/cli/cli-setshow.c b/gdb/cli/cli-setshow.c
similarity index 100%
rename from contrib/gdb/gdb/cli/cli-setshow.c
rename to gdb/cli/cli-setshow.c
diff --git a/contrib/gdb/gdb/cli/cli-setshow.h b/gdb/cli/cli-setshow.h
similarity index 100%
rename from contrib/gdb/gdb/cli/cli-setshow.h
rename to gdb/cli/cli-setshow.h
diff --git a/contrib/gdb/gdb/cli/cli-utils.c b/gdb/cli/cli-utils.c
similarity index 100%
rename from contrib/gdb/gdb/cli/cli-utils.c
rename to gdb/cli/cli-utils.c
diff --git a/contrib/gdb/gdb/cli/cli-utils.h b/gdb/cli/cli-utils.h
similarity index 100%
rename from contrib/gdb/gdb/cli/cli-utils.h
rename to gdb/cli/cli-utils.h
diff --git a/contrib/gdb/gdb/coff-pe-read.c b/gdb/coff-pe-read.c
similarity index 100%
rename from contrib/gdb/gdb/coff-pe-read.c
rename to gdb/coff-pe-read.c
diff --git a/contrib/gdb/gdb/coff-pe-read.h b/gdb/coff-pe-read.h
similarity index 100%
rename from contrib/gdb/gdb/coff-pe-read.h
rename to gdb/coff-pe-read.h
diff --git a/contrib/gdb/gdb/coff-solib.c b/gdb/coff-solib.c
similarity index 100%
rename from contrib/gdb/gdb/coff-solib.c
rename to gdb/coff-solib.c
diff --git a/contrib/gdb/gdb/coff-solib.h b/gdb/coff-solib.h
similarity index 100%
rename from contrib/gdb/gdb/coff-solib.h
rename to gdb/coff-solib.h
diff --git a/contrib/gdb/gdb/coffread.c b/gdb/coffread.c
similarity index 100%
rename from contrib/gdb/gdb/coffread.c
rename to gdb/coffread.c
diff --git a/contrib/gdb/gdb/command.h b/gdb/command.h
similarity index 100%
rename from contrib/gdb/gdb/command.h
rename to gdb/command.h
diff --git a/contrib/gdb/gdb/complaints.c b/gdb/complaints.c
similarity index 100%
rename from contrib/gdb/gdb/complaints.c
rename to gdb/complaints.c
diff --git a/contrib/gdb/gdb/complaints.h b/gdb/complaints.h
similarity index 100%
rename from contrib/gdb/gdb/complaints.h
rename to gdb/complaints.h
diff --git a/contrib/gdb/gdb/completer.c b/gdb/completer.c
similarity index 100%
rename from contrib/gdb/gdb/completer.c
rename to gdb/completer.c
diff --git a/contrib/gdb/gdb/completer.h b/gdb/completer.h
similarity index 100%
rename from contrib/gdb/gdb/completer.h
rename to gdb/completer.h
diff --git a/contrib/gdb/gdb/config/alpha/alpha-osf2.mh b/gdb/config/alpha/alpha-osf2.mh
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/alpha-osf2.mh
rename to gdb/config/alpha/alpha-osf2.mh
diff --git a/contrib/gdb/gdb/config/alpha/alpha-osf3.mh b/gdb/config/alpha/alpha-osf3.mh
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/alpha-osf3.mh
rename to gdb/config/alpha/alpha-osf3.mh
diff --git a/contrib/gdb/gdb/config/alpha/alpha.mt b/gdb/config/alpha/alpha.mt
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/alpha.mt
rename to gdb/config/alpha/alpha.mt
diff --git a/contrib/gdb/gdb/config/alpha/fbsd.mh b/gdb/config/alpha/fbsd.mh
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/fbsd.mh
rename to gdb/config/alpha/fbsd.mh
diff --git a/contrib/gdb/gdb/config/alpha/fbsd.mt b/gdb/config/alpha/fbsd.mt
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/fbsd.mt
rename to gdb/config/alpha/fbsd.mt
diff --git a/contrib/gdb/gdb/config/alpha/nbsd.mh b/gdb/config/alpha/nbsd.mh
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/nbsd.mh
rename to gdb/config/alpha/nbsd.mh
diff --git a/contrib/gdb/gdb/config/alpha/nbsd.mt b/gdb/config/alpha/nbsd.mt
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/nbsd.mt
rename to gdb/config/alpha/nbsd.mt
diff --git a/contrib/gdb/gdb/config/alpha/nm-fbsd.h b/gdb/config/alpha/nm-fbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/nm-fbsd.h
rename to gdb/config/alpha/nm-fbsd.h
diff --git a/contrib/gdb/gdb/config/alpha/nm-nbsd.h b/gdb/config/alpha/nm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/nm-nbsd.h
rename to gdb/config/alpha/nm-nbsd.h
diff --git a/contrib/gdb/gdb/config/alpha/nm-osf.h b/gdb/config/alpha/nm-osf.h
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/nm-osf.h
rename to gdb/config/alpha/nm-osf.h
diff --git a/contrib/gdb/gdb/config/alpha/nm-osf2.h b/gdb/config/alpha/nm-osf2.h
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/nm-osf2.h
rename to gdb/config/alpha/nm-osf2.h
diff --git a/contrib/gdb/gdb/config/alpha/nm-osf3.h b/gdb/config/alpha/nm-osf3.h
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/nm-osf3.h
rename to gdb/config/alpha/nm-osf3.h
diff --git a/contrib/gdb/gdb/config/alpha/tm-alpha.h b/gdb/config/alpha/tm-alpha.h
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/tm-alpha.h
rename to gdb/config/alpha/tm-alpha.h
diff --git a/contrib/gdb/gdb/config/alpha/tm-fbsd.h b/gdb/config/alpha/tm-fbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/tm-fbsd.h
rename to gdb/config/alpha/tm-fbsd.h
diff --git a/contrib/gdb/gdb/config/alpha/tm-nbsd.h b/gdb/config/alpha/tm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/tm-nbsd.h
rename to gdb/config/alpha/tm-nbsd.h
diff --git a/contrib/gdb/gdb/config/alpha/xm-alphaosf.h b/gdb/config/alpha/xm-alphaosf.h
similarity index 100%
rename from contrib/gdb/gdb/config/alpha/xm-alphaosf.h
rename to gdb/config/alpha/xm-alphaosf.h
diff --git a/contrib/gdb/gdb/config/arm/embed.mt b/gdb/config/arm/embed.mt
similarity index 100%
rename from contrib/gdb/gdb/config/arm/embed.mt
rename to gdb/config/arm/embed.mt
diff --git a/contrib/gdb/gdb/config/arm/nbsd.mt b/gdb/config/arm/nbsd.mt
similarity index 100%
rename from contrib/gdb/gdb/config/arm/nbsd.mt
rename to gdb/config/arm/nbsd.mt
diff --git a/contrib/gdb/gdb/config/arm/nbsdaout.mh b/gdb/config/arm/nbsdaout.mh
similarity index 100%
rename from contrib/gdb/gdb/config/arm/nbsdaout.mh
rename to gdb/config/arm/nbsdaout.mh
diff --git a/contrib/gdb/gdb/config/arm/nbsdelf.mh b/gdb/config/arm/nbsdelf.mh
similarity index 100%
rename from contrib/gdb/gdb/config/arm/nbsdelf.mh
rename to gdb/config/arm/nbsdelf.mh
diff --git a/contrib/gdb/gdb/config/arm/nm-nbsd.h b/gdb/config/arm/nm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/arm/nm-nbsd.h
rename to gdb/config/arm/nm-nbsd.h
diff --git a/contrib/gdb/gdb/config/arm/nm-nbsdaout.h b/gdb/config/arm/nm-nbsdaout.h
similarity index 100%
rename from contrib/gdb/gdb/config/arm/nm-nbsdaout.h
rename to gdb/config/arm/nm-nbsdaout.h
diff --git a/contrib/gdb/gdb/config/arm/tm-arm.h b/gdb/config/arm/tm-arm.h
similarity index 100%
rename from contrib/gdb/gdb/config/arm/tm-arm.h
rename to gdb/config/arm/tm-arm.h
diff --git a/contrib/gdb/gdb/config/arm/tm-embed.h b/gdb/config/arm/tm-embed.h
similarity index 100%
rename from contrib/gdb/gdb/config/arm/tm-embed.h
rename to gdb/config/arm/tm-embed.h
diff --git a/contrib/gdb/gdb/config/arm/tm-nbsd.h b/gdb/config/arm/tm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/arm/tm-nbsd.h
rename to gdb/config/arm/tm-nbsd.h
diff --git a/contrib/gdb/gdb/config/arm/tm-wince.h b/gdb/config/arm/tm-wince.h
similarity index 100%
rename from contrib/gdb/gdb/config/arm/tm-wince.h
rename to gdb/config/arm/tm-wince.h
diff --git a/contrib/gdb/gdb/config/arm/wince.mt b/gdb/config/arm/wince.mt
similarity index 100%
rename from contrib/gdb/gdb/config/arm/wince.mt
rename to gdb/config/arm/wince.mt
diff --git a/contrib/gdb/gdb/config/arm/xm-nbsd.h b/gdb/config/arm/xm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/arm/xm-nbsd.h
rename to gdb/config/arm/xm-nbsd.h
diff --git a/contrib/gdb/gdb/config/i386/embed.mt b/gdb/config/i386/embed.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/embed.mt
rename to gdb/config/i386/embed.mt
diff --git a/contrib/gdb/gdb/config/i386/fbsd.mh b/gdb/config/i386/fbsd.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/fbsd.mh
rename to gdb/config/i386/fbsd.mh
diff --git a/contrib/gdb/gdb/config/i386/fbsd.mt b/gdb/config/i386/fbsd.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/fbsd.mt
rename to gdb/config/i386/fbsd.mt
diff --git a/contrib/gdb/gdb/config/i386/fbsd64.mh b/gdb/config/i386/fbsd64.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/fbsd64.mh
rename to gdb/config/i386/fbsd64.mh
diff --git a/contrib/gdb/gdb/config/i386/fbsd64.mt b/gdb/config/i386/fbsd64.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/fbsd64.mt
rename to gdb/config/i386/fbsd64.mt
diff --git a/contrib/gdb/gdb/config/i386/go32.mh b/gdb/config/i386/go32.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/go32.mh
rename to gdb/config/i386/go32.mh
diff --git a/contrib/gdb/gdb/config/i386/go32.mt b/gdb/config/i386/go32.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/go32.mt
rename to gdb/config/i386/go32.mt
diff --git a/contrib/gdb/gdb/config/i386/i386aout.mt b/gdb/config/i386/i386aout.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386aout.mt
rename to gdb/config/i386/i386aout.mt
diff --git a/contrib/gdb/gdb/config/i386/i386gnu.mh b/gdb/config/i386/i386gnu.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386gnu.mh
rename to gdb/config/i386/i386gnu.mh
diff --git a/contrib/gdb/gdb/config/i386/i386gnu.mt b/gdb/config/i386/i386gnu.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386gnu.mt
rename to gdb/config/i386/i386gnu.mt
diff --git a/contrib/gdb/gdb/config/i386/i386lynx.mh b/gdb/config/i386/i386lynx.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386lynx.mh
rename to gdb/config/i386/i386lynx.mh
diff --git a/contrib/gdb/gdb/config/i386/i386lynx.mt b/gdb/config/i386/i386lynx.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386lynx.mt
rename to gdb/config/i386/i386lynx.mt
diff --git a/contrib/gdb/gdb/config/i386/i386nw.mt b/gdb/config/i386/i386nw.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386nw.mt
rename to gdb/config/i386/i386nw.mt
diff --git a/contrib/gdb/gdb/config/i386/i386sco.mh b/gdb/config/i386/i386sco.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386sco.mh
rename to gdb/config/i386/i386sco.mh
diff --git a/contrib/gdb/gdb/config/i386/i386sco4.mh b/gdb/config/i386/i386sco4.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386sco4.mh
rename to gdb/config/i386/i386sco4.mh
diff --git a/contrib/gdb/gdb/config/i386/i386sco5.mh b/gdb/config/i386/i386sco5.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386sco5.mh
rename to gdb/config/i386/i386sco5.mh
diff --git a/contrib/gdb/gdb/config/i386/i386sol2.mh b/gdb/config/i386/i386sol2.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386sol2.mh
rename to gdb/config/i386/i386sol2.mh
diff --git a/contrib/gdb/gdb/config/i386/i386sol2.mt b/gdb/config/i386/i386sol2.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386sol2.mt
rename to gdb/config/i386/i386sol2.mt
diff --git a/contrib/gdb/gdb/config/i386/i386v.mh b/gdb/config/i386/i386v.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386v.mh
rename to gdb/config/i386/i386v.mh
diff --git a/contrib/gdb/gdb/config/i386/i386v.mt b/gdb/config/i386/i386v.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386v.mt
rename to gdb/config/i386/i386v.mt
diff --git a/contrib/gdb/gdb/config/i386/i386v4.mh b/gdb/config/i386/i386v4.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386v4.mh
rename to gdb/config/i386/i386v4.mh
diff --git a/contrib/gdb/gdb/config/i386/i386v42mp.mh b/gdb/config/i386/i386v42mp.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/i386v42mp.mh
rename to gdb/config/i386/i386v42mp.mh
diff --git a/contrib/gdb/gdb/config/i386/nbsd.mt b/gdb/config/i386/nbsd.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nbsd.mt
rename to gdb/config/i386/nbsd.mt
diff --git a/contrib/gdb/gdb/config/i386/nbsd64.mh b/gdb/config/i386/nbsd64.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nbsd64.mh
rename to gdb/config/i386/nbsd64.mh
diff --git a/contrib/gdb/gdb/config/i386/nbsd64.mt b/gdb/config/i386/nbsd64.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nbsd64.mt
rename to gdb/config/i386/nbsd64.mt
diff --git a/contrib/gdb/gdb/config/i386/nbsdaout.mh b/gdb/config/i386/nbsdaout.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nbsdaout.mh
rename to gdb/config/i386/nbsdaout.mh
diff --git a/contrib/gdb/gdb/config/i386/nbsdelf.mh b/gdb/config/i386/nbsdelf.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nbsdelf.mh
rename to gdb/config/i386/nbsdelf.mh
diff --git a/contrib/gdb/gdb/config/i386/ncr3000.mh b/gdb/config/i386/ncr3000.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/ncr3000.mh
rename to gdb/config/i386/ncr3000.mh
diff --git a/contrib/gdb/gdb/config/i386/ncr3000.mt b/gdb/config/i386/ncr3000.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/ncr3000.mt
rename to gdb/config/i386/ncr3000.mt
diff --git a/contrib/gdb/gdb/config/i386/nm-fbsd.h b/gdb/config/i386/nm-fbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-fbsd.h
rename to gdb/config/i386/nm-fbsd.h
diff --git a/contrib/gdb/gdb/config/i386/nm-fbsd64.h b/gdb/config/i386/nm-fbsd64.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-fbsd64.h
rename to gdb/config/i386/nm-fbsd64.h
diff --git a/contrib/gdb/gdb/config/i386/nm-go32.h b/gdb/config/i386/nm-go32.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-go32.h
rename to gdb/config/i386/nm-go32.h
diff --git a/contrib/gdb/gdb/config/i386/nm-i386.h b/gdb/config/i386/nm-i386.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-i386.h
rename to gdb/config/i386/nm-i386.h
diff --git a/contrib/gdb/gdb/config/i386/nm-i386gnu.h b/gdb/config/i386/nm-i386gnu.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-i386gnu.h
rename to gdb/config/i386/nm-i386gnu.h
diff --git a/contrib/gdb/gdb/config/i386/nm-i386lynx.h b/gdb/config/i386/nm-i386lynx.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-i386lynx.h
rename to gdb/config/i386/nm-i386lynx.h
diff --git a/contrib/gdb/gdb/config/i386/nm-i386sco.h b/gdb/config/i386/nm-i386sco.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-i386sco.h
rename to gdb/config/i386/nm-i386sco.h
diff --git a/contrib/gdb/gdb/config/i386/nm-i386sco4.h b/gdb/config/i386/nm-i386sco4.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-i386sco4.h
rename to gdb/config/i386/nm-i386sco4.h
diff --git a/contrib/gdb/gdb/config/i386/nm-i386sco5.h b/gdb/config/i386/nm-i386sco5.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-i386sco5.h
rename to gdb/config/i386/nm-i386sco5.h
diff --git a/contrib/gdb/gdb/config/i386/nm-i386sol2.h b/gdb/config/i386/nm-i386sol2.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-i386sol2.h
rename to gdb/config/i386/nm-i386sol2.h
diff --git a/contrib/gdb/gdb/config/i386/nm-i386v.h b/gdb/config/i386/nm-i386v.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-i386v.h
rename to gdb/config/i386/nm-i386v.h
diff --git a/contrib/gdb/gdb/config/i386/nm-i386v4.h b/gdb/config/i386/nm-i386v4.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-i386v4.h
rename to gdb/config/i386/nm-i386v4.h
diff --git a/contrib/gdb/gdb/config/i386/nm-i386v42mp.h b/gdb/config/i386/nm-i386v42mp.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-i386v42mp.h
rename to gdb/config/i386/nm-i386v42mp.h
diff --git a/contrib/gdb/gdb/config/i386/nm-nbsd.h b/gdb/config/i386/nm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-nbsd.h
rename to gdb/config/i386/nm-nbsd.h
diff --git a/contrib/gdb/gdb/config/i386/nm-nbsdaout.h b/gdb/config/i386/nm-nbsdaout.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-nbsdaout.h
rename to gdb/config/i386/nm-nbsdaout.h
diff --git a/contrib/gdb/gdb/config/i386/nm-nto.h b/gdb/config/i386/nm-nto.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-nto.h
rename to gdb/config/i386/nm-nto.h
diff --git a/contrib/gdb/gdb/config/i386/nm-obsd.h b/gdb/config/i386/nm-obsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nm-obsd.h
rename to gdb/config/i386/nm-obsd.h
diff --git a/contrib/gdb/gdb/config/i386/nto.mh b/gdb/config/i386/nto.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nto.mh
rename to gdb/config/i386/nto.mh
diff --git a/contrib/gdb/gdb/config/i386/nto.mt b/gdb/config/i386/nto.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/nto.mt
rename to gdb/config/i386/nto.mt
diff --git a/contrib/gdb/gdb/config/i386/obsd.mh b/gdb/config/i386/obsd.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/obsd.mh
rename to gdb/config/i386/obsd.mh
diff --git a/contrib/gdb/gdb/config/i386/obsd.mt b/gdb/config/i386/obsd.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/obsd.mt
rename to gdb/config/i386/obsd.mt
diff --git a/contrib/gdb/gdb/config/i386/obsd64.mh b/gdb/config/i386/obsd64.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/obsd64.mh
rename to gdb/config/i386/obsd64.mh
diff --git a/contrib/gdb/gdb/config/i386/obsd64.mt b/gdb/config/i386/obsd64.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/obsd64.mt
rename to gdb/config/i386/obsd64.mt
diff --git a/contrib/gdb/gdb/config/i386/obsdaout.mh b/gdb/config/i386/obsdaout.mh
similarity index 100%
rename from contrib/gdb/gdb/config/i386/obsdaout.mh
rename to gdb/config/i386/obsdaout.mh
diff --git a/contrib/gdb/gdb/config/i386/tm-fbsd.h b/gdb/config/i386/tm-fbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/tm-fbsd.h
rename to gdb/config/i386/tm-fbsd.h
diff --git a/contrib/gdb/gdb/config/i386/tm-go32.h b/gdb/config/i386/tm-go32.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/tm-go32.h
rename to gdb/config/i386/tm-go32.h
diff --git a/contrib/gdb/gdb/config/i386/tm-i386.h b/gdb/config/i386/tm-i386.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/tm-i386.h
rename to gdb/config/i386/tm-i386.h
diff --git a/contrib/gdb/gdb/config/i386/tm-i386lynx.h b/gdb/config/i386/tm-i386lynx.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/tm-i386lynx.h
rename to gdb/config/i386/tm-i386lynx.h
diff --git a/contrib/gdb/gdb/config/i386/tm-i386sol2.h b/gdb/config/i386/tm-i386sol2.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/tm-i386sol2.h
rename to gdb/config/i386/tm-i386sol2.h
diff --git a/contrib/gdb/gdb/config/i386/tm-nbsd.h b/gdb/config/i386/tm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/tm-nbsd.h
rename to gdb/config/i386/tm-nbsd.h
diff --git a/contrib/gdb/gdb/config/i386/tm-nto.h b/gdb/config/i386/tm-nto.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/tm-nto.h
rename to gdb/config/i386/tm-nto.h
diff --git a/contrib/gdb/gdb/config/i386/tm-vxworks.h b/gdb/config/i386/tm-vxworks.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/tm-vxworks.h
rename to gdb/config/i386/tm-vxworks.h
diff --git a/contrib/gdb/gdb/config/i386/vxworks.mt b/gdb/config/i386/vxworks.mt
similarity index 100%
rename from contrib/gdb/gdb/config/i386/vxworks.mt
rename to gdb/config/i386/vxworks.mt
diff --git a/contrib/gdb/gdb/config/i386/xm-go32.h b/gdb/config/i386/xm-go32.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/xm-go32.h
rename to gdb/config/i386/xm-go32.h
diff --git a/contrib/gdb/gdb/config/i386/xm-i386.h b/gdb/config/i386/xm-i386.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/xm-i386.h
rename to gdb/config/i386/xm-i386.h
diff --git a/contrib/gdb/gdb/config/i386/xm-i386sco.h b/gdb/config/i386/xm-i386sco.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/xm-i386sco.h
rename to gdb/config/i386/xm-i386sco.h
diff --git a/contrib/gdb/gdb/config/i386/xm-i386v.h b/gdb/config/i386/xm-i386v.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/xm-i386v.h
rename to gdb/config/i386/xm-i386v.h
diff --git a/contrib/gdb/gdb/config/i386/xm-i386v4.h b/gdb/config/i386/xm-i386v4.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/xm-i386v4.h
rename to gdb/config/i386/xm-i386v4.h
diff --git a/contrib/gdb/gdb/config/i386/xm-nbsd.h b/gdb/config/i386/xm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/i386/xm-nbsd.h
rename to gdb/config/i386/xm-nbsd.h
diff --git a/contrib/gdb/gdb/config/ia64/fbsd.mh b/gdb/config/ia64/fbsd.mh
similarity index 100%
rename from contrib/gdb/gdb/config/ia64/fbsd.mh
rename to gdb/config/ia64/fbsd.mh
diff --git a/contrib/gdb/gdb/config/ia64/fbsd.mt b/gdb/config/ia64/fbsd.mt
similarity index 100%
rename from contrib/gdb/gdb/config/ia64/fbsd.mt
rename to gdb/config/ia64/fbsd.mt
diff --git a/contrib/gdb/gdb/config/ia64/ia64.mt b/gdb/config/ia64/ia64.mt
similarity index 100%
rename from contrib/gdb/gdb/config/ia64/ia64.mt
rename to gdb/config/ia64/ia64.mt
diff --git a/contrib/gdb/gdb/config/ia64/nm-fbsd.h b/gdb/config/ia64/nm-fbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/ia64/nm-fbsd.h
rename to gdb/config/ia64/nm-fbsd.h
diff --git a/contrib/gdb/gdb/config/ia64/tm-fbsd.h b/gdb/config/ia64/tm-fbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/ia64/tm-fbsd.h
rename to gdb/config/ia64/tm-fbsd.h
diff --git a/contrib/gdb/gdb/config/ia64/tm-ia64.h b/gdb/config/ia64/tm-ia64.h
similarity index 100%
rename from contrib/gdb/gdb/config/ia64/tm-ia64.h
rename to gdb/config/ia64/tm-ia64.h
diff --git a/contrib/gdb/gdb/config/mips/decstation.mh b/gdb/config/mips/decstation.mh
similarity index 100%
rename from contrib/gdb/gdb/config/mips/decstation.mh
rename to gdb/config/mips/decstation.mh
diff --git a/contrib/gdb/gdb/config/mips/embed.mt b/gdb/config/mips/embed.mt
similarity index 100%
rename from contrib/gdb/gdb/config/mips/embed.mt
rename to gdb/config/mips/embed.mt
diff --git a/contrib/gdb/gdb/config/mips/littlemips.mh b/gdb/config/mips/littlemips.mh
similarity index 100%
rename from contrib/gdb/gdb/config/mips/littlemips.mh
rename to gdb/config/mips/littlemips.mh
diff --git a/contrib/gdb/gdb/config/mips/mipsv4.mh b/gdb/config/mips/mipsv4.mh
similarity index 100%
rename from contrib/gdb/gdb/config/mips/mipsv4.mh
rename to gdb/config/mips/mipsv4.mh
diff --git a/contrib/gdb/gdb/config/mips/mipsv4.mt b/gdb/config/mips/mipsv4.mt
similarity index 100%
rename from contrib/gdb/gdb/config/mips/mipsv4.mt
rename to gdb/config/mips/mipsv4.mt
diff --git a/contrib/gdb/gdb/config/mips/nbsd.mh b/gdb/config/mips/nbsd.mh
similarity index 100%
rename from contrib/gdb/gdb/config/mips/nbsd.mh
rename to gdb/config/mips/nbsd.mh
diff --git a/contrib/gdb/gdb/config/mips/nbsd.mt b/gdb/config/mips/nbsd.mt
similarity index 100%
rename from contrib/gdb/gdb/config/mips/nbsd.mt
rename to gdb/config/mips/nbsd.mt
diff --git a/contrib/gdb/gdb/config/mips/news-mips.mh b/gdb/config/mips/news-mips.mh
similarity index 100%
rename from contrib/gdb/gdb/config/mips/news-mips.mh
rename to gdb/config/mips/news-mips.mh
diff --git a/contrib/gdb/gdb/config/mips/nm-mips.h b/gdb/config/mips/nm-mips.h
similarity index 100%
rename from contrib/gdb/gdb/config/mips/nm-mips.h
rename to gdb/config/mips/nm-mips.h
diff --git a/contrib/gdb/gdb/config/mips/nm-nbsd.h b/gdb/config/mips/nm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/mips/nm-nbsd.h
rename to gdb/config/mips/nm-nbsd.h
diff --git a/contrib/gdb/gdb/config/mips/nm-news-mips.h b/gdb/config/mips/nm-news-mips.h
similarity index 100%
rename from contrib/gdb/gdb/config/mips/nm-news-mips.h
rename to gdb/config/mips/nm-news-mips.h
diff --git a/contrib/gdb/gdb/config/mips/nm-riscos.h b/gdb/config/mips/nm-riscos.h
similarity index 100%
rename from contrib/gdb/gdb/config/mips/nm-riscos.h
rename to gdb/config/mips/nm-riscos.h
diff --git a/contrib/gdb/gdb/config/mips/riscos.mh b/gdb/config/mips/riscos.mh
similarity index 100%
rename from contrib/gdb/gdb/config/mips/riscos.mh
rename to gdb/config/mips/riscos.mh
diff --git a/contrib/gdb/gdb/config/mips/tm-mips.h b/gdb/config/mips/tm-mips.h
similarity index 100%
rename from contrib/gdb/gdb/config/mips/tm-mips.h
rename to gdb/config/mips/tm-mips.h
diff --git a/contrib/gdb/gdb/config/mips/tm-mipsv4.h b/gdb/config/mips/tm-mipsv4.h
similarity index 100%
rename from contrib/gdb/gdb/config/mips/tm-mipsv4.h
rename to gdb/config/mips/tm-mipsv4.h
diff --git a/contrib/gdb/gdb/config/mips/tm-nbsd.h b/gdb/config/mips/tm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/mips/tm-nbsd.h
rename to gdb/config/mips/tm-nbsd.h
diff --git a/contrib/gdb/gdb/config/mips/tm-vxmips.h b/gdb/config/mips/tm-vxmips.h
similarity index 100%
rename from contrib/gdb/gdb/config/mips/tm-vxmips.h
rename to gdb/config/mips/tm-vxmips.h
diff --git a/contrib/gdb/gdb/config/mips/tm-wince.h b/gdb/config/mips/tm-wince.h
similarity index 100%
rename from contrib/gdb/gdb/config/mips/tm-wince.h
rename to gdb/config/mips/tm-wince.h
diff --git a/contrib/gdb/gdb/config/mips/vxmips.mt b/gdb/config/mips/vxmips.mt
similarity index 100%
rename from contrib/gdb/gdb/config/mips/vxmips.mt
rename to gdb/config/mips/vxmips.mt
diff --git a/contrib/gdb/gdb/config/mips/wince.mt b/gdb/config/mips/wince.mt
similarity index 100%
rename from contrib/gdb/gdb/config/mips/wince.mt
rename to gdb/config/mips/wince.mt
diff --git a/contrib/gdb/gdb/config/mips/xm-mips.h b/gdb/config/mips/xm-mips.h
similarity index 100%
rename from contrib/gdb/gdb/config/mips/xm-mips.h
rename to gdb/config/mips/xm-mips.h
diff --git a/contrib/gdb/gdb/config/mips/xm-mipsv4.h b/gdb/config/mips/xm-mipsv4.h
similarity index 100%
rename from contrib/gdb/gdb/config/mips/xm-mipsv4.h
rename to gdb/config/mips/xm-mipsv4.h
diff --git a/contrib/gdb/gdb/config/mips/xm-riscos.h b/gdb/config/mips/xm-riscos.h
similarity index 100%
rename from contrib/gdb/gdb/config/mips/xm-riscos.h
rename to gdb/config/mips/xm-riscos.h
diff --git a/contrib/gdb/gdb/config/nm-gnu.h b/gdb/config/nm-gnu.h
similarity index 100%
rename from contrib/gdb/gdb/config/nm-gnu.h
rename to gdb/config/nm-gnu.h
diff --git a/contrib/gdb/gdb/config/nm-lynx.h b/gdb/config/nm-lynx.h
similarity index 100%
rename from contrib/gdb/gdb/config/nm-lynx.h
rename to gdb/config/nm-lynx.h
diff --git a/contrib/gdb/gdb/config/nm-nbsd.h b/gdb/config/nm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/nm-nbsd.h
rename to gdb/config/nm-nbsd.h
diff --git a/contrib/gdb/gdb/config/nm-nbsdaout.h b/gdb/config/nm-nbsdaout.h
similarity index 100%
rename from contrib/gdb/gdb/config/nm-nbsdaout.h
rename to gdb/config/nm-nbsdaout.h
diff --git a/contrib/gdb/gdb/config/nm-sysv4.h b/gdb/config/nm-sysv4.h
similarity index 100%
rename from contrib/gdb/gdb/config/nm-sysv4.h
rename to gdb/config/nm-sysv4.h
diff --git a/contrib/gdb/gdb/config/powerpc/fbsd.mh b/gdb/config/powerpc/fbsd.mh
similarity index 100%
rename from contrib/gdb/gdb/config/powerpc/fbsd.mh
rename to gdb/config/powerpc/fbsd.mh
diff --git a/contrib/gdb/gdb/config/powerpc/fbsd.mt b/gdb/config/powerpc/fbsd.mt
similarity index 100%
rename from contrib/gdb/gdb/config/powerpc/fbsd.mt
rename to gdb/config/powerpc/fbsd.mt
diff --git a/contrib/gdb/gdb/config/powerpc/nbsd.mh b/gdb/config/powerpc/nbsd.mh
similarity index 100%
rename from contrib/gdb/gdb/config/powerpc/nbsd.mh
rename to gdb/config/powerpc/nbsd.mh
diff --git a/contrib/gdb/gdb/config/powerpc/nbsd.mt b/gdb/config/powerpc/nbsd.mt
similarity index 100%
rename from contrib/gdb/gdb/config/powerpc/nbsd.mt
rename to gdb/config/powerpc/nbsd.mt
diff --git a/contrib/gdb/gdb/config/powerpc/nm-fbsd.h b/gdb/config/powerpc/nm-fbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/powerpc/nm-fbsd.h
rename to gdb/config/powerpc/nm-fbsd.h
diff --git a/contrib/gdb/gdb/config/powerpc/nm-nbsd.h b/gdb/config/powerpc/nm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/powerpc/nm-nbsd.h
rename to gdb/config/powerpc/nm-nbsd.h
diff --git a/contrib/gdb/gdb/config/powerpc/ppc-eabi.mt b/gdb/config/powerpc/ppc-eabi.mt
similarity index 100%
rename from contrib/gdb/gdb/config/powerpc/ppc-eabi.mt
rename to gdb/config/powerpc/ppc-eabi.mt
diff --git a/contrib/gdb/gdb/config/powerpc/ppc-sim.mt b/gdb/config/powerpc/ppc-sim.mt
similarity index 100%
rename from contrib/gdb/gdb/config/powerpc/ppc-sim.mt
rename to gdb/config/powerpc/ppc-sim.mt
diff --git a/contrib/gdb/gdb/config/powerpc/tm-nbsd.h b/gdb/config/powerpc/tm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/powerpc/tm-nbsd.h
rename to gdb/config/powerpc/tm-nbsd.h
diff --git a/contrib/gdb/gdb/config/powerpc/tm-ppc-eabi.h b/gdb/config/powerpc/tm-ppc-eabi.h
similarity index 100%
rename from contrib/gdb/gdb/config/powerpc/tm-ppc-eabi.h
rename to gdb/config/powerpc/tm-ppc-eabi.h
diff --git a/contrib/gdb/gdb/config/powerpc/tm-ppcle-eabi.h b/gdb/config/powerpc/tm-ppcle-eabi.h
similarity index 100%
rename from contrib/gdb/gdb/config/powerpc/tm-ppcle-eabi.h
rename to gdb/config/powerpc/tm-ppcle-eabi.h
diff --git a/contrib/gdb/gdb/config/powerpc/tm-ppcle-sim.h b/gdb/config/powerpc/tm-ppcle-sim.h
similarity index 100%
rename from contrib/gdb/gdb/config/powerpc/tm-ppcle-sim.h
rename to gdb/config/powerpc/tm-ppcle-sim.h
diff --git a/contrib/gdb/gdb/config/powerpc/tm-vxworks.h b/gdb/config/powerpc/tm-vxworks.h
similarity index 100%
rename from contrib/gdb/gdb/config/powerpc/tm-vxworks.h
rename to gdb/config/powerpc/tm-vxworks.h
diff --git a/contrib/gdb/gdb/config/powerpc/vxworks.mt b/gdb/config/powerpc/vxworks.mt
similarity index 100%
rename from contrib/gdb/gdb/config/powerpc/vxworks.mt
rename to gdb/config/powerpc/vxworks.mt
diff --git a/contrib/gdb/gdb/config/rs6000/nm-rs6000.h b/gdb/config/rs6000/nm-rs6000.h
similarity index 100%
rename from contrib/gdb/gdb/config/rs6000/nm-rs6000.h
rename to gdb/config/rs6000/nm-rs6000.h
diff --git a/contrib/gdb/gdb/config/rs6000/nm-rs6000ly.h b/gdb/config/rs6000/nm-rs6000ly.h
similarity index 100%
rename from contrib/gdb/gdb/config/rs6000/nm-rs6000ly.h
rename to gdb/config/rs6000/nm-rs6000ly.h
diff --git a/contrib/gdb/gdb/config/rs6000/rs6000.mh b/gdb/config/rs6000/rs6000.mh
similarity index 100%
rename from contrib/gdb/gdb/config/rs6000/rs6000.mh
rename to gdb/config/rs6000/rs6000.mh
diff --git a/contrib/gdb/gdb/config/rs6000/rs6000.mt b/gdb/config/rs6000/rs6000.mt
similarity index 100%
rename from contrib/gdb/gdb/config/rs6000/rs6000.mt
rename to gdb/config/rs6000/rs6000.mt
diff --git a/contrib/gdb/gdb/config/rs6000/rs6000lynx.mh b/gdb/config/rs6000/rs6000lynx.mh
similarity index 100%
rename from contrib/gdb/gdb/config/rs6000/rs6000lynx.mh
rename to gdb/config/rs6000/rs6000lynx.mh
diff --git a/contrib/gdb/gdb/config/rs6000/rs6000lynx.mt b/gdb/config/rs6000/rs6000lynx.mt
similarity index 100%
rename from contrib/gdb/gdb/config/rs6000/rs6000lynx.mt
rename to gdb/config/rs6000/rs6000lynx.mt
diff --git a/contrib/gdb/gdb/config/rs6000/tm-rs6000.h b/gdb/config/rs6000/tm-rs6000.h
similarity index 100%
rename from contrib/gdb/gdb/config/rs6000/tm-rs6000.h
rename to gdb/config/rs6000/tm-rs6000.h
diff --git a/contrib/gdb/gdb/config/rs6000/tm-rs6000ly.h b/gdb/config/rs6000/tm-rs6000ly.h
similarity index 100%
rename from contrib/gdb/gdb/config/rs6000/tm-rs6000ly.h
rename to gdb/config/rs6000/tm-rs6000ly.h
diff --git a/contrib/gdb/gdb/config/rs6000/xm-rs6000.h b/gdb/config/rs6000/xm-rs6000.h
similarity index 100%
rename from contrib/gdb/gdb/config/rs6000/xm-rs6000.h
rename to gdb/config/rs6000/xm-rs6000.h
diff --git a/contrib/gdb/gdb/config/s390/s390.mh b/gdb/config/s390/s390.mh
similarity index 100%
rename from contrib/gdb/gdb/config/s390/s390.mh
rename to gdb/config/s390/s390.mh
diff --git a/contrib/gdb/gdb/config/s390/s390.mt b/gdb/config/s390/s390.mt
similarity index 100%
rename from contrib/gdb/gdb/config/s390/s390.mt
rename to gdb/config/s390/s390.mt
diff --git a/contrib/gdb/gdb/config/sparc/fbsd.mh b/gdb/config/sparc/fbsd.mh
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/fbsd.mh
rename to gdb/config/sparc/fbsd.mh
diff --git a/contrib/gdb/gdb/config/sparc/fbsd.mt b/gdb/config/sparc/fbsd.mt
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/fbsd.mt
rename to gdb/config/sparc/fbsd.mt
diff --git a/contrib/gdb/gdb/config/sparc/nbsd.mt b/gdb/config/sparc/nbsd.mt
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/nbsd.mt
rename to gdb/config/sparc/nbsd.mt
diff --git a/contrib/gdb/gdb/config/sparc/nbsd64.mh b/gdb/config/sparc/nbsd64.mh
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/nbsd64.mh
rename to gdb/config/sparc/nbsd64.mh
diff --git a/contrib/gdb/gdb/config/sparc/nbsd64.mt b/gdb/config/sparc/nbsd64.mt
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/nbsd64.mt
rename to gdb/config/sparc/nbsd64.mt
diff --git a/contrib/gdb/gdb/config/sparc/nbsdaout.mh b/gdb/config/sparc/nbsdaout.mh
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/nbsdaout.mh
rename to gdb/config/sparc/nbsdaout.mh
diff --git a/contrib/gdb/gdb/config/sparc/nbsdelf.mh b/gdb/config/sparc/nbsdelf.mh
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/nbsdelf.mh
rename to gdb/config/sparc/nbsdelf.mh
diff --git a/contrib/gdb/gdb/config/sparc/nm-fbsd.h b/gdb/config/sparc/nm-fbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/nm-fbsd.h
rename to gdb/config/sparc/nm-fbsd.h
diff --git a/contrib/gdb/gdb/config/sparc/nm-nbsd.h b/gdb/config/sparc/nm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/nm-nbsd.h
rename to gdb/config/sparc/nm-nbsd.h
diff --git a/contrib/gdb/gdb/config/sparc/nm-nbsdaout.h b/gdb/config/sparc/nm-nbsdaout.h
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/nm-nbsdaout.h
rename to gdb/config/sparc/nm-nbsdaout.h
diff --git a/contrib/gdb/gdb/config/sparc/nm-sol2.h b/gdb/config/sparc/nm-sol2.h
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/nm-sol2.h
rename to gdb/config/sparc/nm-sol2.h
diff --git a/contrib/gdb/gdb/config/sparc/obsd.mt b/gdb/config/sparc/obsd.mt
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/obsd.mt
rename to gdb/config/sparc/obsd.mt
diff --git a/contrib/gdb/gdb/config/sparc/obsd64.mt b/gdb/config/sparc/obsd64.mt
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/obsd64.mt
rename to gdb/config/sparc/obsd64.mt
diff --git a/contrib/gdb/gdb/config/sparc/sol2-64.mt b/gdb/config/sparc/sol2-64.mt
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/sol2-64.mt
rename to gdb/config/sparc/sol2-64.mt
diff --git a/contrib/gdb/gdb/config/sparc/sol2.mh b/gdb/config/sparc/sol2.mh
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/sol2.mh
rename to gdb/config/sparc/sol2.mh
diff --git a/contrib/gdb/gdb/config/sparc/sol2.mt b/gdb/config/sparc/sol2.mt
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/sol2.mt
rename to gdb/config/sparc/sol2.mt
diff --git a/contrib/gdb/gdb/config/sparc/sparc.mt b/gdb/config/sparc/sparc.mt
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/sparc.mt
rename to gdb/config/sparc/sparc.mt
diff --git a/contrib/gdb/gdb/config/sparc/sparc64.mt b/gdb/config/sparc/sparc64.mt
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/sparc64.mt
rename to gdb/config/sparc/sparc64.mt
diff --git a/contrib/gdb/gdb/config/sparc/tm-fbsd.h b/gdb/config/sparc/tm-fbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/tm-fbsd.h
rename to gdb/config/sparc/tm-fbsd.h
diff --git a/contrib/gdb/gdb/config/sparc/tm-nbsd.h b/gdb/config/sparc/tm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/tm-nbsd.h
rename to gdb/config/sparc/tm-nbsd.h
diff --git a/contrib/gdb/gdb/config/sparc/tm-nbsd64.h b/gdb/config/sparc/tm-nbsd64.h
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/tm-nbsd64.h
rename to gdb/config/sparc/tm-nbsd64.h
diff --git a/contrib/gdb/gdb/config/sparc/tm-sol2.h b/gdb/config/sparc/tm-sol2.h
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/tm-sol2.h
rename to gdb/config/sparc/tm-sol2.h
diff --git a/contrib/gdb/gdb/config/sparc/tm-vxworks.h b/gdb/config/sparc/tm-vxworks.h
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/tm-vxworks.h
rename to gdb/config/sparc/tm-vxworks.h
diff --git a/contrib/gdb/gdb/config/sparc/vxworks.mt b/gdb/config/sparc/vxworks.mt
similarity index 100%
rename from contrib/gdb/gdb/config/sparc/vxworks.mt
rename to gdb/config/sparc/vxworks.mt
diff --git a/contrib/gdb/gdb/config/tm-lynx.h b/gdb/config/tm-lynx.h
similarity index 100%
rename from contrib/gdb/gdb/config/tm-lynx.h
rename to gdb/config/tm-lynx.h
diff --git a/contrib/gdb/gdb/config/tm-nto.h b/gdb/config/tm-nto.h
similarity index 100%
rename from contrib/gdb/gdb/config/tm-nto.h
rename to gdb/config/tm-nto.h
diff --git a/contrib/gdb/gdb/config/tm-sunos.h b/gdb/config/tm-sunos.h
similarity index 100%
rename from contrib/gdb/gdb/config/tm-sunos.h
rename to gdb/config/tm-sunos.h
diff --git a/contrib/gdb/gdb/config/tm-sysv4.h b/gdb/config/tm-sysv4.h
similarity index 100%
rename from contrib/gdb/gdb/config/tm-sysv4.h
rename to gdb/config/tm-sysv4.h
diff --git a/contrib/gdb/gdb/config/tm-vxworks.h b/gdb/config/tm-vxworks.h
similarity index 100%
rename from contrib/gdb/gdb/config/tm-vxworks.h
rename to gdb/config/tm-vxworks.h
diff --git a/contrib/gdb/gdb/config/xm-nbsd.h b/gdb/config/xm-nbsd.h
similarity index 100%
rename from contrib/gdb/gdb/config/xm-nbsd.h
rename to gdb/config/xm-nbsd.h
diff --git a/contrib/gdb/gdb/config/xm-sysv4.h b/gdb/config/xm-sysv4.h
similarity index 100%
rename from contrib/gdb/gdb/config/xm-sysv4.h
rename to gdb/config/xm-sysv4.h
diff --git a/contrib/gdb/gdb/copying.awk b/gdb/copying.awk
similarity index 100%
rename from contrib/gdb/gdb/copying.awk
rename to gdb/copying.awk
diff --git a/contrib/gdb/gdb/copying.c b/gdb/copying.c
similarity index 100%
rename from contrib/gdb/gdb/copying.c
rename to gdb/copying.c
diff --git a/contrib/gdb/gdb/core-aout.c b/gdb/core-aout.c
similarity index 100%
rename from contrib/gdb/gdb/core-aout.c
rename to gdb/core-aout.c
diff --git a/contrib/gdb/gdb/core-regset.c b/gdb/core-regset.c
similarity index 100%
rename from contrib/gdb/gdb/core-regset.c
rename to gdb/core-regset.c
diff --git a/contrib/gdb/gdb/corefile.c b/gdb/corefile.c
similarity index 100%
rename from contrib/gdb/gdb/corefile.c
rename to gdb/corefile.c
diff --git a/contrib/gdb/gdb/corelow.c b/gdb/corelow.c
similarity index 100%
rename from contrib/gdb/gdb/corelow.c
rename to gdb/corelow.c
diff --git a/contrib/gdb/gdb/cp-abi.c b/gdb/cp-abi.c
similarity index 100%
rename from contrib/gdb/gdb/cp-abi.c
rename to gdb/cp-abi.c
diff --git a/contrib/gdb/gdb/cp-abi.h b/gdb/cp-abi.h
similarity index 100%
rename from contrib/gdb/gdb/cp-abi.h
rename to gdb/cp-abi.h
diff --git a/contrib/gdb/gdb/cp-namespace.c b/gdb/cp-namespace.c
similarity index 100%
rename from contrib/gdb/gdb/cp-namespace.c
rename to gdb/cp-namespace.c
diff --git a/contrib/gdb/gdb/cp-support.c b/gdb/cp-support.c
similarity index 100%
rename from contrib/gdb/gdb/cp-support.c
rename to gdb/cp-support.c
diff --git a/contrib/gdb/gdb/cp-support.h b/gdb/cp-support.h
similarity index 100%
rename from contrib/gdb/gdb/cp-support.h
rename to gdb/cp-support.h
diff --git a/contrib/gdb/gdb/cp-valprint.c b/gdb/cp-valprint.c
similarity index 100%
rename from contrib/gdb/gdb/cp-valprint.c
rename to gdb/cp-valprint.c
diff --git a/contrib/gdb/gdb/cpu32bug-rom.c b/gdb/cpu32bug-rom.c
similarity index 100%
rename from contrib/gdb/gdb/cpu32bug-rom.c
rename to gdb/cpu32bug-rom.c
diff --git a/contrib/gdb/gdb/dbug-rom.c b/gdb/dbug-rom.c
similarity index 100%
rename from contrib/gdb/gdb/dbug-rom.c
rename to gdb/dbug-rom.c
diff --git a/contrib/gdb/gdb/dbxread.c b/gdb/dbxread.c
similarity index 100%
rename from contrib/gdb/gdb/dbxread.c
rename to gdb/dbxread.c
diff --git a/contrib/gdb/gdb/dcache.c b/gdb/dcache.c
similarity index 100%
rename from contrib/gdb/gdb/dcache.c
rename to gdb/dcache.c
diff --git a/contrib/gdb/gdb/dcache.h b/gdb/dcache.h
similarity index 100%
rename from contrib/gdb/gdb/dcache.h
rename to gdb/dcache.h
diff --git a/contrib/gdb/gdb/defs.h b/gdb/defs.h
similarity index 100%
rename from contrib/gdb/gdb/defs.h
rename to gdb/defs.h
diff --git a/contrib/gdb/gdb/delta68-nat.c b/gdb/delta68-nat.c
similarity index 100%
rename from contrib/gdb/gdb/delta68-nat.c
rename to gdb/delta68-nat.c
diff --git a/contrib/gdb/gdb/demangle.c b/gdb/demangle.c
similarity index 100%
rename from contrib/gdb/gdb/demangle.c
rename to gdb/demangle.c
diff --git a/contrib/gdb/gdb/dictionary.c b/gdb/dictionary.c
similarity index 100%
rename from contrib/gdb/gdb/dictionary.c
rename to gdb/dictionary.c
diff --git a/contrib/gdb/gdb/dictionary.h b/gdb/dictionary.h
similarity index 100%
rename from contrib/gdb/gdb/dictionary.h
rename to gdb/dictionary.h
diff --git a/contrib/gdb/gdb/dink32-rom.c b/gdb/dink32-rom.c
similarity index 100%
rename from contrib/gdb/gdb/dink32-rom.c
rename to gdb/dink32-rom.c
diff --git a/contrib/gdb/gdb/disasm.c b/gdb/disasm.c
similarity index 100%
rename from contrib/gdb/gdb/disasm.c
rename to gdb/disasm.c
diff --git a/contrib/gdb/gdb/disasm.h b/gdb/disasm.h
similarity index 100%
rename from contrib/gdb/gdb/disasm.h
rename to gdb/disasm.h
diff --git a/contrib/gdb/gdb/doc/GDBvn.texi b/gdb/doc/GDBvn.texi
similarity index 100%
rename from contrib/gdb/gdb/doc/GDBvn.texi
rename to gdb/doc/GDBvn.texi
diff --git a/contrib/gdb/gdb/doc/LRS b/gdb/doc/LRS
similarity index 100%
rename from contrib/gdb/gdb/doc/LRS
rename to gdb/doc/LRS
diff --git a/contrib/gdb/gdb/doc/a4rc.sed b/gdb/doc/a4rc.sed
similarity index 100%
rename from contrib/gdb/gdb/doc/a4rc.sed
rename to gdb/doc/a4rc.sed
diff --git a/contrib/gdb/gdb/doc/agentexpr.texi b/gdb/doc/agentexpr.texi
similarity index 100%
rename from contrib/gdb/gdb/doc/agentexpr.texi
rename to gdb/doc/agentexpr.texi
diff --git a/contrib/gdb/gdb/doc/all-cfg.texi b/gdb/doc/all-cfg.texi
similarity index 100%
rename from contrib/gdb/gdb/doc/all-cfg.texi
rename to gdb/doc/all-cfg.texi
diff --git a/contrib/gdb/gdb/doc/annotate.texinfo b/gdb/doc/annotate.texinfo
similarity index 100%
rename from contrib/gdb/gdb/doc/annotate.texinfo
rename to gdb/doc/annotate.texinfo
diff --git a/contrib/gdb/gdb/doc/fdl.texi b/gdb/doc/fdl.texi
similarity index 100%
rename from contrib/gdb/gdb/doc/fdl.texi
rename to gdb/doc/fdl.texi
diff --git a/contrib/gdb/gdb/doc/gdb.info-1 b/gdb/doc/gdb.info-1
similarity index 100%
rename from contrib/gdb/gdb/doc/gdb.info-1
rename to gdb/doc/gdb.info-1
diff --git a/contrib/gdb/gdb/doc/gdb.info-2 b/gdb/doc/gdb.info-2
similarity index 100%
rename from contrib/gdb/gdb/doc/gdb.info-2
rename to gdb/doc/gdb.info-2
diff --git a/contrib/gdb/gdb/doc/gdb.info-3 b/gdb/doc/gdb.info-3
similarity index 100%
rename from contrib/gdb/gdb/doc/gdb.info-3
rename to gdb/doc/gdb.info-3
diff --git a/contrib/gdb/gdb/doc/gdb.texinfo b/gdb/doc/gdb.texinfo
similarity index 100%
rename from contrib/gdb/gdb/doc/gdb.texinfo
rename to gdb/doc/gdb.texinfo
diff --git a/contrib/gdb/gdb/doc/gdbint.texinfo b/gdb/doc/gdbint.texinfo
similarity index 100%
rename from contrib/gdb/gdb/doc/gdbint.texinfo
rename to gdb/doc/gdbint.texinfo
diff --git a/contrib/gdb/gdb/doc/gpl.texi b/gdb/doc/gpl.texi
similarity index 100%
rename from contrib/gdb/gdb/doc/gpl.texi
rename to gdb/doc/gpl.texi
diff --git a/contrib/gdb/gdb/doc/lpsrc.sed b/gdb/doc/lpsrc.sed
similarity index 100%
rename from contrib/gdb/gdb/doc/lpsrc.sed
rename to gdb/doc/lpsrc.sed
diff --git a/contrib/gdb/gdb/doc/observer.texi b/gdb/doc/observer.texi
similarity index 100%
rename from contrib/gdb/gdb/doc/observer.texi
rename to gdb/doc/observer.texi
diff --git a/contrib/gdb/gdb/doc/psrc.sed b/gdb/doc/psrc.sed
similarity index 100%
rename from contrib/gdb/gdb/doc/psrc.sed
rename to gdb/doc/psrc.sed
diff --git a/contrib/gdb/gdb/doc/refcard.tex b/gdb/doc/refcard.tex
similarity index 100%
rename from contrib/gdb/gdb/doc/refcard.tex
rename to gdb/doc/refcard.tex
diff --git a/contrib/gdb/gdb/doc/stabs.texinfo b/gdb/doc/stabs.texinfo
similarity index 100%
rename from contrib/gdb/gdb/doc/stabs.texinfo
rename to gdb/doc/stabs.texinfo
diff --git a/contrib/gdb/gdb/doublest.c b/gdb/doublest.c
similarity index 100%
rename from contrib/gdb/gdb/doublest.c
rename to gdb/doublest.c
diff --git a/contrib/gdb/gdb/doublest.h b/gdb/doublest.h
similarity index 100%
rename from contrib/gdb/gdb/doublest.h
rename to gdb/doublest.h
diff --git a/contrib/gdb/gdb/dpx2-nat.c b/gdb/dpx2-nat.c
similarity index 100%
rename from contrib/gdb/gdb/dpx2-nat.c
rename to gdb/dpx2-nat.c
diff --git a/contrib/gdb/gdb/dsrec.c b/gdb/dsrec.c
similarity index 100%
rename from contrib/gdb/gdb/dsrec.c
rename to gdb/dsrec.c
diff --git a/contrib/gdb/gdb/dummy-frame.c b/gdb/dummy-frame.c
similarity index 100%
rename from contrib/gdb/gdb/dummy-frame.c
rename to gdb/dummy-frame.c
diff --git a/contrib/gdb/gdb/dummy-frame.h b/gdb/dummy-frame.h
similarity index 100%
rename from contrib/gdb/gdb/dummy-frame.h
rename to gdb/dummy-frame.h
diff --git a/contrib/gdb/gdb/dve3900-rom.c b/gdb/dve3900-rom.c
similarity index 100%
rename from contrib/gdb/gdb/dve3900-rom.c
rename to gdb/dve3900-rom.c
diff --git a/contrib/gdb/gdb/dwarf2-frame.c b/gdb/dwarf2-frame.c
similarity index 100%
rename from contrib/gdb/gdb/dwarf2-frame.c
rename to gdb/dwarf2-frame.c
diff --git a/contrib/gdb/gdb/dwarf2-frame.h b/gdb/dwarf2-frame.h
similarity index 100%
rename from contrib/gdb/gdb/dwarf2-frame.h
rename to gdb/dwarf2-frame.h
diff --git a/contrib/gdb/gdb/dwarf2expr.c b/gdb/dwarf2expr.c
similarity index 100%
rename from contrib/gdb/gdb/dwarf2expr.c
rename to gdb/dwarf2expr.c
diff --git a/contrib/gdb/gdb/dwarf2expr.h b/gdb/dwarf2expr.h
similarity index 100%
rename from contrib/gdb/gdb/dwarf2expr.h
rename to gdb/dwarf2expr.h
diff --git a/contrib/gdb/gdb/dwarf2loc.c b/gdb/dwarf2loc.c
similarity index 100%
rename from contrib/gdb/gdb/dwarf2loc.c
rename to gdb/dwarf2loc.c
diff --git a/contrib/gdb/gdb/dwarf2loc.h b/gdb/dwarf2loc.h
similarity index 100%
rename from contrib/gdb/gdb/dwarf2loc.h
rename to gdb/dwarf2loc.h
diff --git a/contrib/gdb/gdb/dwarf2read.c b/gdb/dwarf2read.c
similarity index 100%
rename from contrib/gdb/gdb/dwarf2read.c
rename to gdb/dwarf2read.c
diff --git a/contrib/gdb/gdb/dwarfread.c b/gdb/dwarfread.c
similarity index 100%
rename from contrib/gdb/gdb/dwarfread.c
rename to gdb/dwarfread.c
diff --git a/contrib/gdb/gdb/elfread.c b/gdb/elfread.c
similarity index 100%
rename from contrib/gdb/gdb/elfread.c
rename to gdb/elfread.c
diff --git a/contrib/gdb/gdb/environ.c b/gdb/environ.c
similarity index 100%
rename from contrib/gdb/gdb/environ.c
rename to gdb/environ.c
diff --git a/contrib/gdb/gdb/environ.h b/gdb/environ.h
similarity index 100%
rename from contrib/gdb/gdb/environ.h
rename to gdb/environ.h
diff --git a/contrib/gdb/gdb/eval.c b/gdb/eval.c
similarity index 100%
rename from contrib/gdb/gdb/eval.c
rename to gdb/eval.c
diff --git a/contrib/gdb/gdb/event-loop.c b/gdb/event-loop.c
similarity index 100%
rename from contrib/gdb/gdb/event-loop.c
rename to gdb/event-loop.c
diff --git a/contrib/gdb/gdb/event-loop.h b/gdb/event-loop.h
similarity index 100%
rename from contrib/gdb/gdb/event-loop.h
rename to gdb/event-loop.h
diff --git a/contrib/gdb/gdb/event-top.c b/gdb/event-top.c
similarity index 100%
rename from contrib/gdb/gdb/event-top.c
rename to gdb/event-top.c
diff --git a/contrib/gdb/gdb/event-top.h b/gdb/event-top.h
similarity index 100%
rename from contrib/gdb/gdb/event-top.h
rename to gdb/event-top.h
diff --git a/contrib/gdb/gdb/exc_request.defs b/gdb/exc_request.defs
similarity index 100%
rename from contrib/gdb/gdb/exc_request.defs
rename to gdb/exc_request.defs
diff --git a/contrib/gdb/gdb/exec.c b/gdb/exec.c
similarity index 100%
rename from contrib/gdb/gdb/exec.c
rename to gdb/exec.c
diff --git a/contrib/gdb/gdb/exec.h b/gdb/exec.h
similarity index 100%
rename from contrib/gdb/gdb/exec.h
rename to gdb/exec.h
diff --git a/contrib/gdb/gdb/expprint.c b/gdb/expprint.c
similarity index 100%
rename from contrib/gdb/gdb/expprint.c
rename to gdb/expprint.c
diff --git a/contrib/gdb/gdb/expression.h b/gdb/expression.h
similarity index 100%
rename from contrib/gdb/gdb/expression.h
rename to gdb/expression.h
diff --git a/contrib/gdb/gdb/f-exp.c b/gdb/f-exp.c
similarity index 100%
rename from contrib/gdb/gdb/f-exp.c
rename to gdb/f-exp.c
diff --git a/contrib/gdb/gdb/f-exp.y b/gdb/f-exp.y
similarity index 100%
rename from contrib/gdb/gdb/f-exp.y
rename to gdb/f-exp.y
diff --git a/contrib/gdb/gdb/f-lang.c b/gdb/f-lang.c
similarity index 100%
rename from contrib/gdb/gdb/f-lang.c
rename to gdb/f-lang.c
diff --git a/contrib/gdb/gdb/f-lang.h b/gdb/f-lang.h
similarity index 100%
rename from contrib/gdb/gdb/f-lang.h
rename to gdb/f-lang.h
diff --git a/contrib/gdb/gdb/f-typeprint.c b/gdb/f-typeprint.c
similarity index 100%
rename from contrib/gdb/gdb/f-typeprint.c
rename to gdb/f-typeprint.c
diff --git a/contrib/gdb/gdb/f-valprint.c b/gdb/f-valprint.c
similarity index 100%
rename from contrib/gdb/gdb/f-valprint.c
rename to gdb/f-valprint.c
diff --git a/contrib/gdb/gdb/fbsd-proc.c b/gdb/fbsd-proc.c
similarity index 100%
rename from contrib/gdb/gdb/fbsd-proc.c
rename to gdb/fbsd-proc.c
diff --git a/contrib/gdb/gdb/findvar.c b/gdb/findvar.c
similarity index 100%
rename from contrib/gdb/gdb/findvar.c
rename to gdb/findvar.c
diff --git a/contrib/gdb/gdb/fork-child.c b/gdb/fork-child.c
similarity index 100%
rename from contrib/gdb/gdb/fork-child.c
rename to gdb/fork-child.c
diff --git a/contrib/gdb/gdb/frame-base.c b/gdb/frame-base.c
similarity index 100%
rename from contrib/gdb/gdb/frame-base.c
rename to gdb/frame-base.c
diff --git a/contrib/gdb/gdb/frame-base.h b/gdb/frame-base.h
similarity index 100%
rename from contrib/gdb/gdb/frame-base.h
rename to gdb/frame-base.h
diff --git a/contrib/gdb/gdb/frame-unwind.c b/gdb/frame-unwind.c
similarity index 100%
rename from contrib/gdb/gdb/frame-unwind.c
rename to gdb/frame-unwind.c
diff --git a/contrib/gdb/gdb/frame-unwind.h b/gdb/frame-unwind.h
similarity index 100%
rename from contrib/gdb/gdb/frame-unwind.h
rename to gdb/frame-unwind.h
diff --git a/contrib/gdb/gdb/frame.c b/gdb/frame.c
similarity index 100%
rename from contrib/gdb/gdb/frame.c
rename to gdb/frame.c
diff --git a/contrib/gdb/gdb/frame.h b/gdb/frame.h
similarity index 100%
rename from contrib/gdb/gdb/frame.h
rename to gdb/frame.h
diff --git a/contrib/gdb/gdb/gcore.c b/gdb/gcore.c
similarity index 100%
rename from contrib/gdb/gdb/gcore.c
rename to gdb/gcore.c
diff --git a/contrib/gdb/gdb/gdb-events.c b/gdb/gdb-events.c
similarity index 100%
rename from contrib/gdb/gdb/gdb-events.c
rename to gdb/gdb-events.c
diff --git a/contrib/gdb/gdb/gdb-events.h b/gdb/gdb-events.h
similarity index 100%
rename from contrib/gdb/gdb/gdb-events.h
rename to gdb/gdb-events.h
diff --git a/contrib/gdb/gdb/gdb-events.sh b/gdb/gdb-events.sh
similarity index 100%
rename from contrib/gdb/gdb/gdb-events.sh
rename to gdb/gdb-events.sh
diff --git a/contrib/gdb/gdb/gdb-stabs.h b/gdb/gdb-stabs.h
similarity index 100%
rename from contrib/gdb/gdb/gdb-stabs.h
rename to gdb/gdb-stabs.h
diff --git a/contrib/gdb/gdb/gdb.1 b/gdb/gdb.1
similarity index 100%
rename from contrib/gdb/gdb/gdb.1
rename to gdb/gdb.1
diff --git a/contrib/gdb/gdb/gdb.c b/gdb/gdb.c
similarity index 100%
rename from contrib/gdb/gdb/gdb.c
rename to gdb/gdb.c
diff --git a/contrib/gdb/gdb/gdb.gdb b/gdb/gdb.gdb
similarity index 100%
rename from contrib/gdb/gdb/gdb.gdb
rename to gdb/gdb.gdb
diff --git a/contrib/gdb/gdb/gdb.h b/gdb/gdb.h
similarity index 100%
rename from contrib/gdb/gdb/gdb.h
rename to gdb/gdb.h
diff --git a/contrib/gdb/gdb/gdb_assert.h b/gdb/gdb_assert.h
similarity index 100%
rename from contrib/gdb/gdb/gdb_assert.h
rename to gdb/gdb_assert.h
diff --git a/contrib/gdb/gdb/gdb_curses.h b/gdb/gdb_curses.h
similarity index 100%
rename from contrib/gdb/gdb/gdb_curses.h
rename to gdb/gdb_curses.h
diff --git a/contrib/gdb/gdb/gdb_dirent.h b/gdb/gdb_dirent.h
similarity index 100%
rename from contrib/gdb/gdb/gdb_dirent.h
rename to gdb/gdb_dirent.h
diff --git a/contrib/gdb/gdb/gdb_gcore.sh b/gdb/gdb_gcore.sh
similarity index 100%
rename from contrib/gdb/gdb/gdb_gcore.sh
rename to gdb/gdb_gcore.sh
diff --git a/contrib/gdb/gdb/gdb_indent.sh b/gdb/gdb_indent.sh
similarity index 100%
rename from contrib/gdb/gdb/gdb_indent.sh
rename to gdb/gdb_indent.sh
diff --git a/contrib/gdb/gdb/gdb_locale.h b/gdb/gdb_locale.h
similarity index 100%
rename from contrib/gdb/gdb/gdb_locale.h
rename to gdb/gdb_locale.h
diff --git a/contrib/gdb/gdb/gdb_mbuild.sh b/gdb/gdb_mbuild.sh
similarity index 100%
rename from contrib/gdb/gdb/gdb_mbuild.sh
rename to gdb/gdb_mbuild.sh
diff --git a/contrib/gdb/gdb/gdb_obstack.h b/gdb/gdb_obstack.h
similarity index 100%
rename from contrib/gdb/gdb/gdb_obstack.h
rename to gdb/gdb_obstack.h
diff --git a/contrib/gdb/gdb/gdb_proc_service.h b/gdb/gdb_proc_service.h
similarity index 100%
rename from contrib/gdb/gdb/gdb_proc_service.h
rename to gdb/gdb_proc_service.h
diff --git a/contrib/gdb/gdb/gdb_regex.h b/gdb/gdb_regex.h
similarity index 100%
rename from contrib/gdb/gdb/gdb_regex.h
rename to gdb/gdb_regex.h
diff --git a/contrib/gdb/gdb/gdb_stat.h b/gdb/gdb_stat.h
similarity index 100%
rename from contrib/gdb/gdb/gdb_stat.h
rename to gdb/gdb_stat.h
diff --git a/contrib/gdb/gdb/gdb_string.h b/gdb/gdb_string.h
similarity index 100%
rename from contrib/gdb/gdb/gdb_string.h
rename to gdb/gdb_string.h
diff --git a/contrib/gdb/gdb/gdb_thread_db.h b/gdb/gdb_thread_db.h
similarity index 100%
rename from contrib/gdb/gdb/gdb_thread_db.h
rename to gdb/gdb_thread_db.h
diff --git a/contrib/gdb/gdb/gdb_vfork.h b/gdb/gdb_vfork.h
similarity index 100%
rename from contrib/gdb/gdb/gdb_vfork.h
rename to gdb/gdb_vfork.h
diff --git a/contrib/gdb/gdb/gdb_wait.h b/gdb/gdb_wait.h
similarity index 100%
rename from contrib/gdb/gdb/gdb_wait.h
rename to gdb/gdb_wait.h
diff --git a/contrib/gdb/gdb/gdbarch.c b/gdb/gdbarch.c
similarity index 100%
rename from contrib/gdb/gdb/gdbarch.c
rename to gdb/gdbarch.c
diff --git a/contrib/gdb/gdb/gdbarch.h b/gdb/gdbarch.h
similarity index 100%
rename from contrib/gdb/gdb/gdbarch.h
rename to gdb/gdbarch.h
diff --git a/contrib/gdb/gdb/gdbarch.sh b/gdb/gdbarch.sh
similarity index 100%
rename from contrib/gdb/gdb/gdbarch.sh
rename to gdb/gdbarch.sh
diff --git a/contrib/gdb/gdb/gdbcmd.h b/gdb/gdbcmd.h
similarity index 100%
rename from contrib/gdb/gdb/gdbcmd.h
rename to gdb/gdbcmd.h
diff --git a/contrib/gdb/gdb/gdbcore.h b/gdb/gdbcore.h
similarity index 100%
rename from contrib/gdb/gdb/gdbcore.h
rename to gdb/gdbcore.h
diff --git a/contrib/gdb/gdb/gdbinit.in b/gdb/gdbinit.in
similarity index 100%
rename from contrib/gdb/gdb/gdbinit.in
rename to gdb/gdbinit.in
diff --git a/contrib/gdb/gdb/gdbserver/README b/gdb/gdbserver/README
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/README
rename to gdb/gdbserver/README
diff --git a/contrib/gdb/gdb/gdbserver/acinclude.m4 b/gdb/gdbserver/acinclude.m4
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/acinclude.m4
rename to gdb/gdbserver/acinclude.m4
diff --git a/contrib/gdb/gdb/gdbserver/aclocal.m4 b/gdb/gdbserver/aclocal.m4
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/aclocal.m4
rename to gdb/gdbserver/aclocal.m4
diff --git a/contrib/gdb/gdb/gdbserver/gdbreplay.c b/gdb/gdbserver/gdbreplay.c
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/gdbreplay.c
rename to gdb/gdbserver/gdbreplay.c
diff --git a/contrib/gdb/gdb/gdbserver/gdbserver.1 b/gdb/gdbserver/gdbserver.1
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/gdbserver.1
rename to gdb/gdbserver/gdbserver.1
diff --git a/contrib/gdb/gdb/gdbserver/i387-fp.c b/gdb/gdbserver/i387-fp.c
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/i387-fp.c
rename to gdb/gdbserver/i387-fp.c
diff --git a/contrib/gdb/gdb/gdbserver/i387-fp.h b/gdb/gdbserver/i387-fp.h
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/i387-fp.h
rename to gdb/gdbserver/i387-fp.h
diff --git a/contrib/gdb/gdb/gdbserver/inferiors.c b/gdb/gdbserver/inferiors.c
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/inferiors.c
rename to gdb/gdbserver/inferiors.c
diff --git a/contrib/gdb/gdb/gdbserver/mem-break.c b/gdb/gdbserver/mem-break.c
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/mem-break.c
rename to gdb/gdbserver/mem-break.c
diff --git a/contrib/gdb/gdb/gdbserver/mem-break.h b/gdb/gdbserver/mem-break.h
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/mem-break.h
rename to gdb/gdbserver/mem-break.h
diff --git a/contrib/gdb/gdb/gdbserver/proc-service.c b/gdb/gdbserver/proc-service.c
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/proc-service.c
rename to gdb/gdbserver/proc-service.c
diff --git a/contrib/gdb/gdb/gdbserver/regcache.c b/gdb/gdbserver/regcache.c
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/regcache.c
rename to gdb/gdbserver/regcache.c
diff --git a/contrib/gdb/gdb/gdbserver/regcache.h b/gdb/gdbserver/regcache.h
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/regcache.h
rename to gdb/gdbserver/regcache.h
diff --git a/contrib/gdb/gdb/gdbserver/remote-utils.c b/gdb/gdbserver/remote-utils.c
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/remote-utils.c
rename to gdb/gdbserver/remote-utils.c
diff --git a/contrib/gdb/gdb/gdbserver/server.c b/gdb/gdbserver/server.c
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/server.c
rename to gdb/gdbserver/server.c
diff --git a/contrib/gdb/gdb/gdbserver/server.h b/gdb/gdbserver/server.h
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/server.h
rename to gdb/gdbserver/server.h
diff --git a/contrib/gdb/gdb/gdbserver/target.c b/gdb/gdbserver/target.c
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/target.c
rename to gdb/gdbserver/target.c
diff --git a/contrib/gdb/gdb/gdbserver/target.h b/gdb/gdbserver/target.h
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/target.h
rename to gdb/gdbserver/target.h
diff --git a/contrib/gdb/gdb/gdbserver/terminal.h b/gdb/gdbserver/terminal.h
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/terminal.h
rename to gdb/gdbserver/terminal.h
diff --git a/contrib/gdb/gdb/gdbserver/thread-db.c b/gdb/gdbserver/thread-db.c
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/thread-db.c
rename to gdb/gdbserver/thread-db.c
diff --git a/contrib/gdb/gdb/gdbserver/utils.c b/gdb/gdbserver/utils.c
similarity index 100%
rename from contrib/gdb/gdb/gdbserver/utils.c
rename to gdb/gdbserver/utils.c
diff --git a/contrib/gdb/gdb/gdbthread.h b/gdb/gdbthread.h
similarity index 100%
rename from contrib/gdb/gdb/gdbthread.h
rename to gdb/gdbthread.h
diff --git a/contrib/gdb/gdb/gdbtypes.c b/gdb/gdbtypes.c
similarity index 100%
rename from contrib/gdb/gdb/gdbtypes.c
rename to gdb/gdbtypes.c
diff --git a/contrib/gdb/gdb/gdbtypes.h b/gdb/gdbtypes.h
similarity index 100%
rename from contrib/gdb/gdb/gdbtypes.h
rename to gdb/gdbtypes.h
diff --git a/contrib/gdb/gdb/glibc-tdep.c b/gdb/glibc-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/glibc-tdep.c
rename to gdb/glibc-tdep.c
diff --git a/contrib/gdb/gdb/glibc-tdep.h b/gdb/glibc-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/glibc-tdep.h
rename to gdb/glibc-tdep.h
diff --git a/contrib/gdb/gdb/gnu-nat.c b/gdb/gnu-nat.c
similarity index 100%
rename from contrib/gdb/gdb/gnu-nat.c
rename to gdb/gnu-nat.c
diff --git a/contrib/gdb/gdb/gnu-nat.h b/gdb/gnu-nat.h
similarity index 100%
rename from contrib/gdb/gdb/gnu-nat.h
rename to gdb/gnu-nat.h
diff --git a/contrib/gdb/gdb/gnu-v2-abi.c b/gdb/gnu-v2-abi.c
similarity index 100%
rename from contrib/gdb/gdb/gnu-v2-abi.c
rename to gdb/gnu-v2-abi.c
diff --git a/contrib/gdb/gdb/gnu-v3-abi.c b/gdb/gnu-v3-abi.c
similarity index 100%
rename from contrib/gdb/gdb/gnu-v3-abi.c
rename to gdb/gnu-v3-abi.c
diff --git a/contrib/gdb/gdb/go32-nat.c b/gdb/go32-nat.c
similarity index 100%
rename from contrib/gdb/gdb/go32-nat.c
rename to gdb/go32-nat.c
diff --git a/contrib/gdb/gdb/gregset.h b/gdb/gregset.h
similarity index 100%
rename from contrib/gdb/gdb/gregset.h
rename to gdb/gregset.h
diff --git a/contrib/gdb/gdb/hpacc-abi.c b/gdb/hpacc-abi.c
similarity index 100%
rename from contrib/gdb/gdb/hpacc-abi.c
rename to gdb/hpacc-abi.c
diff --git a/contrib/gdb/gdb/hpread.c b/gdb/hpread.c
similarity index 100%
rename from contrib/gdb/gdb/hpread.c
rename to gdb/hpread.c
diff --git a/contrib/gdb/gdb/i386-nat.c b/gdb/i386-nat.c
similarity index 100%
rename from contrib/gdb/gdb/i386-nat.c
rename to gdb/i386-nat.c
diff --git a/contrib/gdb/gdb/i386-nto-tdep.c b/gdb/i386-nto-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/i386-nto-tdep.c
rename to gdb/i386-nto-tdep.c
diff --git a/contrib/gdb/gdb/i386-sol2-tdep.c b/gdb/i386-sol2-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/i386-sol2-tdep.c
rename to gdb/i386-sol2-tdep.c
diff --git a/contrib/gdb/gdb/i386-stub.c b/gdb/i386-stub.c
similarity index 100%
rename from contrib/gdb/gdb/i386-stub.c
rename to gdb/i386-stub.c
diff --git a/contrib/gdb/gdb/i386-tdep.c b/gdb/i386-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/i386-tdep.c
rename to gdb/i386-tdep.c
diff --git a/contrib/gdb/gdb/i386-tdep.h b/gdb/i386-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/i386-tdep.h
rename to gdb/i386-tdep.h
diff --git a/contrib/gdb/gdb/i386bsd-nat.c b/gdb/i386bsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/i386bsd-nat.c
rename to gdb/i386bsd-nat.c
diff --git a/contrib/gdb/gdb/i386bsd-tdep.c b/gdb/i386bsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/i386bsd-tdep.c
rename to gdb/i386bsd-tdep.c
diff --git a/contrib/gdb/gdb/i386fbsd-nat.c b/gdb/i386fbsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/i386fbsd-nat.c
rename to gdb/i386fbsd-nat.c
diff --git a/contrib/gdb/gdb/i386fbsd-tdep.c b/gdb/i386fbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/i386fbsd-tdep.c
rename to gdb/i386fbsd-tdep.c
diff --git a/contrib/gdb/gdb/i386gnu-nat.c b/gdb/i386gnu-nat.c
similarity index 100%
rename from contrib/gdb/gdb/i386gnu-nat.c
rename to gdb/i386gnu-nat.c
diff --git a/contrib/gdb/gdb/i386gnu-tdep.c b/gdb/i386gnu-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/i386gnu-tdep.c
rename to gdb/i386gnu-tdep.c
diff --git a/contrib/gdb/gdb/i386ly-tdep.c b/gdb/i386ly-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/i386ly-tdep.c
rename to gdb/i386ly-tdep.c
diff --git a/contrib/gdb/gdb/i386nbsd-tdep.c b/gdb/i386nbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/i386nbsd-tdep.c
rename to gdb/i386nbsd-tdep.c
diff --git a/contrib/gdb/gdb/i386obsd-nat.c b/gdb/i386obsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/i386obsd-nat.c
rename to gdb/i386obsd-nat.c
diff --git a/contrib/gdb/gdb/i386obsd-tdep.c b/gdb/i386obsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/i386obsd-tdep.c
rename to gdb/i386obsd-tdep.c
diff --git a/contrib/gdb/gdb/i386v-nat.c b/gdb/i386v-nat.c
similarity index 100%
rename from contrib/gdb/gdb/i386v-nat.c
rename to gdb/i386v-nat.c
diff --git a/contrib/gdb/gdb/i386v4-nat.c b/gdb/i386v4-nat.c
similarity index 100%
rename from contrib/gdb/gdb/i386v4-nat.c
rename to gdb/i386v4-nat.c
diff --git a/contrib/gdb/gdb/i387-tdep.c b/gdb/i387-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/i387-tdep.c
rename to gdb/i387-tdep.c
diff --git a/contrib/gdb/gdb/i387-tdep.h b/gdb/i387-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/i387-tdep.h
rename to gdb/i387-tdep.h
diff --git a/contrib/gdb/gdb/ia64-fbsd-nat.c b/gdb/ia64-fbsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/ia64-fbsd-nat.c
rename to gdb/ia64-fbsd-nat.c
diff --git a/contrib/gdb/gdb/ia64-fbsd-tdep.c b/gdb/ia64-fbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/ia64-fbsd-tdep.c
rename to gdb/ia64-fbsd-tdep.c
diff --git a/contrib/gdb/gdb/ia64-tdep.c b/gdb/ia64-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/ia64-tdep.c
rename to gdb/ia64-tdep.c
diff --git a/contrib/gdb/gdb/ia64-tdep.h b/gdb/ia64-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/ia64-tdep.h
rename to gdb/ia64-tdep.h
diff --git a/contrib/gdb/gdb/inf-loop.c b/gdb/inf-loop.c
similarity index 100%
rename from contrib/gdb/gdb/inf-loop.c
rename to gdb/inf-loop.c
diff --git a/contrib/gdb/gdb/inf-loop.h b/gdb/inf-loop.h
similarity index 100%
rename from contrib/gdb/gdb/inf-loop.h
rename to gdb/inf-loop.h
diff --git a/contrib/gdb/gdb/infcall.c b/gdb/infcall.c
similarity index 100%
rename from contrib/gdb/gdb/infcall.c
rename to gdb/infcall.c
diff --git a/contrib/gdb/gdb/infcall.h b/gdb/infcall.h
similarity index 100%
rename from contrib/gdb/gdb/infcall.h
rename to gdb/infcall.h
diff --git a/contrib/gdb/gdb/infcmd.c b/gdb/infcmd.c
similarity index 100%
rename from contrib/gdb/gdb/infcmd.c
rename to gdb/infcmd.c
diff --git a/contrib/gdb/gdb/inferior.h b/gdb/inferior.h
similarity index 100%
rename from contrib/gdb/gdb/inferior.h
rename to gdb/inferior.h
diff --git a/contrib/gdb/gdb/inflow.c b/gdb/inflow.c
similarity index 100%
rename from contrib/gdb/gdb/inflow.c
rename to gdb/inflow.c
diff --git a/contrib/gdb/gdb/inflow.h b/gdb/inflow.h
similarity index 100%
rename from contrib/gdb/gdb/inflow.h
rename to gdb/inflow.h
diff --git a/contrib/gdb/gdb/infptrace.c b/gdb/infptrace.c
similarity index 100%
rename from contrib/gdb/gdb/infptrace.c
rename to gdb/infptrace.c
diff --git a/contrib/gdb/gdb/infrun.c b/gdb/infrun.c
similarity index 100%
rename from contrib/gdb/gdb/infrun.c
rename to gdb/infrun.c
diff --git a/contrib/gdb/gdb/inftarg.c b/gdb/inftarg.c
similarity index 100%
rename from contrib/gdb/gdb/inftarg.c
rename to gdb/inftarg.c
diff --git a/contrib/gdb/gdb/infttrace.c b/gdb/infttrace.c
similarity index 100%
rename from contrib/gdb/gdb/infttrace.c
rename to gdb/infttrace.c
diff --git a/contrib/gdb/gdb/infttrace.h b/gdb/infttrace.h
similarity index 100%
rename from contrib/gdb/gdb/infttrace.h
rename to gdb/infttrace.h
diff --git a/contrib/gdb/gdb/interps.c b/gdb/interps.c
similarity index 100%
rename from contrib/gdb/gdb/interps.c
rename to gdb/interps.c
diff --git a/contrib/gdb/gdb/interps.h b/gdb/interps.h
similarity index 100%
rename from contrib/gdb/gdb/interps.h
rename to gdb/interps.h
diff --git a/contrib/gdb/gdb/jv-exp.c b/gdb/jv-exp.c
similarity index 100%
rename from contrib/gdb/gdb/jv-exp.c
rename to gdb/jv-exp.c
diff --git a/contrib/gdb/gdb/jv-exp.y b/gdb/jv-exp.y
similarity index 100%
rename from contrib/gdb/gdb/jv-exp.y
rename to gdb/jv-exp.y
diff --git a/contrib/gdb/gdb/jv-lang.c b/gdb/jv-lang.c
similarity index 100%
rename from contrib/gdb/gdb/jv-lang.c
rename to gdb/jv-lang.c
diff --git a/contrib/gdb/gdb/jv-lang.h b/gdb/jv-lang.h
similarity index 100%
rename from contrib/gdb/gdb/jv-lang.h
rename to gdb/jv-lang.h
diff --git a/contrib/gdb/gdb/jv-typeprint.c b/gdb/jv-typeprint.c
similarity index 100%
rename from contrib/gdb/gdb/jv-typeprint.c
rename to gdb/jv-typeprint.c
diff --git a/contrib/gdb/gdb/jv-valprint.c b/gdb/jv-valprint.c
similarity index 100%
rename from contrib/gdb/gdb/jv-valprint.c
rename to gdb/jv-valprint.c
diff --git a/contrib/gdb/gdb/kod-cisco.c b/gdb/kod-cisco.c
similarity index 100%
rename from contrib/gdb/gdb/kod-cisco.c
rename to gdb/kod-cisco.c
diff --git a/contrib/gdb/gdb/kod.c b/gdb/kod.c
similarity index 100%
rename from contrib/gdb/gdb/kod.c
rename to gdb/kod.c
diff --git a/contrib/gdb/gdb/kod.h b/gdb/kod.h
similarity index 100%
rename from contrib/gdb/gdb/kod.h
rename to gdb/kod.h
diff --git a/contrib/gdb/gdb/language.c b/gdb/language.c
similarity index 100%
rename from contrib/gdb/gdb/language.c
rename to gdb/language.c
diff --git a/contrib/gdb/gdb/language.h b/gdb/language.h
similarity index 100%
rename from contrib/gdb/gdb/language.h
rename to gdb/language.h
diff --git a/contrib/gdb/gdb/libunwind-frame.c b/gdb/libunwind-frame.c
similarity index 100%
rename from contrib/gdb/gdb/libunwind-frame.c
rename to gdb/libunwind-frame.c
diff --git a/contrib/gdb/gdb/libunwind-frame.h b/gdb/libunwind-frame.h
similarity index 100%
rename from contrib/gdb/gdb/libunwind-frame.h
rename to gdb/libunwind-frame.h
diff --git a/contrib/gdb/gdb/lin-lwp.c b/gdb/lin-lwp.c
similarity index 100%
rename from contrib/gdb/gdb/lin-lwp.c
rename to gdb/lin-lwp.c
diff --git a/contrib/gdb/gdb/linespec.c b/gdb/linespec.c
similarity index 100%
rename from contrib/gdb/gdb/linespec.c
rename to gdb/linespec.c
diff --git a/contrib/gdb/gdb/linespec.h b/gdb/linespec.h
similarity index 100%
rename from contrib/gdb/gdb/linespec.h
rename to gdb/linespec.h
diff --git a/contrib/gdb/gdb/lynx-nat.c b/gdb/lynx-nat.c
similarity index 100%
rename from contrib/gdb/gdb/lynx-nat.c
rename to gdb/lynx-nat.c
diff --git a/contrib/gdb/gdb/m2-exp.c b/gdb/m2-exp.c
similarity index 100%
rename from contrib/gdb/gdb/m2-exp.c
rename to gdb/m2-exp.c
diff --git a/contrib/gdb/gdb/m2-exp.y b/gdb/m2-exp.y
similarity index 100%
rename from contrib/gdb/gdb/m2-exp.y
rename to gdb/m2-exp.y
diff --git a/contrib/gdb/gdb/m2-lang.c b/gdb/m2-lang.c
similarity index 100%
rename from contrib/gdb/gdb/m2-lang.c
rename to gdb/m2-lang.c
diff --git a/contrib/gdb/gdb/m2-lang.h b/gdb/m2-lang.h
similarity index 100%
rename from contrib/gdb/gdb/m2-lang.h
rename to gdb/m2-lang.h
diff --git a/contrib/gdb/gdb/m2-typeprint.c b/gdb/m2-typeprint.c
similarity index 100%
rename from contrib/gdb/gdb/m2-typeprint.c
rename to gdb/m2-typeprint.c
diff --git a/contrib/gdb/gdb/m2-valprint.c b/gdb/m2-valprint.c
similarity index 100%
rename from contrib/gdb/gdb/m2-valprint.c
rename to gdb/m2-valprint.c
diff --git a/contrib/gdb/gdb/macrocmd.c b/gdb/macrocmd.c
similarity index 100%
rename from contrib/gdb/gdb/macrocmd.c
rename to gdb/macrocmd.c
diff --git a/contrib/gdb/gdb/macroexp.c b/gdb/macroexp.c
similarity index 100%
rename from contrib/gdb/gdb/macroexp.c
rename to gdb/macroexp.c
diff --git a/contrib/gdb/gdb/macroexp.h b/gdb/macroexp.h
similarity index 100%
rename from contrib/gdb/gdb/macroexp.h
rename to gdb/macroexp.h
diff --git a/contrib/gdb/gdb/macroscope.c b/gdb/macroscope.c
similarity index 100%
rename from contrib/gdb/gdb/macroscope.c
rename to gdb/macroscope.c
diff --git a/contrib/gdb/gdb/macroscope.h b/gdb/macroscope.h
similarity index 100%
rename from contrib/gdb/gdb/macroscope.h
rename to gdb/macroscope.h
diff --git a/contrib/gdb/gdb/macrotab.c b/gdb/macrotab.c
similarity index 100%
rename from contrib/gdb/gdb/macrotab.c
rename to gdb/macrotab.c
diff --git a/contrib/gdb/gdb/macrotab.h b/gdb/macrotab.h
similarity index 100%
rename from contrib/gdb/gdb/macrotab.h
rename to gdb/macrotab.h
diff --git a/contrib/gdb/gdb/main.c b/gdb/main.c
similarity index 100%
rename from contrib/gdb/gdb/main.c
rename to gdb/main.c
diff --git a/contrib/gdb/gdb/main.h b/gdb/main.h
similarity index 100%
rename from contrib/gdb/gdb/main.h
rename to gdb/main.h
diff --git a/contrib/gdb/gdb/maint.c b/gdb/maint.c
similarity index 100%
rename from contrib/gdb/gdb/maint.c
rename to gdb/maint.c
diff --git a/contrib/gdb/gdb/mdebugread.c b/gdb/mdebugread.c
similarity index 100%
rename from contrib/gdb/gdb/mdebugread.c
rename to gdb/mdebugread.c
diff --git a/contrib/gdb/gdb/mem-break.c b/gdb/mem-break.c
similarity index 100%
rename from contrib/gdb/gdb/mem-break.c
rename to gdb/mem-break.c
diff --git a/contrib/gdb/gdb/memattr.c b/gdb/memattr.c
similarity index 100%
rename from contrib/gdb/gdb/memattr.c
rename to gdb/memattr.c
diff --git a/contrib/gdb/gdb/memattr.h b/gdb/memattr.h
similarity index 100%
rename from contrib/gdb/gdb/memattr.h
rename to gdb/memattr.h
diff --git a/contrib/gdb/gdb/mi/mi-cmd-break.c b/gdb/mi/mi-cmd-break.c
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-cmd-break.c
rename to gdb/mi/mi-cmd-break.c
diff --git a/contrib/gdb/gdb/mi/mi-cmd-disas.c b/gdb/mi/mi-cmd-disas.c
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-cmd-disas.c
rename to gdb/mi/mi-cmd-disas.c
diff --git a/contrib/gdb/gdb/mi/mi-cmd-env.c b/gdb/mi/mi-cmd-env.c
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-cmd-env.c
rename to gdb/mi/mi-cmd-env.c
diff --git a/contrib/gdb/gdb/mi/mi-cmd-file.c b/gdb/mi/mi-cmd-file.c
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-cmd-file.c
rename to gdb/mi/mi-cmd-file.c
diff --git a/contrib/gdb/gdb/mi/mi-cmd-stack.c b/gdb/mi/mi-cmd-stack.c
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-cmd-stack.c
rename to gdb/mi/mi-cmd-stack.c
diff --git a/contrib/gdb/gdb/mi/mi-cmd-var.c b/gdb/mi/mi-cmd-var.c
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-cmd-var.c
rename to gdb/mi/mi-cmd-var.c
diff --git a/contrib/gdb/gdb/mi/mi-cmds.c b/gdb/mi/mi-cmds.c
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-cmds.c
rename to gdb/mi/mi-cmds.c
diff --git a/contrib/gdb/gdb/mi/mi-cmds.h b/gdb/mi/mi-cmds.h
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-cmds.h
rename to gdb/mi/mi-cmds.h
diff --git a/contrib/gdb/gdb/mi/mi-console.c b/gdb/mi/mi-console.c
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-console.c
rename to gdb/mi/mi-console.c
diff --git a/contrib/gdb/gdb/mi/mi-console.h b/gdb/mi/mi-console.h
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-console.h
rename to gdb/mi/mi-console.h
diff --git a/contrib/gdb/gdb/mi/mi-getopt.c b/gdb/mi/mi-getopt.c
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-getopt.c
rename to gdb/mi/mi-getopt.c
diff --git a/contrib/gdb/gdb/mi/mi-getopt.h b/gdb/mi/mi-getopt.h
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-getopt.h
rename to gdb/mi/mi-getopt.h
diff --git a/contrib/gdb/gdb/mi/mi-interp.c b/gdb/mi/mi-interp.c
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-interp.c
rename to gdb/mi/mi-interp.c
diff --git a/contrib/gdb/gdb/mi/mi-main.c b/gdb/mi/mi-main.c
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-main.c
rename to gdb/mi/mi-main.c
diff --git a/contrib/gdb/gdb/mi/mi-main.h b/gdb/mi/mi-main.h
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-main.h
rename to gdb/mi/mi-main.h
diff --git a/contrib/gdb/gdb/mi/mi-out.c b/gdb/mi/mi-out.c
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-out.c
rename to gdb/mi/mi-out.c
diff --git a/contrib/gdb/gdb/mi/mi-out.h b/gdb/mi/mi-out.h
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-out.h
rename to gdb/mi/mi-out.h
diff --git a/contrib/gdb/gdb/mi/mi-parse.c b/gdb/mi/mi-parse.c
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-parse.c
rename to gdb/mi/mi-parse.c
diff --git a/contrib/gdb/gdb/mi/mi-parse.h b/gdb/mi/mi-parse.h
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-parse.h
rename to gdb/mi/mi-parse.h
diff --git a/contrib/gdb/gdb/mi/mi-symbol-cmds.c b/gdb/mi/mi-symbol-cmds.c
similarity index 100%
rename from contrib/gdb/gdb/mi/mi-symbol-cmds.c
rename to gdb/mi/mi-symbol-cmds.c
diff --git a/contrib/gdb/gdb/minimon.h b/gdb/minimon.h
similarity index 100%
rename from contrib/gdb/gdb/minimon.h
rename to gdb/minimon.h
diff --git a/contrib/gdb/gdb/minsyms.c b/gdb/minsyms.c
similarity index 100%
rename from contrib/gdb/gdb/minsyms.c
rename to gdb/minsyms.c
diff --git a/contrib/gdb/gdb/mips-nat.c b/gdb/mips-nat.c
similarity index 100%
rename from contrib/gdb/gdb/mips-nat.c
rename to gdb/mips-nat.c
diff --git a/contrib/gdb/gdb/mips-tdep.c b/gdb/mips-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/mips-tdep.c
rename to gdb/mips-tdep.c
diff --git a/contrib/gdb/gdb/mips-tdep.h b/gdb/mips-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/mips-tdep.h
rename to gdb/mips-tdep.h
diff --git a/contrib/gdb/gdb/mipsnbsd-nat.c b/gdb/mipsnbsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/mipsnbsd-nat.c
rename to gdb/mipsnbsd-nat.c
diff --git a/contrib/gdb/gdb/mipsnbsd-tdep.c b/gdb/mipsnbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/mipsnbsd-tdep.c
rename to gdb/mipsnbsd-tdep.c
diff --git a/contrib/gdb/gdb/mipsnbsd-tdep.h b/gdb/mipsnbsd-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/mipsnbsd-tdep.h
rename to gdb/mipsnbsd-tdep.h
diff --git a/contrib/gdb/gdb/mipsread.c b/gdb/mipsread.c
similarity index 100%
rename from contrib/gdb/gdb/mipsread.c
rename to gdb/mipsread.c
diff --git a/contrib/gdb/gdb/mipsv4-nat.c b/gdb/mipsv4-nat.c
similarity index 100%
rename from contrib/gdb/gdb/mipsv4-nat.c
rename to gdb/mipsv4-nat.c
diff --git a/contrib/gdb/gdb/monitor.c b/gdb/monitor.c
similarity index 100%
rename from contrib/gdb/gdb/monitor.c
rename to gdb/monitor.c
diff --git a/contrib/gdb/gdb/monitor.h b/gdb/monitor.h
similarity index 100%
rename from contrib/gdb/gdb/monitor.h
rename to gdb/monitor.h
diff --git a/contrib/gdb/gdb/msg.defs b/gdb/msg.defs
similarity index 100%
rename from contrib/gdb/gdb/msg.defs
rename to gdb/msg.defs
diff --git a/contrib/gdb/gdb/msg_reply.defs b/gdb/msg_reply.defs
similarity index 100%
rename from contrib/gdb/gdb/msg_reply.defs
rename to gdb/msg_reply.defs
diff --git a/contrib/gdb/gdb/nbsd-tdep.c b/gdb/nbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/nbsd-tdep.c
rename to gdb/nbsd-tdep.c
diff --git a/contrib/gdb/gdb/nbsd-tdep.h b/gdb/nbsd-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/nbsd-tdep.h
rename to gdb/nbsd-tdep.h
diff --git a/contrib/gdb/gdb/nlmread.c b/gdb/nlmread.c
similarity index 100%
rename from contrib/gdb/gdb/nlmread.c
rename to gdb/nlmread.c
diff --git a/contrib/gdb/gdb/notify.defs b/gdb/notify.defs
similarity index 100%
rename from contrib/gdb/gdb/notify.defs
rename to gdb/notify.defs
diff --git a/contrib/gdb/gdb/nto-procfs.c b/gdb/nto-procfs.c
similarity index 100%
rename from contrib/gdb/gdb/nto-procfs.c
rename to gdb/nto-procfs.c
diff --git a/contrib/gdb/gdb/nto-tdep.c b/gdb/nto-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/nto-tdep.c
rename to gdb/nto-tdep.c
diff --git a/contrib/gdb/gdb/nto-tdep.h b/gdb/nto-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/nto-tdep.h
rename to gdb/nto-tdep.h
diff --git a/contrib/gdb/gdb/objc-exp.c b/gdb/objc-exp.c
similarity index 100%
rename from contrib/gdb/gdb/objc-exp.c
rename to gdb/objc-exp.c
diff --git a/contrib/gdb/gdb/objc-exp.y b/gdb/objc-exp.y
similarity index 100%
rename from contrib/gdb/gdb/objc-exp.y
rename to gdb/objc-exp.y
diff --git a/contrib/gdb/gdb/objc-lang.c b/gdb/objc-lang.c
similarity index 100%
rename from contrib/gdb/gdb/objc-lang.c
rename to gdb/objc-lang.c
diff --git a/contrib/gdb/gdb/objc-lang.h b/gdb/objc-lang.h
similarity index 100%
rename from contrib/gdb/gdb/objc-lang.h
rename to gdb/objc-lang.h
diff --git a/contrib/gdb/gdb/objfiles.c b/gdb/objfiles.c
similarity index 100%
rename from contrib/gdb/gdb/objfiles.c
rename to gdb/objfiles.c
diff --git a/contrib/gdb/gdb/objfiles.h b/gdb/objfiles.h
similarity index 100%
rename from contrib/gdb/gdb/objfiles.h
rename to gdb/objfiles.h
diff --git a/contrib/gdb/gdb/observer.c b/gdb/observer.c
similarity index 100%
rename from contrib/gdb/gdb/observer.c
rename to gdb/observer.c
diff --git a/contrib/gdb/gdb/observer.h b/gdb/observer.h
similarity index 100%
rename from contrib/gdb/gdb/observer.h
rename to gdb/observer.h
diff --git a/contrib/gdb/gdb/ocd.c b/gdb/ocd.c
similarity index 100%
rename from contrib/gdb/gdb/ocd.c
rename to gdb/ocd.c
diff --git a/contrib/gdb/gdb/ocd.h b/gdb/ocd.h
similarity index 100%
rename from contrib/gdb/gdb/ocd.h
rename to gdb/ocd.h
diff --git a/contrib/gdb/gdb/osabi.c b/gdb/osabi.c
similarity index 100%
rename from contrib/gdb/gdb/osabi.c
rename to gdb/osabi.c
diff --git a/contrib/gdb/gdb/osabi.h b/gdb/osabi.h
similarity index 100%
rename from contrib/gdb/gdb/osabi.h
rename to gdb/osabi.h
diff --git a/contrib/gdb/gdb/p-exp.c b/gdb/p-exp.c
similarity index 100%
rename from contrib/gdb/gdb/p-exp.c
rename to gdb/p-exp.c
diff --git a/contrib/gdb/gdb/p-exp.y b/gdb/p-exp.y
similarity index 100%
rename from contrib/gdb/gdb/p-exp.y
rename to gdb/p-exp.y
diff --git a/contrib/gdb/gdb/p-lang.c b/gdb/p-lang.c
similarity index 100%
rename from contrib/gdb/gdb/p-lang.c
rename to gdb/p-lang.c
diff --git a/contrib/gdb/gdb/p-lang.h b/gdb/p-lang.h
similarity index 100%
rename from contrib/gdb/gdb/p-lang.h
rename to gdb/p-lang.h
diff --git a/contrib/gdb/gdb/p-typeprint.c b/gdb/p-typeprint.c
similarity index 100%
rename from contrib/gdb/gdb/p-typeprint.c
rename to gdb/p-typeprint.c
diff --git a/contrib/gdb/gdb/p-valprint.c b/gdb/p-valprint.c
similarity index 100%
rename from contrib/gdb/gdb/p-valprint.c
rename to gdb/p-valprint.c
diff --git a/contrib/gdb/gdb/pa64solib.c b/gdb/pa64solib.c
similarity index 100%
rename from contrib/gdb/gdb/pa64solib.c
rename to gdb/pa64solib.c
diff --git a/contrib/gdb/gdb/pa64solib.h b/gdb/pa64solib.h
similarity index 100%
rename from contrib/gdb/gdb/pa64solib.h
rename to gdb/pa64solib.h
diff --git a/contrib/gdb/gdb/parse.c b/gdb/parse.c
similarity index 100%
rename from contrib/gdb/gdb/parse.c
rename to gdb/parse.c
diff --git a/contrib/gdb/gdb/parser-defs.h b/gdb/parser-defs.h
similarity index 100%
rename from contrib/gdb/gdb/parser-defs.h
rename to gdb/parser-defs.h
diff --git a/contrib/gdb/gdb/ppc-bdm.c b/gdb/ppc-bdm.c
similarity index 100%
rename from contrib/gdb/gdb/ppc-bdm.c
rename to gdb/ppc-bdm.c
diff --git a/contrib/gdb/gdb/ppc-sysv-tdep.c b/gdb/ppc-sysv-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/ppc-sysv-tdep.c
rename to gdb/ppc-sysv-tdep.c
diff --git a/contrib/gdb/gdb/ppc-tdep.h b/gdb/ppc-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/ppc-tdep.h
rename to gdb/ppc-tdep.h
diff --git a/contrib/gdb/gdb/ppcbug-rom.c b/gdb/ppcbug-rom.c
similarity index 100%
rename from contrib/gdb/gdb/ppcbug-rom.c
rename to gdb/ppcbug-rom.c
diff --git a/contrib/gdb/gdb/ppcfbsd-nat.c b/gdb/ppcfbsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/ppcfbsd-nat.c
rename to gdb/ppcfbsd-nat.c
diff --git a/contrib/gdb/gdb/ppcfbsd-tdep.c b/gdb/ppcfbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/ppcfbsd-tdep.c
rename to gdb/ppcfbsd-tdep.c
diff --git a/contrib/gdb/gdb/ppcfbsd-tdep.h b/gdb/ppcfbsd-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/ppcfbsd-tdep.h
rename to gdb/ppcfbsd-tdep.h
diff --git a/contrib/gdb/gdb/ppcnbsd-nat.c b/gdb/ppcnbsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/ppcnbsd-nat.c
rename to gdb/ppcnbsd-nat.c
diff --git a/contrib/gdb/gdb/ppcnbsd-tdep.c b/gdb/ppcnbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/ppcnbsd-tdep.c
rename to gdb/ppcnbsd-tdep.c
diff --git a/contrib/gdb/gdb/ppcnbsd-tdep.h b/gdb/ppcnbsd-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/ppcnbsd-tdep.h
rename to gdb/ppcnbsd-tdep.h
diff --git a/contrib/gdb/gdb/printcmd.c b/gdb/printcmd.c
similarity index 100%
rename from contrib/gdb/gdb/printcmd.c
rename to gdb/printcmd.c
diff --git a/contrib/gdb/gdb/proc-api.c b/gdb/proc-api.c
similarity index 100%
rename from contrib/gdb/gdb/proc-api.c
rename to gdb/proc-api.c
diff --git a/contrib/gdb/gdb/proc-events.c b/gdb/proc-events.c
similarity index 100%
rename from contrib/gdb/gdb/proc-events.c
rename to gdb/proc-events.c
diff --git a/contrib/gdb/gdb/proc-flags.c b/gdb/proc-flags.c
similarity index 100%
rename from contrib/gdb/gdb/proc-flags.c
rename to gdb/proc-flags.c
diff --git a/contrib/gdb/gdb/proc-service.c b/gdb/proc-service.c
similarity index 100%
rename from contrib/gdb/gdb/proc-service.c
rename to gdb/proc-service.c
diff --git a/contrib/gdb/gdb/proc-utils.h b/gdb/proc-utils.h
similarity index 100%
rename from contrib/gdb/gdb/proc-utils.h
rename to gdb/proc-utils.h
diff --git a/contrib/gdb/gdb/proc-why.c b/gdb/proc-why.c
similarity index 100%
rename from contrib/gdb/gdb/proc-why.c
rename to gdb/proc-why.c
diff --git a/contrib/gdb/gdb/process_reply.defs b/gdb/process_reply.defs
similarity index 100%
rename from contrib/gdb/gdb/process_reply.defs
rename to gdb/process_reply.defs
diff --git a/contrib/gdb/gdb/procfs.c b/gdb/procfs.c
similarity index 100%
rename from contrib/gdb/gdb/procfs.c
rename to gdb/procfs.c
diff --git a/contrib/gdb/gdb/regcache.c b/gdb/regcache.c
similarity index 100%
rename from contrib/gdb/gdb/regcache.c
rename to gdb/regcache.c
diff --git a/contrib/gdb/gdb/regcache.h b/gdb/regcache.h
similarity index 100%
rename from contrib/gdb/gdb/regcache.h
rename to gdb/regcache.h
diff --git a/contrib/gdb/gdb/regformats/reg-arm.dat b/gdb/regformats/reg-arm.dat
similarity index 100%
rename from contrib/gdb/gdb/regformats/reg-arm.dat
rename to gdb/regformats/reg-arm.dat
diff --git a/contrib/gdb/gdb/regformats/reg-i386.dat b/gdb/regformats/reg-i386.dat
similarity index 100%
rename from contrib/gdb/gdb/regformats/reg-i386.dat
rename to gdb/regformats/reg-i386.dat
diff --git a/contrib/gdb/gdb/regformats/reg-ia64.dat b/gdb/regformats/reg-ia64.dat
similarity index 100%
rename from contrib/gdb/gdb/regformats/reg-ia64.dat
rename to gdb/regformats/reg-ia64.dat
diff --git a/contrib/gdb/gdb/regformats/reg-m68k.dat b/gdb/regformats/reg-m68k.dat
similarity index 100%
rename from contrib/gdb/gdb/regformats/reg-m68k.dat
rename to gdb/regformats/reg-m68k.dat
diff --git a/contrib/gdb/gdb/regformats/reg-mips.dat b/gdb/regformats/reg-mips.dat
similarity index 100%
rename from contrib/gdb/gdb/regformats/reg-mips.dat
rename to gdb/regformats/reg-mips.dat
diff --git a/contrib/gdb/gdb/regformats/reg-ppc.dat b/gdb/regformats/reg-ppc.dat
similarity index 100%
rename from contrib/gdb/gdb/regformats/reg-ppc.dat
rename to gdb/regformats/reg-ppc.dat
diff --git a/contrib/gdb/gdb/regformats/reg-s390.dat b/gdb/regformats/reg-s390.dat
similarity index 100%
rename from contrib/gdb/gdb/regformats/reg-s390.dat
rename to gdb/regformats/reg-s390.dat
diff --git a/contrib/gdb/gdb/regformats/reg-s390x.dat b/gdb/regformats/reg-s390x.dat
similarity index 100%
rename from contrib/gdb/gdb/regformats/reg-s390x.dat
rename to gdb/regformats/reg-s390x.dat
diff --git a/contrib/gdb/gdb/regformats/reg-sh.dat b/gdb/regformats/reg-sh.dat
similarity index 100%
rename from contrib/gdb/gdb/regformats/reg-sh.dat
rename to gdb/regformats/reg-sh.dat
diff --git a/contrib/gdb/gdb/regformats/reg-x86-64.dat b/gdb/regformats/reg-x86-64.dat
similarity index 100%
rename from contrib/gdb/gdb/regformats/reg-x86-64.dat
rename to gdb/regformats/reg-x86-64.dat
diff --git a/contrib/gdb/gdb/regformats/regdat.sh b/gdb/regformats/regdat.sh
similarity index 100%
rename from contrib/gdb/gdb/regformats/regdat.sh
rename to gdb/regformats/regdat.sh
diff --git a/contrib/gdb/gdb/regformats/regdef.h b/gdb/regformats/regdef.h
similarity index 100%
rename from contrib/gdb/gdb/regformats/regdef.h
rename to gdb/regformats/regdef.h
diff --git a/contrib/gdb/gdb/reggroups.c b/gdb/reggroups.c
similarity index 100%
rename from contrib/gdb/gdb/reggroups.c
rename to gdb/reggroups.c
diff --git a/contrib/gdb/gdb/reggroups.h b/gdb/reggroups.h
similarity index 100%
rename from contrib/gdb/gdb/reggroups.h
rename to gdb/reggroups.h
diff --git a/contrib/gdb/gdb/regset.h b/gdb/regset.h
similarity index 100%
rename from contrib/gdb/gdb/regset.h
rename to gdb/regset.h
diff --git a/contrib/gdb/gdb/remote-e7000.c b/gdb/remote-e7000.c
similarity index 100%
rename from contrib/gdb/gdb/remote-e7000.c
rename to gdb/remote-e7000.c
diff --git a/contrib/gdb/gdb/remote-est.c b/gdb/remote-est.c
similarity index 100%
rename from contrib/gdb/gdb/remote-est.c
rename to gdb/remote-est.c
diff --git a/contrib/gdb/gdb/remote-fileio.c b/gdb/remote-fileio.c
similarity index 100%
rename from contrib/gdb/gdb/remote-fileio.c
rename to gdb/remote-fileio.c
diff --git a/contrib/gdb/gdb/remote-fileio.h b/gdb/remote-fileio.h
similarity index 100%
rename from contrib/gdb/gdb/remote-fileio.h
rename to gdb/remote-fileio.h
diff --git a/contrib/gdb/gdb/remote-hms.c b/gdb/remote-hms.c
similarity index 100%
rename from contrib/gdb/gdb/remote-hms.c
rename to gdb/remote-hms.c
diff --git a/contrib/gdb/gdb/remote-mips.c b/gdb/remote-mips.c
similarity index 100%
rename from contrib/gdb/gdb/remote-mips.c
rename to gdb/remote-mips.c
diff --git a/contrib/gdb/gdb/remote-rdi.c b/gdb/remote-rdi.c
similarity index 100%
rename from contrib/gdb/gdb/remote-rdi.c
rename to gdb/remote-rdi.c
diff --git a/contrib/gdb/gdb/remote-rdp.c b/gdb/remote-rdp.c
similarity index 100%
rename from contrib/gdb/gdb/remote-rdp.c
rename to gdb/remote-rdp.c
diff --git a/contrib/gdb/gdb/remote-sds.c b/gdb/remote-sds.c
similarity index 100%
rename from contrib/gdb/gdb/remote-sds.c
rename to gdb/remote-sds.c
diff --git a/contrib/gdb/gdb/remote-sim.c b/gdb/remote-sim.c
similarity index 100%
rename from contrib/gdb/gdb/remote-sim.c
rename to gdb/remote-sim.c
diff --git a/contrib/gdb/gdb/remote-st.c b/gdb/remote-st.c
similarity index 100%
rename from contrib/gdb/gdb/remote-st.c
rename to gdb/remote-st.c
diff --git a/contrib/gdb/gdb/remote-utils.c b/gdb/remote-utils.c
similarity index 100%
rename from contrib/gdb/gdb/remote-utils.c
rename to gdb/remote-utils.c
diff --git a/contrib/gdb/gdb/remote-utils.h b/gdb/remote-utils.h
similarity index 100%
rename from contrib/gdb/gdb/remote-utils.h
rename to gdb/remote-utils.h
diff --git a/contrib/gdb/gdb/remote-vx.c b/gdb/remote-vx.c
similarity index 100%
rename from contrib/gdb/gdb/remote-vx.c
rename to gdb/remote-vx.c
diff --git a/contrib/gdb/gdb/remote-vx68.c b/gdb/remote-vx68.c
similarity index 100%
rename from contrib/gdb/gdb/remote-vx68.c
rename to gdb/remote-vx68.c
diff --git a/contrib/gdb/gdb/remote-vxmips.c b/gdb/remote-vxmips.c
similarity index 100%
rename from contrib/gdb/gdb/remote-vxmips.c
rename to gdb/remote-vxmips.c
diff --git a/contrib/gdb/gdb/remote-vxsparc.c b/gdb/remote-vxsparc.c
similarity index 100%
rename from contrib/gdb/gdb/remote-vxsparc.c
rename to gdb/remote-vxsparc.c
diff --git a/contrib/gdb/gdb/remote.c b/gdb/remote.c
similarity index 100%
rename from contrib/gdb/gdb/remote.c
rename to gdb/remote.c
diff --git a/contrib/gdb/gdb/remote.h b/gdb/remote.h
similarity index 100%
rename from contrib/gdb/gdb/remote.h
rename to gdb/remote.h
diff --git a/contrib/gdb/gdb/reply_mig_hack.awk b/gdb/reply_mig_hack.awk
similarity index 100%
rename from contrib/gdb/gdb/reply_mig_hack.awk
rename to gdb/reply_mig_hack.awk
diff --git a/contrib/gdb/gdb/rom68k-rom.c b/gdb/rom68k-rom.c
similarity index 100%
rename from contrib/gdb/gdb/rom68k-rom.c
rename to gdb/rom68k-rom.c
diff --git a/contrib/gdb/gdb/rs6000-nat.c b/gdb/rs6000-nat.c
similarity index 100%
rename from contrib/gdb/gdb/rs6000-nat.c
rename to gdb/rs6000-nat.c
diff --git a/contrib/gdb/gdb/rs6000-tdep.c b/gdb/rs6000-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/rs6000-tdep.c
rename to gdb/rs6000-tdep.c
diff --git a/contrib/gdb/gdb/s390-nat.c b/gdb/s390-nat.c
similarity index 100%
rename from contrib/gdb/gdb/s390-nat.c
rename to gdb/s390-nat.c
diff --git a/contrib/gdb/gdb/s390-tdep.c b/gdb/s390-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/s390-tdep.c
rename to gdb/s390-tdep.c
diff --git a/contrib/gdb/gdb/s390-tdep.h b/gdb/s390-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/s390-tdep.h
rename to gdb/s390-tdep.h
diff --git a/contrib/gdb/gdb/scm-exp.c b/gdb/scm-exp.c
similarity index 100%
rename from contrib/gdb/gdb/scm-exp.c
rename to gdb/scm-exp.c
diff --git a/contrib/gdb/gdb/scm-lang.c b/gdb/scm-lang.c
similarity index 100%
rename from contrib/gdb/gdb/scm-lang.c
rename to gdb/scm-lang.c
diff --git a/contrib/gdb/gdb/scm-lang.h b/gdb/scm-lang.h
similarity index 100%
rename from contrib/gdb/gdb/scm-lang.h
rename to gdb/scm-lang.h
diff --git a/contrib/gdb/gdb/scm-tags.h b/gdb/scm-tags.h
similarity index 100%
rename from contrib/gdb/gdb/scm-tags.h
rename to gdb/scm-tags.h
diff --git a/contrib/gdb/gdb/scm-valprint.c b/gdb/scm-valprint.c
similarity index 100%
rename from contrib/gdb/gdb/scm-valprint.c
rename to gdb/scm-valprint.c
diff --git a/contrib/gdb/gdb/sentinel-frame.c b/gdb/sentinel-frame.c
similarity index 100%
rename from contrib/gdb/gdb/sentinel-frame.c
rename to gdb/sentinel-frame.c
diff --git a/contrib/gdb/gdb/sentinel-frame.h b/gdb/sentinel-frame.h
similarity index 100%
rename from contrib/gdb/gdb/sentinel-frame.h
rename to gdb/sentinel-frame.h
diff --git a/contrib/gdb/gdb/ser-e7kpc.c b/gdb/ser-e7kpc.c
similarity index 100%
rename from contrib/gdb/gdb/ser-e7kpc.c
rename to gdb/ser-e7kpc.c
diff --git a/contrib/gdb/gdb/ser-go32.c b/gdb/ser-go32.c
similarity index 100%
rename from contrib/gdb/gdb/ser-go32.c
rename to gdb/ser-go32.c
diff --git a/contrib/gdb/gdb/ser-pipe.c b/gdb/ser-pipe.c
similarity index 100%
rename from contrib/gdb/gdb/ser-pipe.c
rename to gdb/ser-pipe.c
diff --git a/contrib/gdb/gdb/ser-tcp.c b/gdb/ser-tcp.c
similarity index 100%
rename from contrib/gdb/gdb/ser-tcp.c
rename to gdb/ser-tcp.c
diff --git a/contrib/gdb/gdb/ser-unix.c b/gdb/ser-unix.c
similarity index 100%
rename from contrib/gdb/gdb/ser-unix.c
rename to gdb/ser-unix.c
diff --git a/contrib/gdb/gdb/ser-unix.h b/gdb/ser-unix.h
similarity index 100%
rename from contrib/gdb/gdb/ser-unix.h
rename to gdb/ser-unix.h
diff --git a/contrib/gdb/gdb/serial.c b/gdb/serial.c
similarity index 100%
rename from contrib/gdb/gdb/serial.c
rename to gdb/serial.c
diff --git a/contrib/gdb/gdb/serial.h b/gdb/serial.h
similarity index 100%
rename from contrib/gdb/gdb/serial.h
rename to gdb/serial.h
diff --git a/contrib/gdb/gdb/signals/signals.c b/gdb/signals/signals.c
similarity index 100%
rename from contrib/gdb/gdb/signals/signals.c
rename to gdb/signals/signals.c
diff --git a/contrib/gdb/gdb/sim-regno.h b/gdb/sim-regno.h
similarity index 100%
rename from contrib/gdb/gdb/sim-regno.h
rename to gdb/sim-regno.h
diff --git a/contrib/gdb/gdb/sol-thread.c b/gdb/sol-thread.c
similarity index 100%
rename from contrib/gdb/gdb/sol-thread.c
rename to gdb/sol-thread.c
diff --git a/contrib/gdb/gdb/solib-legacy.c b/gdb/solib-legacy.c
similarity index 100%
rename from contrib/gdb/gdb/solib-legacy.c
rename to gdb/solib-legacy.c
diff --git a/contrib/gdb/gdb/solib-osf.c b/gdb/solib-osf.c
similarity index 100%
rename from contrib/gdb/gdb/solib-osf.c
rename to gdb/solib-osf.c
diff --git a/contrib/gdb/gdb/solib-sunos.c b/gdb/solib-sunos.c
similarity index 100%
rename from contrib/gdb/gdb/solib-sunos.c
rename to gdb/solib-sunos.c
diff --git a/contrib/gdb/gdb/solib-svr4.c b/gdb/solib-svr4.c
similarity index 100%
rename from contrib/gdb/gdb/solib-svr4.c
rename to gdb/solib-svr4.c
diff --git a/contrib/gdb/gdb/solib-svr4.h b/gdb/solib-svr4.h
similarity index 100%
rename from contrib/gdb/gdb/solib-svr4.h
rename to gdb/solib-svr4.h
diff --git a/contrib/gdb/gdb/solib.c b/gdb/solib.c
similarity index 100%
rename from contrib/gdb/gdb/solib.c
rename to gdb/solib.c
diff --git a/contrib/gdb/gdb/solib.h b/gdb/solib.h
similarity index 100%
rename from contrib/gdb/gdb/solib.h
rename to gdb/solib.h
diff --git a/contrib/gdb/gdb/solist.h b/gdb/solist.h
similarity index 100%
rename from contrib/gdb/gdb/solist.h
rename to gdb/solist.h
diff --git a/contrib/gdb/gdb/somread.c b/gdb/somread.c
similarity index 100%
rename from contrib/gdb/gdb/somread.c
rename to gdb/somread.c
diff --git a/contrib/gdb/gdb/somsolib.c b/gdb/somsolib.c
similarity index 100%
rename from contrib/gdb/gdb/somsolib.c
rename to gdb/somsolib.c
diff --git a/contrib/gdb/gdb/somsolib.h b/gdb/somsolib.h
similarity index 100%
rename from contrib/gdb/gdb/somsolib.h
rename to gdb/somsolib.h
diff --git a/contrib/gdb/gdb/source.c b/gdb/source.c
similarity index 100%
rename from contrib/gdb/gdb/source.c
rename to gdb/source.c
diff --git a/contrib/gdb/gdb/source.h b/gdb/source.h
similarity index 100%
rename from contrib/gdb/gdb/source.h
rename to gdb/source.h
diff --git a/contrib/gdb/gdb/sparc-nat.c b/gdb/sparc-nat.c
similarity index 100%
rename from contrib/gdb/gdb/sparc-nat.c
rename to gdb/sparc-nat.c
diff --git a/contrib/gdb/gdb/sparc-nat.h b/gdb/sparc-nat.h
similarity index 100%
rename from contrib/gdb/gdb/sparc-nat.h
rename to gdb/sparc-nat.h
diff --git a/contrib/gdb/gdb/sparc-sol2-nat.c b/gdb/sparc-sol2-nat.c
similarity index 100%
rename from contrib/gdb/gdb/sparc-sol2-nat.c
rename to gdb/sparc-sol2-nat.c
diff --git a/contrib/gdb/gdb/sparc-sol2-tdep.c b/gdb/sparc-sol2-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/sparc-sol2-tdep.c
rename to gdb/sparc-sol2-tdep.c
diff --git a/contrib/gdb/gdb/sparc-stub.c b/gdb/sparc-stub.c
similarity index 100%
rename from contrib/gdb/gdb/sparc-stub.c
rename to gdb/sparc-stub.c
diff --git a/contrib/gdb/gdb/sparc-tdep.c b/gdb/sparc-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/sparc-tdep.c
rename to gdb/sparc-tdep.c
diff --git a/contrib/gdb/gdb/sparc-tdep.h b/gdb/sparc-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/sparc-tdep.h
rename to gdb/sparc-tdep.h
diff --git a/contrib/gdb/gdb/sparc64-nat.c b/gdb/sparc64-nat.c
similarity index 100%
rename from contrib/gdb/gdb/sparc64-nat.c
rename to gdb/sparc64-nat.c
diff --git a/contrib/gdb/gdb/sparc64-sol2-tdep.c b/gdb/sparc64-sol2-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/sparc64-sol2-tdep.c
rename to gdb/sparc64-sol2-tdep.c
diff --git a/contrib/gdb/gdb/sparc64-tdep.c b/gdb/sparc64-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/sparc64-tdep.c
rename to gdb/sparc64-tdep.c
diff --git a/contrib/gdb/gdb/sparc64-tdep.h b/gdb/sparc64-tdep.h
similarity index 100%
rename from contrib/gdb/gdb/sparc64-tdep.h
rename to gdb/sparc64-tdep.h
diff --git a/contrib/gdb/gdb/sparc64fbsd-nat.c b/gdb/sparc64fbsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/sparc64fbsd-nat.c
rename to gdb/sparc64fbsd-nat.c
diff --git a/contrib/gdb/gdb/sparc64fbsd-tdep.c b/gdb/sparc64fbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/sparc64fbsd-tdep.c
rename to gdb/sparc64fbsd-tdep.c
diff --git a/contrib/gdb/gdb/sparc64nbsd-nat.c b/gdb/sparc64nbsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/sparc64nbsd-nat.c
rename to gdb/sparc64nbsd-nat.c
diff --git a/contrib/gdb/gdb/sparc64nbsd-tdep.c b/gdb/sparc64nbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/sparc64nbsd-tdep.c
rename to gdb/sparc64nbsd-tdep.c
diff --git a/contrib/gdb/gdb/sparc64obsd-tdep.c b/gdb/sparc64obsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/sparc64obsd-tdep.c
rename to gdb/sparc64obsd-tdep.c
diff --git a/contrib/gdb/gdb/sparcnbsd-nat.c b/gdb/sparcnbsd-nat.c
similarity index 100%
rename from contrib/gdb/gdb/sparcnbsd-nat.c
rename to gdb/sparcnbsd-nat.c
diff --git a/contrib/gdb/gdb/sparcnbsd-tdep.c b/gdb/sparcnbsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/sparcnbsd-tdep.c
rename to gdb/sparcnbsd-tdep.c
diff --git a/contrib/gdb/gdb/sparcobsd-tdep.c b/gdb/sparcobsd-tdep.c
similarity index 100%
rename from contrib/gdb/gdb/sparcobsd-tdep.c
rename to gdb/sparcobsd-tdep.c
diff --git a/contrib/gdb/gdb/srec.h b/gdb/srec.h
similarity index 100%
rename from contrib/gdb/gdb/srec.h
rename to gdb/srec.h
diff --git a/contrib/gdb/gdb/stabsread.c b/gdb/stabsread.c
similarity index 100%
rename from contrib/gdb/gdb/stabsread.c
rename to gdb/stabsread.c
diff --git a/contrib/gdb/gdb/stabsread.h b/gdb/stabsread.h
similarity index 100%
rename from contrib/gdb/gdb/stabsread.h
rename to gdb/stabsread.h
diff --git a/contrib/gdb/gdb/stack.c b/gdb/stack.c
similarity index 100%
rename from contrib/gdb/gdb/stack.c
rename to gdb/stack.c
diff --git a/contrib/gdb/gdb/stack.h b/gdb/stack.h
similarity index 100%
rename from contrib/gdb/gdb/stack.h
rename to gdb/stack.h
diff --git a/contrib/gdb/gdb/standalone.c b/gdb/standalone.c
similarity index 100%
rename from contrib/gdb/gdb/standalone.c
rename to gdb/standalone.c
diff --git a/contrib/gdb/gdb/std-regs.c b/gdb/std-regs.c
similarity index 100%
rename from contrib/gdb/gdb/std-regs.c
rename to gdb/std-regs.c
diff --git a/contrib/gdb/gdb/stop-gdb.c b/gdb/stop-gdb.c
similarity index 100%
rename from contrib/gdb/gdb/stop-gdb.c
rename to gdb/stop-gdb.c
diff --git a/contrib/gdb/gdb/sun3-nat.c b/gdb/sun3-nat.c
similarity index 100%
rename from contrib/gdb/gdb/sun3-nat.c
rename to gdb/sun3-nat.c
diff --git a/contrib/gdb/gdb/symfile.c b/gdb/symfile.c
similarity index 100%
rename from contrib/gdb/gdb/symfile.c
rename to gdb/symfile.c
diff --git a/contrib/gdb/gdb/symfile.h b/gdb/symfile.h
similarity index 100%
rename from contrib/gdb/gdb/symfile.h
rename to gdb/symfile.h
diff --git a/contrib/gdb/gdb/symmisc.c b/gdb/symmisc.c
similarity index 100%
rename from contrib/gdb/gdb/symmisc.c
rename to gdb/symmisc.c
diff --git a/contrib/gdb/gdb/symtab.c b/gdb/symtab.c
similarity index 100%
rename from contrib/gdb/gdb/symtab.c
rename to gdb/symtab.c
diff --git a/contrib/gdb/gdb/symtab.h b/gdb/symtab.h
similarity index 100%
rename from contrib/gdb/gdb/symtab.h
rename to gdb/symtab.h
diff --git a/contrib/gdb/gdb/target.c b/gdb/target.c
similarity index 100%
rename from contrib/gdb/gdb/target.c
rename to gdb/target.c
diff --git a/contrib/gdb/gdb/target.h b/gdb/target.h
similarity index 100%
rename from contrib/gdb/gdb/target.h
rename to gdb/target.h
diff --git a/contrib/gdb/gdb/terminal.h b/gdb/terminal.h
similarity index 100%
rename from contrib/gdb/gdb/terminal.h
rename to gdb/terminal.h
diff --git a/contrib/gdb/gdb/thread-db.c b/gdb/thread-db.c
similarity index 100%
rename from contrib/gdb/gdb/thread-db.c
rename to gdb/thread-db.c
diff --git a/contrib/gdb/gdb/thread.c b/gdb/thread.c
similarity index 100%
rename from contrib/gdb/gdb/thread.c
rename to gdb/thread.c
diff --git a/contrib/gdb/gdb/top.c b/gdb/top.c
similarity index 100%
rename from contrib/gdb/gdb/top.c
rename to gdb/top.c
diff --git a/contrib/gdb/gdb/top.h b/gdb/top.h
similarity index 100%
rename from contrib/gdb/gdb/top.h
rename to gdb/top.h
diff --git a/contrib/gdb/gdb/tracepoint.c b/gdb/tracepoint.c
similarity index 100%
rename from contrib/gdb/gdb/tracepoint.c
rename to gdb/tracepoint.c
diff --git a/contrib/gdb/gdb/tracepoint.h b/gdb/tracepoint.h
similarity index 100%
rename from contrib/gdb/gdb/tracepoint.h
rename to gdb/tracepoint.h
diff --git a/contrib/gdb/gdb/trad-frame.c b/gdb/trad-frame.c
similarity index 100%
rename from contrib/gdb/gdb/trad-frame.c
rename to gdb/trad-frame.c
diff --git a/contrib/gdb/gdb/trad-frame.h b/gdb/trad-frame.h
similarity index 100%
rename from contrib/gdb/gdb/trad-frame.h
rename to gdb/trad-frame.h
diff --git a/contrib/gdb/gdb/tui/tui-command.c b/gdb/tui/tui-command.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-command.c
rename to gdb/tui/tui-command.c
diff --git a/contrib/gdb/gdb/tui/tui-command.h b/gdb/tui/tui-command.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-command.h
rename to gdb/tui/tui-command.h
diff --git a/contrib/gdb/gdb/tui/tui-data.c b/gdb/tui/tui-data.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-data.c
rename to gdb/tui/tui-data.c
diff --git a/contrib/gdb/gdb/tui/tui-data.h b/gdb/tui/tui-data.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-data.h
rename to gdb/tui/tui-data.h
diff --git a/contrib/gdb/gdb/tui/tui-disasm.c b/gdb/tui/tui-disasm.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-disasm.c
rename to gdb/tui/tui-disasm.c
diff --git a/contrib/gdb/gdb/tui/tui-disasm.h b/gdb/tui/tui-disasm.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-disasm.h
rename to gdb/tui/tui-disasm.h
diff --git a/contrib/gdb/gdb/tui/tui-file.c b/gdb/tui/tui-file.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-file.c
rename to gdb/tui/tui-file.c
diff --git a/contrib/gdb/gdb/tui/tui-file.h b/gdb/tui/tui-file.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-file.h
rename to gdb/tui/tui-file.h
diff --git a/contrib/gdb/gdb/tui/tui-hooks.c b/gdb/tui/tui-hooks.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-hooks.c
rename to gdb/tui/tui-hooks.c
diff --git a/contrib/gdb/gdb/tui/tui-hooks.h b/gdb/tui/tui-hooks.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-hooks.h
rename to gdb/tui/tui-hooks.h
diff --git a/contrib/gdb/gdb/tui/tui-interp.c b/gdb/tui/tui-interp.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-interp.c
rename to gdb/tui/tui-interp.c
diff --git a/contrib/gdb/gdb/tui/tui-io.c b/gdb/tui/tui-io.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-io.c
rename to gdb/tui/tui-io.c
diff --git a/contrib/gdb/gdb/tui/tui-io.h b/gdb/tui/tui-io.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-io.h
rename to gdb/tui/tui-io.h
diff --git a/contrib/gdb/gdb/tui/tui-layout.c b/gdb/tui/tui-layout.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-layout.c
rename to gdb/tui/tui-layout.c
diff --git a/contrib/gdb/gdb/tui/tui-layout.h b/gdb/tui/tui-layout.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-layout.h
rename to gdb/tui/tui-layout.h
diff --git a/contrib/gdb/gdb/tui/tui-main.c b/gdb/tui/tui-main.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-main.c
rename to gdb/tui/tui-main.c
diff --git a/contrib/gdb/gdb/tui/tui-out.c b/gdb/tui/tui-out.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-out.c
rename to gdb/tui/tui-out.c
diff --git a/contrib/gdb/gdb/tui/tui-regs.c b/gdb/tui/tui-regs.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-regs.c
rename to gdb/tui/tui-regs.c
diff --git a/contrib/gdb/gdb/tui/tui-regs.h b/gdb/tui/tui-regs.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-regs.h
rename to gdb/tui/tui-regs.h
diff --git a/contrib/gdb/gdb/tui/tui-source.c b/gdb/tui/tui-source.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-source.c
rename to gdb/tui/tui-source.c
diff --git a/contrib/gdb/gdb/tui/tui-source.h b/gdb/tui/tui-source.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-source.h
rename to gdb/tui/tui-source.h
diff --git a/contrib/gdb/gdb/tui/tui-stack.c b/gdb/tui/tui-stack.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-stack.c
rename to gdb/tui/tui-stack.c
diff --git a/contrib/gdb/gdb/tui/tui-stack.h b/gdb/tui/tui-stack.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-stack.h
rename to gdb/tui/tui-stack.h
diff --git a/contrib/gdb/gdb/tui/tui-win.c b/gdb/tui/tui-win.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-win.c
rename to gdb/tui/tui-win.c
diff --git a/contrib/gdb/gdb/tui/tui-win.h b/gdb/tui/tui-win.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-win.h
rename to gdb/tui/tui-win.h
diff --git a/contrib/gdb/gdb/tui/tui-windata.c b/gdb/tui/tui-windata.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-windata.c
rename to gdb/tui/tui-windata.c
diff --git a/contrib/gdb/gdb/tui/tui-windata.h b/gdb/tui/tui-windata.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-windata.h
rename to gdb/tui/tui-windata.h
diff --git a/contrib/gdb/gdb/tui/tui-wingeneral.c b/gdb/tui/tui-wingeneral.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-wingeneral.c
rename to gdb/tui/tui-wingeneral.c
diff --git a/contrib/gdb/gdb/tui/tui-wingeneral.h b/gdb/tui/tui-wingeneral.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-wingeneral.h
rename to gdb/tui/tui-wingeneral.h
diff --git a/contrib/gdb/gdb/tui/tui-winsource.c b/gdb/tui/tui-winsource.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-winsource.c
rename to gdb/tui/tui-winsource.c
diff --git a/contrib/gdb/gdb/tui/tui-winsource.h b/gdb/tui/tui-winsource.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui-winsource.h
rename to gdb/tui/tui-winsource.h
diff --git a/contrib/gdb/gdb/tui/tui.c b/gdb/tui/tui.c
similarity index 100%
rename from contrib/gdb/gdb/tui/tui.c
rename to gdb/tui/tui.c
diff --git a/contrib/gdb/gdb/tui/tui.h b/gdb/tui/tui.h
similarity index 100%
rename from contrib/gdb/gdb/tui/tui.h
rename to gdb/tui/tui.h
diff --git a/contrib/gdb/gdb/typeprint.c b/gdb/typeprint.c
similarity index 100%
rename from contrib/gdb/gdb/typeprint.c
rename to gdb/typeprint.c
diff --git a/contrib/gdb/gdb/typeprint.h b/gdb/typeprint.h
similarity index 100%
rename from contrib/gdb/gdb/typeprint.h
rename to gdb/typeprint.h
diff --git a/contrib/gdb/gdb/ui-file.c b/gdb/ui-file.c
similarity index 100%
rename from contrib/gdb/gdb/ui-file.c
rename to gdb/ui-file.c
diff --git a/contrib/gdb/gdb/ui-file.h b/gdb/ui-file.h
similarity index 100%
rename from contrib/gdb/gdb/ui-file.h
rename to gdb/ui-file.h
diff --git a/contrib/gdb/gdb/ui-out.c b/gdb/ui-out.c
similarity index 100%
rename from contrib/gdb/gdb/ui-out.c
rename to gdb/ui-out.c
diff --git a/contrib/gdb/gdb/ui-out.h b/gdb/ui-out.h
similarity index 100%
rename from contrib/gdb/gdb/ui-out.h
rename to gdb/ui-out.h
diff --git a/contrib/gdb/gdb/user-regs.c b/gdb/user-regs.c
similarity index 100%
rename from contrib/gdb/gdb/user-regs.c
rename to gdb/user-regs.c
diff --git a/contrib/gdb/gdb/user-regs.h b/gdb/user-regs.h
similarity index 100%
rename from contrib/gdb/gdb/user-regs.h
rename to gdb/user-regs.h
diff --git a/contrib/gdb/gdb/utils.c b/gdb/utils.c
similarity index 100%
rename from contrib/gdb/gdb/utils.c
rename to gdb/utils.c
diff --git a/contrib/gdb/gdb/uw-thread.c b/gdb/uw-thread.c
similarity index 100%
rename from contrib/gdb/gdb/uw-thread.c
rename to gdb/uw-thread.c
diff --git a/contrib/gdb/gdb/valarith.c b/gdb/valarith.c
similarity index 100%
rename from contrib/gdb/gdb/valarith.c
rename to gdb/valarith.c
diff --git a/contrib/gdb/gdb/valops.c b/gdb/valops.c
similarity index 100%
rename from contrib/gdb/gdb/valops.c
rename to gdb/valops.c
diff --git a/contrib/gdb/gdb/valprint.c b/gdb/valprint.c
similarity index 100%
rename from contrib/gdb/gdb/valprint.c
rename to gdb/valprint.c
diff --git a/contrib/gdb/gdb/valprint.h b/gdb/valprint.h
similarity index 100%
rename from contrib/gdb/gdb/valprint.h
rename to gdb/valprint.h
diff --git a/contrib/gdb/gdb/value.h b/gdb/value.h
similarity index 100%
rename from contrib/gdb/gdb/value.h
rename to gdb/value.h
diff --git a/contrib/gdb/gdb/values.c b/gdb/values.c
similarity index 100%
rename from contrib/gdb/gdb/values.c
rename to gdb/values.c
diff --git a/contrib/gdb/gdb/varobj.c b/gdb/varobj.c
similarity index 100%
rename from contrib/gdb/gdb/varobj.c
rename to gdb/varobj.c
diff --git a/contrib/gdb/gdb/varobj.h b/gdb/varobj.h
similarity index 100%
rename from contrib/gdb/gdb/varobj.h
rename to gdb/varobj.h
diff --git a/contrib/gdb/gdb/version.h b/gdb/version.h
similarity index 100%
rename from contrib/gdb/gdb/version.h
rename to gdb/version.h
diff --git a/contrib/gdb/gdb/version.in b/gdb/version.in
similarity index 100%
rename from contrib/gdb/gdb/version.in
rename to gdb/version.in
diff --git a/contrib/gdb/gdb/win32-nat.c b/gdb/win32-nat.c
similarity index 100%
rename from contrib/gdb/gdb/win32-nat.c
rename to gdb/win32-nat.c
diff --git a/contrib/gdb/gdb/wince-stub.c b/gdb/wince-stub.c
similarity index 100%
rename from contrib/gdb/gdb/wince-stub.c
rename to gdb/wince-stub.c
diff --git a/contrib/gdb/gdb/wince-stub.h b/gdb/wince-stub.h
similarity index 100%
rename from contrib/gdb/gdb/wince-stub.h
rename to gdb/wince-stub.h
diff --git a/contrib/gdb/gdb/wince.c b/gdb/wince.c
similarity index 100%
rename from contrib/gdb/gdb/wince.c
rename to gdb/wince.c
diff --git a/contrib/gdb/gdb/wrapper.c b/gdb/wrapper.c
similarity index 100%
rename from contrib/gdb/gdb/wrapper.c
rename to gdb/wrapper.c
diff --git a/contrib/gdb/gdb/wrapper.h b/gdb/wrapper.h
similarity index 100%
rename from contrib/gdb/gdb/wrapper.h
rename to gdb/wrapper.h
diff --git a/contrib/gdb/gdb/xcoffread.c b/gdb/xcoffread.c
similarity index 100%
rename from contrib/gdb/gdb/xcoffread.c
rename to gdb/xcoffread.c
diff --git a/contrib/gdb/gdb/xcoffsolib.c b/gdb/xcoffsolib.c
similarity index 100%
rename from contrib/gdb/gdb/xcoffsolib.c
rename to gdb/xcoffsolib.c
diff --git a/contrib/gdb/gdb/xcoffsolib.h b/gdb/xcoffsolib.h
similarity index 100%
rename from contrib/gdb/gdb/xcoffsolib.h
rename to gdb/xcoffsolib.h
diff --git a/contrib/gdb/gdb/xmodem.c b/gdb/xmodem.c
similarity index 100%
rename from contrib/gdb/gdb/xmodem.c
rename to gdb/xmodem.c
diff --git a/contrib/gdb/gdb/xmodem.h b/gdb/xmodem.h
similarity index 100%
rename from contrib/gdb/gdb/xmodem.h
rename to gdb/xmodem.h
diff --git a/contrib/gdb/gettext.m4 b/gettext.m4
similarity index 100%
rename from contrib/gdb/gettext.m4
rename to gettext.m4
diff --git a/contrib/gdb/include/COPYING b/include/COPYING
similarity index 100%
rename from contrib/gdb/include/COPYING
rename to include/COPYING
diff --git a/contrib/gdb/include/MAINTAINERS b/include/MAINTAINERS
similarity index 100%
rename from contrib/gdb/include/MAINTAINERS
rename to include/MAINTAINERS
diff --git a/contrib/gdb/include/alloca-conf.h b/include/alloca-conf.h
similarity index 100%
rename from contrib/gdb/include/alloca-conf.h
rename to include/alloca-conf.h
diff --git a/contrib/gdb/include/ansidecl.h b/include/ansidecl.h
similarity index 100%
rename from contrib/gdb/include/ansidecl.h
rename to include/ansidecl.h
diff --git a/contrib/gdb/include/bfdlink.h b/include/bfdlink.h
similarity index 100%
rename from contrib/gdb/include/bfdlink.h
rename to include/bfdlink.h
diff --git a/contrib/gdb/include/bin-bugs.h b/include/bin-bugs.h
similarity index 100%
rename from contrib/gdb/include/bin-bugs.h
rename to include/bin-bugs.h
diff --git a/contrib/gdb/include/bout.h b/include/bout.h
similarity index 100%
rename from contrib/gdb/include/bout.h
rename to include/bout.h
diff --git a/contrib/gdb/include/demangle.h b/include/demangle.h
similarity index 100%
rename from contrib/gdb/include/demangle.h
rename to include/demangle.h
diff --git a/contrib/gdb/include/dis-asm.h b/include/dis-asm.h
similarity index 100%
rename from contrib/gdb/include/dis-asm.h
rename to include/dis-asm.h
diff --git a/contrib/gdb/include/dyn-string.h b/include/dyn-string.h
similarity index 100%
rename from contrib/gdb/include/dyn-string.h
rename to include/dyn-string.h
diff --git a/contrib/gdb/include/fibheap.h b/include/fibheap.h
similarity index 100%
rename from contrib/gdb/include/fibheap.h
rename to include/fibheap.h
diff --git a/contrib/gdb/include/filenames.h b/include/filenames.h
similarity index 100%
rename from contrib/gdb/include/filenames.h
rename to include/filenames.h
diff --git a/contrib/gdb/include/floatformat.h b/include/floatformat.h
similarity index 100%
rename from contrib/gdb/include/floatformat.h
rename to include/floatformat.h
diff --git a/contrib/gdb/include/fnmatch.h b/include/fnmatch.h
similarity index 100%
rename from contrib/gdb/include/fnmatch.h
rename to include/fnmatch.h
diff --git a/contrib/gdb/include/fopen-bin.h b/include/fopen-bin.h
similarity index 100%
rename from contrib/gdb/include/fopen-bin.h
rename to include/fopen-bin.h
diff --git a/contrib/gdb/include/fopen-same.h b/include/fopen-same.h
similarity index 100%
rename from contrib/gdb/include/fopen-same.h
rename to include/fopen-same.h
diff --git a/contrib/gdb/include/fopen-vms.h b/include/fopen-vms.h
similarity index 100%
rename from contrib/gdb/include/fopen-vms.h
rename to include/fopen-vms.h
diff --git a/contrib/gdb/include/gdb/callback.h b/include/gdb/callback.h
similarity index 100%
rename from contrib/gdb/include/gdb/callback.h
rename to include/gdb/callback.h
diff --git a/contrib/gdb/include/gdb/fileio.h b/include/gdb/fileio.h
similarity index 100%
rename from contrib/gdb/include/gdb/fileio.h
rename to include/gdb/fileio.h
diff --git a/contrib/gdb/include/gdb/remote-sim.h b/include/gdb/remote-sim.h
similarity index 100%
rename from contrib/gdb/include/gdb/remote-sim.h
rename to include/gdb/remote-sim.h
diff --git a/contrib/gdb/include/gdb/signals.h b/include/gdb/signals.h
similarity index 100%
rename from contrib/gdb/include/gdb/signals.h
rename to include/gdb/signals.h
diff --git a/contrib/gdb/include/gdb/sim-arm.h b/include/gdb/sim-arm.h
similarity index 100%
rename from contrib/gdb/include/gdb/sim-arm.h
rename to include/gdb/sim-arm.h
diff --git a/contrib/gdb/include/gdb/sim-d10v.h b/include/gdb/sim-d10v.h
similarity index 100%
rename from contrib/gdb/include/gdb/sim-d10v.h
rename to include/gdb/sim-d10v.h
diff --git a/contrib/gdb/include/gdb/sim-frv.h b/include/gdb/sim-frv.h
similarity index 100%
rename from contrib/gdb/include/gdb/sim-frv.h
rename to include/gdb/sim-frv.h
diff --git a/contrib/gdb/include/gdb/sim-h8300.h b/include/gdb/sim-h8300.h
similarity index 100%
rename from contrib/gdb/include/gdb/sim-h8300.h
rename to include/gdb/sim-h8300.h
diff --git a/contrib/gdb/include/gdb/sim-sh.h b/include/gdb/sim-sh.h
similarity index 100%
rename from contrib/gdb/include/gdb/sim-sh.h
rename to include/gdb/sim-sh.h
diff --git a/contrib/gdb/include/gdbm.h b/include/gdbm.h
similarity index 100%
rename from contrib/gdb/include/gdbm.h
rename to include/gdbm.h
diff --git a/contrib/gdb/include/getopt.h b/include/getopt.h
similarity index 100%
rename from contrib/gdb/include/getopt.h
rename to include/getopt.h
diff --git a/contrib/gdb/include/hashtab.h b/include/hashtab.h
similarity index 100%
rename from contrib/gdb/include/hashtab.h
rename to include/hashtab.h
diff --git a/contrib/gdb/include/hp-symtab.h b/include/hp-symtab.h
similarity index 100%
rename from contrib/gdb/include/hp-symtab.h
rename to include/hp-symtab.h
diff --git a/contrib/gdb/include/ieee.h b/include/ieee.h
similarity index 100%
rename from contrib/gdb/include/ieee.h
rename to include/ieee.h
diff --git a/contrib/gdb/include/libiberty.h b/include/libiberty.h
similarity index 100%
rename from contrib/gdb/include/libiberty.h
rename to include/libiberty.h
diff --git a/contrib/gdb/include/md5.h b/include/md5.h
similarity index 100%
rename from contrib/gdb/include/md5.h
rename to include/md5.h
diff --git a/contrib/gdb/include/oasys.h b/include/oasys.h
similarity index 100%
rename from contrib/gdb/include/oasys.h
rename to include/oasys.h
diff --git a/contrib/gdb/include/objalloc.h b/include/objalloc.h
similarity index 100%
rename from contrib/gdb/include/objalloc.h
rename to include/objalloc.h
diff --git a/contrib/gdb/include/obstack.h b/include/obstack.h
similarity index 100%
rename from contrib/gdb/include/obstack.h
rename to include/obstack.h
diff --git a/contrib/gdb/include/os9k.h b/include/os9k.h
similarity index 100%
rename from contrib/gdb/include/os9k.h
rename to include/os9k.h
diff --git a/contrib/gdb/include/partition.h b/include/partition.h
similarity index 100%
rename from contrib/gdb/include/partition.h
rename to include/partition.h
diff --git a/contrib/gdb/include/progress.h b/include/progress.h
similarity index 100%
rename from contrib/gdb/include/progress.h
rename to include/progress.h
diff --git a/contrib/gdb/include/safe-ctype.h b/include/safe-ctype.h
similarity index 100%
rename from contrib/gdb/include/safe-ctype.h
rename to include/safe-ctype.h
diff --git a/contrib/gdb/include/sort.h b/include/sort.h
similarity index 100%
rename from contrib/gdb/include/sort.h
rename to include/sort.h
diff --git a/contrib/gdb/include/splay-tree.h b/include/splay-tree.h
similarity index 100%
rename from contrib/gdb/include/splay-tree.h
rename to include/splay-tree.h
diff --git a/contrib/gdb/include/symcat.h b/include/symcat.h
similarity index 100%
rename from contrib/gdb/include/symcat.h
rename to include/symcat.h
diff --git a/contrib/gdb/include/ternary.h b/include/ternary.h
similarity index 100%
rename from contrib/gdb/include/ternary.h
rename to include/ternary.h
diff --git a/contrib/gdb/include/xregex.h b/include/xregex.h
similarity index 100%
rename from contrib/gdb/include/xregex.h
rename to include/xregex.h
diff --git a/contrib/gdb/include/xregex2.h b/include/xregex2.h
similarity index 100%
rename from contrib/gdb/include/xregex2.h
rename to include/xregex2.h
diff --git a/contrib/gdb/include/xtensa-isa-internal.h b/include/xtensa-isa-internal.h
similarity index 100%
rename from contrib/gdb/include/xtensa-isa-internal.h
rename to include/xtensa-isa-internal.h
diff --git a/contrib/gdb/include/xtensa-isa.h b/include/xtensa-isa.h
similarity index 100%
rename from contrib/gdb/include/xtensa-isa.h
rename to include/xtensa-isa.h
diff --git a/contrib/gdb/install-sh b/install-sh
similarity index 100%
rename from contrib/gdb/install-sh
rename to install-sh
diff --git a/contrib/gdb/libtool.m4 b/libtool.m4
similarity index 100%
rename from contrib/gdb/libtool.m4
rename to libtool.m4
diff --git a/contrib/gdb/ltcf-c.sh b/ltcf-c.sh
similarity index 100%
rename from contrib/gdb/ltcf-c.sh
rename to ltcf-c.sh
diff --git a/contrib/gdb/ltcf-cxx.sh b/ltcf-cxx.sh
similarity index 100%
rename from contrib/gdb/ltcf-cxx.sh
rename to ltcf-cxx.sh
diff --git a/contrib/gdb/ltcf-gcj.sh b/ltcf-gcj.sh
similarity index 100%
rename from contrib/gdb/ltcf-gcj.sh
rename to ltcf-gcj.sh
diff --git a/contrib/gdb/ltconfig b/ltconfig
similarity index 100%
rename from contrib/gdb/ltconfig
rename to ltconfig
diff --git a/contrib/gdb/ltmain.sh b/ltmain.sh
similarity index 100%
rename from contrib/gdb/ltmain.sh
rename to ltmain.sh
diff --git a/contrib/gdb/md5.sum b/md5.sum
similarity index 100%
rename from contrib/gdb/md5.sum
rename to md5.sum
diff --git a/contrib/gdb/missing b/missing
similarity index 100%
rename from contrib/gdb/missing
rename to missing
diff --git a/contrib/gdb/mkinstalldirs b/mkinstalldirs
similarity index 100%
rename from contrib/gdb/mkinstalldirs
rename to mkinstalldirs
diff --git a/contrib/gdb/move-if-change b/move-if-change
similarity index 100%
rename from contrib/gdb/move-if-change
rename to move-if-change
diff --git a/contrib/gdb/src-release b/src-release
similarity index 100%
rename from contrib/gdb/src-release
rename to src-release
diff --git a/contrib/gdb/symlink-tree b/symlink-tree
similarity index 100%
rename from contrib/gdb/symlink-tree
rename to symlink-tree
diff --git a/contrib/gdb/ylwrap b/ylwrap
similarity index 100%
rename from contrib/gdb/ylwrap
rename to ylwrap

From f460f3b974e37adaa3a9dacc3b3ab94d30b65f11 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Tue, 14 Oct 2014 12:04:50 +0000
Subject: [PATCH 002/279] Clear VM_RC_LIST.

Sponsored by:	The FreeBSD Foundation
---
 release/tools/azure.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release/tools/azure.conf b/release/tools/azure.conf
index da205bb1d30..8f16acc0d27 100644
--- a/release/tools/azure.conf
+++ b/release/tools/azure.conf
@@ -11,4 +11,4 @@ export VM_EXTRA_PACKAGES=
 # Set to a list of third-party software to enable in rc.conf(5).
 # Example:
 #export VM_RC_LIST="apache24"
-export VM_RC_LIST="apache24"
+export VM_RC_LIST=

From 60b7edb5a7fe57f8f1c70038d833dd445c83a0c6 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Tue, 14 Oct 2014 12:13:43 +0000
Subject: [PATCH 003/279] Fix signal list to trigger umount(8).

Sponsored by:	The FreeBSD Foundation
---
 release/amd64/mk-azure.sh | 2 +-
 release/i386/mk-azure.sh  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/release/amd64/mk-azure.sh b/release/amd64/mk-azure.sh
index b691d54f6a6..4ba0bfae0e1 100755
--- a/release/amd64/mk-azure.sh
+++ b/release/amd64/mk-azure.sh
@@ -67,7 +67,7 @@ vm_create_azure() {
 			usage
 	fi
 
-	trap "umount ${DESTDIR}/dev ${DESTDIR}" EXIT
+	trap "umount ${DESTDIR}/dev ${DESTDIR}" INT QUIT TRAP ABRT TERM
 
 	i=0
 	mkdir -p ${DESTDIR}
diff --git a/release/i386/mk-azure.sh b/release/i386/mk-azure.sh
index b691d54f6a6..4ba0bfae0e1 100755
--- a/release/i386/mk-azure.sh
+++ b/release/i386/mk-azure.sh
@@ -67,7 +67,7 @@ vm_create_azure() {
 			usage
 	fi
 
-	trap "umount ${DESTDIR}/dev ${DESTDIR}" EXIT
+	trap "umount ${DESTDIR}/dev ${DESTDIR}" INT QUIT TRAP ABRT TERM
 
 	i=0
 	mkdir -p ${DESTDIR}

From f38d42ba6338870de7d0a3ab873fc1768480fcef Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Tue, 14 Oct 2014 15:18:22 +0000
Subject: [PATCH 004/279] Output an informational message when mkimg(1) runs,
 so it does not appear that the process has stopped while waiting for a 'y/n'
 response when waagent is deprovisioned.

Sponsored by:	The FreeBSD Foundation
---
 release/amd64/mk-azure.sh | 2 ++
 release/i386/mk-azure.sh  | 2 ++
 2 files changed, 4 insertions(+)

diff --git a/release/amd64/mk-azure.sh b/release/amd64/mk-azure.sh
index 4ba0bfae0e1..45b70ae73a1 100755
--- a/release/amd64/mk-azure.sh
+++ b/release/amd64/mk-azure.sh
@@ -130,6 +130,8 @@ vm_create_azure() {
 		sleep 1
 	done
 
+	echo "Creating image...  Please wait."
+
 	mkimg -f vhdf -s gpt \
 		-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
 		-p freebsd-swap/swapfs::1G \
diff --git a/release/i386/mk-azure.sh b/release/i386/mk-azure.sh
index 4ba0bfae0e1..45b70ae73a1 100755
--- a/release/i386/mk-azure.sh
+++ b/release/i386/mk-azure.sh
@@ -130,6 +130,8 @@ vm_create_azure() {
 		sleep 1
 	done
 
+	echo "Creating image...  Please wait."
+
 	mkimg -f vhdf -s gpt \
 		-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
 		-p freebsd-swap/swapfs::1G \

From 6513e474ac5b1450691119b31343805f1d1266fc Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Wed, 29 Oct 2014 14:57:30 +0000
Subject: [PATCH 005/279] Move virtual machine / cloud provider targets and
 options from release/Makefile to their own Makefile.

Sponsored by:	The FreeBSD Foundation
---
 release/Makefile    | 51 +---------------------------------------
 release/Makefile.vm | 57 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 58 insertions(+), 50 deletions(-)
 create mode 100644 release/Makefile.vm

diff --git a/release/Makefile b/release/Makefile
index 8a517f81024..5c425e48f0a 100644
--- a/release/Makefile
+++ b/release/Makefile
@@ -97,12 +97,6 @@ IMAGES+=	memstick.img
 IMAGES+=	mini-memstick.img
 .endif
 
-VMTARGETS=	vm-base vm-image
-VMFORMATS?=	vhd vmdk qcow2 raw
-VMSIZE?=	20G
-VMBASE?=	vm
-AZURECONF?=	${.CURDIR}/tools/azure.conf
-
 CLEANFILES=	packagesystem *.txz MANIFEST system ${IMAGES}
 .if defined(WITH_COMPRESSED_IMAGES) && !empty(WITH_COMPRESSED_IMAGES)
 . for I in ${IMAGES}
@@ -112,22 +106,7 @@ CLEANFILES+=	${I}.xz
 .if defined(WITH_DVD) && !empty(WITH_DVD)
 CLEANFILES+=	pkg-stage
 .endif
-.if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
-CLEANFILES+=	${VMBASE}.img
-. for FORMAT in ${VMFORMATS}
-CLEANFILES+=	${VMBASE}.${FORMAT}
-. endfor
-.endif
 CLEANDIRS=	dist ftp release bootonly dvd
-.if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
-CLEANDIRS+=	${VMTARGETS}
-.endif
-.if exists(${.CURDIR}/${TARGET}/mk-azure.sh)
-CLEANFILES+=	${OSRELEASE}.vhd \
-		${OSRELEASE}.vhd.raw \
-		azure.img
-CLEANDIRS+=	vm-azure
-.endif
 beforeclean:
 	chflags -R noschg .
 .include <bsd.obj.mk>
@@ -324,32 +303,4 @@ install:
 		${DESTDIR}/vmimages/CHECKSUM.MD5
 .endif
 
-vm-base:
-.if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
-. if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
-	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		${.CURDIR}/${TARGET}/mk-vmimage.sh ${.TARGET} \
-		${VMBASE}.img ${WORLDDIR} ${.OBJDIR}/${.TARGET} ${VMSIZE}
-. endif
-.endif
-	touch ${.TARGET}
-
-vm-image: vm-base
-.if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
-. if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
-.  for FORMAT in ${VMFORMATS}
-	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		${.CURDIR}/${TARGET}/mk-vmimage.sh ${.TARGET} \
-		${VMBASE}.img ${FORMAT} ${VMBASE}.${FORMAT}
-.  endfor
-. endif
-.endif
-	touch ${.TARGET}
-
-vm-azure:
-.if exists(${.CURDIR}/${TARGET}/mk-azure.sh)
-	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} AZURECONF=${AZURECONF} \
-		${.CURDIR}/${TARGET}/mk-azure.sh ${.TARGET} azure.img \
-		${WORLDDIR} ${.TARGET} ${VMSIZE} ${OSRELEASE}.vhd
-.endif
-	touch ${.TARGET}
+.include "${.CURDIR}/Makefile.vm"
diff --git a/release/Makefile.vm b/release/Makefile.vm
new file mode 100644
index 00000000000..1d1f64223f5
--- /dev/null
+++ b/release/Makefile.vm
@@ -0,0 +1,57 @@
+#
+# $FreeBSD$
+#
+#
+# Makefile for building virtual machine and cloud provider disk images.
+#
+
+VMTARGETS=	vm-base vm-image
+VMFORMATS?=	vhd vmdk qcow2 raw
+VMSIZE?=	20G
+VMBASE?=	vm
+AZURECONF?=	${.CURDIR}/tools/azure.conf
+
+.if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
+CLEANDIRS+=	${VMTARGETS}
+CLEANFILES+=	${VMBASE}.img
+. for FORMAT in ${VMFORMATS}
+CLEANFILES+=	${VMBASE}.${FORMAT}
+. endfor
+.endif
+
+.if exists(${.CURDIR}/${TARGET}/mk-azure.sh)
+CLEANFILES+=	${OSRELEASE}.vhd \
+		${OSRELEASE}.vhd.raw \
+		azure.img
+CLEANDIRS+=	vm-azure
+.endif
+
+vm-base:
+.if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
+. if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
+	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
+		${.CURDIR}/${TARGET}/mk-vmimage.sh ${.TARGET} \
+		${VMBASE}.img ${WORLDDIR} ${.OBJDIR}/${.TARGET} ${VMSIZE}
+. endif
+.endif
+	touch ${.TARGET}
+
+vm-image: vm-base
+.if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
+. if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
+.  for FORMAT in ${VMFORMATS}
+	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
+		${.CURDIR}/${TARGET}/mk-vmimage.sh ${.TARGET} \
+		${VMBASE}.img ${FORMAT} ${VMBASE}.${FORMAT}
+.  endfor
+. endif
+.endif
+	touch ${.TARGET}
+
+vm-azure:
+.if exists(${.CURDIR}/${TARGET}/mk-azure.sh)
+	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} AZURECONF=${AZURECONF} \
+		${.CURDIR}/${TARGET}/mk-azure.sh ${.TARGET} azure.img \
+		${WORLDDIR} ${.TARGET} ${VMSIZE} ${OSRELEASE}.vhd
+.endif
+	touch ${.TARGET}

From d08ce983a7bbe104dfc78c920a86985ae2fd60cc Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Wed, 29 Oct 2014 15:52:17 +0000
Subject: [PATCH 006/279] Add glue to allow enabling building cloud provider VM
 images by default.

When WITH_CLOUDWARE is not empty, add CLOUDTARGETS
to the release/Makefile 'release' target.

CLOUDTARGETS is generated from the contents of
CLOUDWARE, which should be a list of all supported
target providers.

Sponsored by:	The FreeBSD Foundation
---
 release/Makefile    |  3 +++
 release/Makefile.vm | 24 ++++++++++++++++--------
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/release/Makefile b/release/Makefile
index 5c425e48f0a..5edd4d0f4f6 100644
--- a/release/Makefile
+++ b/release/Makefile
@@ -269,6 +269,9 @@ release:
 .if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
 	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} ${VMTARGETS}
 .endif
+.if defined(WITH_CLOUDWARE) && !empty(WITH_CLOUDWARE) && !empty(CLOUDWARE)
+	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} ${CLOUDTARGETS}
+.endif
 
 install:
 .if defined(DESTDIR) && !empty(DESTDIR)
diff --git a/release/Makefile.vm b/release/Makefile.vm
index 1d1f64223f5..82eb7d4ece4 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -9,7 +9,22 @@ VMTARGETS=	vm-base vm-image
 VMFORMATS?=	vhd vmdk qcow2 raw
 VMSIZE?=	20G
 VMBASE?=	vm
-AZURECONF?=	${.CURDIR}/tools/azure.conf
+
+CLOUDWARE?=	AZURE
+AZURE_FORMAT=	vhd
+
+.if defined(WITH_CLOUDWARE) && !empty(WITH_CLOUDWARE) && !empty(CLOUDWARE)
+. for _CW in ${CLOUDWARE}
+CLOUDTARGETS+=	vm-${_CW:tl}
+CLEANDIRS+=	vm-${_CW:tl}
+CLEANFILES+=	${_CW:tl}.img \
+		${_CW:tl}.${${_CW:tu}_FORMAT} \
+		${_CW:tl}.${${_CW:tu}_FORMAT}.raw
+.  if exists(${.CURDIR}/tools/${_CW:tl}.conf) && !defined(${_CW:tu}CONF)
+${_CW:tu}CONF?=	${.CURDIR}/tools/${_CW:tl}.conf
+.  endif
+. endfor
+.endif
 
 .if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
 CLEANDIRS+=	${VMTARGETS}
@@ -19,13 +34,6 @@ CLEANFILES+=	${VMBASE}.${FORMAT}
 . endfor
 .endif
 
-.if exists(${.CURDIR}/${TARGET}/mk-azure.sh)
-CLEANFILES+=	${OSRELEASE}.vhd \
-		${OSRELEASE}.vhd.raw \
-		azure.img
-CLEANDIRS+=	vm-azure
-.endif
-
 vm-base:
 .if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
 . if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)

From f27438c75256f550d9327bf6d3915fc83fd34620 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Wed, 29 Oct 2014 16:18:29 +0000
Subject: [PATCH 007/279] Avoid hard-coding the Azure image file format. While
 here, avoid using OSRELEASE for the output file name.

Sponsored by:	The FreeBSD Foundation
---
 release/Makefile.vm       | 6 ++++--
 release/amd64/mk-azure.sh | 2 +-
 release/i386/mk-azure.sh  | 2 +-
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/release/Makefile.vm b/release/Makefile.vm
index 82eb7d4ece4..ce7f0eab922 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -11,7 +11,7 @@ VMSIZE?=	20G
 VMBASE?=	vm
 
 CLOUDWARE?=	AZURE
-AZURE_FORMAT=	vhd
+AZURE_FORMAT=	vhdf
 
 .if defined(WITH_CLOUDWARE) && !empty(WITH_CLOUDWARE) && !empty(CLOUDWARE)
 . for _CW in ${CLOUDWARE}
@@ -20,6 +20,7 @@ CLEANDIRS+=	vm-${_CW:tl}
 CLEANFILES+=	${_CW:tl}.img \
 		${_CW:tl}.${${_CW:tu}_FORMAT} \
 		${_CW:tl}.${${_CW:tu}_FORMAT}.raw
+${_CW:tu}IMAGE=	${_CW:tl}.${${_CW:tu}_FORMAT}
 .  if exists(${.CURDIR}/tools/${_CW:tl}.conf) && !defined(${_CW:tu}CONF)
 ${_CW:tu}CONF?=	${.CURDIR}/tools/${_CW:tl}.conf
 .  endif
@@ -59,7 +60,8 @@ vm-image: vm-base
 vm-azure:
 .if exists(${.CURDIR}/${TARGET}/mk-azure.sh)
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} AZURECONF=${AZURECONF} \
+		AZURE_FORMAT=${AZURE_FORMAT} \
 		${.CURDIR}/${TARGET}/mk-azure.sh ${.TARGET} azure.img \
-		${WORLDDIR} ${.TARGET} ${VMSIZE} ${OSRELEASE}.vhd
+		${WORLDDIR} ${.TARGET} ${VMSIZE} ${AZUREIMAGE}
 .endif
 	touch ${.TARGET}
diff --git a/release/amd64/mk-azure.sh b/release/amd64/mk-azure.sh
index 45b70ae73a1..43c92040ed3 100755
--- a/release/amd64/mk-azure.sh
+++ b/release/amd64/mk-azure.sh
@@ -132,7 +132,7 @@ vm_create_azure() {
 
 	echo "Creating image...  Please wait."
 
-	mkimg -f vhdf -s gpt \
+	mkimg -f ${AZURE_FORMAT} -s gpt \
 		-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
 		-p freebsd-swap/swapfs::1G \
 		-p freebsd-ufs/rootfs:=${VMBASE} \
diff --git a/release/i386/mk-azure.sh b/release/i386/mk-azure.sh
index 45b70ae73a1..43c92040ed3 100755
--- a/release/i386/mk-azure.sh
+++ b/release/i386/mk-azure.sh
@@ -132,7 +132,7 @@ vm_create_azure() {
 
 	echo "Creating image...  Please wait."
 
-	mkimg -f vhdf -s gpt \
+	mkimg -f ${AZURE_FORMAT} -s gpt \
 		-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
 		-p freebsd-swap/swapfs::1G \
 		-p freebsd-ufs/rootfs:=${VMBASE} \

From 461f70f64addf73babe6a0c5ed1f808fb88ac197 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Wed, 29 Oct 2014 16:20:49 +0000
Subject: [PATCH 008/279] Remove a few vestiges of passing an exit code to
 panic().

Sponsored by:	The FreeBSD Foundation
---
 release/amd64/mk-azure.sh | 4 ++--
 release/i386/mk-azure.sh  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/release/amd64/mk-azure.sh b/release/amd64/mk-azure.sh
index 43c92040ed3..ac89258996f 100755
--- a/release/amd64/mk-azure.sh
+++ b/release/amd64/mk-azure.sh
@@ -78,7 +78,7 @@ vm_create_azure() {
 	mount /dev/${mddev} ${DESTDIR}
 	make -C ${WORLDDIR} DESTDIR=$(realpath ${DESTDIR}) \
 		installworld installkernel distribution || \
-		panic 1 "\n\nCannot install the base system to ${DESTDIR}."
+		panic "\n\nCannot install the base system to ${DESTDIR}."
 	mount -t devfs devfs ${DESTDIR}/dev
 	chroot ${DESTDIR} /usr/bin/newaliases
 	echo '# Custom /etc/fstab for FreeBSD VM images' \
@@ -125,7 +125,7 @@ vm_create_azure() {
 			# This should never happen.  But, it has happened.
 			msg="Cannot umount(8) ${DESTDIR}\n"
 			msg="${msg}Something has gone horribly wrong."
-			panic 1 "${msg}"
+			panic "${msg}"
 		fi
 		sleep 1
 	done
diff --git a/release/i386/mk-azure.sh b/release/i386/mk-azure.sh
index 43c92040ed3..ac89258996f 100755
--- a/release/i386/mk-azure.sh
+++ b/release/i386/mk-azure.sh
@@ -78,7 +78,7 @@ vm_create_azure() {
 	mount /dev/${mddev} ${DESTDIR}
 	make -C ${WORLDDIR} DESTDIR=$(realpath ${DESTDIR}) \
 		installworld installkernel distribution || \
-		panic 1 "\n\nCannot install the base system to ${DESTDIR}."
+		panic "\n\nCannot install the base system to ${DESTDIR}."
 	mount -t devfs devfs ${DESTDIR}/dev
 	chroot ${DESTDIR} /usr/bin/newaliases
 	echo '# Custom /etc/fstab for FreeBSD VM images' \
@@ -125,7 +125,7 @@ vm_create_azure() {
 			# This should never happen.  But, it has happened.
 			msg="Cannot umount(8) ${DESTDIR}\n"
 			msg="${msg}Something has gone horribly wrong."
-			panic 1 "${msg}"
+			panic "${msg}"
 		fi
 		sleep 1
 	done

From 573e7cca4b79ea8fb0e194d719bd66f2d6a95f64 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Wed, 29 Oct 2014 17:04:09 +0000
Subject: [PATCH 009/279] Initial commit providing a mechanism to create
 openstack images as part of the release build.

This mimics the way Microsoft Azure images are
built, with the addition of installing the
net/cloud-init package and adding a (commented)
rc.conf(5) entry for cloudinit.

Sponsored by:	The FreeBSD Foundation
---
 release/Makefile.vm           |  14 +++-
 release/amd64/mk-openstack.sh | 153 ++++++++++++++++++++++++++++++++++
 release/i386/mk-openstack.sh  | 153 ++++++++++++++++++++++++++++++++++
 release/tools/openstack.conf  |  10 +++
 4 files changed, 329 insertions(+), 1 deletion(-)
 create mode 100755 release/amd64/mk-openstack.sh
 create mode 100755 release/i386/mk-openstack.sh
 create mode 100644 release/tools/openstack.conf

diff --git a/release/Makefile.vm b/release/Makefile.vm
index ce7f0eab922..0ef7fc8ef2e 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -10,8 +10,10 @@ VMFORMATS?=	vhd vmdk qcow2 raw
 VMSIZE?=	20G
 VMBASE?=	vm
 
-CLOUDWARE?=	AZURE
+CLOUDWARE?=	AZURE \
+		OPENSTACK
 AZURE_FORMAT=	vhdf
+OPENSTACK_FORMAT=qcow2
 
 .if defined(WITH_CLOUDWARE) && !empty(WITH_CLOUDWARE) && !empty(CLOUDWARE)
 . for _CW in ${CLOUDWARE}
@@ -65,3 +67,13 @@ vm-azure:
 		${WORLDDIR} ${.TARGET} ${VMSIZE} ${AZUREIMAGE}
 .endif
 	touch ${.TARGET}
+
+vm-openstack:
+.if exists(${.CURDIR}/${TARGET}/mk-openstack.sh)
+	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
+		OPENSTACKCONF=${OPENSTACKCONF}
+		OPENSTACK_FORMAT=${OPENSTACK_FORMAT} \
+		${.CURDIR}/${TARGET}/mk-openstack.sh ${.TARGET} openstack.img \
+		${WORLDDIR} ${.TARGET} ${VMSIZE} ${OPENSTACKIMAGE}
+.endif
+	touch ${.TARGET}
diff --git a/release/amd64/mk-openstack.sh b/release/amd64/mk-openstack.sh
new file mode 100755
index 00000000000..32e0f4cbb5f
--- /dev/null
+++ b/release/amd64/mk-openstack.sh
@@ -0,0 +1,153 @@
+#!/bin/sh
+#-
+# Copyright (c) 2014 The FreeBSD Foundation
+# All rights reserved.
+#
+# This software was developed by Glen Barber under sponsorship
+# from the FreeBSD Foundation.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# mk-openstack.sh: Create virtual machine disk images for Openstack
+#
+# $FreeBSD$
+#
+
+export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin"
+
+usage() {
+	echo "Usage:"
+	echo -n "$(basename ${0}) vm-openstack <base image>"
+	echo " <source tree> <dest dir> <disk image size> <vm image name>"
+	exit 1
+}
+
+panic() {
+	msg="${@}"
+	printf "${msg}\n"
+	if [ ! -z "${mddev}" ]; then
+		mdconfig -d -u ${mddev}
+	fi
+	# Do not allow one failure case to chain through any remaining image
+	# builds.
+	exit 0
+}
+
+vm_create_openstack() {
+	# Arguments:
+	# vm-openstack <base image> <source tree> <dest dir> <disk image size>
+	#	<vm image name>
+
+	VMBASE="${1}"
+	WORLDDIR="${2}"
+	DESTDIR="${3}"
+	VMSIZE="${4}"
+	VMIMAGE="${5}"
+
+	if [ -z "${VMBASE}" -o -z "${WORLDDIR}" -o -z "${DESTDIR}" \
+		-o -z "${VMSIZE}" -o -z "${VMIMAGE}" ]; then
+			usage
+	fi
+
+	trap "umount ${DESTDIR}/dev ${DESTDIR}" INT QUIT TRAP ABRT TERM
+
+	i=0
+	mkdir -p ${DESTDIR}
+	truncate -s ${VMSIZE} ${VMBASE}
+	mddev=$(mdconfig -f ${VMBASE})
+	newfs -j /dev/${mddev}
+	mkdir -p ${DESTDIR}
+	mount /dev/${mddev} ${DESTDIR}
+	make -C ${WORLDDIR} DESTDIR=$(realpath ${DESTDIR}) \
+		installworld installkernel distribution || \
+		panic "\n\nCannot install the base system to ${DESTDIR}."
+	mount -t devfs devfs ${DESTDIR}/dev
+	chroot ${DESTDIR} /usr/bin/newaliases
+	echo '# Custom /etc/fstab for FreeBSD VM images' \
+		> ${DESTDIR}/etc/fstab
+	echo '/dev/gpt/rootfs	/	ufs	rw	2	2' \
+		>> ${DESTDIR}/etc/fstab
+	echo '/dev/gpt/swapfs	none	swap	sw	0	0' \
+		>> ${DESTDIR}/etc/fstab
+
+	chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart
+	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg bootstrap -y
+	if [ ! -z "${VM_EXTRA_PACKAGES}" ]; then
+	        chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg install -y \
+	                ${VM_EXTRA_PACKAGES}
+	fi
+
+	rm -f ${DESTDIR}/etc/resolv.conf
+	echo 'sshd_enable="YES"' > ${DESTDIR}/etc/rc.conf
+	echo 'ifconfig_DEFAULT="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf
+
+	if [ ! -z "${VM_RC_LIST}" ]; then
+		for _rcvar in ${VM_RC_LIST}; do
+			echo ${_rcvar}_enable="YES" >> ${DESTDIR}/etc/rc.conf
+		done
+	fi
+
+	sync
+
+	while ! umount ${DESTDIR}/dev ${DESTDIR}; do
+		i=$(( $i + 1 ))
+		if [ $i -ge 10 ]; then
+			# This should never happen.  But, it has happened.
+			msg="Cannot umount(8) ${DESTDIR}\n"
+			msg="${msg}Something has gone horribly wrong."
+			panic "${msg}"
+		fi
+		sleep 1
+	done
+
+	echo "Creating image...  Please wait."
+
+	mkimg -f ${OPENSTACK_FORMAT} -s gpt \
+		-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
+		-p freebsd-swap/swapfs::1G \
+		-p freebsd-ufs/rootfs:=${VMBASE} \
+		-o ${VMIMAGE}.raw
+
+	return 0
+}
+
+main() {
+	cmd="${1}"
+	shift 1
+
+	if [ -e "${OPENSTACKCONF}" -a ! -c "${OPENSTACKCONF}" ]; then
+		. ${OPENSTACKCONF}
+	fi
+
+	case ${cmd} in
+		vm-openstack)
+			eval vm_create_openstack "$@" || return 0
+			;;
+		*|\?)
+			usage
+			;;
+	esac
+
+	return 0
+}
+
+main "$@"
diff --git a/release/i386/mk-openstack.sh b/release/i386/mk-openstack.sh
new file mode 100755
index 00000000000..32e0f4cbb5f
--- /dev/null
+++ b/release/i386/mk-openstack.sh
@@ -0,0 +1,153 @@
+#!/bin/sh
+#-
+# Copyright (c) 2014 The FreeBSD Foundation
+# All rights reserved.
+#
+# This software was developed by Glen Barber under sponsorship
+# from the FreeBSD Foundation.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# mk-openstack.sh: Create virtual machine disk images for Openstack
+#
+# $FreeBSD$
+#
+
+export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin"
+
+usage() {
+	echo "Usage:"
+	echo -n "$(basename ${0}) vm-openstack <base image>"
+	echo " <source tree> <dest dir> <disk image size> <vm image name>"
+	exit 1
+}
+
+panic() {
+	msg="${@}"
+	printf "${msg}\n"
+	if [ ! -z "${mddev}" ]; then
+		mdconfig -d -u ${mddev}
+	fi
+	# Do not allow one failure case to chain through any remaining image
+	# builds.
+	exit 0
+}
+
+vm_create_openstack() {
+	# Arguments:
+	# vm-openstack <base image> <source tree> <dest dir> <disk image size>
+	#	<vm image name>
+
+	VMBASE="${1}"
+	WORLDDIR="${2}"
+	DESTDIR="${3}"
+	VMSIZE="${4}"
+	VMIMAGE="${5}"
+
+	if [ -z "${VMBASE}" -o -z "${WORLDDIR}" -o -z "${DESTDIR}" \
+		-o -z "${VMSIZE}" -o -z "${VMIMAGE}" ]; then
+			usage
+	fi
+
+	trap "umount ${DESTDIR}/dev ${DESTDIR}" INT QUIT TRAP ABRT TERM
+
+	i=0
+	mkdir -p ${DESTDIR}
+	truncate -s ${VMSIZE} ${VMBASE}
+	mddev=$(mdconfig -f ${VMBASE})
+	newfs -j /dev/${mddev}
+	mkdir -p ${DESTDIR}
+	mount /dev/${mddev} ${DESTDIR}
+	make -C ${WORLDDIR} DESTDIR=$(realpath ${DESTDIR}) \
+		installworld installkernel distribution || \
+		panic "\n\nCannot install the base system to ${DESTDIR}."
+	mount -t devfs devfs ${DESTDIR}/dev
+	chroot ${DESTDIR} /usr/bin/newaliases
+	echo '# Custom /etc/fstab for FreeBSD VM images' \
+		> ${DESTDIR}/etc/fstab
+	echo '/dev/gpt/rootfs	/	ufs	rw	2	2' \
+		>> ${DESTDIR}/etc/fstab
+	echo '/dev/gpt/swapfs	none	swap	sw	0	0' \
+		>> ${DESTDIR}/etc/fstab
+
+	chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart
+	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg bootstrap -y
+	if [ ! -z "${VM_EXTRA_PACKAGES}" ]; then
+	        chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg install -y \
+	                ${VM_EXTRA_PACKAGES}
+	fi
+
+	rm -f ${DESTDIR}/etc/resolv.conf
+	echo 'sshd_enable="YES"' > ${DESTDIR}/etc/rc.conf
+	echo 'ifconfig_DEFAULT="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf
+
+	if [ ! -z "${VM_RC_LIST}" ]; then
+		for _rcvar in ${VM_RC_LIST}; do
+			echo ${_rcvar}_enable="YES" >> ${DESTDIR}/etc/rc.conf
+		done
+	fi
+
+	sync
+
+	while ! umount ${DESTDIR}/dev ${DESTDIR}; do
+		i=$(( $i + 1 ))
+		if [ $i -ge 10 ]; then
+			# This should never happen.  But, it has happened.
+			msg="Cannot umount(8) ${DESTDIR}\n"
+			msg="${msg}Something has gone horribly wrong."
+			panic "${msg}"
+		fi
+		sleep 1
+	done
+
+	echo "Creating image...  Please wait."
+
+	mkimg -f ${OPENSTACK_FORMAT} -s gpt \
+		-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
+		-p freebsd-swap/swapfs::1G \
+		-p freebsd-ufs/rootfs:=${VMBASE} \
+		-o ${VMIMAGE}.raw
+
+	return 0
+}
+
+main() {
+	cmd="${1}"
+	shift 1
+
+	if [ -e "${OPENSTACKCONF}" -a ! -c "${OPENSTACKCONF}" ]; then
+		. ${OPENSTACKCONF}
+	fi
+
+	case ${cmd} in
+		vm-openstack)
+			eval vm_create_openstack "$@" || return 0
+			;;
+		*|\?)
+			usage
+			;;
+	esac
+
+	return 0
+}
+
+main "$@"
diff --git a/release/tools/openstack.conf b/release/tools/openstack.conf
new file mode 100644
index 00000000000..faa6ee24541
--- /dev/null
+++ b/release/tools/openstack.conf
@@ -0,0 +1,10 @@
+#!/bin/sh
+#
+# $FreeBSD$
+#
+
+# Set to a list of packages to install.
+export VM_EXTRA_PACKAGES="net/cloud-init"
+
+# Set to a list of third-party software to enable in rc.conf(5).
+export VM_RC_LIST="#cloudinit"

From 6943fb9ff22b4ac77624d54ed488d2259f03fd52 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Wed, 29 Oct 2014 19:44:34 +0000
Subject: [PATCH 010/279] Fix output file name for openstack images. No further
 conversion is necessary for this VM file target, so there is no need to
 append the '.raw' suffix here.

Sponsored by:	The FreeBSD Foundation
---
 release/amd64/mk-openstack.sh | 2 +-
 release/i386/mk-openstack.sh  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/release/amd64/mk-openstack.sh b/release/amd64/mk-openstack.sh
index 32e0f4cbb5f..863860b520e 100755
--- a/release/amd64/mk-openstack.sh
+++ b/release/amd64/mk-openstack.sh
@@ -125,7 +125,7 @@ vm_create_openstack() {
 		-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
 		-p freebsd-swap/swapfs::1G \
 		-p freebsd-ufs/rootfs:=${VMBASE} \
-		-o ${VMIMAGE}.raw
+		-o ${VMIMAGE}
 
 	return 0
 }
diff --git a/release/i386/mk-openstack.sh b/release/i386/mk-openstack.sh
index 32e0f4cbb5f..863860b520e 100755
--- a/release/i386/mk-openstack.sh
+++ b/release/i386/mk-openstack.sh
@@ -125,7 +125,7 @@ vm_create_openstack() {
 		-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
 		-p freebsd-swap/swapfs::1G \
 		-p freebsd-ufs/rootfs:=${VMBASE} \
-		-o ${VMIMAGE}.raw
+		-o ${VMIMAGE}
 
 	return 0
 }

From 35d8a6aa427dba1e3cbcc5b5125f1ec4b45b7733 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Sat, 1 Nov 2014 20:41:47 +0000
Subject: [PATCH 011/279] Uncomment the cloudinit rc.conf(5) line.

Sponsored by:	The FreeBSD Foundation
---
 release/tools/openstack.conf | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release/tools/openstack.conf b/release/tools/openstack.conf
index faa6ee24541..b43ad76b6da 100644
--- a/release/tools/openstack.conf
+++ b/release/tools/openstack.conf
@@ -7,4 +7,4 @@
 export VM_EXTRA_PACKAGES="net/cloud-init"
 
 # Set to a list of third-party software to enable in rc.conf(5).
-export VM_RC_LIST="#cloudinit"
+export VM_RC_LIST="cloudinit"

From f61371e0feb38a9f044138098f41a7a89631f291 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Mon, 3 Nov 2014 23:47:00 +0000
Subject: [PATCH 012/279] Add line continuation so OPENSTACKCONF is actually
 included in the env(1).

Sponsored by:	The FreeBSD Foundation
---
 release/Makefile.vm | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release/Makefile.vm b/release/Makefile.vm
index 0ef7fc8ef2e..db3ac541654 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -71,7 +71,7 @@ vm-azure:
 vm-openstack:
 .if exists(${.CURDIR}/${TARGET}/mk-openstack.sh)
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		OPENSTACKCONF=${OPENSTACKCONF}
+		OPENSTACKCONF=${OPENSTACKCONF} \
 		OPENSTACK_FORMAT=${OPENSTACK_FORMAT} \
 		${.CURDIR}/${TARGET}/mk-openstack.sh ${.TARGET} openstack.img \
 		${WORLDDIR} ${.TARGET} ${VMSIZE} ${OPENSTACKIMAGE}

From 7d2ef96183ea3e29543eca16bf5774b054046373 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Mon, 3 Nov 2014 23:59:53 +0000
Subject: [PATCH 013/279] Add a 'vm-cloudware' target, used to drive all
 targets in CLOUDTARGETS.

Sponsored by:	The FreeBSD Foundation
---
 release/Makefile.vm | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/release/Makefile.vm b/release/Makefile.vm
index db3ac541654..73042f72bf2 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -59,6 +59,8 @@ vm-image: vm-base
 .endif
 	touch ${.TARGET}
 
+vm-cloudware:	${CLOUDTARGETS}
+
 vm-azure:
 .if exists(${.CURDIR}/${TARGET}/mk-azure.sh)
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} AZURECONF=${AZURECONF} \

From 21f93ded5aca3c7c2f41018ce2d700557f113779 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Tue, 4 Nov 2014 00:02:23 +0000
Subject: [PATCH 014/279] Add examples for WITH_CLOUDWARE to
 release.conf.sample. Add WITH_CLOUDWARE evaluation to RELEASE_RMAKEFLAGS.

Sponsored by:	The FreeBSD Foundation
---
 release/release.conf.sample | 8 ++++++++
 release/release.sh          | 7 ++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/release/release.conf.sample b/release/release.conf.sample
index c79ed9f8700..f6cfadc6c85 100644
--- a/release/release.conf.sample
+++ b/release/release.conf.sample
@@ -98,3 +98,11 @@ PORTBRANCH="ports/head@rHEAD"
 ## image formats to create.  Valid values are listed in the mkimg(1)
 ## manual page, as well as 'mkimg --formats' output.
 #VMFORMATS="vhdf vmdk qcow2 raw"
+
+## Set to a non-empty value to build virtual machine images for various
+## cloud providers as part of the release build.
+#WITH_CLOUDWARE=
+
+## If WITH_CLOUDWARE is set to a non-empty value, this is a list of providers
+## to create disk images.
+#CLOUDWARE="AZURE OPENSTACK"
diff --git a/release/release.sh b/release/release.sh
index 87b36974070..616ae96df7a 100755
--- a/release/release.sh
+++ b/release/release.sh
@@ -94,6 +94,10 @@ WITH_COMPRESSED_IMAGES=
 WITH_VMIMAGES=
 WITH_COMPRESSED_VMIMAGES=
 
+# Set to non-empty value to build virtual machine images for various
+# cloud providers as part of the release.
+WITH_CLOUDWARE=
+
 usage() {
 	echo "Usage: $0 [-c release.conf]"
 	exit 1
@@ -174,7 +178,8 @@ CHROOT_DMAKEFLAGS="${CONF_FILES}"
 RELEASE_WMAKEFLAGS="${MAKE_FLAGS} ${WORLD_FLAGS} ${ARCH_FLAGS} ${CONF_FILES}"
 RELEASE_KMAKEFLAGS="${MAKE_FLAGS} ${KERNEL_FLAGS} KERNCONF=\"${KERNEL}\" ${ARCH_FLAGS} ${CONF_FILES}"
 RELEASE_RMAKEFLAGS="${ARCH_FLAGS} KERNCONF=\"${KERNEL}\" ${CONF_FILES} \
-	${DOCPORTS} WITH_DVD=${WITH_DVD} WITH_VMIMAGES=${WITH_VMIMAGES}"
+	${DOCPORTS} WITH_DVD=${WITH_DVD} WITH_VMIMAGES=${WITH_VMIMAGES} \
+	WITH_CLOUDWARE=${WITH_CLOUDWARE}"
 
 # Force src checkout if configured
 FORCE_SRC_KEY=

From 1e7c1f17428a0b342afd622fad341a389bed7a3f Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Wed, 5 Nov 2014 13:22:19 +0000
Subject: [PATCH 015/279] Initial rewrite to consolidate VM image build scripts
 into one. There may be some very sharp edges here while refactoring.

 - Move amd64/mk-vmimage.sh -> scripts/mk-vmimage.sh.
 - Remove vm-base target from Makefile.vm.
 - In vm-image target, use getopts flags for argument passing.
 - Create tools/vmimage.subr, containing default and prototype
   for the following functions that are used to drive the build,
   run in this order:

   vm_install_base()
   vm_extra_install_base()
   vm_extra_install_packages()
   vm_extra_install_ports()
   vm_extra_enable_services()
   vm_extra_pre_umount()
   vm_create_disk()
   vm_extra_create_disk()

 - In tools/azure.conf, override:

   vm_extra_install_base()
   vm_extra_pre_umount()
   vm_extra_create_disk()

 - In tools/openstack.conf, override:

   vm_extra_install_base()
   vm_extra_pre_umount()

Sponsored by:	The FreeBSD Foundation
---
 release/Makefile.vm           |  41 ++++---
 release/amd64/mk-openstack.sh | 153 --------------------------
 release/amd64/mk-vmimage.sh   | 197 ----------------------------------
 release/i386/mk-azure.sh      | 173 -----------------------------
 release/i386/mk-openstack.sh  | 153 --------------------------
 release/i386/mk-vmimage.sh    | 197 ----------------------------------
 release/scripts/mk-vmimage.sh | 102 ++++++++++++++++++
 release/tools/azure.conf      |  37 +++++++
 release/tools/openstack.conf  |  16 +++
 release/tools/vmimage.subr    | 150 ++++++++++++++++++++++++++
 10 files changed, 323 insertions(+), 896 deletions(-)
 delete mode 100755 release/amd64/mk-openstack.sh
 delete mode 100755 release/amd64/mk-vmimage.sh
 delete mode 100755 release/i386/mk-azure.sh
 delete mode 100755 release/i386/mk-openstack.sh
 delete mode 100755 release/i386/mk-vmimage.sh
 create mode 100755 release/scripts/mk-vmimage.sh
 create mode 100644 release/tools/vmimage.subr

diff --git a/release/Makefile.vm b/release/Makefile.vm
index 73042f72bf2..c37a4ad759d 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -5,7 +5,7 @@
 # Makefile for building virtual machine and cloud provider disk images.
 #
 
-VMTARGETS=	vm-base vm-image
+VMTARGETS=	vm-image
 VMFORMATS?=	vhd vmdk qcow2 raw
 VMSIZE?=	20G
 VMBASE?=	vm
@@ -37,23 +37,17 @@ CLEANFILES+=	${VMBASE}.${FORMAT}
 . endfor
 .endif
 
-vm-base:
-.if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
-. if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
-	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		${.CURDIR}/${TARGET}/mk-vmimage.sh ${.TARGET} \
-		${VMBASE}.img ${WORLDDIR} ${.OBJDIR}/${.TARGET} ${VMSIZE}
-. endif
-.endif
-	touch ${.TARGET}
+vm-base:	vm-image
 
-vm-image: vm-base
+vm-image:
 .if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
 . if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
 .  for FORMAT in ${VMFORMATS}
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		${.CURDIR}/${TARGET}/mk-vmimage.sh ${.TARGET} \
-		${VMBASE}.img ${FORMAT} ${VMBASE}.${FORMAT}
+		${.CURDIR}/${TARGET}/mk-vmimage.sh \
+		-C ${.CURDIR}/tools/vmimage.subr -d ${.TARGET} \
+		-i ${VMBASE}.img -s ${VMSIZE} -f ${FORMAT} \
+		-S ${WORLDDIR} -o ${VMBASE}.${FORMAT}
 .  endfor
 . endif
 .endif
@@ -62,20 +56,21 @@ vm-image: vm-base
 vm-cloudware:	${CLOUDTARGETS}
 
 vm-azure:
-.if exists(${.CURDIR}/${TARGET}/mk-azure.sh)
-	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} AZURECONF=${AZURECONF} \
-		AZURE_FORMAT=${AZURE_FORMAT} \
-		${.CURDIR}/${TARGET}/mk-azure.sh ${.TARGET} azure.img \
-		${WORLDDIR} ${.TARGET} ${VMSIZE} ${AZUREIMAGE}
+.if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
+	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
+		${.CURDIR}/${TARGET}/mk-vmimage.sh \
+		-C ${.CURDIR}/tools/vmimage.subr -d ${.TARGET} \
+		-i azure.img -s ${VMSIZE} -f ${AZURE_FORMAT} \
+		-S ${WORLDDIR} -o ${AZUREIMAGE} -c ${AZURECONF}
 .endif
 	touch ${.TARGET}
 
 vm-openstack:
-.if exists(${.CURDIR}/${TARGET}/mk-openstack.sh)
+.if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		OPENSTACKCONF=${OPENSTACKCONF} \
-		OPENSTACK_FORMAT=${OPENSTACK_FORMAT} \
-		${.CURDIR}/${TARGET}/mk-openstack.sh ${.TARGET} openstack.img \
-		${WORLDDIR} ${.TARGET} ${VMSIZE} ${OPENSTACKIMAGE}
+		${.CURDIR}/${TARGET}/mk-vmimage.sh \
+		-C ${.CURDIR}/tools/vmimage.subr -d ${.TARGET} \
+		-i openstack.img -s ${VMSIZE} -f ${OPENSTACK_FORMAT} \
+		-S ${WORLDDIR} -o ${OPENSTACKIMAGE} -c ${OPENSTACKCONF}
 .endif
 	touch ${.TARGET}
diff --git a/release/amd64/mk-openstack.sh b/release/amd64/mk-openstack.sh
deleted file mode 100755
index 863860b520e..00000000000
--- a/release/amd64/mk-openstack.sh
+++ /dev/null
@@ -1,153 +0,0 @@
-#!/bin/sh
-#-
-# Copyright (c) 2014 The FreeBSD Foundation
-# All rights reserved.
-#
-# This software was developed by Glen Barber under sponsorship
-# from the FreeBSD Foundation.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
-#
-# mk-openstack.sh: Create virtual machine disk images for Openstack
-#
-# $FreeBSD$
-#
-
-export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin"
-
-usage() {
-	echo "Usage:"
-	echo -n "$(basename ${0}) vm-openstack <base image>"
-	echo " <source tree> <dest dir> <disk image size> <vm image name>"
-	exit 1
-}
-
-panic() {
-	msg="${@}"
-	printf "${msg}\n"
-	if [ ! -z "${mddev}" ]; then
-		mdconfig -d -u ${mddev}
-	fi
-	# Do not allow one failure case to chain through any remaining image
-	# builds.
-	exit 0
-}
-
-vm_create_openstack() {
-	# Arguments:
-	# vm-openstack <base image> <source tree> <dest dir> <disk image size>
-	#	<vm image name>
-
-	VMBASE="${1}"
-	WORLDDIR="${2}"
-	DESTDIR="${3}"
-	VMSIZE="${4}"
-	VMIMAGE="${5}"
-
-	if [ -z "${VMBASE}" -o -z "${WORLDDIR}" -o -z "${DESTDIR}" \
-		-o -z "${VMSIZE}" -o -z "${VMIMAGE}" ]; then
-			usage
-	fi
-
-	trap "umount ${DESTDIR}/dev ${DESTDIR}" INT QUIT TRAP ABRT TERM
-
-	i=0
-	mkdir -p ${DESTDIR}
-	truncate -s ${VMSIZE} ${VMBASE}
-	mddev=$(mdconfig -f ${VMBASE})
-	newfs -j /dev/${mddev}
-	mkdir -p ${DESTDIR}
-	mount /dev/${mddev} ${DESTDIR}
-	make -C ${WORLDDIR} DESTDIR=$(realpath ${DESTDIR}) \
-		installworld installkernel distribution || \
-		panic "\n\nCannot install the base system to ${DESTDIR}."
-	mount -t devfs devfs ${DESTDIR}/dev
-	chroot ${DESTDIR} /usr/bin/newaliases
-	echo '# Custom /etc/fstab for FreeBSD VM images' \
-		> ${DESTDIR}/etc/fstab
-	echo '/dev/gpt/rootfs	/	ufs	rw	2	2' \
-		>> ${DESTDIR}/etc/fstab
-	echo '/dev/gpt/swapfs	none	swap	sw	0	0' \
-		>> ${DESTDIR}/etc/fstab
-
-	chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart
-	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg bootstrap -y
-	if [ ! -z "${VM_EXTRA_PACKAGES}" ]; then
-	        chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg install -y \
-	                ${VM_EXTRA_PACKAGES}
-	fi
-
-	rm -f ${DESTDIR}/etc/resolv.conf
-	echo 'sshd_enable="YES"' > ${DESTDIR}/etc/rc.conf
-	echo 'ifconfig_DEFAULT="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf
-
-	if [ ! -z "${VM_RC_LIST}" ]; then
-		for _rcvar in ${VM_RC_LIST}; do
-			echo ${_rcvar}_enable="YES" >> ${DESTDIR}/etc/rc.conf
-		done
-	fi
-
-	sync
-
-	while ! umount ${DESTDIR}/dev ${DESTDIR}; do
-		i=$(( $i + 1 ))
-		if [ $i -ge 10 ]; then
-			# This should never happen.  But, it has happened.
-			msg="Cannot umount(8) ${DESTDIR}\n"
-			msg="${msg}Something has gone horribly wrong."
-			panic "${msg}"
-		fi
-		sleep 1
-	done
-
-	echo "Creating image...  Please wait."
-
-	mkimg -f ${OPENSTACK_FORMAT} -s gpt \
-		-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
-		-p freebsd-swap/swapfs::1G \
-		-p freebsd-ufs/rootfs:=${VMBASE} \
-		-o ${VMIMAGE}
-
-	return 0
-}
-
-main() {
-	cmd="${1}"
-	shift 1
-
-	if [ -e "${OPENSTACKCONF}" -a ! -c "${OPENSTACKCONF}" ]; then
-		. ${OPENSTACKCONF}
-	fi
-
-	case ${cmd} in
-		vm-openstack)
-			eval vm_create_openstack "$@" || return 0
-			;;
-		*|\?)
-			usage
-			;;
-	esac
-
-	return 0
-}
-
-main "$@"
diff --git a/release/amd64/mk-vmimage.sh b/release/amd64/mk-vmimage.sh
deleted file mode 100755
index b3ffd232f84..00000000000
--- a/release/amd64/mk-vmimage.sh
+++ /dev/null
@@ -1,197 +0,0 @@
-#!/bin/sh
-#-
-# Copyright (c) 2014 The FreeBSD Foundation
-# All rights reserved.
-#
-# This software was developed by Glen Barber under sponsorship
-# from the FreeBSD Foundation.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
-#
-# mk-vmimage.sh: Create virtual machine disk images in various formats.
-#
-# $FreeBSD$
-#
-
-PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
-export PATH
-
-usage_vm_base() {
-	echo -n "$(basename ${0}) vm-base <base image> <source tree>"
-	echo	" <dest dir> <disk image size>"
-	return 0
-}
-
-usage_vm_image() {
-	echo -n "$(basename ${0}) vm-image <base image> <image format>"
-	echo	" <output image>"
-	return 0
-}
-
-usage() {
-	echo "Usage:"
-	echo "$(basename ${0}) [vm-base|vm-image] [...]"
-	echo
-	usage_vm_base
-	echo
-	usage_vm_image
-	exit 1
-}
-
-panic() {
-	msg="${@}"
-	printf "${msg}\n"
-	if [ ! -z "${mddev}" ]; then
-		mdconfig -d -u ${mddev}
-	fi
-	case ${cmd} in
-		vm-base)
-			# If the vm-base target fails, the vm-image target
-			# cannot possibly succeed.  Touch the .TARGET file
-			# so it is not attempted.
-			touch vm-image
-			;;
-		*)
-			# FALLTHROUGH
-			;;
-	esac
-	# Do not allow one failure case to chain through any remaining image
-	# builds.
-	return 1
-}
-
-vm_create_baseimage() {
-	# Creates the UFS root filesystem for the virtual machine disk,
-	# written to the formatted disk image with mkimg(1).
-	#
-	# Arguments:
-	# vm-base <base image> <source tree> <dest dir> <disk image size>
-
-	VMBASE="${1}"
-	WORLDDIR="${2}"
-	DESTDIR="${3}"
-	VMSIZE="${4}"
-
-	if [ -z "${VMBASE}" -o -z "${WORLDDIR}" -o -z "${DESTDIR}" \
-		-o -z "${VMSIZE}" ]; then
-			usage
-	fi
-
-	i=0
-	mkdir -p ${DESTDIR}
-	truncate -s ${VMSIZE} ${VMBASE}
-	mddev=$(mdconfig -f ${VMBASE})
-	newfs -j /dev/${mddev}
-	mount /dev/${mddev} ${DESTDIR}
-	cd ${WORLDDIR} && \
-		make DESTDIR=${DESTDIR} \
-		installworld installkernel distribution || \
-		panic "\n\nCannot install the base system to ${DESTDIR}."
-	chroot ${DESTDIR} /usr/bin/newaliases
-	echo '# Custom /etc/fstab for FreeBSD VM images' \
-		> ${DESTDIR}/etc/fstab
-	echo '/dev/gpt/rootfs	/	ufs	rw	2	2' \
-		>> ${DESTDIR}/etc/fstab
-	echo '/dev/gpt/swapfs	none	swap	sw	0	0' \
-		>> ${DESTDIR}/etc/fstab
-	sync
-	while ! umount ${DESTDIR}; do
-		i=$(( $i + 1 ))
-		if [ $i -ge 10 ]; then
-			# This should never happen.  But, it has happened.
-			msg="Cannot umount(8) ${DESTDIR}\n"
-			msg="${msg}Something has gone horribly wrong."
-			panic "${msg}"
-		fi
-		sleep 1
-	done
-
-	return 0
-}
-
-vm_create_vmdisk() {
-	# Creates the virtual machine disk image from the raw disk image.
-	#
-	# Arguments:
-	# vm-image <base image> <image format> <output image>"
-
-	VMBASE="${1}"
-	FORMAT="${2}"
-	VMIMAGE="${3}"
-
-	if [ -z "${VMBASE}" -o -z "${FORMAT}" -o -z "${VMIMAGE}" ]; then
-		usage
-	fi
-
-	mkimg_version=$(mkimg --version 2>/dev/null | awk '{print $2}')
-
-	# We need mkimg(1) '--version' output, at minimum, to be able to
-	# tell what virtual machine disk image formats are available.
-	# Bail if mkimg(1) reports an empty '--version' value.
-	if [ -z "${mkimg_version}" ]; then
-		msg="Cannot determine mkimg(1) version.\n"
-		msg="${msg}Cannot continue without a known mkimg(1) version."
-		panic "${msg}"
-	fi
-
-	if ! mkimg --formats 2>/dev/null | grep -q ${FORMAT}; then
-		panic "'${FORMAT}' is not supported by this mkimg(1).\n"
-	fi
-
-	case ${FORMAT} in
-		vhd)
-			mkimg_format=vhdf
-			;;
-		*)
-			mkimg_format=${FORMAT}
-			;;
-	esac
-
-	mkimg -f ${mkimg_format} -s gpt \
-		-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
-		-p freebsd-swap/swapfs::1G \
-		-p freebsd-ufs/rootfs:=${VMBASE} \
-		-o ${VMIMAGE}
-
-	return 0
-}
-
-main() {
-	cmd="${1}"
-	shift 1
-
-	case ${cmd} in
-		vm-base)
-			eval vm_create_baseimage "$@" || return 0
-			;;
-		vm-image)
-			eval vm_create_vmdisk "$@" || return 0
-			;;
-		*|\?)
-			usage
-			;;
-	esac
-
-	return 0
-}
-
-main "$@"
diff --git a/release/i386/mk-azure.sh b/release/i386/mk-azure.sh
deleted file mode 100755
index ac89258996f..00000000000
--- a/release/i386/mk-azure.sh
+++ /dev/null
@@ -1,173 +0,0 @@
-#!/bin/sh
-#-
-# Copyright (c) 2014 The FreeBSD Foundation
-# All rights reserved.
-#
-# This software was developed by Glen Barber under sponsorship
-# from the FreeBSD Foundation.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
-#
-# mk-azure.sh: Create virtual machine disk images for Microsoft Azure
-#
-# $FreeBSD$
-#
-
-export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin"
-
-usage() {
-	echo "Usage:"
-	echo -n "$(basename ${0}) vm-azure <base image>"
-	echo " <source tree> <dest dir> <disk image size> <vm image name>"
-	exit 1
-}
-
-panic() {
-	msg="${@}"
-	printf "${msg}\n"
-	if [ ! -z "${mddev}" ]; then
-		mdconfig -d -u ${mddev}
-	fi
-	# Do not allow one failure case to chain through any remaining image
-	# builds.
-	exit 0
-}
-
-vm_create_azure() {
-	# Arguments:
-	# vm-azure <base image> <source tree> <dest dir> <disk image size> <vm image name>
-
-	VMBASE="${1}"
-	WORLDDIR="${2}"
-	DESTDIR="${3}"
-	VMSIZE="${4}"
-	VMIMAGE="${5}"
-
-	if [ -z "${VMBASE}" -o -z "${WORLDDIR}" -o -z "${DESTDIR}" \
-		-o -z "${VMSIZE}" -o -z "${VMIMAGE}" ]; then
-			usage
-	fi
-
-	trap "umount ${DESTDIR}/dev ${DESTDIR}" INT QUIT TRAP ABRT TERM
-
-	i=0
-	mkdir -p ${DESTDIR}
-	truncate -s ${VMSIZE} ${VMBASE}
-	mddev=$(mdconfig -f ${VMBASE})
-	newfs -j /dev/${mddev}
-	mkdir -p ${DESTDIR}
-	mount /dev/${mddev} ${DESTDIR}
-	make -C ${WORLDDIR} DESTDIR=$(realpath ${DESTDIR}) \
-		installworld installkernel distribution || \
-		panic "\n\nCannot install the base system to ${DESTDIR}."
-	mount -t devfs devfs ${DESTDIR}/dev
-	chroot ${DESTDIR} /usr/bin/newaliases
-	echo '# Custom /etc/fstab for FreeBSD VM images' \
-		> ${DESTDIR}/etc/fstab
-	echo '/dev/gpt/rootfs	/	ufs	rw	2	2' \
-		>> ${DESTDIR}/etc/fstab
-	# Although a swap partition is created, it is not used in Azure.
-	echo '#/dev/gpt/swapfs	none	swap	sw	0	0' \
-		>> ${DESTDIR}/etc/fstab
-
-	chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart
-	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg bootstrap -y
-	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg install -y \
-	        python python2 python27 py27-asn1 sudo bash
-	if [ ! -z "${VM_EXTRA_PACKAGES}" ]; then
-	        chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg install -y \
-	                ${VM_EXTRA_PACKAGES}
-	fi
-
-	fetch -o ${DESTDIR}/usr/sbin/waagent \
-		http://people.freebsd.org/~gjb/waagent
-	chmod +x ${DESTDIR}/usr/sbin/waagent
-	rm -f ${DESTDIR}/etc/resolv.conf
-	chroot ${DESTDIR} /usr/sbin/waagent -verbose -install
-	yes | chroot ${DESTDIR} /usr/sbin/waagent -deprovision
-	echo 'sshd_enable="YES"' > ${DESTDIR}/etc/rc.conf
-	echo 'ifconfig_hn0="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf
-	echo 'waagent_enable="YES"' >> ${DESTDIR}/etc/rc.conf
-
-	echo 'console="comconsole vidconsole"' >> ${DESTDIR}/boot/loader.conf
-	echo 'comconsole_speed="115200"' >> ${DESTDIR}/boot/loader.conf
-
-	if [ ! -z "${VM_RC_LIST}" ]; then
-		for _rcvar in ${VM_RC_LIST}; do
-			echo ${_rcvar}_enable="YES" >> ${DESTDIR}/etc/rc.conf
-		done
-	fi
-
-	sync
-
-	while ! umount ${DESTDIR}/dev ${DESTDIR}; do
-		i=$(( $i + 1 ))
-		if [ $i -ge 10 ]; then
-			# This should never happen.  But, it has happened.
-			msg="Cannot umount(8) ${DESTDIR}\n"
-			msg="${msg}Something has gone horribly wrong."
-			panic "${msg}"
-		fi
-		sleep 1
-	done
-
-	echo "Creating image...  Please wait."
-
-	mkimg -f ${AZURE_FORMAT} -s gpt \
-		-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
-		-p freebsd-swap/swapfs::1G \
-		-p freebsd-ufs/rootfs:=${VMBASE} \
-		-o ${VMIMAGE}.raw
-
-	if [ ! -x "/usr/local/bin/qemu-img" ]; then
-		env ASSUME_ALWAYS_YES=yes pkg install -y emulators/qemu-devel
-	fi
-
-	size=$(qemu-img info -f raw --output json ${VMIMAGE}.raw | awk '/virtual-size/ {print $2}' | tr -d ',')
-	size=$(( ( ${size} / ( 1024 * 1024 ) + 1 ) * ( 1024 * 1024 ) ))
-	qemu-img resize ${VMIMAGE}.raw ${size}
-	qemu-img convert -f raw -o subformat=fixed -O vpc ${VMIMAGE}.raw ${VMIMAGE}
-
-	return 0
-}
-
-main() {
-	cmd="${1}"
-	shift 1
-
-	if [ -e "${AZURECONF}" -a ! -c "${AZURECONF}" ]; then
-		. ${AZURECONF}
-	fi
-
-	case ${cmd} in
-		vm-azure)
-			eval vm_create_azure "$@" || return 0
-			;;
-		*|\?)
-			usage
-			;;
-	esac
-
-	return 0
-}
-
-main "$@"
diff --git a/release/i386/mk-openstack.sh b/release/i386/mk-openstack.sh
deleted file mode 100755
index 863860b520e..00000000000
--- a/release/i386/mk-openstack.sh
+++ /dev/null
@@ -1,153 +0,0 @@
-#!/bin/sh
-#-
-# Copyright (c) 2014 The FreeBSD Foundation
-# All rights reserved.
-#
-# This software was developed by Glen Barber under sponsorship
-# from the FreeBSD Foundation.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
-#
-# mk-openstack.sh: Create virtual machine disk images for Openstack
-#
-# $FreeBSD$
-#
-
-export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin"
-
-usage() {
-	echo "Usage:"
-	echo -n "$(basename ${0}) vm-openstack <base image>"
-	echo " <source tree> <dest dir> <disk image size> <vm image name>"
-	exit 1
-}
-
-panic() {
-	msg="${@}"
-	printf "${msg}\n"
-	if [ ! -z "${mddev}" ]; then
-		mdconfig -d -u ${mddev}
-	fi
-	# Do not allow one failure case to chain through any remaining image
-	# builds.
-	exit 0
-}
-
-vm_create_openstack() {
-	# Arguments:
-	# vm-openstack <base image> <source tree> <dest dir> <disk image size>
-	#	<vm image name>
-
-	VMBASE="${1}"
-	WORLDDIR="${2}"
-	DESTDIR="${3}"
-	VMSIZE="${4}"
-	VMIMAGE="${5}"
-
-	if [ -z "${VMBASE}" -o -z "${WORLDDIR}" -o -z "${DESTDIR}" \
-		-o -z "${VMSIZE}" -o -z "${VMIMAGE}" ]; then
-			usage
-	fi
-
-	trap "umount ${DESTDIR}/dev ${DESTDIR}" INT QUIT TRAP ABRT TERM
-
-	i=0
-	mkdir -p ${DESTDIR}
-	truncate -s ${VMSIZE} ${VMBASE}
-	mddev=$(mdconfig -f ${VMBASE})
-	newfs -j /dev/${mddev}
-	mkdir -p ${DESTDIR}
-	mount /dev/${mddev} ${DESTDIR}
-	make -C ${WORLDDIR} DESTDIR=$(realpath ${DESTDIR}) \
-		installworld installkernel distribution || \
-		panic "\n\nCannot install the base system to ${DESTDIR}."
-	mount -t devfs devfs ${DESTDIR}/dev
-	chroot ${DESTDIR} /usr/bin/newaliases
-	echo '# Custom /etc/fstab for FreeBSD VM images' \
-		> ${DESTDIR}/etc/fstab
-	echo '/dev/gpt/rootfs	/	ufs	rw	2	2' \
-		>> ${DESTDIR}/etc/fstab
-	echo '/dev/gpt/swapfs	none	swap	sw	0	0' \
-		>> ${DESTDIR}/etc/fstab
-
-	chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart
-	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg bootstrap -y
-	if [ ! -z "${VM_EXTRA_PACKAGES}" ]; then
-	        chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg install -y \
-	                ${VM_EXTRA_PACKAGES}
-	fi
-
-	rm -f ${DESTDIR}/etc/resolv.conf
-	echo 'sshd_enable="YES"' > ${DESTDIR}/etc/rc.conf
-	echo 'ifconfig_DEFAULT="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf
-
-	if [ ! -z "${VM_RC_LIST}" ]; then
-		for _rcvar in ${VM_RC_LIST}; do
-			echo ${_rcvar}_enable="YES" >> ${DESTDIR}/etc/rc.conf
-		done
-	fi
-
-	sync
-
-	while ! umount ${DESTDIR}/dev ${DESTDIR}; do
-		i=$(( $i + 1 ))
-		if [ $i -ge 10 ]; then
-			# This should never happen.  But, it has happened.
-			msg="Cannot umount(8) ${DESTDIR}\n"
-			msg="${msg}Something has gone horribly wrong."
-			panic "${msg}"
-		fi
-		sleep 1
-	done
-
-	echo "Creating image...  Please wait."
-
-	mkimg -f ${OPENSTACK_FORMAT} -s gpt \
-		-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
-		-p freebsd-swap/swapfs::1G \
-		-p freebsd-ufs/rootfs:=${VMBASE} \
-		-o ${VMIMAGE}
-
-	return 0
-}
-
-main() {
-	cmd="${1}"
-	shift 1
-
-	if [ -e "${OPENSTACKCONF}" -a ! -c "${OPENSTACKCONF}" ]; then
-		. ${OPENSTACKCONF}
-	fi
-
-	case ${cmd} in
-		vm-openstack)
-			eval vm_create_openstack "$@" || return 0
-			;;
-		*|\?)
-			usage
-			;;
-	esac
-
-	return 0
-}
-
-main "$@"
diff --git a/release/i386/mk-vmimage.sh b/release/i386/mk-vmimage.sh
deleted file mode 100755
index b3ffd232f84..00000000000
--- a/release/i386/mk-vmimage.sh
+++ /dev/null
@@ -1,197 +0,0 @@
-#!/bin/sh
-#-
-# Copyright (c) 2014 The FreeBSD Foundation
-# All rights reserved.
-#
-# This software was developed by Glen Barber under sponsorship
-# from the FreeBSD Foundation.
-#
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions
-# are met:
-# 1. Redistributions of source code must retain the above copyright
-#    notice, this list of conditions and the following disclaimer.
-# 2. Redistributions in binary form must reproduce the above copyright
-#    notice, this list of conditions and the following disclaimer in the
-#    documentation and/or other materials provided with the distribution.
-#
-# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
-# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
-# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
-# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
-# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
-# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
-# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
-# SUCH DAMAGE.
-#
-# mk-vmimage.sh: Create virtual machine disk images in various formats.
-#
-# $FreeBSD$
-#
-
-PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
-export PATH
-
-usage_vm_base() {
-	echo -n "$(basename ${0}) vm-base <base image> <source tree>"
-	echo	" <dest dir> <disk image size>"
-	return 0
-}
-
-usage_vm_image() {
-	echo -n "$(basename ${0}) vm-image <base image> <image format>"
-	echo	" <output image>"
-	return 0
-}
-
-usage() {
-	echo "Usage:"
-	echo "$(basename ${0}) [vm-base|vm-image] [...]"
-	echo
-	usage_vm_base
-	echo
-	usage_vm_image
-	exit 1
-}
-
-panic() {
-	msg="${@}"
-	printf "${msg}\n"
-	if [ ! -z "${mddev}" ]; then
-		mdconfig -d -u ${mddev}
-	fi
-	case ${cmd} in
-		vm-base)
-			# If the vm-base target fails, the vm-image target
-			# cannot possibly succeed.  Touch the .TARGET file
-			# so it is not attempted.
-			touch vm-image
-			;;
-		*)
-			# FALLTHROUGH
-			;;
-	esac
-	# Do not allow one failure case to chain through any remaining image
-	# builds.
-	return 1
-}
-
-vm_create_baseimage() {
-	# Creates the UFS root filesystem for the virtual machine disk,
-	# written to the formatted disk image with mkimg(1).
-	#
-	# Arguments:
-	# vm-base <base image> <source tree> <dest dir> <disk image size>
-
-	VMBASE="${1}"
-	WORLDDIR="${2}"
-	DESTDIR="${3}"
-	VMSIZE="${4}"
-
-	if [ -z "${VMBASE}" -o -z "${WORLDDIR}" -o -z "${DESTDIR}" \
-		-o -z "${VMSIZE}" ]; then
-			usage
-	fi
-
-	i=0
-	mkdir -p ${DESTDIR}
-	truncate -s ${VMSIZE} ${VMBASE}
-	mddev=$(mdconfig -f ${VMBASE})
-	newfs -j /dev/${mddev}
-	mount /dev/${mddev} ${DESTDIR}
-	cd ${WORLDDIR} && \
-		make DESTDIR=${DESTDIR} \
-		installworld installkernel distribution || \
-		panic "\n\nCannot install the base system to ${DESTDIR}."
-	chroot ${DESTDIR} /usr/bin/newaliases
-	echo '# Custom /etc/fstab for FreeBSD VM images' \
-		> ${DESTDIR}/etc/fstab
-	echo '/dev/gpt/rootfs	/	ufs	rw	2	2' \
-		>> ${DESTDIR}/etc/fstab
-	echo '/dev/gpt/swapfs	none	swap	sw	0	0' \
-		>> ${DESTDIR}/etc/fstab
-	sync
-	while ! umount ${DESTDIR}; do
-		i=$(( $i + 1 ))
-		if [ $i -ge 10 ]; then
-			# This should never happen.  But, it has happened.
-			msg="Cannot umount(8) ${DESTDIR}\n"
-			msg="${msg}Something has gone horribly wrong."
-			panic "${msg}"
-		fi
-		sleep 1
-	done
-
-	return 0
-}
-
-vm_create_vmdisk() {
-	# Creates the virtual machine disk image from the raw disk image.
-	#
-	# Arguments:
-	# vm-image <base image> <image format> <output image>"
-
-	VMBASE="${1}"
-	FORMAT="${2}"
-	VMIMAGE="${3}"
-
-	if [ -z "${VMBASE}" -o -z "${FORMAT}" -o -z "${VMIMAGE}" ]; then
-		usage
-	fi
-
-	mkimg_version=$(mkimg --version 2>/dev/null | awk '{print $2}')
-
-	# We need mkimg(1) '--version' output, at minimum, to be able to
-	# tell what virtual machine disk image formats are available.
-	# Bail if mkimg(1) reports an empty '--version' value.
-	if [ -z "${mkimg_version}" ]; then
-		msg="Cannot determine mkimg(1) version.\n"
-		msg="${msg}Cannot continue without a known mkimg(1) version."
-		panic "${msg}"
-	fi
-
-	if ! mkimg --formats 2>/dev/null | grep -q ${FORMAT}; then
-		panic "'${FORMAT}' is not supported by this mkimg(1).\n"
-	fi
-
-	case ${FORMAT} in
-		vhd)
-			mkimg_format=vhdf
-			;;
-		*)
-			mkimg_format=${FORMAT}
-			;;
-	esac
-
-	mkimg -f ${mkimg_format} -s gpt \
-		-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
-		-p freebsd-swap/swapfs::1G \
-		-p freebsd-ufs/rootfs:=${VMBASE} \
-		-o ${VMIMAGE}
-
-	return 0
-}
-
-main() {
-	cmd="${1}"
-	shift 1
-
-	case ${cmd} in
-		vm-base)
-			eval vm_create_baseimage "$@" || return 0
-			;;
-		vm-image)
-			eval vm_create_vmdisk "$@" || return 0
-			;;
-		*|\?)
-			usage
-			;;
-	esac
-
-	return 0
-}
-
-main "$@"
diff --git a/release/scripts/mk-vmimage.sh b/release/scripts/mk-vmimage.sh
new file mode 100755
index 00000000000..b5065b66ab2
--- /dev/null
+++ b/release/scripts/mk-vmimage.sh
@@ -0,0 +1,102 @@
+#!/bin/sh
+#-
+# Copyright (c) 2014 The FreeBSD Foundation
+# All rights reserved.
+#
+# This software was developed by Glen Barber under sponsorship
+# from the FreeBSD Foundation.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+# SUCH DAMAGE.
+#
+# mk-vmimage.sh: Create virtual machine disk images in various formats.
+#
+# $FreeBSD$
+#
+
+main() {
+	local arg
+	while getopts "C:c:d:f:i:o:s:S:" arg; do
+		case "${arg}" in
+			C)
+				VMBUILDCONF="${OPTARG}"
+				;;
+			c)
+				VMCONFIG="${OPTARG}"
+				;;
+			d)
+				DESTDIR="${OPTARG}"
+				;;
+			f)
+				VMFORMAT="${OPTARG}"
+				;;
+			i)
+				VMBASE="${VMBASE}"
+				;;
+			o)
+				VMIMAGE="${OPTARG}"
+				;;
+			s)
+				VMSIZE="${OPTARG}"
+				;;
+			S)
+				WORLDDIR="${OPTARG}"
+				;;
+			*)
+				;;
+		esac
+	done
+	shift $(( ${OPTIND} - 1))
+
+	if [ -z "${VMBASE}" -o \
+		-z "${WORLDDIR}" -o \
+		-z "${DESTDIR}" -o \
+		-z "${VMSIZE}" -o \
+		-z "${VMIMAGE}" -o \
+		-z "${VMCONFIG}" ];
+	then
+		usage
+	fi
+
+	if [ -z "${VMBUILDCONF}" ] || [ ! -e "${VMBUILDCONF}" ]; then
+		echo "Must provide the path to vmimage.subr."
+		return 1
+	fi
+
+	. "${VMBUILDCONF}"
+
+	if [ ! -z "${VMCONFIG}" ] && [ -e "${VMCONFIG}" ]; then
+		. "${VMCONFIG}"
+	fi
+
+	vm_install_base
+	vm_extra_install_base
+	vm_extra_install_packages
+	vm_extra_install_ports
+	vm_extra_enable_services
+	vm_extra_pre_umount
+	vm_create_disk
+	vm_extra_create_disk
+
+	return 0
+}
+
+main "$@"
diff --git a/release/tools/azure.conf b/release/tools/azure.conf
index 8f16acc0d27..16c207da324 100644
--- a/release/tools/azure.conf
+++ b/release/tools/azure.conf
@@ -12,3 +12,40 @@ export VM_EXTRA_PACKAGES=
 # Example:
 #export VM_RC_LIST="apache24"
 export VM_RC_LIST=
+
+vm_extra_install_base() {
+	fetch -o ${DESTDIR}/usr/sbin/waagent \
+		http://people.freebsd.org/~gjb/waagent
+	chmod +x ${DESTDIR}/usr/sbin/waagent
+	rm -f ${DESTDIR}/etc/resolv.conf
+
+	return 0
+}
+
+vm_extra_pre_umount() {
+	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg install -y \
+		python python2 python27 py27-asn1 sudo bash
+	chroot ${DESTDIR} /usr/sbin/waagent -verbose -install
+	yes | chroot ${DESTDIR} /usr/sbin/waagent -deprovision
+	echo 'sshd_enable="YES"' >> ${DESTDIR}/etc/rc.conf
+	echo 'ifconfig_hn0="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf
+	echo 'waagent_enable="YES"' >> ${DESTDIR}/etc/rc.conf
+	echo 'console="comconsole vidconsole"' >> ${DESTDIR}/boot/loader.conf
+	echo 'comconsole_speed="115200"' >> ${DESTDIR}/boot/loader.conf
+
+	return 0
+}
+
+vm_extra_create_disk() {
+	if [ ! -x "/usr/local/bin/qemu-img" ]; then
+		env ASSUME_ALWAYS_YES=yes pkg install -y emulators/qemu-devel
+	fi
+
+	mv ${VMIMAGE} ${VMIMAGE}.raw
+	size=$(qemu-img info -f raw --output json ${VMIMAGE}.raw | awk '/virtual-size/ {print $2}' | tr -d ',')
+	size=$(( ( ${size} / ( 1024 * 1024 ) + 1 ) * ( 1024 * 1024 ) ))
+	qemu-img resize ${VMIMAGE}.raw ${size}
+	qemu-img convert -f raw -o subformat=fixed -O vpc ${VMIMAGE}.raw ${VMIMAGE}
+
+	return 0
+}
diff --git a/release/tools/openstack.conf b/release/tools/openstack.conf
index b43ad76b6da..6aa17493f97 100644
--- a/release/tools/openstack.conf
+++ b/release/tools/openstack.conf
@@ -8,3 +8,19 @@ export VM_EXTRA_PACKAGES="net/cloud-init"
 
 # Set to a list of third-party software to enable in rc.conf(5).
 export VM_RC_LIST="cloudinit"
+
+vm_extra_install_base() {
+	fetch -o ${DESTDIR}/usr/sbin/waagent \
+		http://people.freebsd.org/~gjb/waagent
+	chmod +x ${DESTDIR}/usr/sbin/waagent
+	rm -f ${DESTDIR}/etc/resolv.conf
+
+	return 0
+}
+
+vm_extra_pre_umount() {
+	echo 'sshd_enable="YES"' >> ${DESTDIR}/etc/rc.conf
+	echo 'ifconfig_DEFAULT="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf
+
+	return 0
+}
diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
new file mode 100644
index 00000000000..622e0fc967e
--- /dev/null
+++ b/release/tools/vmimage.subr
@@ -0,0 +1,150 @@
+#!/bin/sh
+#
+# $FreeBSD$
+#
+#
+# Common functions for virtual machine image build scripts.
+#
+
+export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
+trap "cleanup" INT QUIT TRAP ABRT TERM
+
+mkimg_bootcode="/boot/pmbr"
+mkimg_partitions="-p freebsd-boot/bootfs:=/boot/gptboot"
+mkimg_partitions="${mkimg_partitions} -p freebsd-swap/swapfs::1G"
+mkimg_partitions="${mkimg_partitions} freebsd-ufs/rootfs:=${VMBASE}"
+
+usage() {
+	echo "${0} usage:"
+	echo "${@}"
+	return 1
+}
+
+err() {
+	printf "${@}\n"
+	cleanup
+	return 1
+}
+
+cleanup() {
+	if [ ! -z "${mddev}" ]; then
+		mdconfig -d -u ${mddev}
+	fi
+	umount ${DESTDIR}/dev
+	umount ${DESTDIR}
+
+	return 0
+}
+
+vm_create_base() {
+	# Creates the UFS root filesystem for the virtual machine disk,
+	# written to the formatted disk image with mkimg(1).
+
+	mkdir -p ${DESTDIR}
+	truncate -s ${VMSIZE} ${VMBASE}
+	mddev=$(mdconfig -f ${VMBASE})
+	newfs -j /dev/${mddev}
+	mount /dev/${mddev} ${DESTDIR}
+
+	return 0
+}
+
+vm_install_base() {
+	# Installs the FreeBSD userland/kernel to the virtual machine disk.
+
+	cd ${WORLDDIR} && \
+		make DESTDIR=${DESTDIR} \
+		installworld installkernel distribution || \
+		err "\n\nCannot install the base system to ${DESTDIR}."
+
+	echo '# Custom /etc/fstab for FreeBSD VM images' \
+		> ${DESTDIR}/etc/fstab
+	echo '/dev/gpt/rootfs   /       ufs     rw      1       1' \
+		>> ${DESTDIR}/etc/fstab
+	echo '/dev/gpt/swapfs  none    swap    sw      0       0' \
+		>> ${DESTDIR}/etc/fstab
+
+	chroot ${DESTDIR} /usr/bin/newaliases
+	chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart
+
+	return 0
+}
+
+vm_extra_install_base() {
+	# Prototype.  When overridden, runs extra post-installworld commands
+	# as needed, based on the target virtual machine image or cloud
+	# provider image target.
+
+	return 0
+}
+
+vm_extra_enable_services() {
+	if [ ! -z "${VM_RC_LIST}" ]; then
+		for _rcvar in ${VM_RC_LIST}; do
+			echo ${_rcvar}_enable="YES" >> ${DESTDIR}/etc/rc.conf
+		done
+	fi
+
+	return 0
+}
+
+vm_extra_install_packages() {
+	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
+		/usr/sbin/pkg bootstrap -y
+	if [ ! -z "${VM_EXTRA_PACKAGES}" ]; then
+		chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
+			/usr/sbin/pkg install -y ${VM_EXTRA_PACKAGES}
+	fi
+
+	return 0
+}
+
+vm_extra_install_ports() {
+	# Prototype.  When overridden, installs additional ports within the
+	# virtual machine environment.
+
+	return 0
+}
+
+vm_umount_base() {
+	i=0
+	sync
+	while ! umount ${DESTDIR}/dev ${DESTDIR}; do
+		i=$(( $i + 1 ))
+		if [ $i -ge 10 ]; then
+			# This should never happen.  But, it has happened.
+			msg="Cannot umount(8) ${DESTDIR}\n"
+			msg="${msg}Something has gone horribly wrong."
+			err "${msg}"
+		fi
+		sleep 1
+	done
+
+	return 0
+}
+
+vm_create_disk() {
+	if [ -z "${mkimg_paritions}" ]; then
+		err "No partition types specified.  Skipping."
+		return 1
+	fi
+	echo "Creating image...  Please wait."
+	echo
+	mkimg -f ${mkimg_format} -s ${mkimg_scheme} \
+		${mkimg_bootcode} \
+		${mkimg_partitions} \
+		${mkimg_outfile}
+
+	mkimg 	-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
+		-p freebsd-swap/swapfs::1G \
+		-p freebsd-ufs/rootfs:=${VMBASE} \
+		-o ${VMIMAGE}.raw
+
+	return 0
+}
+
+vm_extra_create_disk() {
+
+	return 0
+}
+

From efeb11a772e65d88562140c9f0cf82504bdb1079 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Fri, 7 Nov 2014 01:48:12 +0000
Subject: [PATCH 016/279] Add write_partition_layout() used to populate the
 final image.

Fix duplicated mkimg(1) call in vm_create_disk().

Add primitive (untested) PowerPC/PowerPC64 VM image
support.

Note: As it is currently written, the /boot/pmbr and
/boot/{gptboot,boot1.hfs} use the build host and not
the target build.  Fixing this is likely going to be
a hack in itself.

Sponsored by:	The FreeBSD Foundation
---
 release/tools/vmimage.subr | 38 ++++++++++++++++++++++++++------------
 1 file changed, 26 insertions(+), 12 deletions(-)

diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index 622e0fc967e..3cf19ff360e 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -9,10 +9,31 @@
 export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
 trap "cleanup" INT QUIT TRAP ABRT TERM
 
-mkimg_bootcode="/boot/pmbr"
-mkimg_partitions="-p freebsd-boot/bootfs:=/boot/gptboot"
-mkimg_partitions="${mkimg_partitions} -p freebsd-swap/swapfs::1G"
-mkimg_partitions="${mkimg_partitions} freebsd-ufs/rootfs:=${VMBASE}"
+write_partition_layout() {
+
+	case "${TARGET}:${TARGET_ARCH}" in
+		amd64:amd64 | i386:i386)
+			mkimg -f gpt -b /boot/pmbr \
+				-p freebsd-boot/bootfs:=/boot/gptboot \
+				-p freebsd-swap/swapfs::1G \
+				-p freebsd-ufs/rootfs:=${VMBASE}
+				-o ${VMIMAGE}
+			;;
+		powerpc:powerpc*)
+			mkimg -f apm \
+				-p freebsd-boot/bootfs:=/boot/boot1.hfs \
+				-p freebsd-swap/swapfs::1G \
+				-p freebsd-ufs/rootfs:=${VMBASE}
+				-o ${VMIMAGE}
+			;;
+		*)
+			# ENOTSUPP
+			return 1
+			;;
+	esac
+
+	return 0
+}
 
 usage() {
 	echo "${0} usage:"
@@ -130,15 +151,8 @@ vm_create_disk() {
 	fi
 	echo "Creating image...  Please wait."
 	echo
-	mkimg -f ${mkimg_format} -s ${mkimg_scheme} \
-		${mkimg_bootcode} \
-		${mkimg_partitions} \
-		${mkimg_outfile}
 
-	mkimg 	-b /boot/pmbr -p freebsd-boot/bootfs:=/boot/gptboot \
-		-p freebsd-swap/swapfs::1G \
-		-p freebsd-ufs/rootfs:=${VMBASE} \
-		-o ${VMIMAGE}.raw
+	write_partition_layout || return 1
 
 	return 0
 }

From 87f6c4ac27a42eb100a3c7e696d2ae718b6c039d Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Sat, 8 Nov 2014 12:23:50 +0000
Subject: [PATCH 017/279] Return if vm_create_disk() is unsuccessful.

Sponsored by:	The FreeBSD Foundation
---
 release/scripts/mk-vmimage.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release/scripts/mk-vmimage.sh b/release/scripts/mk-vmimage.sh
index b5065b66ab2..27c48b72b91 100755
--- a/release/scripts/mk-vmimage.sh
+++ b/release/scripts/mk-vmimage.sh
@@ -93,7 +93,7 @@ main() {
 	vm_extra_install_ports
 	vm_extra_enable_services
 	vm_extra_pre_umount
-	vm_create_disk
+	vm_create_disk || return 0
 	vm_extra_create_disk
 
 	return 0

From b6dd8a62c4ae49b3bc425d0559431de20268917b Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Sat, 8 Nov 2014 12:40:59 +0000
Subject: [PATCH 018/279] Add CLEANFILES entry for VM targets

Sponsored by:	The FreeBSD Foundation
---
 release/Makefile.vm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/release/Makefile.vm b/release/Makefile.vm
index c37a4ad759d..30f38682b46 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -20,6 +20,7 @@ OPENSTACK_FORMAT=qcow2
 CLOUDTARGETS+=	vm-${_CW:tl}
 CLEANDIRS+=	vm-${_CW:tl}
 CLEANFILES+=	${_CW:tl}.img \
+		vm-${_CW:tl} \
 		${_CW:tl}.${${_CW:tu}_FORMAT} \
 		${_CW:tl}.${${_CW:tu}_FORMAT}.raw
 ${_CW:tu}IMAGE=	${_CW:tl}.${${_CW:tu}_FORMAT}

From a02bfc832740f019d863a87d6efe2594a4e903c8 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Sat, 8 Nov 2014 12:45:02 +0000
Subject: [PATCH 019/279] Add vm_extra_pre_umount() prototype to vmimage.subr.

Sponsored by:	The FreeBSD Foundation
---
 release/tools/vmimage.subr | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index 3cf19ff360e..7e54a7f3311 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -127,6 +127,13 @@ vm_extra_install_ports() {
 	return 0
 }
 
+vm_extra_pre_umount() {
+	# Prototype.  When overridden, installs additional ports within the
+	# virtual machine environment.
+
+	return 0
+}
+
 vm_umount_base() {
 	i=0
 	sync

From 1c34bf95522f9245605fafcc9df3936f5211014e Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Sat, 8 Nov 2014 12:45:35 +0000
Subject: [PATCH 020/279] Fix DESTDIR for installworld, and make sure it is
 created before use.

Sponsored by:	The FreeBSD Foundation
---
 release/Makefile.vm | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/release/Makefile.vm b/release/Makefile.vm
index 30f38682b46..eb167defa5f 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -44,9 +44,10 @@ vm-image:
 .if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
 . if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
 .  for FORMAT in ${VMFORMATS}
+	mkdir -p ${.OBJDIR}/${.TARGET}
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
 		${.CURDIR}/${TARGET}/mk-vmimage.sh \
-		-C ${.CURDIR}/tools/vmimage.subr -d ${.TARGET} \
+		-C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} \
 		-i ${VMBASE}.img -s ${VMSIZE} -f ${FORMAT} \
 		-S ${WORLDDIR} -o ${VMBASE}.${FORMAT}
 .  endfor
@@ -58,9 +59,10 @@ vm-cloudware:	${CLOUDTARGETS}
 
 vm-azure:
 .if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
+	mkdir -p ${.OBJDIR}/${.TARGET}
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
 		${.CURDIR}/${TARGET}/mk-vmimage.sh \
-		-C ${.CURDIR}/tools/vmimage.subr -d ${.TARGET} \
+		-C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} \
 		-i azure.img -s ${VMSIZE} -f ${AZURE_FORMAT} \
 		-S ${WORLDDIR} -o ${AZUREIMAGE} -c ${AZURECONF}
 .endif
@@ -68,9 +70,10 @@ vm-azure:
 
 vm-openstack:
 .if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
+	mkdir -p ${.OBJDIR}/${.TARGET}
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
 		${.CURDIR}/${TARGET}/mk-vmimage.sh \
-		-C ${.CURDIR}/tools/vmimage.subr -d ${.TARGET} \
+		-C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} \
 		-i openstack.img -s ${VMSIZE} -f ${OPENSTACK_FORMAT} \
 		-S ${WORLDDIR} -o ${OPENSTACKIMAGE} -c ${OPENSTACKCONF}
 .endif

From 144d107fcd0859d2133a35d26de0559bb3b7c957 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Sat, 8 Nov 2014 12:47:21 +0000
Subject: [PATCH 021/279] Move usage() from vmimage.subr to mk-vmimage.sh, in
 case vmimage.subr has not been sourced.

Sponsored by:	The FreeBSD Foundation
---
 release/scripts/mk-vmimage.sh | 6 ++++++
 release/tools/vmimage.subr    | 6 ------
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/release/scripts/mk-vmimage.sh b/release/scripts/mk-vmimage.sh
index 27c48b72b91..0dd330fa98d 100755
--- a/release/scripts/mk-vmimage.sh
+++ b/release/scripts/mk-vmimage.sh
@@ -32,6 +32,12 @@
 # $FreeBSD$
 #
 
+usage() {
+	echo "${0} usage:"
+	echo "${@}"
+	return 1
+}
+
 main() {
 	local arg
 	while getopts "C:c:d:f:i:o:s:S:" arg; do
diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index 7e54a7f3311..62273a528f1 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -35,12 +35,6 @@ write_partition_layout() {
 	return 0
 }
 
-usage() {
-	echo "${0} usage:"
-	echo "${@}"
-	return 1
-}
-
 err() {
 	printf "${@}\n"
 	cleanup

From 07fe767eb842e52868613d9347baad709a4533db Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Sat, 8 Nov 2014 12:59:32 +0000
Subject: [PATCH 022/279] Spell 'OPTARG' correctly. Actually call
 vm_create_base().

Sponsored by:	The FreeBSD Foundation
---
 release/scripts/mk-vmimage.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/release/scripts/mk-vmimage.sh b/release/scripts/mk-vmimage.sh
index 0dd330fa98d..7df7e3d10fb 100755
--- a/release/scripts/mk-vmimage.sh
+++ b/release/scripts/mk-vmimage.sh
@@ -55,7 +55,7 @@ main() {
 				VMFORMAT="${OPTARG}"
 				;;
 			i)
-				VMBASE="${VMBASE}"
+				VMBASE="${OPTARG}"
 				;;
 			o)
 				VMIMAGE="${OPTARG}"
@@ -93,6 +93,7 @@ main() {
 		. "${VMCONFIG}"
 	fi
 
+	vm_create_base
 	vm_install_base
 	vm_extra_install_base
 	vm_extra_install_packages

From 38d59ac708a5f97be0d539379ca725490a9a6f21 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Sat, 8 Nov 2014 13:49:18 +0000
Subject: [PATCH 023/279] Fix line continuation in write_partition_layout().
 Remove variable test that is no longer needed.

Sponsored by:	The FreeBSD Foundation
---
 release/tools/vmimage.subr | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index 62273a528f1..b6a91daa5fb 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -16,14 +16,14 @@ write_partition_layout() {
 			mkimg -f gpt -b /boot/pmbr \
 				-p freebsd-boot/bootfs:=/boot/gptboot \
 				-p freebsd-swap/swapfs::1G \
-				-p freebsd-ufs/rootfs:=${VMBASE}
+				-p freebsd-ufs/rootfs:=${VMBASE} \
 				-o ${VMIMAGE}
 			;;
 		powerpc:powerpc*)
 			mkimg -f apm \
 				-p freebsd-boot/bootfs:=/boot/boot1.hfs \
 				-p freebsd-swap/swapfs::1G \
-				-p freebsd-ufs/rootfs:=${VMBASE}
+				-p freebsd-ufs/rootfs:=${VMBASE} \
 				-o ${VMIMAGE}
 			;;
 		*)
@@ -146,10 +146,6 @@ vm_umount_base() {
 }
 
 vm_create_disk() {
-	if [ -z "${mkimg_paritions}" ]; then
-		err "No partition types specified.  Skipping."
-		return 1
-	fi
 	echo "Creating image...  Please wait."
 	echo
 

From 7d565d608f2aed33566ed8e6dc371e42b626686e Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Sat, 8 Nov 2014 13:49:59 +0000
Subject: [PATCH 024/279] Fix scheme flag to mkimg(1).

Sponsored by:	The FreeBSD Foundation
---
 release/tools/vmimage.subr | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index b6a91daa5fb..49829942993 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -13,14 +13,14 @@ write_partition_layout() {
 
 	case "${TARGET}:${TARGET_ARCH}" in
 		amd64:amd64 | i386:i386)
-			mkimg -f gpt -b /boot/pmbr \
+			mkimg -s gpt -b /boot/pmbr \
 				-p freebsd-boot/bootfs:=/boot/gptboot \
 				-p freebsd-swap/swapfs::1G \
 				-p freebsd-ufs/rootfs:=${VMBASE} \
 				-o ${VMIMAGE}
 			;;
 		powerpc:powerpc*)
-			mkimg -f apm \
+			mkimg -s apm \
 				-p freebsd-boot/bootfs:=/boot/boot1.hfs \
 				-p freebsd-swap/swapfs::1G \
 				-p freebsd-ufs/rootfs:=${VMBASE} \

From e6f3489c367f9bd0e9a6d58364eb1c2bcba38af1 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Sat, 8 Nov 2014 16:26:17 +0000
Subject: [PATCH 025/279] mount(8) and umount(8) devfs(5) as needed.

Sponsored by:	The FreeBSD Foundation
---
 release/tools/vmimage.subr | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index 49829942993..64015b47061 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -79,8 +79,11 @@ vm_install_base() {
 	echo '/dev/gpt/swapfs  none    swap    sw      0       0' \
 		>> ${DESTDIR}/etc/fstab
 
+	mkdir -p ${DESTDIR}/dev
+	mount -t devfs devfs ${DESTDIR}/dev
 	chroot ${DESTDIR} /usr/bin/newaliases
 	chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart
+	umount ${DESTDIR}/dev
 
 	return 0
 }
@@ -104,12 +107,15 @@ vm_extra_enable_services() {
 }
 
 vm_extra_install_packages() {
+	mkdir -p ${DESTDIR}/dev
+	mount -t devfs devfs ${DESTDIR}/dev
 	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
 		/usr/sbin/pkg bootstrap -y
 	if [ ! -z "${VM_EXTRA_PACKAGES}" ]; then
 		chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
 			/usr/sbin/pkg install -y ${VM_EXTRA_PACKAGES}
 	fi
+	umount ${DESTDIR}/dev
 
 	return 0
 }

From 26bb2a6066639c76eeee9b32509dae17250d69da Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Sat, 8 Nov 2014 16:44:45 +0000
Subject: [PATCH 026/279] Change path for mk-vmimage.sh from ${TARGET}/ to
 scripts/ now that it is consolidated into one file.

Fix paths for the base image and output disk image
files.

Sponsored by:	The FreeBSD Foundation
---
 release/Makefile.vm | 28 +++++++++++-----------------
 1 file changed, 11 insertions(+), 17 deletions(-)

diff --git a/release/Makefile.vm b/release/Makefile.vm
index eb167defa5f..d7fc18f941d 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -42,39 +42,33 @@ vm-base:	vm-image
 
 vm-image:
 .if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
-. if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
-.  for FORMAT in ${VMFORMATS}
+. for FORMAT in ${VMFORMATS}
 	mkdir -p ${.OBJDIR}/${.TARGET}
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		${.CURDIR}/${TARGET}/mk-vmimage.sh \
+		${.CURDIR}/scripts/mk-vmimage.sh \
 		-C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} \
-		-i ${VMBASE}.img -s ${VMSIZE} -f ${FORMAT} \
-		-S ${WORLDDIR} -o ${VMBASE}.${FORMAT}
-.  endfor
-. endif
+		-i ${.OBJDIR}/${VMBASE}.img -s ${VMSIZE} -f ${FORMAT} \
+		-S ${WORLDDIR} -o ${.OBJDIR}/${VMBASE}.${FORMAT}
+. endfor
 .endif
 	touch ${.TARGET}
 
 vm-cloudware:	${CLOUDTARGETS}
 
 vm-azure:
-.if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
 	mkdir -p ${.OBJDIR}/${.TARGET}
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		${.CURDIR}/${TARGET}/mk-vmimage.sh \
+		${.CURDIR}/scripts/mk-vmimage.sh \
 		-C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} \
-		-i azure.img -s ${VMSIZE} -f ${AZURE_FORMAT} \
-		-S ${WORLDDIR} -o ${AZUREIMAGE} -c ${AZURECONF}
-.endif
+		-i ${.OBJDIR}/azure.img -s ${VMSIZE} -f ${AZURE_FORMAT} \
+		-S ${WORLDDIR} -o ${.OBJDIR}/${AZUREIMAGE} -c ${AZURECONF}
 	touch ${.TARGET}
 
 vm-openstack:
-.if exists(${.CURDIR}/${TARGET}/mk-vmimage.sh)
 	mkdir -p ${.OBJDIR}/${.TARGET}
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		${.CURDIR}/${TARGET}/mk-vmimage.sh \
+		${.CURDIR}/scripts/mk-vmimage.sh \
 		-C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} \
-		-i openstack.img -s ${VMSIZE} -f ${OPENSTACK_FORMAT} \
-		-S ${WORLDDIR} -o ${OPENSTACKIMAGE} -c ${OPENSTACKCONF}
-.endif
+		-i ${.OBJDIR}/openstack.img -s ${VMSIZE} -f ${OPENSTACK_FORMAT} \
+		-S ${WORLDDIR} -o ${.OBJDIR}/${OPENSTACKIMAGE} -c ${OPENSTACKCONF}
 	touch ${.TARGET}

From 742aade4627297094225acbd82b6d736c22e7230 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Sat, 8 Nov 2014 16:52:07 +0000
Subject: [PATCH 027/279] Call cleanup() after everything is done.

Sponsored by:	The FreeBSD Foundation
---
 release/scripts/mk-vmimage.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/release/scripts/mk-vmimage.sh b/release/scripts/mk-vmimage.sh
index 7df7e3d10fb..05a64cc3f92 100755
--- a/release/scripts/mk-vmimage.sh
+++ b/release/scripts/mk-vmimage.sh
@@ -102,6 +102,7 @@ main() {
 	vm_extra_pre_umount
 	vm_create_disk || return 0
 	vm_extra_create_disk
+	cleanup
 
 	return 0
 }

From f43c861f3814db97dea6b54a6c84a281e303200e Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Mon, 10 Nov 2014 21:10:50 +0000
Subject: [PATCH 028/279] Remove a stray directory from CLEANFILES.

Sponsored by:	The FreeBSD Foundation
---
 release/Makefile.vm | 1 -
 1 file changed, 1 deletion(-)

diff --git a/release/Makefile.vm b/release/Makefile.vm
index d7fc18f941d..d362324379f 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -20,7 +20,6 @@ OPENSTACK_FORMAT=qcow2
 CLOUDTARGETS+=	vm-${_CW:tl}
 CLEANDIRS+=	vm-${_CW:tl}
 CLEANFILES+=	${_CW:tl}.img \
-		vm-${_CW:tl} \
 		${_CW:tl}.${${_CW:tu}_FORMAT} \
 		${_CW:tl}.${${_CW:tu}_FORMAT}.raw
 ${_CW:tu}IMAGE=	${_CW:tl}.${${_CW:tu}_FORMAT}

From 9d3df3bdb7367f7d0ba2a98d5fd2db7c85823743 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Fri, 14 Nov 2014 22:29:33 +0000
Subject: [PATCH 029/279] Set the boot partition type to 'apple-boot' for
 powerpc.

Submitted by:	jhibbits
Sponsored by:	The FreeBSD Foundation
---
 release/tools/vmimage.subr | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index 64015b47061..bc2a7fb1f50 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -21,7 +21,7 @@ write_partition_layout() {
 			;;
 		powerpc:powerpc*)
 			mkimg -s apm \
-				-p freebsd-boot/bootfs:=/boot/boot1.hfs \
+				-p apple-boot/bootfs:=/boot/boot1.hfs \
 				-p freebsd-swap/swapfs::1G \
 				-p freebsd-ufs/rootfs:=${VMBASE} \
 				-o ${VMIMAGE}

From 91333d6a2da25901490044d87e633ac5dba8cb0c Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Wed, 19 Nov 2014 20:19:53 +0000
Subject: [PATCH 030/279] In vm_install_base(), copy the host resolv.conf into
 the build chroot before attempting to do anything that requires working DNS
 (i.e., pkg bootstrap).

In vm_extra_pre_umount(), remove the resolv.conf before
the disk image is unmounted from the backing md(4).

Reported by:	cperciva
Sponsored by:	The FreeBSD Foundation
---
 release/tools/azure.conf     | 2 ++
 release/tools/openstack.conf | 2 ++
 release/tools/vmimage.subr   | 4 ++++
 3 files changed, 8 insertions(+)

diff --git a/release/tools/azure.conf b/release/tools/azure.conf
index 16c207da324..3e4a4864adf 100644
--- a/release/tools/azure.conf
+++ b/release/tools/azure.conf
@@ -33,6 +33,8 @@ vm_extra_pre_umount() {
 	echo 'console="comconsole vidconsole"' >> ${DESTDIR}/boot/loader.conf
 	echo 'comconsole_speed="115200"' >> ${DESTDIR}/boot/loader.conf
 
+	rm -f ${DESTDIR}/etc/resolv.conf
+
 	return 0
 }
 
diff --git a/release/tools/openstack.conf b/release/tools/openstack.conf
index 6aa17493f97..3e0e857e9d8 100644
--- a/release/tools/openstack.conf
+++ b/release/tools/openstack.conf
@@ -22,5 +22,7 @@ vm_extra_pre_umount() {
 	echo 'sshd_enable="YES"' >> ${DESTDIR}/etc/rc.conf
 	echo 'ifconfig_DEFAULT="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf
 
+	rm -f ${DESTDIR}/etc/resolv.conf
+
 	return 0
 }
diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index bc2a7fb1f50..00a0a2e23da 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -85,6 +85,8 @@ vm_install_base() {
 	chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart
 	umount ${DESTDIR}/dev
 
+	cp /etc/resolv.conf ${DESTDIR}/etc/resolv.conf
+
 	return 0
 }
 
@@ -131,6 +133,8 @@ vm_extra_pre_umount() {
 	# Prototype.  When overridden, installs additional ports within the
 	# virtual machine environment.
 
+	rm -f ${DESTDIR}/etc/resolv.conf
+
 	return 0
 }
 

From 7d97075f02527bcf29318143c2fc3650be79ee07 Mon Sep 17 00:00:00 2001
From: Colin Percival <cperciva@FreeBSD.org>
Date: Wed, 19 Nov 2014 22:17:22 +0000
Subject: [PATCH 031/279] Silence errors when umounting the chroot's /dev,
 since it probably doesn't exist when we're running this.

Unmount filesystems before attempting to destroy the md which holds them.
---
 release/tools/vmimage.subr | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index 00a0a2e23da..3e1ddce66b1 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -42,11 +42,11 @@ err() {
 }
 
 cleanup() {
+	umount ${DESTDIR}/dev 2>/dev/null
+	umount ${DESTDIR}
 	if [ ! -z "${mddev}" ]; then
 		mdconfig -d -u ${mddev}
 	fi
-	umount ${DESTDIR}/dev
-	umount ${DESTDIR}
 
 	return 0
 }

From 3b4252413d2c175ee3678e69c6e8095e57060fee Mon Sep 17 00:00:00 2001
From: Colin Percival <cperciva@FreeBSD.org>
Date: Thu, 20 Nov 2014 00:16:55 +0000
Subject: [PATCH 032/279] Unmount filesystem and destroy md before we read the
 vnode from disk and package it into a disk image.  Otherwise we end up
 packaging an unclean filesystem.

---
 release/scripts/mk-vmimage.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release/scripts/mk-vmimage.sh b/release/scripts/mk-vmimage.sh
index 05a64cc3f92..bf0e1325dde 100755
--- a/release/scripts/mk-vmimage.sh
+++ b/release/scripts/mk-vmimage.sh
@@ -100,9 +100,9 @@ main() {
 	vm_extra_install_ports
 	vm_extra_enable_services
 	vm_extra_pre_umount
+	cleanup
 	vm_create_disk || return 0
 	vm_extra_create_disk
-	cleanup
 
 	return 0
 }

From 48096f81ed910e33371f0a2fb5212c06c9c707c4 Mon Sep 17 00:00:00 2001
From: Colin Percival <cperciva@FreeBSD.org>
Date: Thu, 20 Nov 2014 03:46:35 +0000
Subject: [PATCH 033/279] Merge duplicative vm-CLOUDTYPE targets before
 additional duplication gets added by the impending arrival of ec2 and gcloud.

---
 release/Makefile.vm | 27 +++++++++------------------
 1 file changed, 9 insertions(+), 18 deletions(-)

diff --git a/release/Makefile.vm b/release/Makefile.vm
index d362324379f..bb5311fdc66 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -26,6 +26,15 @@ ${_CW:tu}IMAGE=	${_CW:tl}.${${_CW:tu}_FORMAT}
 .  if exists(${.CURDIR}/tools/${_CW:tl}.conf) && !defined(${_CW:tu}CONF)
 ${_CW:tu}CONF?=	${.CURDIR}/tools/${_CW:tl}.conf
 .  endif
+
+vm-${_CW:tl}:
+	mkdir -p ${.OBJDIR}/${.TARGET}
+	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
+		${.CURDIR}/scripts/mk-vmimage.sh \
+		-C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} \
+		-i ${.OBJDIR}/${_CW:tl}.img -s ${VMSIZE} -f ${${_CW}_FORMAT} \
+		-S ${WORLDDIR} -o ${.OBJDIR}/${${_CW}IMAGE} -c ${${_CW}CONF}
+	touch ${.TARGET}
 . endfor
 .endif
 
@@ -53,21 +62,3 @@ vm-image:
 	touch ${.TARGET}
 
 vm-cloudware:	${CLOUDTARGETS}
-
-vm-azure:
-	mkdir -p ${.OBJDIR}/${.TARGET}
-	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		${.CURDIR}/scripts/mk-vmimage.sh \
-		-C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} \
-		-i ${.OBJDIR}/azure.img -s ${VMSIZE} -f ${AZURE_FORMAT} \
-		-S ${WORLDDIR} -o ${.OBJDIR}/${AZUREIMAGE} -c ${AZURECONF}
-	touch ${.TARGET}
-
-vm-openstack:
-	mkdir -p ${.OBJDIR}/${.TARGET}
-	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		${.CURDIR}/scripts/mk-vmimage.sh \
-		-C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} \
-		-i ${.OBJDIR}/openstack.img -s ${VMSIZE} -f ${OPENSTACK_FORMAT} \
-		-S ${WORLDDIR} -o ${.OBJDIR}/${OPENSTACKIMAGE} -c ${OPENSTACKCONF}
-	touch ${.TARGET}

From 4e7bc9f0f7b329ce952e0faa4674d143ae2568ac Mon Sep 17 00:00:00 2001
From: Colin Percival <cperciva@FreeBSD.org>
Date: Fri, 21 Nov 2014 01:53:40 +0000
Subject: [PATCH 034/279] Add NOSWAP option which can be set by a vmimage.conf
 file to specify that no swap space should be created in the image.  This will
 be used by EC2 builds, since FreeBSD/EC2 allocates swap space on "ephemeral"
 disks which are physically attached to the Xen host node.

---
 release/tools/vmimage.subr | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index 3e1ddce66b1..c7805721d2a 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -10,19 +10,22 @@ export PATH="/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin:/usr/local/sbin"
 trap "cleanup" INT QUIT TRAP ABRT TERM
 
 write_partition_layout() {
+	if [ -z "${NOSWAP}" ]; then
+		SWAPOPT="-p freebsd-swap/swapfs::1G"
+	fi
 
 	case "${TARGET}:${TARGET_ARCH}" in
 		amd64:amd64 | i386:i386)
 			mkimg -s gpt -b /boot/pmbr \
 				-p freebsd-boot/bootfs:=/boot/gptboot \
-				-p freebsd-swap/swapfs::1G \
+				${SWAPOPT} \
 				-p freebsd-ufs/rootfs:=${VMBASE} \
 				-o ${VMIMAGE}
 			;;
 		powerpc:powerpc*)
 			mkimg -s apm \
 				-p apple-boot/bootfs:=/boot/boot1.hfs \
-				-p freebsd-swap/swapfs::1G \
+				${SWAPOPT} \
 				-p freebsd-ufs/rootfs:=${VMBASE} \
 				-o ${VMIMAGE}
 			;;
@@ -76,8 +79,10 @@ vm_install_base() {
 		> ${DESTDIR}/etc/fstab
 	echo '/dev/gpt/rootfs   /       ufs     rw      1       1' \
 		>> ${DESTDIR}/etc/fstab
-	echo '/dev/gpt/swapfs  none    swap    sw      0       0' \
-		>> ${DESTDIR}/etc/fstab
+	if [ -z "${NOSWAP}" ]; then
+		echo '/dev/gpt/swapfs  none    swap    sw      0       0' \
+			>> ${DESTDIR}/etc/fstab
+	fi
 
 	mkdir -p ${DESTDIR}/dev
 	mount -t devfs devfs ${DESTDIR}/dev

From 32dbc829820b1b03bf2e27d204cb3fee8d072da9 Mon Sep 17 00:00:00 2001
From: Colin Percival <cperciva@FreeBSD.org>
Date: Fri, 21 Nov 2014 02:13:12 +0000
Subject: [PATCH 035/279] Change how packages are installed into VM images:
 Rather than chrooting into the image and running 'pkg install' from there,
 use 'pkg fetch' to download packages into a temporary location and then 'pkg
 add' to install them into the image.

This simplifies the code by avoiding the need to copy /etc/resolv.conf
into the image and then delete it later, and makes it possible to cross
build (e.g., to create an amd64 image when running on i386 hardware; or
in the future for building disk images for embedded platforms).

Because pkg was implicitly installed when VM_EXTRA_PACKAGES was non-empty,
add it to VM_EXTRA_PACKAGES in azure.conf and openstack.conf to maintain
the current behaviour.

By default repo-FreeBSD.sqlite is copied into the image, (a) to match
previous behaviour, where the file would be downloaded by the chrooted
pkg invocation; and (b) because it may be useful for testing purposes,
e.g., to see why a package didn't get installed.  Because this file is
large (46 MB) and not likely to be useful in -RELEASE images which are
being launched into Clouds several months later, it can be disabled by
setting NOREPOSQLITE.

As far as I know this commit does not change the disk images produced in
any filesystem-visible way.
---
 release/tools/azure.conf     |  5 +----
 release/tools/openstack.conf |  5 +----
 release/tools/vmimage.subr   | 21 ++++++++++-----------
 3 files changed, 12 insertions(+), 19 deletions(-)

diff --git a/release/tools/azure.conf b/release/tools/azure.conf
index 3e4a4864adf..46b678cd5c2 100644
--- a/release/tools/azure.conf
+++ b/release/tools/azure.conf
@@ -5,7 +5,7 @@
 
 # Set to a list of packages to install.
 # Example:
-#export VM_EXTRA_PACKAGES="www/apache24"
+#export VM_EXTRA_PACKAGES="www/apache24 ports-mgmt/pkg"
 export VM_EXTRA_PACKAGES=
 
 # Set to a list of third-party software to enable in rc.conf(5).
@@ -17,7 +17,6 @@ vm_extra_install_base() {
 	fetch -o ${DESTDIR}/usr/sbin/waagent \
 		http://people.freebsd.org/~gjb/waagent
 	chmod +x ${DESTDIR}/usr/sbin/waagent
-	rm -f ${DESTDIR}/etc/resolv.conf
 
 	return 0
 }
@@ -33,8 +32,6 @@ vm_extra_pre_umount() {
 	echo 'console="comconsole vidconsole"' >> ${DESTDIR}/boot/loader.conf
 	echo 'comconsole_speed="115200"' >> ${DESTDIR}/boot/loader.conf
 
-	rm -f ${DESTDIR}/etc/resolv.conf
-
 	return 0
 }
 
diff --git a/release/tools/openstack.conf b/release/tools/openstack.conf
index 3e0e857e9d8..ed88ae2972f 100644
--- a/release/tools/openstack.conf
+++ b/release/tools/openstack.conf
@@ -4,7 +4,7 @@
 #
 
 # Set to a list of packages to install.
-export VM_EXTRA_PACKAGES="net/cloud-init"
+export VM_EXTRA_PACKAGES="net/cloud-init ports-mgmt/pkg"
 
 # Set to a list of third-party software to enable in rc.conf(5).
 export VM_RC_LIST="cloudinit"
@@ -13,7 +13,6 @@ vm_extra_install_base() {
 	fetch -o ${DESTDIR}/usr/sbin/waagent \
 		http://people.freebsd.org/~gjb/waagent
 	chmod +x ${DESTDIR}/usr/sbin/waagent
-	rm -f ${DESTDIR}/etc/resolv.conf
 
 	return 0
 }
@@ -22,7 +21,5 @@ vm_extra_pre_umount() {
 	echo 'sshd_enable="YES"' >> ${DESTDIR}/etc/rc.conf
 	echo 'ifconfig_DEFAULT="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf
 
-	rm -f ${DESTDIR}/etc/resolv.conf
-
 	return 0
 }
diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index c7805721d2a..63e39066fea 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -90,8 +90,6 @@ vm_install_base() {
 	chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart
 	umount ${DESTDIR}/dev
 
-	cp /etc/resolv.conf ${DESTDIR}/etc/resolv.conf
-
 	return 0
 }
 
@@ -114,15 +112,18 @@ vm_extra_enable_services() {
 }
 
 vm_extra_install_packages() {
-	mkdir -p ${DESTDIR}/dev
-	mount -t devfs devfs ${DESTDIR}/dev
-	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
-		/usr/sbin/pkg bootstrap -y
 	if [ ! -z "${VM_EXTRA_PACKAGES}" ]; then
-		chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
-			/usr/sbin/pkg install -y ${VM_EXTRA_PACKAGES}
+		PKGSDIR=`mktemp -d`
+		ABI=`/usr/sbin/pkg -c ${DESTDIR} config abi`
+		/usr/sbin/pkg -o ABI=${ABI} fetch -o ${PKGSDIR} -d -y ${VM_EXTRA_PACKAGES}
+		for PKG in ${PKGSDIR}/All/*; do
+			/usr/sbin/pkg -c ${DESTDIR} add -M - < ${PKG}
+		done
+		rm -r ${PKGSDIR}
+		if [ -z "${NOREPOSQLITE}" ]; then
+			cp /var/db/pkg/repo-FreeBSD.sqlite ${DESTDIR}/var/db/pkg
+		fi
 	fi
-	umount ${DESTDIR}/dev
 
 	return 0
 }
@@ -138,8 +139,6 @@ vm_extra_pre_umount() {
 	# Prototype.  When overridden, installs additional ports within the
 	# virtual machine environment.
 
-	rm -f ${DESTDIR}/etc/resolv.conf
-
 	return 0
 }
 

From 30a20c89bf905f464bf62a3904e74802fb86b80d Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Fri, 21 Nov 2014 02:30:37 +0000
Subject: [PATCH 036/279] Revert r274773, after I *specifically* objected to
 this change numerous times.

Sponsored by:	The FreeBSD Foundation
---
 release/tools/azure.conf     |  5 ++++-
 release/tools/openstack.conf |  5 ++++-
 release/tools/vmimage.subr   | 21 +++++++++++----------
 3 files changed, 19 insertions(+), 12 deletions(-)

diff --git a/release/tools/azure.conf b/release/tools/azure.conf
index 46b678cd5c2..3e4a4864adf 100644
--- a/release/tools/azure.conf
+++ b/release/tools/azure.conf
@@ -5,7 +5,7 @@
 
 # Set to a list of packages to install.
 # Example:
-#export VM_EXTRA_PACKAGES="www/apache24 ports-mgmt/pkg"
+#export VM_EXTRA_PACKAGES="www/apache24"
 export VM_EXTRA_PACKAGES=
 
 # Set to a list of third-party software to enable in rc.conf(5).
@@ -17,6 +17,7 @@ vm_extra_install_base() {
 	fetch -o ${DESTDIR}/usr/sbin/waagent \
 		http://people.freebsd.org/~gjb/waagent
 	chmod +x ${DESTDIR}/usr/sbin/waagent
+	rm -f ${DESTDIR}/etc/resolv.conf
 
 	return 0
 }
@@ -32,6 +33,8 @@ vm_extra_pre_umount() {
 	echo 'console="comconsole vidconsole"' >> ${DESTDIR}/boot/loader.conf
 	echo 'comconsole_speed="115200"' >> ${DESTDIR}/boot/loader.conf
 
+	rm -f ${DESTDIR}/etc/resolv.conf
+
 	return 0
 }
 
diff --git a/release/tools/openstack.conf b/release/tools/openstack.conf
index ed88ae2972f..3e0e857e9d8 100644
--- a/release/tools/openstack.conf
+++ b/release/tools/openstack.conf
@@ -4,7 +4,7 @@
 #
 
 # Set to a list of packages to install.
-export VM_EXTRA_PACKAGES="net/cloud-init ports-mgmt/pkg"
+export VM_EXTRA_PACKAGES="net/cloud-init"
 
 # Set to a list of third-party software to enable in rc.conf(5).
 export VM_RC_LIST="cloudinit"
@@ -13,6 +13,7 @@ vm_extra_install_base() {
 	fetch -o ${DESTDIR}/usr/sbin/waagent \
 		http://people.freebsd.org/~gjb/waagent
 	chmod +x ${DESTDIR}/usr/sbin/waagent
+	rm -f ${DESTDIR}/etc/resolv.conf
 
 	return 0
 }
@@ -21,5 +22,7 @@ vm_extra_pre_umount() {
 	echo 'sshd_enable="YES"' >> ${DESTDIR}/etc/rc.conf
 	echo 'ifconfig_DEFAULT="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf
 
+	rm -f ${DESTDIR}/etc/resolv.conf
+
 	return 0
 }
diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index 63e39066fea..c7805721d2a 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -90,6 +90,8 @@ vm_install_base() {
 	chroot ${DESTDIR} /etc/rc.d/ldconfig forcestart
 	umount ${DESTDIR}/dev
 
+	cp /etc/resolv.conf ${DESTDIR}/etc/resolv.conf
+
 	return 0
 }
 
@@ -112,18 +114,15 @@ vm_extra_enable_services() {
 }
 
 vm_extra_install_packages() {
+	mkdir -p ${DESTDIR}/dev
+	mount -t devfs devfs ${DESTDIR}/dev
+	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
+		/usr/sbin/pkg bootstrap -y
 	if [ ! -z "${VM_EXTRA_PACKAGES}" ]; then
-		PKGSDIR=`mktemp -d`
-		ABI=`/usr/sbin/pkg -c ${DESTDIR} config abi`
-		/usr/sbin/pkg -o ABI=${ABI} fetch -o ${PKGSDIR} -d -y ${VM_EXTRA_PACKAGES}
-		for PKG in ${PKGSDIR}/All/*; do
-			/usr/sbin/pkg -c ${DESTDIR} add -M - < ${PKG}
-		done
-		rm -r ${PKGSDIR}
-		if [ -z "${NOREPOSQLITE}" ]; then
-			cp /var/db/pkg/repo-FreeBSD.sqlite ${DESTDIR}/var/db/pkg
-		fi
+		chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
+			/usr/sbin/pkg install -y ${VM_EXTRA_PACKAGES}
 	fi
+	umount ${DESTDIR}/dev
 
 	return 0
 }
@@ -139,6 +138,8 @@ vm_extra_pre_umount() {
 	# Prototype.  When overridden, installs additional ports within the
 	# virtual machine environment.
 
+	rm -f ${DESTDIR}/etc/resolv.conf
+
 	return 0
 }
 

From a4900ff5a54937813a00feb0f3f157a0c432dc3d Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Mon, 24 Nov 2014 02:34:01 +0000
Subject: [PATCH 037/279] In vm_extra_install_packages(), only bootstrap pkg(8)
 if VM_EXTRA_PACKAGES is empty.

In vm_extra_pre_umount(), cleanup downloaded packages
if pkg(8) was bootstrapped earlier.

Inspired by:	cperciva
Sponsored by:	The FreeBSD Foundation
---
 release/tools/vmimage.subr | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index c7805721d2a..3ea505619be 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -114,14 +114,15 @@ vm_extra_enable_services() {
 }
 
 vm_extra_install_packages() {
+	if [ -z "${VM_EXTRA_PACKAGES}" ]; then
+		return 0
+	fi
 	mkdir -p ${DESTDIR}/dev
 	mount -t devfs devfs ${DESTDIR}/dev
 	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
 		/usr/sbin/pkg bootstrap -y
-	if [ ! -z "${VM_EXTRA_PACKAGES}" ]; then
 		chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
 			/usr/sbin/pkg install -y ${VM_EXTRA_PACKAGES}
-	fi
 	umount ${DESTDIR}/dev
 
 	return 0
@@ -138,6 +139,10 @@ vm_extra_pre_umount() {
 	# Prototype.  When overridden, installs additional ports within the
 	# virtual machine environment.
 
+	if [ -e ${DESTDIR}/usr/local/sbin/pkg ]; then
+		chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
+			/usr/local/sbin/pkg clean -y -a
+	fi
 	rm -f ${DESTDIR}/etc/resolv.conf
 
 	return 0

From e5b4ea4d4ebec42ddea93abcc39615b0926bf0f2 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Mon, 24 Nov 2014 02:36:43 +0000
Subject: [PATCH 038/279] Fix indentation nit.

Sponsored by:	The FreeBSD Foundation
---
 release/tools/vmimage.subr | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index 3ea505619be..2fd70fffea5 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -121,8 +121,8 @@ vm_extra_install_packages() {
 	mount -t devfs devfs ${DESTDIR}/dev
 	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
 		/usr/sbin/pkg bootstrap -y
-		chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
-			/usr/sbin/pkg install -y ${VM_EXTRA_PACKAGES}
+	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes \
+		/usr/sbin/pkg install -y ${VM_EXTRA_PACKAGES}
 	umount ${DESTDIR}/dev
 
 	return 0

From c05228ba1c8fc8fe2e9cb19da4d04bf282900999 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Mon, 8 Dec 2014 07:43:02 +0000
Subject: [PATCH 039/279] - Document why usr.bin/vi needs to be built as part
 of bootstrap-tools - Do not build usr.bin/vi as part of bootstrap-tools if
 NO_SHARE is defined   as share/termcap will be skipped if NO_SHARE is
 defined; that being said,   something needs to be done for dealing with make
 distribute as it will be   built via make distribute if not done during make
 buildworld. - Add -DNO_SHARE to NOFUN to deal with make xdev because of the
 fact that it   implements a partial make buildworld/make installworld, but
 because it   doesn't define everything, not all install paths exist in
 expected stages,   which results in build failures with usr.bin/vi in
 bootstrap-tools because   "WORLDTMP" is not fully populated when
 _xb-bootstrap-tools is run.

  tl;dr: this is enough to just make sure that make xdev isn't broken on this
  branch when MK_VI == no and make delete-old has been run on the build host
---
 Makefile.inc1 | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 9e5b9801ebc..8f921e2b1ae 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -1388,6 +1388,8 @@ build-tools: .MAKE
 		${MAKE} DIRPRFX=${_tool}/ depend && \
 		${MAKE} DIRPRFX=${_tool}/ all
 .endfor
+	# usr.bin/vi is required to process files in share/termcap
+.if !defined(NO_SHARE)
 .for _tool in \
     usr.bin/vi
 	${_+_}@${ECHODIR} "===> ${_tool} (obj,depend,all,install)"; \
@@ -1397,6 +1399,7 @@ build-tools: .MAKE
 		${MAKE} DIRPRFX=${_tool}/ all MK_MAN=no SUBDIR= && \
 		${MAKE} DIRPRFX=${_tool}/ install MK_MAN=no SUBDIR= DESTDIR=${WORLDTMP}/
 .endfor
+.endif
 
 #
 # kernel-tools: Build kernel-building tools
@@ -2014,7 +2017,7 @@ NOFUN=-DNO_FSCHG MK_HTML=no MK_INFO=no -DNO_LINT \
 	MK_KERBEROS=no MK_RESCUE=no MK_TESTS=no MK_WARNS=no \
 	MK_VI=no \
 	TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-	CPUTYPE=${XDEV_CPUTYPE}
+	CPUTYPE=${XDEV_CPUTYPE} -DNO_SHARE
 
 XDDIR=${TARGET_ARCH}-freebsd
 XDTP?=/usr/${XDDIR}

From b43a681b2a4d512e09cff3b091a29ebece4611c8 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Wed, 10 Dec 2014 05:59:29 +0000
Subject: [PATCH 040/279] Move MK_BHYVE/MK_HYPERV to architecture specific
 sections in Makefiles instead of introducing an architecture specific section
 in src.opts.mk

Make it clear in the documentation that the options only affect certain
architectures to reduce user confusion if the knobs are tweaked

A better method for abstracting this out will be done in the future in a
different commit

Discussed with/Requested by: imp
---
 lib/Makefile                      | 54 ++++++++++++++++---------------
 share/examples/Makefile           | 12 ++++---
 share/man/man4/Makefile           |  8 ++---
 share/mk/src.opts.mk              | 16 ++-------
 sys/conf/kern.opts.mk             | 16 ++-------
 sys/modules/Makefile              | 15 ++++-----
 tools/build/options/WITHOUT_BHYVE |  2 ++
 tools/build/options/WITH_HYPERV   |  2 ++
 8 files changed, 54 insertions(+), 71 deletions(-)

diff --git a/lib/Makefile b/lib/Makefile
index 0d757c0ce3c..c794d89777a 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -173,10 +173,6 @@ _libngatm=	libngatm
 _libauditd=	libauditd
 .endif
 
-.if ${MK_BHYVE} != "no"
-_libvmmapi=	libvmmapi
-.endif
-
 .if ${MK_BLUETOOTH} != "no"
 _libbluetooth=	libbluetooth
 _libsdp=	libsdp
@@ -261,28 +257,6 @@ _libypclnt=	libypclnt
 _libevent=	libevent
 .endif
 
-.if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64"
-_libsmb=	libsmb
-_libvgl=	libvgl
-_libproc=	libproc
-_librtld_db=	librtld_db
-.endif
-
-.if ${MACHINE_CPUARCH} == "mips"
-_libproc=	libproc
-_librtld_db=	librtld_db
-.endif
-
-.if ${MACHINE_CPUARCH} == "powerpc"
-_libproc=	libproc
-_librtld_db=	librtld_db
-_libsmb=	libsmb
-.endif
-
-.if ${MACHINE_CPUARCH} == "sparc64"
-_libsmb=	libsmb
-.endif
-
 .if ${MK_OPENSSL} != "no"
 _libmp=		libmp
 .endif
@@ -326,6 +300,34 @@ _libusbhid=	libusbhid
 _libusb=	libusb
 .endif
 
+.if ${MACHINE_CPUARCH} == "amd64"
+.if ${MK_BHYVE} != "no"
+_libvmmapi=	libvmmapi
+.endif
+.endif
+
+.if ${MACHINE_CPUARCH} == "i386" || ${MACHINE_CPUARCH} == "amd64"
+_libsmb=	libsmb
+_libvgl=	libvgl
+_libproc=	libproc
+_librtld_db=	librtld_db
+.endif
+
+.if ${MACHINE_CPUARCH} == "mips"
+_libproc=	libproc
+_librtld_db=	librtld_db
+.endif
+
+.if ${MACHINE_CPUARCH} == "powerpc"
+_libproc=	libproc
+_librtld_db=	librtld_db
+_libsmb=	libsmb
+.endif
+
+.if ${MACHINE_CPUARCH} == "sparc64"
+_libsmb=	libsmb
+.endif
+
 .if !defined(LIBRARIES_ONLY)
 afterinstall:
 	${INSTALL_SYMLINK} ../include ${DESTDIR}/usr/lib/include
diff --git a/share/examples/Makefile b/share/examples/Makefile
index a0bd647c19e..bf5b955a7a1 100644
--- a/share/examples/Makefile
+++ b/share/examples/Makefile
@@ -184,11 +184,6 @@ NO_OBJ=
 # in environments where it's not possible to keep /sys publicly readable)
 SHARED?=	copies
 
-.if ${MK_BHYVE} != "no"
-LDIRS+=		bhyve
-XFILES+=	bhyve/vmrun.sh
-.endif
-
 .if ${MK_HAST} != "no"
 LDIRS+=		hast
 XFILES+=	hast/ucarp.sh \
@@ -219,6 +214,13 @@ XFILES+=	libusb20/Makefile \
 		libusb20/control.c
 .endif
 
+.if ${MACHINE_CPUARCH} == "amd64"
+.if ${MK_BHYVE} != "no"
+LDIRS+=		bhyve
+XFILES+=	bhyve/vmrun.sh
+.endif
+.endif
+
 beforeinstall: ${SHARED} etc-examples
 .ORDER: ${SHARED} etc-examples
 
diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile
index 5fd160852eb..d14c111cf29 100644
--- a/share/man/man4/Makefile
+++ b/share/man/man4/Makefile
@@ -729,6 +729,10 @@ MLINKS+=qlxge.4 if_qlxge.4
 MLINKS+=qlxgb.4 if_qlxgb.4
 MLINKS+=qlxgbe.4 if_qlxgbe.4
 MLINKS+=sfxge.4 if_sfxge.4
+
+.if ${MK_BHYVE} != "no"
+_bhyve.4=	bhyve.4
+.endif
 .endif
 
 .if ${MACHINE_CPUARCH} == "mips"
@@ -739,10 +743,6 @@ _nvram2env.4=	nvram2env.4
 SUBDIR=	man4.${MACHINE_CPUARCH}
 .endif
 
-.if ${MK_BHYVE} != "no"
-_bhyve.4=	bhyve.4
-.endif
-
 .if ${MK_CCD} != "no"
 _ccd.4=		ccd.4
 .endif
diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk
index cf4ff002bc3..a87e4890207 100644
--- a/share/mk/src.opts.mk
+++ b/share/mk/src.opts.mk
@@ -53,6 +53,7 @@ __DEFAULT_YES_OPTIONS = \
     AUDIT \
     AUTHPF \
     AUTOFS \
+    BHYVE \
     BINUTILS \
     BINUTILS_BOOTSTRAP \
     BLUETOOTH \
@@ -101,6 +102,7 @@ __DEFAULT_YES_OPTIONS = \
     GROFF \
     HAST \
     HTML \
+    HYPERV \
     ICONV \
     INET \
     INET6 \
@@ -234,20 +236,6 @@ __DEFAULT_NO_OPTIONS+=CLANG_IS_CC CLANG CLANG_BOOTSTRAP
 __DEFAULT_YES_OPTIONS+=GCC GCC_BOOTSTRAP GNUCXX
 .endif
 
-# bhyve is only supported on amd64
-.if ${__T} == "amd64"
-__DEFAULT_YES_OPTIONS+=BHYVE
-.else
-MK_BHYVE:=      no
-.endif
-
-# hyperv is only supported on amd64 and i386/i386
-.if ${__T} == "amd64" || (${__T} == "i386" && ${__TT} == "i386")
-__DEFAULT_YES_OPTIONS+=HYPERV
-.else
-MK_HYPERV:=	no
-.endif
-
 .include <bsd.mkopt.mk>
 
 #
diff --git a/sys/conf/kern.opts.mk b/sys/conf/kern.opts.mk
index eeb79f74c15..0fac2199684 100644
--- a/sys/conf/kern.opts.mk
+++ b/sys/conf/kern.opts.mk
@@ -24,11 +24,13 @@
 
 __DEFAULT_YES_OPTIONS = \
     AUTOFS \
+    BHYVE \
     BLUETOOTH \
     CCD \
     CDDL \
     CRYPT \
     FORMAT_EXTENSIONS \
+    HYPERV \
     ISCSI \
     INET \
     INET6 \
@@ -69,20 +71,6 @@ __TT=${TARGET}
 __TT=${MACHINE}
 .endif
 
-# bhyve is only supported on amd64
-.if ${__T} == "amd64"
-__DEFAULT_YES_OPTIONS+=BHYVE
-.else
-MK_BHYVE:=      no
-.endif
-
-# hyperv is only supported on amd64 and i386/i386
-.if ${__T} == "amd64" || (${__T} == "i386" && ${__TT} == "i386")
-__DEFAULT_YES_OPTIONS+=HYPERV
-.else
-MK_HYPERV:=	no
-.endif
-
 # expanded inline from bsd.mkopt.mk to avoid share/mk dependency
 
 # Those that default to yes
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index 479f86bbf76..aa8e6bf0a4c 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -388,10 +388,6 @@ SUBDIR=	\
 _autofs=	autofs
 .endif
 
-.if ${MK_BHYVE} != "no" || defined(ALL_MODULES)
-_vmm=		vmm
-.endif
-
 .if ${MK_CRYPT} != "no" || defined(ALL_MODULES)
 .if exists(${.CURDIR}/../opencrypto)
 _crypto=	crypto
@@ -402,10 +398,6 @@ _random=	random
 .endif
 .endif
 
-.if ${MK_HYPERV} != "no" || defined(ALL_MODULES)
-_hyperv=	hyperv
-.endif
-
 .if (${MK_INET_SUPPORT} != "no" || ${MK_INET6_SUPPORT} != "no") || \
 	defined(ALL_MODULES)
 _carp=		carp
@@ -505,6 +497,9 @@ _et=		et
 _exca=		exca
 _ext2fs=	ext2fs
 _filemon=	filemon
+.if ${MK_HYPERV} != "no" || defined(ALL_MODULES)
+_hyperv=	hyperv
+.endif
 _i2c=		i2c
 .if ${MK_OFED} != "no" || defined(ALL_MODULES)
 _ibcore=        ibcore
@@ -625,6 +620,10 @@ _qlxge=		qlxge
 _qlxgb=		qlxgb
 _qlxgbe=	qlxgbe
 _sfxge=		sfxge
+
+.if ${MK_BHYVE} != "no" || defined(ALL_MODULES)
+_vmm=		vmm
+.endif
 .endif
 
 .if ${MACHINE_CPUARCH} == "i386"
diff --git a/tools/build/options/WITHOUT_BHYVE b/tools/build/options/WITHOUT_BHYVE
index f60fd17ddba..99420b60485 100644
--- a/tools/build/options/WITHOUT_BHYVE
+++ b/tools/build/options/WITHOUT_BHYVE
@@ -2,3 +2,5 @@
 Set to not build or install
 .Xr bhyve 8 ,
 associated utilities, and examples.
+.Pp
+This option only affects amd64/amd64
diff --git a/tools/build/options/WITH_HYPERV b/tools/build/options/WITH_HYPERV
index a6bdf72ff3e..75753c48bb8 100644
--- a/tools/build/options/WITH_HYPERV
+++ b/tools/build/options/WITH_HYPERV
@@ -1,2 +1,4 @@
 .\" $FreeBSD$
 Set to build and install HyperV utilities.
+.Pp
+This option only affects amd64/amd64 and i386/i386

From 3554444aa12c983ccf1e613d88b85375659cfb9f Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Wed, 10 Dec 2014 06:02:31 +0000
Subject: [PATCH 041/279] Add periods to the end of the lines noting
 architecture caveats for *BHYVE/*HYPERV

---
 tools/build/options/WITHOUT_BHYVE | 2 +-
 tools/build/options/WITH_HYPERV   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/build/options/WITHOUT_BHYVE b/tools/build/options/WITHOUT_BHYVE
index 99420b60485..b80a72697a2 100644
--- a/tools/build/options/WITHOUT_BHYVE
+++ b/tools/build/options/WITHOUT_BHYVE
@@ -3,4 +3,4 @@ Set to not build or install
 .Xr bhyve 8 ,
 associated utilities, and examples.
 .Pp
-This option only affects amd64/amd64
+This option only affects amd64/amd64.
diff --git a/tools/build/options/WITH_HYPERV b/tools/build/options/WITH_HYPERV
index 75753c48bb8..d6ef179412b 100644
--- a/tools/build/options/WITH_HYPERV
+++ b/tools/build/options/WITH_HYPERV
@@ -1,4 +1,4 @@
 .\" $FreeBSD$
 Set to build and install HyperV utilities.
 .Pp
-This option only affects amd64/amd64 and i386/i386
+This option only affects amd64/amd64 and i386/i386.

From acf5ac8780f653e55d1f1907a911d0eb9b38c010 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Wed, 10 Dec 2014 06:33:24 +0000
Subject: [PATCH 042/279] Add a description for WITHOUT_CCD

---
 tools/build/options/WITHOUT_CCD | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 tools/build/options/WITHOUT_CCD

diff --git a/tools/build/options/WITHOUT_CCD b/tools/build/options/WITHOUT_CCD
new file mode 100644
index 00000000000..c55a59de1e1
--- /dev/null
+++ b/tools/build/options/WITHOUT_CCD
@@ -0,0 +1,4 @@
+.\" $FreeBSD$
+Set to not build
+.Xr ccd 8
+and related utilities.

From 989966a8f30ba249620b00b4ea79866199dc74dd Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Wed, 10 Dec 2014 20:43:17 +0000
Subject: [PATCH 043/279] Remove building usr.bin/vi as part of build-tools per
 r275687

---
 Makefile.inc1 | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 56c68ed677d..f2f2b39b43c 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -1393,18 +1393,6 @@ build-tools: .MAKE
 		${MAKE} DIRPRFX=${_tool}/ depend && \
 		${MAKE} DIRPRFX=${_tool}/ all
 .endfor
-	# usr.bin/vi is required to process files in share/termcap
-.if !defined(NO_SHARE)
-.for _tool in \
-    usr.bin/vi
-	${_+_}@${ECHODIR} "===> ${_tool} (obj,depend,all,install)"; \
-		cd ${.CURDIR}/${_tool} && \
-		${MAKE} DIRPRFX=${_tool}/ obj && \
-		${MAKE} DIRPRFX=${_tool}/ depend MK_MAN=no SUBDIR= && \
-		${MAKE} DIRPRFX=${_tool}/ all MK_MAN=no SUBDIR= && \
-		${MAKE} DIRPRFX=${_tool}/ install MK_MAN=no SUBDIR= DESTDIR=${WORLDTMP}/
-.endfor
-.endif
 
 #
 # kernel-tools: Build kernel-building tools

From eb1f96f4ca2b0f8cfbb477e64f23143e6a8a2375 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Wed, 17 Dec 2014 01:48:02 +0000
Subject: [PATCH 044/279] Do not enable DEBUG_FILES in stages 1. through 3. of
 the build

Blindly building debugging symbols into the toolchain slows down builds
considerably and eats up a lot more resources than required, both in
terms of memory and disk I/O

This change has no noticeable effect for those who do not build with
MK_DEBUG_FILES=no (which is the default)

Sponsored by: EMC / Isilon Storage Division
---
 Makefile.inc1 | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index f2f2b39b43c..a965b6d97e9 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -261,7 +261,8 @@ BMAKE=		MAKEOBJDIRPREFIX=${WORLDTMP} \
 		MK_HTML=no MK_INFO=no NO_LINT=yes MK_MAN=no \
 		-DNO_PIC MK_PROFILE=no -DNO_SHARED \
 		-DNO_CPU_CFLAGS MK_WARNS=no MK_CTF=no \
-		MK_CLANG_FULL=no MK_LLDB=no MK_TESTS=no
+		MK_CLANG_FULL=no MK_LLDB=no MK_TESTS=no \
+		MK_DEBUG_FILES=no
 
 # build-tools stage
 TMAKE=		MAKEOBJDIRPREFIX=${OBJTREE} \
@@ -273,12 +274,13 @@ TMAKE=		MAKEOBJDIRPREFIX=${OBJTREE} \
 		-DNO_LINT \
 		-DNO_CPU_CFLAGS \
 		MK_WARNS=no MK_CTF=no MK_CLANG_FULL=no MK_LLDB=no MK_MAN=no \
-		MK_SHAREDOCS=no MK_TESTS=no
+		MK_DEBUG_FILES=no \
+		MK_SHAREDOCS=no MK_TESTS=no \
 
 # cross-tools stage
 XMAKE=		TOOLS_PREFIX=${WORLDTMP} ${BMAKE} \
 		TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
-		MK_GDB=no MK_TESTS=no
+		MK_DEBUG_FILES=no MK_GDB=no MK_TESTS=no
 
 # kernel-tools stage
 KTMAKEENV=	INSTALL="sh ${.CURDIR}/tools/install.sh" \
@@ -1471,7 +1473,7 @@ NXBMAKE=	${NXBENV} ${MAKE} \
 		TBLGEN=${OBJTREE}/nxb-bin/usr/bin/tblgen \
 		CLANG_TBLGEN=${OBJTREE}/nxb-bin/usr/bin/clang-tblgen \
 		MACHINE=${TARGET} MACHINE_ARCH=${TARGET_ARCH} \
-		MK_GDB=no MK_TESTS=no \
+		MK_DEBUG_FILES=no MK_GDB=no MK_TESTS=no \
 		SSP_CFLAGS= \
 		MK_HTML=no MK_INFO=no NO_LINT=yes MK_MAN=no \
 		-DNO_PIC MK_PROFILE=no -DNO_SHARED \
@@ -2008,6 +2010,7 @@ XDEV_CPUTYPE?=${TARGET_CPUTYPE}
 NOFUN=-DNO_FSCHG MK_HTML=no MK_INFO=no -DNO_LINT \
 	MK_MAN=no MK_NLS=no MK_PROFILE=no \
 	MK_KERBEROS=no MK_RESCUE=no MK_TESTS=no MK_WARNS=no \
+	MK_DEBUG_FILES=no \
 	MK_VI=no \
 	TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
 	CPUTYPE=${XDEV_CPUTYPE} -DNO_SHARE

From 4e816302f35ae2df07c048744ac56f40d18111d4 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Wed, 17 Dec 2014 01:48:32 +0000
Subject: [PATCH 045/279] Remove leftover MK_VI == no from previous work that's
 no longer needed

---
 Makefile.inc1 | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index a965b6d97e9..b642eaeaaa4 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -2011,7 +2011,6 @@ NOFUN=-DNO_FSCHG MK_HTML=no MK_INFO=no -DNO_LINT \
 	MK_MAN=no MK_NLS=no MK_PROFILE=no \
 	MK_KERBEROS=no MK_RESCUE=no MK_TESTS=no MK_WARNS=no \
 	MK_DEBUG_FILES=no \
-	MK_VI=no \
 	TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
 	CPUTYPE=${XDEV_CPUTYPE} -DNO_SHARE
 

From f0fc03360f0561baeb4552a92dd3cf1e46153fc3 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Thu, 18 Dec 2014 18:35:47 +0000
Subject: [PATCH 046/279] Try building a little less groff for build-tools

---
 Makefile.inc1 | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index b642eaeaaa4..1c89323bf44 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -1236,7 +1236,8 @@ _gperf=		gnu/usr.bin/gperf
 .endif
 
 .if ${MK_GROFF} != "no"
-_groff=		gnu/usr.bin/groff
+_groff=		gnu/usr.bin/groff/src/libs \
+		gnu/usr.bin/groff/src/roff/groff
 .endif
 
 .if ${MK_VT} != "no"

From 543028670530848ed9b2eb16bc942a3ce65d75b7 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Thu, 18 Dec 2014 19:02:25 +0000
Subject: [PATCH 047/279] Conditionally build ex, routed, rtquery, and vi
 support into rescue, based on their respective MK_ flags

---
 rescue/rescue/Makefile | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/rescue/rescue/Makefile b/rescue/rescue/Makefile
index e08ec2953f1..c984449881e 100644
--- a/rescue/rescue/Makefile
+++ b/rescue/rescue/Makefile
@@ -97,7 +97,7 @@ CRUNCH_PROGS_sbin= badsect					\
 	mount_msdosfs mount_nfs mount_nullfs			\
 	mount_udf mount_unionfs newfs				\
 	newfs_msdos nos-tun ping reboot				\
-	restore rcorder route routed rtquery rtsol savecore	\
+	restore rcorder route rtsol savecore	\
 	spppcontrol swapon sysctl tunefs umount
 
 .if ${MK_ATM} != "no"
@@ -111,6 +111,9 @@ CRUNCH_PROGS_sbin+= ping6
 .if ${MK_IPFILTER} != "no"
 CRUNCH_PROGS_sbin+= ipf
 .endif
+.if ${MK_ROUTED} != "no"
+CRUNCH_PROGS_sbin+= routed rtquery
+.endif
 .if ${MK_ZFS} != "no"
 CRUNCH_PROGS_sbin+= zfs
 CRUNCH_PROGS_sbin+= zpool
@@ -199,8 +202,10 @@ CRUNCH_LIBS+= -lcrypto
 .endif
 CRUNCH_LIBS+= -lmd
 
+.if ${MK_VI} != "no"
 CRUNCH_PROGS_usr.bin+= vi
 CRUNCH_ALIAS_vi= ex
+.endif
 
 CRUNCH_PROGS_usr.bin+= id
 CRUNCH_ALIAS_id= groups whoami

From 3c24213ac24f9400664a85380cd1df022ef549e1 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Thu, 18 Dec 2014 21:22:23 +0000
Subject: [PATCH 048/279] Parallelize bootstrap-tools

Based on a patch submitted via -arch:
https://lists.freebsd.org/pipermail/freebsd-arch/2014-December/016493.html

MFC after: 1 week
Submitted by: Jia-Shiun Li <jiashiun@gmail.com>
Sponsored by: EMC / Isilon Storage Division
---
 Makefile.inc1 | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 1944cbe25a1..20a6b7d0c5b 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -1235,6 +1235,8 @@ legacy:
 #
 # bootstrap-tools: Build tools needed for compatibility
 #
+_bt=		_bootstrap-tools	
+
 .if ${MK_GAMES} != "no"
 _strfile=	games/fortune/strfile
 .endif
@@ -1246,6 +1248,8 @@ _gperf=		gnu/usr.bin/gperf
 .if ${MK_GROFF} != "no"
 _groff=		gnu/usr.bin/groff/src/libs \
 		gnu/usr.bin/groff/src/roff/groff
+
+${_bt}-gnu/usr.bin/groff/src/roff/groff: ${_bt}-gnu/usr.bin/groff/src/libs
 .endif
 
 .if ${MK_VT} != "no"
@@ -1259,11 +1263,15 @@ _sed=		usr.bin/sed
 .if ${BOOTSTRAPPING} < 1000002
 _m4=		lib/libohash \
 		usr.bin/m4
+
+${_bt}-usr.bin/m4: ${_bt}-lib/libohash
 .endif
 
 .if ${BOOTSTRAPPING} < 1000013
 _yacc=		lib/liby \
 		usr.bin/yacc
+
+${_bt}-usr.bin/yacc: ${_bt}-lib/liby
 .endif
 
 .if ${BOOTSTRAPPING} < 1000014
@@ -1273,6 +1281,8 @@ _crunch=	usr.sbin/crunch
 .if ${BOOTSTRAPPING} < 1000026
 _nmtree=	lib/libnetbsd \
 		usr.sbin/nmtree
+
+${_bt}-usr.sbin/nmtree: ${_bt}-lib/libnetbsd
 .endif
 
 .if ${BOOTSTRAPPING} < 1000027
@@ -1299,6 +1309,9 @@ _clang_tblgen= \
 	lib/clang/libllvmtablegen \
 	usr.bin/clang/tblgen \
 	usr.bin/clang/clang-tblgen
+
+${_bt}-usr.bin/clang/clang-tblgen: ${_bt}-lib/clang/libllvmtablegen ${_bt}-lib/clang/libllvmsupport
+${_bt}-usr.bin/clang/tblgen: ${_bt}-lib/clang/libllvmtablegen ${_bt}-lib/clang/libllvmsupport
 .endif
 
 # dtrace tools are required for older bootstrap env and cross-build
@@ -1324,12 +1337,16 @@ _kerberos5_bootstrap_tools= \
 	kerberos5/tools/asn1_compile \
 	kerberos5/tools/slc \
 	usr.bin/compile_et
+
+${_bt}-kerberos5/tools/slc: ${_bt}-kerberos5/lib/libroken
+${_bt}-kerberos5/tools/asn1_compile: ${_bt}-kerberos5/lib/libroken
 .endif
 
+bootstrap-tools: .PHONY
+
 #	Please document (add comment) why something is in 'bootstrap-tools'.
 #	Try to bound the building of the bootstrap-tool to just the
 #	FreeBSD versions that need the tool built at this stage of the build.
-bootstrap-tools: .MAKE
 .for _tool in \
     ${_clang_tblgen} \
     ${_kerberos5_bootstrap_tools} \
@@ -1354,12 +1371,15 @@ bootstrap-tools: .MAKE
     ${_crunch} \
     ${_nmtree} \
     ${_vtfontcvt}
+${_bt}-${_tool}: .PHONY .MAKE
 	${_+_}@${ECHODIR} "===> ${_tool} (obj,depend,all,install)"; \
 		cd ${.CURDIR}/${_tool} && \
 		${MAKE} DIRPRFX=${_tool}/ obj && \
 		${MAKE} DIRPRFX=${_tool}/ depend && \
 		${MAKE} DIRPRFX=${_tool}/ all && \
 		${MAKE} DIRPRFX=${_tool}/ DESTDIR=${MAKEOBJDIRPREFIX}/legacy install
+
+bootstrap-tools: ${_bt}-${_tool}
 .endfor
 
 #

From 6bb8b90bc60330c7fa9c811ea3cc98bd60ca0142 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Fri, 19 Dec 2014 00:34:10 +0000
Subject: [PATCH 049/279] Put all of gnu/usr.bin/groff back

Forgot about those pesky definitions files..
---
 Makefile.inc1 | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 20a6b7d0c5b..991e7b950d0 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -1246,10 +1246,7 @@ _gperf=		gnu/usr.bin/gperf
 .endif
 
 .if ${MK_GROFF} != "no"
-_groff=		gnu/usr.bin/groff/src/libs \
-		gnu/usr.bin/groff/src/roff/groff
-
-${_bt}-gnu/usr.bin/groff/src/roff/groff: ${_bt}-gnu/usr.bin/groff/src/libs
+_groff=		gnu/usr.bin/groff
 .endif
 
 .if ${MK_VT} != "no"

From ee1108eba87a8d7e9c88602a85fd57b4e3a379b8 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Wed, 14 Jan 2015 14:49:05 +0000
Subject: [PATCH 050/279] In vm_extra_install_base(), do not install waagent in
 the openstack image, because it is not used.  This appears to be a copy
 mistake.

Remove vm_extra_install_base() from the openstack.conf
entirely, since it does not need to be overridden.

Sponsored by:	The FreeBSD Foundation
---
 release/tools/openstack.conf | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/release/tools/openstack.conf b/release/tools/openstack.conf
index 3e0e857e9d8..a88e0f8e37d 100644
--- a/release/tools/openstack.conf
+++ b/release/tools/openstack.conf
@@ -9,15 +9,6 @@ export VM_EXTRA_PACKAGES="net/cloud-init"
 # Set to a list of third-party software to enable in rc.conf(5).
 export VM_RC_LIST="cloudinit"
 
-vm_extra_install_base() {
-	fetch -o ${DESTDIR}/usr/sbin/waagent \
-		http://people.freebsd.org/~gjb/waagent
-	chmod +x ${DESTDIR}/usr/sbin/waagent
-	rm -f ${DESTDIR}/etc/resolv.conf
-
-	return 0
-}
-
 vm_extra_pre_umount() {
 	echo 'sshd_enable="YES"' >> ${DESTDIR}/etc/rc.conf
 	echo 'ifconfig_DEFAULT="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf

From ab7c3d12db7f490a23c1b94d80696d2aefb4ed12 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Wed, 14 Jan 2015 15:23:58 +0000
Subject: [PATCH 051/279] Enable the textmode console by default for VM images,
 since there is no way to tell if the environment will be able to use the
 graphics-mode console.

Sponsored by:	The FreeBSD Foundation
---
 release/tools/vmimage.subr | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index 2fd70fffea5..52530e8397f 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -84,6 +84,12 @@ vm_install_base() {
 			>> ${DESTDIR}/etc/fstab
 	fi
 
+	# Set hw.vga.textmode=1, with the assumption that the hypervisor
+	# will not be capable of using the graphics console mode. 
+	echo '# Comment the next line to enable graphical console mode' \
+		>> ${DESTDIR}/boot/loader.conf
+	echo 'hw.vga.textmode=1' >> ${DESTDIR}/boot/loader.conf
+
 	mkdir -p ${DESTDIR}/dev
 	mount -t devfs devfs ${DESTDIR}/dev
 	chroot ${DESTDIR} /usr/bin/newaliases

From 9af30cb24cb1ddb3f822bd0914bfc206a6fb3187 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Wed, 14 Jan 2015 15:45:18 +0000
Subject: [PATCH 052/279] Enable password-less sudo for openstack images.

Sponsored by:	The FreeBSD Foundation
---
 release/tools/openstack.conf | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/release/tools/openstack.conf b/release/tools/openstack.conf
index a88e0f8e37d..3eb7b1d3199 100644
--- a/release/tools/openstack.conf
+++ b/release/tools/openstack.conf
@@ -13,6 +13,10 @@ vm_extra_pre_umount() {
 	echo 'sshd_enable="YES"' >> ${DESTDIR}/etc/rc.conf
 	echo 'ifconfig_DEFAULT="SYNCDHCP"' >> ${DESTDIR}/etc/rc.conf
 
+	# Openstack wants sudo(8) usable by default without a password.
+	echo 'ALL ALL=(ALL) NOPASSWD:ALL' >> \
+		${DESTDIR}/usr/local/etc/sudoers.d/cloud-init
+
 	rm -f ${DESTDIR}/etc/resolv.conf
 
 	return 0

From b4ff8f11d4e26722acf407ba5d548be3b46460c9 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Wed, 14 Jan 2015 16:27:43 +0000
Subject: [PATCH 053/279] Update the VM_EXTRA_PACKAGES list for the openstack
 images.

The documentation suggests doing a "just fetch this
and run it"-style bootstrap, from which the list of
dependencies was obtained (in github, at:
pellaeon/bsd-cloudinit-installer)

There is one Python dependency unmet, oslo.config,
which is not in the Ports Collection.

Sponsored by:	The FreeBSD Foundation
---
 release/tools/openstack.conf | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/release/tools/openstack.conf b/release/tools/openstack.conf
index 3eb7b1d3199..e5bfd418704 100644
--- a/release/tools/openstack.conf
+++ b/release/tools/openstack.conf
@@ -4,7 +4,9 @@
 #
 
 # Set to a list of packages to install.
-export VM_EXTRA_PACKAGES="net/cloud-init"
+export VM_EXTRA_PACKAGES="net/cloud-init devel/py-pbr devel/py-iso8601 \
+net/py-eventlet net/py-netaddr comms/py-serial devel/py-six \
+devel/py-babel net/py-oauth net/py-netifaces"
 
 # Set to a list of third-party software to enable in rc.conf(5).
 export VM_RC_LIST="cloudinit"

From 5e8706887539e6efc12953fd419182b7a1f9aa05 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Wed, 14 Jan 2015 16:42:54 +0000
Subject: [PATCH 054/279] Add a comment to note that setting hw.vga.textmode=1
 is temporary.

Sponsored by:	The FreeBSD Foundation
---
 release/tools/vmimage.subr | 1 +
 1 file changed, 1 insertion(+)

diff --git a/release/tools/vmimage.subr b/release/tools/vmimage.subr
index 52530e8397f..e200d0af6ff 100644
--- a/release/tools/vmimage.subr
+++ b/release/tools/vmimage.subr
@@ -86,6 +86,7 @@ vm_install_base() {
 
 	# Set hw.vga.textmode=1, with the assumption that the hypervisor
 	# will not be capable of using the graphics console mode. 
+	# XXX: Revert this before merging to head.
 	echo '# Comment the next line to enable graphical console mode' \
 		>> ${DESTDIR}/boot/loader.conf
 	echo 'hw.vga.textmode=1' >> ${DESTDIR}/boot/loader.conf

From 0cdabd97fb65726a1489fa365e8227eac023df92 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Fri, 16 Jan 2015 15:37:07 +0000
Subject: [PATCH 055/279] Remove vm_extra_install_base() for the Azure image,
 now that the waagent exists in the ports tree.

Add sysutils/azure-agent to the VM_EXTRA_PACKAGES list.

In vm_extra_pre_umount(), remove the explicit pkg(8) install
list, as dependencies are resolved by sysutils/azure-agent.

Sponsored by:	The FreeBSD Foundation
---
 release/tools/azure.conf | 13 +------------
 1 file changed, 1 insertion(+), 12 deletions(-)

diff --git a/release/tools/azure.conf b/release/tools/azure.conf
index 3e4a4864adf..922f8d9d406 100644
--- a/release/tools/azure.conf
+++ b/release/tools/azure.conf
@@ -6,25 +6,14 @@
 # Set to a list of packages to install.
 # Example:
 #export VM_EXTRA_PACKAGES="www/apache24"
-export VM_EXTRA_PACKAGES=
+export VM_EXTRA_PACKAGES="sysutils/azure-agent"
 
 # Set to a list of third-party software to enable in rc.conf(5).
 # Example:
 #export VM_RC_LIST="apache24"
 export VM_RC_LIST=
 
-vm_extra_install_base() {
-	fetch -o ${DESTDIR}/usr/sbin/waagent \
-		http://people.freebsd.org/~gjb/waagent
-	chmod +x ${DESTDIR}/usr/sbin/waagent
-	rm -f ${DESTDIR}/etc/resolv.conf
-
-	return 0
-}
-
 vm_extra_pre_umount() {
-	chroot ${DESTDIR} env ASSUME_ALWAYS_YES=yes /usr/sbin/pkg install -y \
-		python python2 python27 py27-asn1 sudo bash
 	chroot ${DESTDIR} /usr/sbin/waagent -verbose -install
 	yes | chroot ${DESTDIR} /usr/sbin/waagent -deprovision
 	echo 'sshd_enable="YES"' >> ${DESTDIR}/etc/rc.conf

From 99e908c904c8dd9487ec3fb702fa7024ea00f86e Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Fri, 16 Jan 2015 17:05:35 +0000
Subject: [PATCH 056/279] Add a 'list-cloudware' target to print the list of
 supported CLOUDWARE values and a description.

Add the AZURE_DESC and OPENSTACK_DESC descriptions.

Sponsored by:	The FreeBSD Foundation
---
 release/Makefile.vm | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/release/Makefile.vm b/release/Makefile.vm
index bb5311fdc66..f28881eca7a 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -13,7 +13,9 @@ VMBASE?=	vm
 CLOUDWARE?=	AZURE \
 		OPENSTACK
 AZURE_FORMAT=	vhdf
+AZURE_DESC=	Microsoft Azure platform image
 OPENSTACK_FORMAT=qcow2
+OPENSTACK_DESC=	OpenStack platform image
 
 .if defined(WITH_CLOUDWARE) && !empty(WITH_CLOUDWARE) && !empty(CLOUDWARE)
 . for _CW in ${CLOUDWARE}
@@ -62,3 +64,10 @@ vm-image:
 	touch ${.TARGET}
 
 vm-cloudware:	${CLOUDTARGETS}
+
+list-cloudware:
+.if !empty(CLOUDWARE)
+. for _CW in ${CLOUDWARE}
+	@${ECHO} "${_CW:tu}: ${${_CW:tu}_DESC}"
+. endfor
+.endif

From b513fff0b2f34490c0eade88c51f613e1f0a849d Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Fri, 16 Jan 2015 17:07:35 +0000
Subject: [PATCH 057/279] Update release(7):

 - Add a "CLOUD HOSTING MACHINE IMAGES" section,
   documenting the CLOUDWARE and WITH_CLOUDWARE
   make(1) environment variables.

 - Document the vm-cloudware and list-cloudware
   targets.

 - Add release/Makefile.vm, release/tools/*.conf
   and release/tools/vmimage.subr to FILES.

Sponsored by:	The FreeBSD Foundation
---
 share/man/man7/release.7 | 57 +++++++++++++++++++++++++++++++++++++++-
 1 file changed, 56 insertions(+), 1 deletion(-)

diff --git a/share/man/man7/release.7 b/share/man/man7/release.7
index 93aec89618a..1284915de69 100644
--- a/share/man/man7/release.7
+++ b/share/man/man7/release.7
@@ -24,7 +24,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd October 2, 2014
+.Dd January 16, 2015
 .Dt RELEASE 7
 .Os
 .Sh NAME
@@ -406,6 +406,47 @@ See
 for valid format values
 .Pq requires version 20140927 or later .
 .El
+.Sh CLOUD HOSTING MACHINE IMAGES
+The
+.Fx
+release build tools support building virtual machine images for various
+cloud hosting providers, each with their own specific configuration to
+include support for each hosting provider by default.
+.Pp
+The following
+.Xr make 1
+environment variables are supported:
+.Pp
+.Bl -tag -width Ev
+.It Va CLOUDWARE
+Set to a list of one or more cloud hosting providers, enclosed in quotes.
+Requires
+.Va WITH_CLOUDWARE
+to also be set.
+.It Va WITH_CLOUDWARE
+Set to a non-empty value to enable building virtual machine images
+for various cloud hosting providers.
+Requires
+.Va CLOUDWARE
+to also be set.
+.El
+.Pp
+Additionally, the
+.Va CLOUDWARE
+and
+.Va WITH_CLOUDWARE
+variables can be added to
+.Pa release.conf ,
+and used in conjunction with
+.Pa release.sh .
+.Pp
+For a list of supported
+.Va CLOUDWARE
+values, run:
+.Bd -literal -offset indent
+cd /usr/src
+make -C release list-cloudware
+.Ed
 .Sh MAKEFILE TARGETS
 The release makefile
 .Pq Pa src/release/Makefile
@@ -470,6 +511,17 @@ target requires the
 .Va WITH_VMIMAGES
 .Xr make 1
 envirionment variable to be set to a non-null value.
+.It Cm vm-cloudware
+Builds
+.Fx
+virtual machine images for various cloud hosting providers.
+See
+.Qq CLOUD HOSTING MACHINE IMAGES
+for implementation details.
+.It Cm list-cloudware
+Displays the list of valid
+.Va CLOUDWARE
+values.
 .El
 .Pp
 Major subtargets called by targets above:
@@ -576,8 +628,11 @@ Typically, one only needs to set
 .It Pa /usr/src/Makefile
 .It Pa /usr/src/Makefile.inc1
 .It Pa /usr/src/release/Makefile
+.It Pa /usr/src/release/Makefile.vm
 .It Pa /usr/src/release/release.sh
 .It Pa /usr/src/release/release.conf.sample
+.It Pa /usr/src/release/tools/*.conf
+.It Pa /usr/src/release/tools/vmimage.subr
 .El
 .Sh EXAMPLES
 The following sequence of commands can be used to build a

From 85311c29b012a1660d679ce1fa17841f13f8ee08 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Fri, 16 Jan 2015 17:40:30 +0000
Subject: [PATCH 058/279] Add 'list-vmtargets' target, which produces a list of
 all supported VM and cloud provider images.

Add VHD_DESC, VMDK_DESC, QCOW2_DESC, RAW_DESC image
descriptions.

Format the output to make a bit more readable.

Update release(7) to document the list-vmtargets target.

Sponsored by:	The FreeBSD Foundation
---
 release/Makefile.vm      | 16 +++++++++++++++-
 share/man/man7/release.7 | 16 ++++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/release/Makefile.vm b/release/Makefile.vm
index f28881eca7a..4652c164a76 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -10,6 +10,11 @@ VMFORMATS?=	vhd vmdk qcow2 raw
 VMSIZE?=	20G
 VMBASE?=	vm
 
+VHD_DESC=	Azure, VirtualPC, Hyper-V, Xen disk image
+VMDK_DESC=	VMWare, VirtualBox disk image
+QCOW2_DESC=	Qemu, KVM disk image
+RAW_DESC=	Unformatted raw disk image
+
 CLOUDWARE?=	AZURE \
 		OPENSTACK
 AZURE_FORMAT=	vhdf
@@ -65,9 +70,18 @@ vm-image:
 
 vm-cloudware:	${CLOUDTARGETS}
 
+list-vmtargets:	list-cloudware
+	@${ECHO}
+	@${ECHO} "Supported virtual machine disk image formats:"
+.for FORMAT in ${VMFORMATS:tu}
+	@${ECHO} "  ${FORMAT:tl}: ${${FORMAT}_DESC}"
+.endfor
+
 list-cloudware:
 .if !empty(CLOUDWARE)
+	@${ECHO}
+	@${ECHO} "Supported cloud hosting provider images:"
 . for _CW in ${CLOUDWARE}
-	@${ECHO} "${_CW:tu}: ${${_CW:tu}_DESC}"
+	@${ECHO} "  ${_CW:tu}: ${${_CW:tu}_DESC}"
 . endfor
 .endif
diff --git a/share/man/man7/release.7 b/share/man/man7/release.7
index 1284915de69..5cd91b68285 100644
--- a/share/man/man7/release.7
+++ b/share/man/man7/release.7
@@ -406,6 +406,16 @@ See
 for valid format values
 .Pq requires version 20140927 or later .
 .El
+.Pp
+For a list of supported
+.Va VMFORMATS
+values
+.Pq including cloud hosting provider formats
+along with a brief description, run:
+.Bd -literal -offset indent
+cd /usr/src
+make -C release list-vmtargets
+.Ed
 .Sh CLOUD HOSTING MACHINE IMAGES
 The
 .Fx
@@ -522,6 +532,12 @@ for implementation details.
 Displays the list of valid
 .Va CLOUDWARE
 values.
+.It Cm list-vmtargets
+Displays the list of valid
+.Va VMFORMAT
+and
+.Va CLOUDWARE
+values.
 .El
 .Pp
 Major subtargets called by targets above:

From 5f87b8f5f9e0754c86ed3e2778eeff6c4656d6ea Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Fri, 16 Jan 2015 19:27:19 +0000
Subject: [PATCH 059/279] Add initial support for the GCE (Google Compute
 Engine) cloud hosting provider image.

Many thanks to swills@ for his work on getting this to
this point.

Submitted by:	swills
Sponsored by:	The FreeBSD Foundation
---
 release/Makefile.vm    |  3 ++
 release/tools/gce.conf | 96 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 99 insertions(+)
 create mode 100644 release/tools/gce.conf

diff --git a/release/Makefile.vm b/release/Makefile.vm
index 4652c164a76..5c052fa04e7 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -16,9 +16,12 @@ QCOW2_DESC=	Qemu, KVM disk image
 RAW_DESC=	Unformatted raw disk image
 
 CLOUDWARE?=	AZURE \
+		GCE \
 		OPENSTACK
 AZURE_FORMAT=	vhdf
 AZURE_DESC=	Microsoft Azure platform image
+GCE_FORMAT=	raw
+GCE_DESC=	Google Compute Engine image
 OPENSTACK_FORMAT=qcow2
 OPENSTACK_DESC=	OpenStack platform image
 
diff --git a/release/tools/gce.conf b/release/tools/gce.conf
new file mode 100644
index 00000000000..bccf1f45fa8
--- /dev/null
+++ b/release/tools/gce.conf
@@ -0,0 +1,96 @@
+#!/bin/sh
+#
+# $FreeBSD$
+#
+
+# Set to a list of packages to install.
+export VM_EXTRA_PACKAGES="firstboot-freebsd-update firstboot-pkgs google-cloud-sdk google-daemon panicmail sudo firstboot-growfs google-startup-scripts"
+
+# Set to a list of third-party software to enable in rc.conf(5).
+export VM_RC_LIST="google_accounts_manager ntpd"
+
+vm_extra_install_base() {
+	echo 'search google.internal' > ${DESTDIR}/etc/resolv.conf
+	echo 'nameserver 169.254.169.254' >> ${DESTDIR}/etc/resolv.conf
+	echo 'nameserver 8.8.8.8' >> ${DESTDIR}/etc/resolv.conf
+}
+
+vm_extra_pre_umount() {
+        cat << EOF >> ${DESTDIR}/etc/rc.conf
+console="comconsole"
+dumpdev="AUTO"
+ifconfig_vtnet0="SYNCDHCP mtu 1460"
+ntpd_sync_on_start="YES"
+ntpd_enable="YES"
+sshd_enable="YES"
+google_accounts_manager_enable="YES"
+#disabled until I can figure out why the reboot for updates is hanging
+#firstboot_freebsd_update_enable="YES"
+#firstboot_pkgs_enable="YES"
+# need to fill in something here
+#firstboot_pkgs_list=""
+panicmail_autosubmit="YES"
+firstboot_growfs_enable="YES"
+google_startup_enable="YES"
+EOF
+
+        cat << EOF >> ${DESTDIR}/boot/loader.conf
+autoboot_delay="-1"
+beastie_disable="YES"
+loader_logo="none"
+hw.memtest.tests="0"
+console="comconsole"
+hw.vtnet.mq_disable=1
+kern.timecounter.hardware=ACPI-safe
+aesni_load="YES"
+nvme_load="YES"
+EOF
+
+	echo '169.254.169.254 metadata.google.internal metadata' > \
+		${DESTDIR}/etc/hosts
+
+        # overwrite ntp.conf
+        cat << EOF > ${DESTDIR}/etc/ntp.conf
+server metadata.google.internal iburst
+
+restrict default kod nomodify notrap nopeer noquery
+restrict -6 default kod nomodify notrap nopeer noquery
+
+restrict 127.0.0.1
+restrict -6 ::1
+restrict 127.127.1.0
+EOF
+
+        cat << EOF >> ${DESTDIR}/etc/syslog.conf
+*.err;kern.warning;auth.notice;mail.crit                /dev/console
+EOF
+
+        cat << EOF >> ${DESTDIR}/etc/ssh/sshd_config
+ChallengeResponseAuthentication no
+X11Forwarding no
+AcceptEnv LANG
+Ciphers aes128-ctr,aes192-ctr,aes256-ctr,arcfour256,arcfour128,aes128-cbc,3des-cbc
+AllowAgentForwarding no
+ClientAliveInterval 420
+EOF
+
+        cat << EOF >> ${DESTDIR}/etc/crontab
+0       3       *       *       *       root    /usr/sbin/freebsd-update cron
+EOF
+
+        cat << EOF >> ${DESTDIR}/etc/sysctl.conf
+net.inet.icmp.drop_redirect=1
+net.inet.ip.redirect=0
+net.inet.tcp.blackhole=2
+net.inet.udp.blackhole=1
+kern.ipc.somaxconn=1024
+debug.trace_on_panic=1
+debug.debugger_on_panic=0
+EOF
+
+        sed -E -i '' 's/^([^#].*[[:space:]])on/\1off/' ${DESTDIR}/etc/ttys
+
+        touch ${DESTDIR}/firstboot
+
+	return 0
+}

From 0700a396cb52143194d38cb683a5a23f36a894fd Mon Sep 17 00:00:00 2001
From: Allan Jude <allanjude@FreeBSD.org>
Date: Sun, 18 Jan 2015 17:25:41 +0000
Subject: [PATCH 060/279] Fix minor syntax and grammar errors in the markup of
 the ee(1) man page

Differential Revision:	https://reviews.freebsd.org/D1552
Submitted by:	bcallah@openbsd.org (original)
Approved by:	wblock (mentor)
MFC after:	1 week
Sponsored by:	ScaleEngine Inc.
---
 contrib/ee/ee.1 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/contrib/ee/ee.1 b/contrib/ee/ee.1
index b66c8aae17a..d6558a14c6f 100644
--- a/contrib/ee/ee.1
+++ b/contrib/ee/ee.1
@@ -329,8 +329,8 @@ A window showing the keyboard operations that can be performed can be
 displayed or not.
 .IP "\fBemacs keys\fR"
 Control keys may be given bindings similar to emacs, or not.
-.IP "\f16 bit characters\fR"
-Toggles whether sixteen bit characters are handled as one 16-bit quantities or 
+.IP "\fB16 bit characters\fR"
+Toggles whether sixteen bit characters are handled as one 16-bit quantity or 
 two 8-bit quantities.  This works primarily with the Chinese Big 5 code set.
 .RE
 .PP
@@ -461,7 +461,7 @@ Turns off display of eight bit characters (they are displayed as their decimal
 value inside angle brackets, e.g., "<220>").
 .IP \fB16bit\fR
 Turns on handling of 16-bit characters.
-.IP \fbno16bit\fR
+.IP \fBno16bit\fR
 Turns off handling of 16-bit characters.
 .IP \fBemacs\fR
 Turns on emacs key bindings.

From 6ae52b27b6955cf609e794e15681b392d720ad97 Mon Sep 17 00:00:00 2001
From: Adrian Chadd <adrian@FreeBSD.org>
Date: Sun, 18 Jan 2015 17:43:00 +0000
Subject: [PATCH 061/279] Oops - use the correct argument order for
 ar9300_set_beacon().

(It's only an issue in AP/adhoc modes. But, still. Grr.)
---
 sys/contrib/dev/ath/ath_hal/ar9300/ar9300_freebsd.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sys/contrib/dev/ath/ath_hal/ar9300/ar9300_freebsd.c b/sys/contrib/dev/ath/ath_hal/ar9300/ar9300_freebsd.c
index 3846dcb2bce..315086c2d42 100644
--- a/sys/contrib/dev/ath/ath_hal/ar9300/ar9300_freebsd.c
+++ b/sys/contrib/dev/ath/ath_hal/ar9300/ar9300_freebsd.c
@@ -606,8 +606,8 @@ ar9300_freebsd_beacon_init(struct ath_hal *ah, uint32_t next_beacon,
     uint32_t beacon_period)
 {
 
-	ar9300_beacon_init(ah, AH_PRIVATE(ah)->ah_opmode,
-	    next_beacon, beacon_period);
+	ar9300_beacon_init(ah, next_beacon, beacon_period,
+	    AH_PRIVATE(ah)->ah_opmode);
 }
 
 HAL_BOOL

From b2bdc62a95a0241981d79c46eb79e3208eeec5f8 Mon Sep 17 00:00:00 2001
From: Adrian Chadd <adrian@FreeBSD.org>
Date: Sun, 18 Jan 2015 18:06:40 +0000
Subject: [PATCH 062/279] Refactor / restructure the RSS code into generic,
 IPv4 and IPv6 specific bits.

The motivation here is to eventually teach netisr and potentially
other networking subsystems a bit more about how RSS work queues / buckets
are configured so things have a hope of auto-configuring in the future.

* net/rss_config.[ch] takes care of the generic bits for doing
  configuration, hash function selection, etc;
* topelitz.[ch] is now in net/ rather than netinet/;
* (and would be in libkern if it didn't directly include RSS_KEYSIZE;
  that's a later thing to fix up.)
* netinet/in_rss.[ch] now just contains the IPv4 specific methods;
* and netinet/in6_rss.[ch] now just contains the IPv6 specific methods.

This should have no functional impact on anyone currently using
the RSS support.

Differential Revision:	D1383
Reviewed by:	gnn, jfv (intel driver bits)
---
 sys/conf/files                  |   6 +-
 sys/dev/e1000/if_igb.c          |   6 +-
 sys/dev/ixgbe/ixgbe.c           |   2 +-
 sys/dev/ixl/ixl_txrx.c          |   2 +-
 sys/net/if_ethersubr.c          |   2 +-
 sys/net/rss_config.c            | 558 +++++++++++++++++++++++++++++++
 sys/net/rss_config.h            | 123 +++++++
 sys/{netinet => net}/toeplitz.c |   4 +-
 sys/{netinet => net}/toeplitz.h |   0
 sys/netinet/in_pcb.c            |   2 +-
 sys/netinet/in_pcbgroup.c       |   3 +
 sys/netinet/in_rss.c            | 570 +-------------------------------
 sys/netinet/in_rss.h            |  85 -----
 sys/netinet/ip_input.c          |   1 +
 sys/netinet/ip_output.c         |   1 +
 sys/netinet/tcp_timer.c         |   1 +
 sys/netinet/udp_usrreq.c        |   1 +
 sys/netinet6/in6_pcbgroup.c     |   4 +-
 sys/netinet6/in6_rss.c          | 103 ++++++
 sys/netinet6/in6_rss.h          |  45 +++
 sys/netinet6/ip6_output.c       |   3 +-
 sys/netinet6/udp6_usrreq.c      |   3 +-
 22 files changed, 870 insertions(+), 655 deletions(-)
 create mode 100644 sys/net/rss_config.c
 create mode 100644 sys/net/rss_config.h
 rename sys/{netinet => net}/toeplitz.c (97%)
 rename sys/{netinet => net}/toeplitz.h (100%)
 create mode 100644 sys/netinet6/in6_rss.c
 create mode 100644 sys/netinet6/in6_rss.h

diff --git a/sys/conf/files b/sys/conf/files
index 22a20d88e1a..5895cce88ab 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -3270,9 +3270,11 @@ net/radix_mpath.c		standard
 net/raw_cb.c			standard
 net/raw_usrreq.c		standard
 net/route.c			standard
+net/rss_config.c		optional inet rss | inet6 rss
 net/rtsock.c			standard
 net/slcompress.c		optional netgraph_vjc | sppp | \
 					 netgraph_sppp
+net/toeplitz.c			optional inet rss | inet6 rss
 net/vnet.c			optional vimage
 net/zlib.c			optional crypto | geom_uzip | ipsec | \
 					 mxge | netgraph_deflate | \
@@ -3422,7 +3424,7 @@ netinet/in_pcb.c		optional inet | inet6
 netinet/in_pcbgroup.c		optional inet pcbgroup | inet6 pcbgroup
 netinet/in_proto.c		optional inet | inet6
 netinet/in_rmx.c		optional inet
-netinet/in_rss.c		optional inet rss | inet6 rss
+netinet/in_rss.c		optional inet rss
 netinet/ip_divert.c		optional inet ipdivert ipfirewall
 netinet/ip_ecn.c		optional inet | inet6
 netinet/ip_encap.c		optional inet | inet6
@@ -3465,7 +3467,6 @@ netinet/tcp_syncache.c		optional inet | inet6
 netinet/tcp_timer.c		optional inet | inet6
 netinet/tcp_timewait.c		optional inet | inet6
 netinet/tcp_usrreq.c		optional inet | inet6
-netinet/toeplitz.c		optional inet rss | inet6 rss
 netinet/udp_usrreq.c		optional inet | inet6
 netinet/libalias/alias.c	optional libalias inet | netgraph_nat inet
 netinet/libalias/alias_db.c	optional libalias inet | netgraph_nat inet
@@ -3485,6 +3486,7 @@ netinet6/in6_pcb.c		optional inet6
 netinet6/in6_pcbgroup.c		optional inet6 pcbgroup
 netinet6/in6_proto.c		optional inet6
 netinet6/in6_rmx.c		optional inet6
+netinet6/in6_rss.c		optional inet6 rss
 netinet6/in6_src.c		optional inet6
 netinet6/ip6_forward.c		optional inet6
 netinet6/ip6_gre.c		optional gre inet6
diff --git a/sys/dev/e1000/if_igb.c b/sys/dev/e1000/if_igb.c
index 6e598c4bba6..c875945845f 100644
--- a/sys/dev/e1000/if_igb.c
+++ b/sys/dev/e1000/if_igb.c
@@ -73,6 +73,9 @@
 #include <net/if_arp.h>
 #include <net/if_dl.h>
 #include <net/if_media.h>
+#ifdef	RSS
+#include <net/rss_config.h>
+#endif
 
 #include <net/if_types.h>
 #include <net/if_vlan_var.h>
@@ -85,9 +88,6 @@
 #include <netinet/tcp.h>
 #include <netinet/tcp_lro.h>
 #include <netinet/udp.h>
-#ifdef	RSS
-#include <netinet/in_rss.h>
-#endif
 
 #include <machine/in_cksum.h>
 #include <dev/led/led.h>
diff --git a/sys/dev/ixgbe/ixgbe.c b/sys/dev/ixgbe/ixgbe.c
index bd9407e3a7a..7231c560b34 100644
--- a/sys/dev/ixgbe/ixgbe.c
+++ b/sys/dev/ixgbe/ixgbe.c
@@ -39,7 +39,7 @@
 #include "ixgbe.h"
 
 #ifdef	RSS
-#include <netinet/in_rss.h>
+#include <net/rss_config.h>
 #endif
 
 /*********************************************************************
diff --git a/sys/dev/ixl/ixl_txrx.c b/sys/dev/ixl/ixl_txrx.c
index 0ba9685a07a..6a6ab7c9ccc 100755
--- a/sys/dev/ixl/ixl_txrx.c
+++ b/sys/dev/ixl/ixl_txrx.c
@@ -1378,7 +1378,7 @@ static inline int
 ixl_ptype_to_hash(u8 ptype)
 {
         struct i40e_rx_ptype_decoded	decoded;
-	u8				ex = 0
+	u8				ex = 0;
 
 	decoded = decode_rx_desc_ptype(ptype);
 	ex = decoded.outer_frag;
diff --git a/sys/net/if_ethersubr.c b/sys/net/if_ethersubr.c
index 9ba1f636063..7227788ac08 100644
--- a/sys/net/if_ethersubr.c
+++ b/sys/net/if_ethersubr.c
@@ -63,6 +63,7 @@
 #include <net/if_vlan_var.h>
 #include <net/if_llatbl.h>
 #include <net/pfil.h>
+#include <net/rss_config.h>
 #include <net/vnet.h>
 
 #include <netpfil/pf/pf_mtag.h>
@@ -71,7 +72,6 @@
 #include <netinet/in.h>
 #include <netinet/in_var.h>
 #include <netinet/if_ether.h>
-#include <netinet/in_rss.h>
 #include <netinet/ip_carp.h>
 #include <netinet/ip_var.h>
 #endif
diff --git a/sys/net/rss_config.c b/sys/net/rss_config.c
new file mode 100644
index 00000000000..55f4952fbec
--- /dev/null
+++ b/sys/net/rss_config.c
@@ -0,0 +1,558 @@
+/*-
+ * Copyright (c) 2010-2011 Juniper Networks, Inc.
+ * All rights reserved.
+ *
+ * This software was developed by Robert N. M. Watson under contract
+ * to Juniper Networks, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+__FBSDID("$FreeBSD$");
+
+#include "opt_inet6.h"
+#include "opt_pcbgroup.h"
+
+#ifndef PCBGROUP
+#error "options RSS depends on options PCBGROUP"
+#endif
+
+#include <sys/param.h>
+#include <sys/mbuf.h>
+#include <sys/socket.h>
+#include <sys/priv.h>
+#include <sys/kernel.h>
+#include <sys/smp.h>
+#include <sys/sysctl.h>
+#include <sys/sbuf.h>
+
+#include <net/if.h>
+#include <net/if_var.h>
+#include <net/netisr.h>
+#include <net/rss_config.h>
+#include <net/toeplitz.h>
+
+#if 0
+#include <netinet/in.h>
+#include <netinet/in_pcb.h>
+#include <netinet/in_rss.h>
+#include <netinet/in_var.h>
+
+/* for software rss hash support */
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+#endif
+
+/*-
+ * Operating system parts of receiver-side scaling (RSS), which allows
+ * network cards to direct flows to particular receive queues based on hashes
+ * of header tuples.  This implementation aligns RSS buckets with connection
+ * groups at the TCP/IP layer, so each bucket is associated with exactly one
+ * group.  As a result, the group lookup structures (and lock) should have an
+ * effective affinity with exactly one CPU.
+ *
+ * Network device drivers needing to configure RSS will query this framework
+ * for parameters, such as the current RSS key, hashing policies, number of
+ * bits, and indirection table mapping hashes to buckets and CPUs.  They may
+ * provide their own supplementary information, such as queue<->CPU bindings.
+ * It is the responsibility of the network device driver to inject packets
+ * into the stack on as close to the right CPU as possible, if playing by RSS
+ * rules.
+ *
+ * TODO:
+ *
+ * - Synchronization for rss_key and other future-configurable parameters.
+ * - Event handler drivers can register to pick up RSS configuration changes.
+ * - Should we allow rss_basecpu to be configured?
+ * - Randomize key on boot.
+ * - IPv6 support.
+ * - Statistics on how often there's a misalignment between hardware
+ *   placement and pcbgroup expectations.
+ */
+
+SYSCTL_DECL(_net_inet);
+SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering");
+
+/*
+ * Toeplitz is the only required hash function in the RSS spec, so use it by
+ * default.
+ */
+static u_int	rss_hashalgo = RSS_HASH_TOEPLITZ;
+SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0,
+    "RSS hash algorithm");
+
+/*
+ * Size of the indirection table; at most 128 entries per the RSS spec.  We
+ * size it to at least 2 times the number of CPUs by default to allow useful
+ * rebalancing.  If not set explicitly with a loader tunable, we tune based
+ * on the number of CPUs present.
+ *
+ * XXXRW: buckets might be better to use for the tunable than bits.
+ */
+static u_int	rss_bits;
+SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0,
+    "RSS bits");
+
+static u_int	rss_mask;
+SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0,
+    "RSS mask");
+
+static const u_int	rss_maxbits = RSS_MAXBITS;
+SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD,
+    __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits");
+
+/*
+ * RSS's own count of the number of CPUs it could be using for processing.
+ * Bounded to 64 by RSS constants.
+ */
+static u_int	rss_ncpus;
+SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0,
+    "Number of CPUs available to RSS");
+
+#define	RSS_MAXCPUS	(1 << (RSS_MAXBITS - 1))
+static const u_int	rss_maxcpus = RSS_MAXCPUS;
+SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD,
+    __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used");
+
+/*
+ * Variable exists just for reporting rss_bits in a user-friendly way.
+ */
+static u_int	rss_buckets;
+SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0,
+    "RSS buckets");
+
+/*
+ * Base CPU number; devices will add this to all CPU numbers returned by the
+ * RSS indirection table.  Currently unmodifable in FreeBSD.
+ */
+static const u_int	rss_basecpu;
+SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD,
+    __DECONST(int *, &rss_basecpu), 0, "RSS base CPU");
+
+/*
+ * RSS secret key, intended to prevent attacks on load-balancing.  Its
+ * effectiveness may be limited by algorithm choice and available entropy
+ * during the boot.
+ *
+ * XXXRW: And that we don't randomize it yet!
+ *
+ * This is the default Microsoft RSS specification key which is also
+ * the Chelsio T5 firmware default key.
+ */
+static uint8_t rss_key[RSS_KEYSIZE] = {
+	0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
+	0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
+	0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
+	0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
+	0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
+};
+
+/*
+ * RSS hash->CPU table, which maps hashed packet headers to particular CPUs.
+ * Drivers may supplement this table with a seperate CPU<->queue table when
+ * programming devices.
+ */
+struct rss_table_entry {
+	uint8_t		rte_cpu;	/* CPU affinity of bucket. */
+};
+static struct rss_table_entry	rss_table[RSS_TABLE_MAXLEN];
+
+static void
+rss_init(__unused void *arg)
+{
+	u_int i;
+	u_int cpuid;
+
+	/*
+	 * Validate tunables, coerce to sensible values.
+	 */
+	switch (rss_hashalgo) {
+	case RSS_HASH_TOEPLITZ:
+	case RSS_HASH_NAIVE:
+		break;
+
+	default:
+		printf("%s: invalid RSS hashalgo %u, coercing to %u",
+		    __func__, rss_hashalgo, RSS_HASH_TOEPLITZ);
+		rss_hashalgo = RSS_HASH_TOEPLITZ;
+	}
+
+	/*
+	 * Count available CPUs.
+	 *
+	 * XXXRW: Note incorrect assumptions regarding contiguity of this set
+	 * elsewhere.
+	 */
+	rss_ncpus = 0;
+	for (i = 0; i <= mp_maxid; i++) {
+		if (CPU_ABSENT(i))
+			continue;
+		rss_ncpus++;
+	}
+	if (rss_ncpus > RSS_MAXCPUS)
+		rss_ncpus = RSS_MAXCPUS;
+
+	/*
+	 * Tune RSS table entries to be no less than 2x the number of CPUs
+	 * -- unless we're running uniprocessor, in which case there's not
+	 * much point in having buckets to rearrange for load-balancing!
+	 */
+	if (rss_ncpus > 1) {
+		if (rss_bits == 0)
+			rss_bits = fls(rss_ncpus - 1) + 1;
+
+		/*
+		 * Microsoft limits RSS table entries to 128, so apply that
+		 * limit to both auto-detected CPU counts and user-configured
+		 * ones.
+		 */
+		if (rss_bits == 0 || rss_bits > RSS_MAXBITS) {
+			printf("%s: RSS bits %u not valid, coercing to  %u",
+			    __func__, rss_bits, RSS_MAXBITS);
+			rss_bits = RSS_MAXBITS;
+		}
+
+		/*
+		 * Figure out how many buckets to use; warn if less than the
+		 * number of configured CPUs, although this is not a fatal
+		 * problem.
+		 */
+		rss_buckets = (1 << rss_bits);
+		if (rss_buckets < rss_ncpus)
+			printf("%s: WARNING: rss_buckets (%u) less than "
+			    "rss_ncpus (%u)\n", __func__, rss_buckets,
+			    rss_ncpus);
+		rss_mask = rss_buckets - 1;
+	} else {
+		rss_bits = 0;
+		rss_buckets = 1;
+		rss_mask = 0;
+	}
+
+	/*
+	 * Set up initial CPU assignments: round-robin by default.
+	 */
+	cpuid = CPU_FIRST();
+	for (i = 0; i < rss_buckets; i++) {
+		rss_table[i].rte_cpu = cpuid;
+		cpuid = CPU_NEXT(cpuid);
+	}
+
+	/*
+	 * Randomize rrs_key.
+	 *
+	 * XXXRW: Not yet.  If nothing else, will require an rss_isbadkey()
+	 * loop to check for "bad" RSS keys.
+	 */
+}
+SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL);
+
+static uint32_t
+rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen,
+    const uint8_t *data)
+{
+	uint32_t v;
+	u_int i;
+
+	v = 0;
+	for (i = 0; i < keylen; i++)
+		v += key[i];
+	for (i = 0; i < datalen; i++)
+		v += data[i];
+	return (v);
+}
+
+uint32_t
+rss_hash(u_int datalen, const uint8_t *data)
+{
+ 
+	switch (rss_hashalgo) {
+	case RSS_HASH_TOEPLITZ:
+		return (toeplitz_hash(sizeof(rss_key), rss_key, datalen,
+		    data));
+
+	case RSS_HASH_NAIVE:
+		return (rss_naive_hash(sizeof(rss_key), rss_key, datalen,
+		    data));
+
+	default:
+		panic("%s: unsupported/unknown hashalgo %d", __func__,
+		    rss_hashalgo);
+	}
+}
+
+/*
+ * Query the number of RSS bits in use.
+ */
+u_int
+rss_getbits(void)
+{
+
+	return (rss_bits);
+}
+
+/*
+ * Query the RSS bucket associated with an RSS hash.
+ */
+u_int
+rss_getbucket(u_int hash)
+{
+
+	return (hash & rss_mask);
+}
+
+/*
+ * Query the RSS layer bucket associated with the given
+ * entry in the RSS hash space.
+ *
+ * The RSS indirection table is 0 .. rss_buckets-1,
+ * covering the low 'rss_bits' of the total 128 slot
+ * RSS indirection table.  So just mask off rss_bits and
+ * return that.
+ *
+ * NIC drivers can then iterate over the 128 slot RSS
+ * indirection table and fetch which RSS bucket to
+ * map it to.  This will typically be a CPU queue
+ */
+u_int
+rss_get_indirection_to_bucket(u_int index)
+{
+
+	return (index & rss_mask);
+}
+
+/*
+ * Query the RSS CPU associated with an RSS bucket.
+ */
+u_int
+rss_getcpu(u_int bucket)
+{
+
+	return (rss_table[bucket].rte_cpu);
+}
+
+/*
+ * netisr CPU affinity lookup given just the hash and hashtype.
+ */
+u_int
+rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type)
+{
+
+	switch (hash_type) {
+	case M_HASHTYPE_RSS_IPV4:
+	case M_HASHTYPE_RSS_TCP_IPV4:
+	case M_HASHTYPE_RSS_UDP_IPV4:
+	case M_HASHTYPE_RSS_IPV6:
+	case M_HASHTYPE_RSS_TCP_IPV6:
+	case M_HASHTYPE_RSS_UDP_IPV6:
+		return (rss_getcpu(rss_getbucket(hash_val)));
+	default:
+		return (NETISR_CPUID_NONE);
+	}
+}
+
+/*
+ * Query the RSS bucket associated with the given hash value and
+ * type.
+ */
+int
+rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id)
+{
+
+	switch (hash_type) {
+	case M_HASHTYPE_RSS_IPV4:
+	case M_HASHTYPE_RSS_TCP_IPV4:
+	case M_HASHTYPE_RSS_UDP_IPV4:
+	case M_HASHTYPE_RSS_IPV6:
+	case M_HASHTYPE_RSS_TCP_IPV6:
+	case M_HASHTYPE_RSS_UDP_IPV6:
+		*bucket_id = rss_getbucket(hash_val);
+		return (0);
+	default:
+		return (-1);
+	}
+}
+
+/*
+ * netisr CPU affinity lookup routine for use by protocols.
+ */
+struct mbuf *
+rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
+{
+
+	M_ASSERTPKTHDR(m);
+	*cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
+	return (m);
+}
+
+int
+rss_m2bucket(struct mbuf *m, uint32_t *bucket_id)
+{
+
+	M_ASSERTPKTHDR(m);
+
+	return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
+	    bucket_id));
+}
+
+/*
+ * Query the RSS hash algorithm.
+ */
+u_int
+rss_gethashalgo(void)
+{
+
+	return (rss_hashalgo);
+}
+
+/*
+ * Query the current RSS key; likely to be used by device drivers when
+ * configuring hardware RSS.  Caller must pass an array of size RSS_KEYSIZE.
+ *
+ * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing?
+ */
+void
+rss_getkey(uint8_t *key)
+{
+
+	bcopy(rss_key, key, sizeof(rss_key));
+}
+
+/*
+ * Query the number of buckets; this may be used by both network device
+ * drivers, which will need to populate hardware shadows of the software
+ * indirection table, and the network stack itself (such as when deciding how
+ * many connection groups to allocate).
+ */
+u_int
+rss_getnumbuckets(void)
+{
+
+	return (rss_buckets);
+}
+
+/*
+ * Query the number of CPUs in use by RSS; may be useful to device drivers
+ * trying to figure out how to map a larger number of CPUs into a smaller
+ * number of receive queues.
+ */
+u_int
+rss_getnumcpus(void)
+{
+
+	return (rss_ncpus);
+}
+
+/*
+ * Return the supported RSS hash configuration.
+ *
+ * NICs should query this to determine what to configure in their redirection
+ * matching table.
+ */
+inline u_int
+rss_gethashconfig(void)
+{
+
+	/* Return 4-tuple for TCP; 2-tuple for others */
+	/*
+	 * UDP may fragment more often than TCP and thus we'll end up with
+	 * NICs returning 2-tuple fragments.
+	 * udp_init() and udplite_init() both currently initialise things
+	 * as 2-tuple.
+	 * So for now disable UDP 4-tuple hashing until all of the other
+	 * pieces are in place.
+	 */
+	return (
+	    RSS_HASHTYPE_RSS_IPV4
+	|    RSS_HASHTYPE_RSS_TCP_IPV4
+	|    RSS_HASHTYPE_RSS_IPV6
+	|    RSS_HASHTYPE_RSS_TCP_IPV6
+	|    RSS_HASHTYPE_RSS_IPV6_EX
+	|    RSS_HASHTYPE_RSS_TCP_IPV6_EX
+#if 0
+	|    RSS_HASHTYPE_RSS_UDP_IPV4
+	|    RSS_HASHTYPE_RSS_UDP_IPV4_EX
+	|    RSS_HASHTYPE_RSS_UDP_IPV6
+	|    RSS_HASHTYPE_RSS_UDP_IPV6_EX
+#endif
+	);
+}
+
+/*
+ * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want
+ * it appearing in debugging output unnecessarily.
+ */
+static int
+sysctl_rss_key(SYSCTL_HANDLER_ARGS)
+{
+	uint8_t temp_rss_key[RSS_KEYSIZE];
+	int error;
+
+	error = priv_check(req->td, PRIV_NETINET_HASHKEY);
+	if (error)
+		return (error);
+
+	bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key));
+	error = sysctl_handle_opaque(oidp, temp_rss_key,
+	    sizeof(temp_rss_key), req);
+	if (error)
+		return (error);
+	if (req->newptr != NULL) {
+		/* XXXRW: Not yet. */
+		return (EINVAL);
+	}
+	return (0);
+}
+SYSCTL_PROC(_net_inet_rss, OID_AUTO, key,
+    CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key,
+    "", "RSS keying material");
+
+static int
+sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)
+{
+	struct sbuf *sb;
+	int error;
+	int i;
+
+	error = 0;
+	error = sysctl_wire_old_buffer(req, 0);
+	if (error != 0)
+		return (error);
+	sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
+	if (sb == NULL)
+		return (ENOMEM);
+	for (i = 0; i < rss_buckets; i++) {
+		sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ",
+		    i,
+		    rss_getcpu(i));
+	}
+	error = sbuf_finish(sb);
+	sbuf_delete(sb);
+
+	return (error);
+}
+SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping,
+    CTLTYPE_STRING | CTLFLAG_RD, NULL, 0,
+    sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping");
diff --git a/sys/net/rss_config.h b/sys/net/rss_config.h
new file mode 100644
index 00000000000..37d82ae1542
--- /dev/null
+++ b/sys/net/rss_config.h
@@ -0,0 +1,123 @@
+/*-
+ * Copyright (c) 2010-2011 Juniper Networks, Inc.
+ * All rights reserved.
+ *
+ * This software was developed by Robert N. M. Watson under contract
+ * to Juniper Networks, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _NET_RSS_CONFIG_H_
+#define	_NET_RSS_CONFIG_H_
+
+#include <netinet/in.h>		/* in_addr_t */
+
+/*
+ * Supported RSS hash functions.
+ */
+#define	RSS_HASH_NAIVE		0x00000001	/* Poor but fast hash. */
+#define	RSS_HASH_TOEPLITZ	0x00000002	/* Required by RSS. */
+#define	RSS_HASH_CRC32		0x00000004	/* Future; some NICs do it. */
+
+#define	RSS_HASH_MASK		(RSS_HASH_NAIVE | RSS_HASH_TOEPLITZ)
+
+/*
+ * Instances of struct inpcbinfo declare an RSS hash type indicating what
+ * header fields are covered.
+ */
+#define	RSS_HASHFIELDS_NONE		0
+#define	RSS_HASHFIELDS_4TUPLE		1
+#define	RSS_HASHFIELDS_2TUPLE		2
+
+/*
+ * Define RSS representations of the M_HASHTYPE_* values, representing
+ * which particular bits are supported.  The NICs can then use this to
+ * calculate which hash types to enable and which not to enable.
+ *
+ * The fact that these line up with M_HASHTYPE_* is not to be relied
+ * upon.
+ */
+#define	RSS_HASHTYPE_RSS_IPV4		(1 << 1)	/* IPv4 2-tuple */
+#define	RSS_HASHTYPE_RSS_TCP_IPV4	(1 << 2)	/* TCPv4 4-tuple */
+#define	RSS_HASHTYPE_RSS_IPV6		(1 << 3)	/* IPv6 2-tuple */
+#define	RSS_HASHTYPE_RSS_TCP_IPV6	(1 << 4)	/* TCPv6 4-tuple */
+#define	RSS_HASHTYPE_RSS_IPV6_EX	(1 << 5)	/* IPv6 2-tuple + ext hdrs */
+#define	RSS_HASHTYPE_RSS_TCP_IPV6_EX	(1 << 6)	/* TCPv6 4-tiple + ext hdrs */
+#define	RSS_HASHTYPE_RSS_UDP_IPV4	(1 << 7)	/* IPv4 UDP 4-tuple */
+#define	RSS_HASHTYPE_RSS_UDP_IPV4_EX	(1 << 8)	/* IPv4 UDP 4-tuple + ext hdrs */
+#define	RSS_HASHTYPE_RSS_UDP_IPV6	(1 << 9)	/* IPv6 UDP 4-tuple */
+#define	RSS_HASHTYPE_RSS_UDP_IPV6_EX	(1 << 10)	/* IPv6 UDP 4-tuple + ext hdrs */
+
+/*
+ * Compile-time limits on the size of the indirection table.
+ */
+#define	RSS_MAXBITS	7
+#define	RSS_TABLE_MAXLEN	(1 << RSS_MAXBITS)
+
+/*
+ * Maximum key size used throughout.  It's OK for hardware to use only the
+ * first 16 bytes, which is all that's required for IPv4.
+ */
+#define	RSS_KEYSIZE	40
+
+/*
+ * For RSS hash methods that do a software hash on an mbuf, the packet
+ * direction (ingress / egress) is required.
+ *
+ * The default direction (INGRESS) is the "receive into the NIC" - ie,
+ * what the hardware is hashing on.
+ */
+#define	RSS_HASH_PKT_INGRESS	0
+#define	RSS_HASH_PKT_EGRESS	1
+
+/*
+ * Device driver interfaces to query RSS properties that must be programmed
+ * into hardware.
+ */
+u_int	rss_getbits(void);
+u_int	rss_getbucket(u_int hash);
+u_int	rss_get_indirection_to_bucket(u_int index);
+u_int	rss_getcpu(u_int bucket);
+void	rss_getkey(uint8_t *key);
+u_int	rss_gethashalgo(void);
+u_int	rss_getnumbuckets(void);
+u_int	rss_getnumcpus(void);
+u_int	rss_gethashconfig(void);
+
+/*
+ * Hash calculation functions.
+ */
+uint32_t	rss_hash(u_int datalen, const uint8_t *data);
+
+/*
+ * Network stack interface to query desired CPU affinity of a packet.
+ */
+struct mbuf * rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid);
+u_int	rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type);
+int	rss_hash2bucket(uint32_t hash_val, uint32_t hash_type,
+	    uint32_t *bucket_id);
+int	rss_m2bucket(struct mbuf *m, uint32_t *bucket_id);
+
+#endif /* !_NET_RSS_CONFIG_H_ */
diff --git a/sys/netinet/toeplitz.c b/sys/net/toeplitz.c
similarity index 97%
rename from sys/netinet/toeplitz.c
rename to sys/net/toeplitz.c
index 84d3a5955a7..4328ec4468a 100644
--- a/sys/netinet/toeplitz.c
+++ b/sys/net/toeplitz.c
@@ -29,8 +29,8 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/types.h>
 
-#include <netinet/in_rss.h>
-#include <netinet/toeplitz.h>
+#include <net/rss_config.h>
+#include <net/toeplitz.h>
 
 #include <sys/systm.h>
 
diff --git a/sys/netinet/toeplitz.h b/sys/net/toeplitz.h
similarity index 100%
rename from sys/netinet/toeplitz.h
rename to sys/net/toeplitz.h
diff --git a/sys/netinet/in_pcb.c b/sys/netinet/in_pcb.c
index 6bec7efc256..f8f290558ba 100644
--- a/sys/netinet/in_pcb.c
+++ b/sys/netinet/in_pcb.c
@@ -71,12 +71,12 @@ __FBSDID("$FreeBSD$");
 #include <net/if_var.h>
 #include <net/if_types.h>
 #include <net/route.h>
+#include <net/rss_config.h>
 #include <net/vnet.h>
 
 #if defined(INET) || defined(INET6)
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
-#include <netinet/in_rss.h>
 #include <netinet/ip_var.h>
 #include <netinet/tcp_var.h>
 #include <netinet/udp.h>
diff --git a/sys/netinet/in_pcbgroup.c b/sys/netinet/in_pcbgroup.c
index 4157290382d..2d3c1366afd 100644
--- a/sys/netinet/in_pcbgroup.c
+++ b/sys/netinet/in_pcbgroup.c
@@ -42,7 +42,10 @@ __FBSDID("$FreeBSD$");
 #include <sys/smp.h>
 #include <sys/socketvar.h>
 
+#include <net/rss_config.h>
+
 #include <netinet/in.h>
+
 #include <netinet/in_pcb.h>
 #include <netinet/in_rss.h>
 #ifdef INET6
diff --git a/sys/netinet/in_rss.c b/sys/netinet/in_rss.c
index 336c64f57c7..087d8467686 100644
--- a/sys/netinet/in_rss.c
+++ b/sys/netinet/in_rss.c
@@ -50,257 +50,18 @@ __FBSDID("$FreeBSD$");
 #include <net/if.h>
 #include <net/if_var.h>
 #include <net/netisr.h>
+#include <net/rss_config.h>
 
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
 #include <netinet/in_rss.h>
 #include <netinet/in_var.h>
-#include <netinet/toeplitz.h>
 
 /* for software rss hash support */
 #include <netinet/ip.h>
 #include <netinet/tcp.h>
 #include <netinet/udp.h>
 
-/*-
- * Operating system parts of receiver-side scaling (RSS), which allows
- * network cards to direct flows to particular receive queues based on hashes
- * of header tuples.  This implementation aligns RSS buckets with connection
- * groups at the TCP/IP layer, so each bucket is associated with exactly one
- * group.  As a result, the group lookup structures (and lock) should have an
- * effective affinity with exactly one CPU.
- *
- * Network device drivers needing to configure RSS will query this framework
- * for parameters, such as the current RSS key, hashing policies, number of
- * bits, and indirection table mapping hashes to buckets and CPUs.  They may
- * provide their own supplementary information, such as queue<->CPU bindings.
- * It is the responsibility of the network device driver to inject packets
- * into the stack on as close to the right CPU as possible, if playing by RSS
- * rules.
- *
- * TODO:
- *
- * - Synchronization for rss_key and other future-configurable parameters.
- * - Event handler drivers can register to pick up RSS configuration changes.
- * - Should we allow rss_basecpu to be configured?
- * - Randomize key on boot.
- * - IPv6 support.
- * - Statistics on how often there's a misalignment between hardware
- *   placement and pcbgroup expectations.
- */
-
-SYSCTL_NODE(_net_inet, OID_AUTO, rss, CTLFLAG_RW, 0, "Receive-side steering");
-
-/*
- * Toeplitz is the only required hash function in the RSS spec, so use it by
- * default.
- */
-static u_int	rss_hashalgo = RSS_HASH_TOEPLITZ;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, hashalgo, CTLFLAG_RDTUN, &rss_hashalgo, 0,
-    "RSS hash algorithm");
-
-/*
- * Size of the indirection table; at most 128 entries per the RSS spec.  We
- * size it to at least 2 times the number of CPUs by default to allow useful
- * rebalancing.  If not set explicitly with a loader tunable, we tune based
- * on the number of CPUs present.
- *
- * XXXRW: buckets might be better to use for the tunable than bits.
- */
-static u_int	rss_bits;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, bits, CTLFLAG_RDTUN, &rss_bits, 0,
-    "RSS bits");
-
-static u_int	rss_mask;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, mask, CTLFLAG_RD, &rss_mask, 0,
-    "RSS mask");
-
-static const u_int	rss_maxbits = RSS_MAXBITS;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, maxbits, CTLFLAG_RD,
-    __DECONST(int *, &rss_maxbits), 0, "RSS maximum bits");
-
-/*
- * RSS's own count of the number of CPUs it could be using for processing.
- * Bounded to 64 by RSS constants.
- */
-static u_int	rss_ncpus;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, ncpus, CTLFLAG_RD, &rss_ncpus, 0,
-    "Number of CPUs available to RSS");
-
-#define	RSS_MAXCPUS	(1 << (RSS_MAXBITS - 1))
-static const u_int	rss_maxcpus = RSS_MAXCPUS;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, maxcpus, CTLFLAG_RD,
-    __DECONST(int *, &rss_maxcpus), 0, "RSS maximum CPUs that can be used");
-
-/*
- * Variable exists just for reporting rss_bits in a user-friendly way.
- */
-static u_int	rss_buckets;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, buckets, CTLFLAG_RD, &rss_buckets, 0,
-    "RSS buckets");
-
-/*
- * Base CPU number; devices will add this to all CPU numbers returned by the
- * RSS indirection table.  Currently unmodifable in FreeBSD.
- */
-static const u_int	rss_basecpu;
-SYSCTL_INT(_net_inet_rss, OID_AUTO, basecpu, CTLFLAG_RD,
-    __DECONST(int *, &rss_basecpu), 0, "RSS base CPU");
-
-/*
- * RSS secret key, intended to prevent attacks on load-balancing.  Its
- * effectiveness may be limited by algorithm choice and available entropy
- * during the boot.
- *
- * XXXRW: And that we don't randomize it yet!
- *
- * This is the default Microsoft RSS specification key which is also
- * the Chelsio T5 firmware default key.
- */
-static uint8_t rss_key[RSS_KEYSIZE] = {
-	0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2,
-	0x41, 0x67, 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0,
-	0xd0, 0xca, 0x2b, 0xcb, 0xae, 0x7b, 0x30, 0xb4,
-	0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30, 0xf2, 0x0c,
-	0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa,
-};
-
-/*
- * RSS hash->CPU table, which maps hashed packet headers to particular CPUs.
- * Drivers may supplement this table with a seperate CPU<->queue table when
- * programming devices.
- */
-struct rss_table_entry {
-	uint8_t		rte_cpu;	/* CPU affinity of bucket. */
-};
-static struct rss_table_entry	rss_table[RSS_TABLE_MAXLEN];
-
-static inline u_int rss_gethashconfig_local(void);
-
-static void
-rss_init(__unused void *arg)
-{
-	u_int i;
-	u_int cpuid;
-
-	/*
-	 * Validate tunables, coerce to sensible values.
-	 */
-	switch (rss_hashalgo) {
-	case RSS_HASH_TOEPLITZ:
-	case RSS_HASH_NAIVE:
-		break;
-
-	default:
-		printf("%s: invalid RSS hashalgo %u, coercing to %u",
-		    __func__, rss_hashalgo, RSS_HASH_TOEPLITZ);
-		rss_hashalgo = RSS_HASH_TOEPLITZ;
-	}
-
-	/*
-	 * Count available CPUs.
-	 *
-	 * XXXRW: Note incorrect assumptions regarding contiguity of this set
-	 * elsewhere.
-	 */
-	rss_ncpus = 0;
-	for (i = 0; i <= mp_maxid; i++) {
-		if (CPU_ABSENT(i))
-			continue;
-		rss_ncpus++;
-	}
-	if (rss_ncpus > RSS_MAXCPUS)
-		rss_ncpus = RSS_MAXCPUS;
-
-	/*
-	 * Tune RSS table entries to be no less than 2x the number of CPUs
-	 * -- unless we're running uniprocessor, in which case there's not
-	 * much point in having buckets to rearrange for load-balancing!
-	 */
-	if (rss_ncpus > 1) {
-		if (rss_bits == 0)
-			rss_bits = fls(rss_ncpus - 1) + 1;
-
-		/*
-		 * Microsoft limits RSS table entries to 128, so apply that
-		 * limit to both auto-detected CPU counts and user-configured
-		 * ones.
-		 */
-		if (rss_bits == 0 || rss_bits > RSS_MAXBITS) {
-			printf("%s: RSS bits %u not valid, coercing to  %u",
-			    __func__, rss_bits, RSS_MAXBITS);
-			rss_bits = RSS_MAXBITS;
-		}
-
-		/*
-		 * Figure out how many buckets to use; warn if less than the
-		 * number of configured CPUs, although this is not a fatal
-		 * problem.
-		 */
-		rss_buckets = (1 << rss_bits);
-		if (rss_buckets < rss_ncpus)
-			printf("%s: WARNING: rss_buckets (%u) less than "
-			    "rss_ncpus (%u)\n", __func__, rss_buckets,
-			    rss_ncpus);
-		rss_mask = rss_buckets - 1;
-	} else {
-		rss_bits = 0;
-		rss_buckets = 1;
-		rss_mask = 0;
-	}
-
-	/*
-	 * Set up initial CPU assignments: round-robin by default.
-	 */
-	cpuid = CPU_FIRST();
-	for (i = 0; i < rss_buckets; i++) {
-		rss_table[i].rte_cpu = cpuid;
-		cpuid = CPU_NEXT(cpuid);
-	}
-
-	/*
-	 * Randomize rrs_key.
-	 *
-	 * XXXRW: Not yet.  If nothing else, will require an rss_isbadkey()
-	 * loop to check for "bad" RSS keys.
-	 */
-}
-SYSINIT(rss_init, SI_SUB_SOFTINTR, SI_ORDER_SECOND, rss_init, NULL);
-
-static uint32_t
-rss_naive_hash(u_int keylen, const uint8_t *key, u_int datalen,
-    const uint8_t *data)
-{
-	uint32_t v;
-	u_int i;
-
-	v = 0;
-	for (i = 0; i < keylen; i++)
-		v += key[i];
-	for (i = 0; i < datalen; i++)
-		v += data[i];
-	return (v);
-}
-
-static uint32_t
-rss_hash(u_int datalen, const uint8_t *data)
-{
- 
-	switch (rss_hashalgo) {
-	case RSS_HASH_TOEPLITZ:
-		return (toeplitz_hash(sizeof(rss_key), rss_key, datalen,
-		    data));
-
-	case RSS_HASH_NAIVE:
-		return (rss_naive_hash(sizeof(rss_key), rss_key, datalen,
-		    data));
-
-	default:
-		panic("%s: unsupported/unknown hashalgo %d", __func__,
-		    rss_hashalgo);
-	}
-}
-
 /*
  * Hash an IPv4 2-tuple.
  */
@@ -341,162 +102,6 @@ rss_hash_ip4_4tuple(struct in_addr src, u_short srcport, struct in_addr dst,
 	return (rss_hash(datalen, data));
 }
 
-#ifdef INET6
-/*
- * Hash an IPv6 2-tuple.
- */
-uint32_t
-rss_hash_ip6_2tuple(const struct in6_addr *src, const struct in6_addr *dst)
-{
-	uint8_t data[sizeof(*src) + sizeof(*dst)];
-	u_int datalen;
-
-	datalen = 0;
-	bcopy(src, &data[datalen], sizeof(*src));
-	datalen += sizeof(*src);
-	bcopy(dst, &data[datalen], sizeof(*dst));
-	datalen += sizeof(*dst);
-	return (rss_hash(datalen, data));
-}
-
-/*
- * Hash an IPv6 4-tuple.
- */
-uint32_t
-rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport,
-    const struct in6_addr *dst, u_short dstport)
-{
-	uint8_t data[sizeof(*src) + sizeof(*dst) + sizeof(srcport) +
-	    sizeof(dstport)];
-	u_int datalen;
-
-	datalen = 0;
-	bcopy(src, &data[datalen], sizeof(*src));
-	datalen += sizeof(*src);
-	bcopy(dst, &data[datalen], sizeof(*dst));
-	datalen += sizeof(*dst);
-	bcopy(&srcport, &data[datalen], sizeof(srcport));
-	datalen += sizeof(srcport);
-	bcopy(&dstport, &data[datalen], sizeof(dstport));
-	datalen += sizeof(dstport);
-	return (rss_hash(datalen, data));
-}
-#endif /* INET6 */
-
-/*
- * Query the number of RSS bits in use.
- */
-u_int
-rss_getbits(void)
-{
-
-	return (rss_bits);
-}
-
-/*
- * Query the RSS bucket associated with an RSS hash.
- */
-u_int
-rss_getbucket(u_int hash)
-{
-
-	return (hash & rss_mask);
-}
-
-/*
- * Query the RSS layer bucket associated with the given
- * entry in the RSS hash space.
- *
- * The RSS indirection table is 0 .. rss_buckets-1,
- * covering the low 'rss_bits' of the total 128 slot
- * RSS indirection table.  So just mask off rss_bits and
- * return that.
- *
- * NIC drivers can then iterate over the 128 slot RSS
- * indirection table and fetch which RSS bucket to
- * map it to.  This will typically be a CPU queue
- */
-u_int
-rss_get_indirection_to_bucket(u_int index)
-{
-
-	return (index & rss_mask);
-}
-
-/*
- * Query the RSS CPU associated with an RSS bucket.
- */
-u_int
-rss_getcpu(u_int bucket)
-{
-
-	return (rss_table[bucket].rte_cpu);
-}
-
-/*
- * netisr CPU affinity lookup given just the hash and hashtype.
- */
-u_int
-rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type)
-{
-
-	switch (hash_type) {
-	case M_HASHTYPE_RSS_IPV4:
-	case M_HASHTYPE_RSS_TCP_IPV4:
-	case M_HASHTYPE_RSS_UDP_IPV4:
-	case M_HASHTYPE_RSS_IPV6:
-	case M_HASHTYPE_RSS_TCP_IPV6:
-	case M_HASHTYPE_RSS_UDP_IPV6:
-		return (rss_getcpu(rss_getbucket(hash_val)));
-	default:
-		return (NETISR_CPUID_NONE);
-	}
-}
-
-/*
- * Query the RSS bucket associated with the given hash value and
- * type.
- */
-int
-rss_hash2bucket(uint32_t hash_val, uint32_t hash_type, uint32_t *bucket_id)
-{
-
-	switch (hash_type) {
-	case M_HASHTYPE_RSS_IPV4:
-	case M_HASHTYPE_RSS_TCP_IPV4:
-	case M_HASHTYPE_RSS_UDP_IPV4:
-	case M_HASHTYPE_RSS_IPV6:
-	case M_HASHTYPE_RSS_TCP_IPV6:
-	case M_HASHTYPE_RSS_UDP_IPV6:
-		*bucket_id = rss_getbucket(hash_val);
-		return (0);
-	default:
-		return (-1);
-	}
-}
-
-/*
- * netisr CPU affinity lookup routine for use by protocols.
- */
-struct mbuf *
-rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
-{
-
-	M_ASSERTPKTHDR(m);
-	*cpuid = rss_hash2cpuid(m->m_pkthdr.flowid, M_HASHTYPE_GET(m));
-	return (m);
-}
-
-int
-rss_m2bucket(struct mbuf *m, uint32_t *bucket_id)
-{
-
-	M_ASSERTPKTHDR(m);
-
-	return(rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
-	    bucket_id));
-}
-
 /*
  * Calculate an appropriate ipv4 2-tuple or 4-tuple given the given
  * IPv4 source/destination address, UDP or TCP source/destination ports
@@ -522,18 +127,18 @@ rss_proto_software_hash_v4(struct in_addr s, struct in_addr d,
 	 * identifier.
 	 */
 	if ((proto == IPPROTO_TCP) &&
-	    (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
+	    (rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
 		hash = rss_hash_ip4_4tuple(s, sp, d, dp);
 		*hashval = hash;
 		*hashtype = M_HASHTYPE_RSS_TCP_IPV4;
 		return (0);
 	} else if ((proto == IPPROTO_UDP) &&
-	    (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
+	    (rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
 		hash = rss_hash_ip4_4tuple(s, sp, d, dp);
 		*hashval = hash;
 		*hashtype = M_HASHTYPE_RSS_UDP_IPV4;
 		return (0);
-	} else if (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) {
+	} else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) {
 		/* RSS doesn't hash on other protocols like SCTP; so 2-tuple */
 		hash = rss_hash_ip4_2tuple(s, d);
 		*hashval = hash;
@@ -625,7 +230,7 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
 	if (flowtype != M_HASHTYPE_NONE) {
 		switch (proto) {
 		case IPPROTO_UDP:
-			if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) &&
+			if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4) &&
 			    (flowtype == M_HASHTYPE_RSS_UDP_IPV4) &&
 			    (is_frag == 0)) {
 				return (1);
@@ -634,14 +239,14 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
 			 * Only allow 2-tuple for UDP frames if we don't also
 			 * support 4-tuple for UDP.
 			 */
-			if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) &&
-			    ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) == 0) &&
+			if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) &&
+			    ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4) == 0) &&
 			    flowtype == M_HASHTYPE_RSS_IPV4) {
 				return (1);
 			}
 			break;
 		case IPPROTO_TCP:
-			if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) &&
+			if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4) &&
 			    (flowtype == M_HASHTYPE_RSS_TCP_IPV4) &&
 			    (is_frag == 0)) {
 				return (1);
@@ -650,14 +255,14 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
 			 * Only allow 2-tuple for TCP frames if we don't also
 			 * support 2-tuple for TCP.
 			 */
-			if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) &&
-			    ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) == 0) &&
+			if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) &&
+			    ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4) == 0) &&
 			    flowtype == M_HASHTYPE_RSS_IPV4) {
 				return (1);
 			}
 			break;
 		default:
-			if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) &&
+			if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) &&
 			    flowtype == M_HASHTYPE_RSS_IPV4) {
 				return (1);
 			}
@@ -671,7 +276,7 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
 	 * XXX TODO: does the hardware hash on 4-tuple if IP
 	 *    options are present?
 	 */
-	if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_TCP_IPV4) &&
+	if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4) &&
 	    (proto == IPPROTO_TCP) &&
 	    (is_frag == 0)) {
 		if (m->m_len < iphlen + sizeof(struct tcphdr)) {
@@ -685,7 +290,7 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
 		    proto,
 		    hashval,
 		    hashtype);
-	} else if ((rss_gethashconfig_local() & RSS_HASHTYPE_RSS_UDP_IPV4) &&
+	} else if ((rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4) &&
 	    (proto == IPPROTO_UDP) &&
 	    (is_frag == 0)) {
 		uh = (struct udphdr *)((caddr_t)ip + iphlen);
@@ -699,7 +304,7 @@ rss_mbuf_software_hash_v4(const struct mbuf *m, int dir, uint32_t *hashval,
 		    proto,
 		    hashval,
 		    hashtype);
-	} else if (rss_gethashconfig_local() & RSS_HASHTYPE_RSS_IPV4) {
+	} else if (rss_gethashconfig() & RSS_HASHTYPE_RSS_IPV4) {
 		/* Default to 2-tuple hash */
 		return rss_proto_software_hash_v4(ip->ip_src, ip->ip_dst,
 		    0,	/* source port */
@@ -750,150 +355,3 @@ rss_soft_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid)
 	}
 	return (m);
 }
-
-/*
- * Query the RSS hash algorithm.
- */
-u_int
-rss_gethashalgo(void)
-{
-
-	return (rss_hashalgo);
-}
-
-/*
- * Query the current RSS key; likely to be used by device drivers when
- * configuring hardware RSS.  Caller must pass an array of size RSS_KEYSIZE.
- *
- * XXXRW: Perhaps we should do the accept-a-length-and-truncate thing?
- */
-void
-rss_getkey(uint8_t *key)
-{
-
-	bcopy(rss_key, key, sizeof(rss_key));
-}
-
-/*
- * Query the number of buckets; this may be used by both network device
- * drivers, which will need to populate hardware shadows of the software
- * indirection table, and the network stack itself (such as when deciding how
- * many connection groups to allocate).
- */
-u_int
-rss_getnumbuckets(void)
-{
-
-	return (rss_buckets);
-}
-
-/*
- * Query the number of CPUs in use by RSS; may be useful to device drivers
- * trying to figure out how to map a larger number of CPUs into a smaller
- * number of receive queues.
- */
-u_int
-rss_getnumcpus(void)
-{
-
-	return (rss_ncpus);
-}
-
-static inline u_int
-rss_gethashconfig_local(void)
-{
-
-	/* Return 4-tuple for TCP; 2-tuple for others */
-	/*
-	 * UDP may fragment more often than TCP and thus we'll end up with
-	 * NICs returning 2-tuple fragments.
-	 * udp_init() and udplite_init() both currently initialise things
-	 * as 2-tuple.
-	 * So for now disable UDP 4-tuple hashing until all of the other
-	 * pieces are in place.
-	 */
-	return (
-	    RSS_HASHTYPE_RSS_IPV4
-	|    RSS_HASHTYPE_RSS_TCP_IPV4
-	|    RSS_HASHTYPE_RSS_IPV6
-	|    RSS_HASHTYPE_RSS_TCP_IPV6
-	|    RSS_HASHTYPE_RSS_IPV6_EX
-	|    RSS_HASHTYPE_RSS_TCP_IPV6_EX
-#if 0
-	|    RSS_HASHTYPE_RSS_UDP_IPV4
-	|    RSS_HASHTYPE_RSS_UDP_IPV4_EX
-	|    RSS_HASHTYPE_RSS_UDP_IPV6
-	|    RSS_HASHTYPE_RSS_UDP_IPV6_EX
-#endif
-	);
-}
-
-/*
- * Return the supported RSS hash configuration.
- *
- * NICs should query this to determine what to configure in their redirection
- * matching table.
- */
-u_int
-rss_gethashconfig(void)
-{
-
-	return (rss_gethashconfig_local());
-}
-
-/*
- * XXXRW: Confirm that sysctl -a won't dump this keying material, don't want
- * it appearing in debugging output unnecessarily.
- */
-static int
-sysctl_rss_key(SYSCTL_HANDLER_ARGS)
-{
-	uint8_t temp_rss_key[RSS_KEYSIZE];
-	int error;
-
-	error = priv_check(req->td, PRIV_NETINET_HASHKEY);
-	if (error)
-		return (error);
-
-	bcopy(rss_key, temp_rss_key, sizeof(temp_rss_key));
-	error = sysctl_handle_opaque(oidp, temp_rss_key,
-	    sizeof(temp_rss_key), req);
-	if (error)
-		return (error);
-	if (req->newptr != NULL) {
-		/* XXXRW: Not yet. */
-		return (EINVAL);
-	}
-	return (0);
-}
-SYSCTL_PROC(_net_inet_rss, OID_AUTO, key,
-    CTLTYPE_OPAQUE | CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 0, sysctl_rss_key,
-    "", "RSS keying material");
-
-static int
-sysctl_rss_bucket_mapping(SYSCTL_HANDLER_ARGS)
-{
-	struct sbuf *sb;
-	int error;
-	int i;
-
-	error = 0;
-	error = sysctl_wire_old_buffer(req, 0);
-	if (error != 0)
-		return (error);
-	sb = sbuf_new_for_sysctl(NULL, NULL, 512, req);
-	if (sb == NULL)
-		return (ENOMEM);
-	for (i = 0; i < rss_buckets; i++) {
-		sbuf_printf(sb, "%s%d:%d", i == 0 ? "" : " ",
-		    i,
-		    rss_getcpu(i));
-	}
-	error = sbuf_finish(sb);
-	sbuf_delete(sb);
-
-	return (error);
-}
-SYSCTL_PROC(_net_inet_rss, OID_AUTO, bucket_mapping,
-    CTLTYPE_STRING | CTLFLAG_RD, NULL, 0,
-    sysctl_rss_bucket_mapping, "", "RSS bucket -> CPU mapping");
diff --git a/sys/netinet/in_rss.h b/sys/netinet/in_rss.h
index 90d28ed5309..f493eab9a46 100644
--- a/sys/netinet/in_rss.h
+++ b/sys/netinet/in_rss.h
@@ -34,97 +34,12 @@
 
 #include <netinet/in.h>		/* in_addr_t */
 
-/*
- * Supported RSS hash functions.
- */
-#define	RSS_HASH_NAIVE		0x00000001	/* Poor but fast hash. */
-#define	RSS_HASH_TOEPLITZ	0x00000002	/* Required by RSS. */
-#define	RSS_HASH_CRC32		0x00000004	/* Future; some NICs do it. */
-
-#define	RSS_HASH_MASK		(RSS_HASH_NAIVE | RSS_HASH_TOEPLITZ)
-
-/*
- * Instances of struct inpcbinfo declare an RSS hash type indicating what
- * header fields are covered.
- */
-#define	RSS_HASHFIELDS_NONE		0
-#define	RSS_HASHFIELDS_4TUPLE		1
-#define	RSS_HASHFIELDS_2TUPLE		2
-
-/*
- * Define RSS representations of the M_HASHTYPE_* values, representing
- * which particular bits are supported.  The NICs can then use this to
- * calculate which hash types to enable and which not to enable.
- *
- * The fact that these line up with M_HASHTYPE_* is not to be relied
- * upon.
- */
-#define	RSS_HASHTYPE_RSS_IPV4		(1 << 1)	/* IPv4 2-tuple */
-#define	RSS_HASHTYPE_RSS_TCP_IPV4	(1 << 2)	/* TCPv4 4-tuple */
-#define	RSS_HASHTYPE_RSS_IPV6		(1 << 3)	/* IPv6 2-tuple */
-#define	RSS_HASHTYPE_RSS_TCP_IPV6	(1 << 4)	/* TCPv6 4-tuple */
-#define	RSS_HASHTYPE_RSS_IPV6_EX	(1 << 5)	/* IPv6 2-tuple + ext hdrs */
-#define	RSS_HASHTYPE_RSS_TCP_IPV6_EX	(1 << 6)	/* TCPv6 4-tiple + ext hdrs */
-#define	RSS_HASHTYPE_RSS_UDP_IPV4	(1 << 7)	/* IPv4 UDP 4-tuple */
-#define	RSS_HASHTYPE_RSS_UDP_IPV4_EX	(1 << 8)	/* IPv4 UDP 4-tuple + ext hdrs */
-#define	RSS_HASHTYPE_RSS_UDP_IPV6	(1 << 9)	/* IPv6 UDP 4-tuple */
-#define	RSS_HASHTYPE_RSS_UDP_IPV6_EX	(1 << 10)	/* IPv6 UDP 4-tuple + ext hdrs */
-
-/*
- * Compile-time limits on the size of the indirection table.
- */
-#define	RSS_MAXBITS	7
-#define	RSS_TABLE_MAXLEN	(1 << RSS_MAXBITS)
-
-/*
- * Maximum key size used throughout.  It's OK for hardware to use only the
- * first 16 bytes, which is all that's required for IPv4.
- */
-#define	RSS_KEYSIZE	40
-
-/*
- * For RSS hash methods that do a software hash on an mbuf, the packet
- * direction (ingress / egress) is required.
- *
- * The default direction (INGRESS) is the "receive into the NIC" - ie,
- * what the hardware is hashing on.
- */
-#define	RSS_HASH_PKT_INGRESS	0
-#define	RSS_HASH_PKT_EGRESS	1
-
-/*
- * Device driver interfaces to query RSS properties that must be programmed
- * into hardware.
- */
-u_int	rss_getbits(void);
-u_int	rss_getbucket(u_int hash);
-u_int	rss_get_indirection_to_bucket(u_int index);
-u_int	rss_getcpu(u_int bucket);
-void	rss_getkey(uint8_t *key);
-u_int	rss_gethashalgo(void);
-u_int	rss_getnumbuckets(void);
-u_int	rss_getnumcpus(void);
-u_int	rss_gethashconfig(void);
-
 /*
  * Network stack interface to generate a hash for a protocol tuple.
  */
 uint32_t	rss_hash_ip4_4tuple(struct in_addr src, u_short srcport,
 		    struct in_addr dst, u_short dstport);
 uint32_t	rss_hash_ip4_2tuple(struct in_addr src, struct in_addr dst);
-uint32_t	rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport,
-		    const struct in6_addr *dst, u_short dstport);
-uint32_t	rss_hash_ip6_2tuple(const struct in6_addr *src,
-		    const struct in6_addr *dst);
-
-/*
- * Network stack interface to query desired CPU affinity of a packet.
- */
-struct mbuf	*rss_m2cpuid(struct mbuf *m, uintptr_t source, u_int *cpuid);
-u_int		rss_hash2cpuid(uint32_t hash_val, uint32_t hash_type);
-int		rss_hash2bucket(uint32_t hash_val, uint32_t hash_type,
-		uint32_t *bucket_id);
-int		rss_m2bucket(struct mbuf *m, uint32_t *bucket_id);
 
 /*
  * Functions to calculate a software RSS hash for a given mbuf or
diff --git a/sys/netinet/ip_input.c b/sys/netinet/ip_input.c
index adbc449de07..9ca6f700099 100644
--- a/sys/netinet/ip_input.c
+++ b/sys/netinet/ip_input.c
@@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$");
 #include <net/if_dl.h>
 #include <net/route.h>
 #include <net/netisr.h>
+#include <net/rss_config.h>
 #include <net/vnet.h>
 
 #include <netinet/in.h>
diff --git a/sys/netinet/ip_output.c b/sys/netinet/ip_output.c
index 275c29d1ea4..ccd5e898da9 100644
--- a/sys/netinet/ip_output.c
+++ b/sys/netinet/ip_output.c
@@ -65,6 +65,7 @@ __FBSDID("$FreeBSD$");
 #ifdef RADIX_MPATH
 #include <net/radix_mpath.h>
 #endif
+#include <net/rss_config.h>
 #include <net/vnet.h>
 
 #include <netinet/in.h>
diff --git a/sys/netinet/tcp_timer.c b/sys/netinet/tcp_timer.c
index 5d379ea38df..6cd0ffd239c 100644
--- a/sys/netinet/tcp_timer.c
+++ b/sys/netinet/tcp_timer.c
@@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
 
 #include <net/if.h>
 #include <net/route.h>
+#include <net/rss_config.h>
 #include <net/vnet.h>
 #include <net/netisr.h>
 
diff --git a/sys/netinet/udp_usrreq.c b/sys/netinet/udp_usrreq.c
index 482ee151038..d13821fbb5c 100644
--- a/sys/netinet/udp_usrreq.c
+++ b/sys/netinet/udp_usrreq.c
@@ -70,6 +70,7 @@ __FBSDID("$FreeBSD$");
 #include <net/if.h>
 #include <net/if_var.h>
 #include <net/route.h>
+#include <net/rss_config.h>
 
 #include <netinet/in.h>
 #include <netinet/in_kdtrace.h>
diff --git a/sys/netinet6/in6_pcbgroup.c b/sys/netinet6/in6_pcbgroup.c
index 45773ed055e..694de99f312 100644
--- a/sys/netinet6/in6_pcbgroup.c
+++ b/sys/netinet6/in6_pcbgroup.c
@@ -37,11 +37,13 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/mbuf.h>
 
+#include <net/rss_config.h>
+
 #include <netinet/in.h>
 #include <netinet/in_pcb.h>
-#include <netinet/in_rss.h>
 #ifdef INET6
 #include <netinet6/in6_pcb.h>
+#include <netinet6/in6_rss.h>
 #endif /* INET6 */
 
 /*
diff --git a/sys/netinet6/in6_rss.c b/sys/netinet6/in6_rss.c
new file mode 100644
index 00000000000..40d54d94c9e
--- /dev/null
+++ b/sys/netinet6/in6_rss.c
@@ -0,0 +1,103 @@
+/*-
+ * Copyright (c) 2010-2011 Juniper Networks, Inc.
+ * All rights reserved.
+ *
+ * This software was developed by Robert N. M. Watson under contract
+ * to Juniper Networks, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+
+__FBSDID("$FreeBSD$");
+
+#include "opt_inet6.h"
+#include "opt_pcbgroup.h"
+
+#ifndef PCBGROUP
+#error "options RSS depends on options PCBGROUP"
+#endif
+
+#include <sys/param.h>
+#include <sys/mbuf.h>
+#include <sys/socket.h>
+#include <sys/priv.h>
+#include <sys/kernel.h>
+#include <sys/smp.h>
+#include <sys/sysctl.h>
+#include <sys/sbuf.h>
+
+#include <net/if.h>
+#include <net/if_var.h>
+#include <net/netisr.h>
+#include <net/rss_config.h>
+
+#include <netinet/in.h>
+#include <netinet/in_pcb.h>
+#include <netinet6/in6_rss.h>
+#include <netinet/in_var.h>
+
+/* for software rss hash support */
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+
+/*
+ * Hash an IPv6 2-tuple.
+ */
+uint32_t
+rss_hash_ip6_2tuple(const struct in6_addr *src, const struct in6_addr *dst)
+{
+	uint8_t data[sizeof(*src) + sizeof(*dst)];
+	u_int datalen;
+
+	datalen = 0;
+	bcopy(src, &data[datalen], sizeof(*src));
+	datalen += sizeof(*src);
+	bcopy(dst, &data[datalen], sizeof(*dst));
+	datalen += sizeof(*dst);
+	return (rss_hash(datalen, data));
+}
+
+/*
+ * Hash an IPv6 4-tuple.
+ */
+uint32_t
+rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport,
+    const struct in6_addr *dst, u_short dstport)
+{
+	uint8_t data[sizeof(*src) + sizeof(*dst) + sizeof(srcport) +
+	    sizeof(dstport)];
+	u_int datalen;
+
+	datalen = 0;
+	bcopy(src, &data[datalen], sizeof(*src));
+	datalen += sizeof(*src);
+	bcopy(dst, &data[datalen], sizeof(*dst));
+	datalen += sizeof(*dst);
+	bcopy(&srcport, &data[datalen], sizeof(srcport));
+	datalen += sizeof(srcport);
+	bcopy(&dstport, &data[datalen], sizeof(dstport));
+	datalen += sizeof(dstport);
+	return (rss_hash(datalen, data));
+}
diff --git a/sys/netinet6/in6_rss.h b/sys/netinet6/in6_rss.h
new file mode 100644
index 00000000000..b215b5b80a0
--- /dev/null
+++ b/sys/netinet6/in6_rss.h
@@ -0,0 +1,45 @@
+/*-
+ * Copyright (c) 2010-2011 Juniper Networks, Inc.
+ * All rights reserved.
+ *
+ * This software was developed by Robert N. M. Watson under contract
+ * to Juniper Networks, Inc.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _NETINET6_IN6_RSS_H_
+#define	_NETINET6_IN6_RSS_H_
+
+#include <netinet/in.h>		/* in_addr_t */
+
+/*
+ * Network stack interface to generate a hash for a protocol tuple.
+ */
+uint32_t	rss_hash_ip6_4tuple(const struct in6_addr *src, u_short srcport,
+		    const struct in6_addr *dst, u_short dstport);
+uint32_t	rss_hash_ip6_2tuple(const struct in6_addr *src,
+		    const struct in6_addr *dst);
+
+#endif /* !_NETINET6_IN6_RSS_H_ */
diff --git a/sys/netinet6/ip6_output.c b/sys/netinet6/ip6_output.c
index 32308e49fde..a3474d34dc3 100644
--- a/sys/netinet6/ip6_output.c
+++ b/sys/netinet6/ip6_output.c
@@ -91,6 +91,7 @@ __FBSDID("$FreeBSD$");
 #include <net/netisr.h>
 #include <net/route.h>
 #include <net/pfil.h>
+#include <net/rss_config.h>
 #include <net/vnet.h>
 
 #include <netinet/in.h>
@@ -103,7 +104,7 @@ __FBSDID("$FreeBSD$");
 #include <netinet/in_pcb.h>
 #include <netinet/tcp_var.h>
 #include <netinet6/nd6.h>
-#include <netinet/in_rss.h>
+#include <netinet6/in6_rss.h>
 
 #ifdef IPSEC
 #include <netipsec/ipsec.h>
diff --git a/sys/netinet6/udp6_usrreq.c b/sys/netinet6/udp6_usrreq.c
index fc6b6a0a810..ab4dc7a8ce3 100644
--- a/sys/netinet6/udp6_usrreq.c
+++ b/sys/netinet6/udp6_usrreq.c
@@ -97,6 +97,7 @@ __FBSDID("$FreeBSD$");
 #include <net/if_var.h>
 #include <net/if_types.h>
 #include <net/route.h>
+#include <net/rss_config.h>
 
 #include <netinet/in.h>
 #include <netinet/in_kdtrace.h>
@@ -112,11 +113,11 @@ __FBSDID("$FreeBSD$");
 #include <netinet/udp.h>
 #include <netinet/udp_var.h>
 #include <netinet/udplite.h>
-#include <netinet/in_rss.h>
 
 #include <netinet6/ip6protosw.h>
 #include <netinet6/ip6_var.h>
 #include <netinet6/in6_pcb.h>
+#include <netinet6/in6_rss.h>
 #include <netinet6/udp6_var.h>
 #include <netinet6/scope6_var.h>
 

From e751d176f3bdce759a3b092b414c0279c0f3abf5 Mon Sep 17 00:00:00 2001
From: Steven Hartland <smh@FreeBSD.org>
Date: Sun, 18 Jan 2015 18:25:12 +0000
Subject: [PATCH 063/279] Fix bsdinstall when working with geli boot disks

PR:		196790
Differential Revision:	https://reviews.freebsd.org/D566
Submitted by:	Michael Gmelin <grembo@FreeBSD.org>
MFC after:	2 weeks
Sponsored by:	Multiplay
---
 usr.sbin/bsdinstall/scripts/zfsboot | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/usr.sbin/bsdinstall/scripts/zfsboot b/usr.sbin/bsdinstall/scripts/zfsboot
index 673961aef98..5bdf94fa0a9 100755
--- a/usr.sbin/bsdinstall/scripts/zfsboot
+++ b/usr.sbin/bsdinstall/scripts/zfsboot
@@ -1077,6 +1077,9 @@ zfs_create_boot()
 			boot_vdevs="$boot_vdevs $disk$bootpart"
 		fi
 		zroot_vdevs="$zroot_vdevs $disk$targetpart"
+		if [ "$ZFSBOOT_GELI_ENCRYPTION" ]; then
+			zroot_vdevs="$zroot_vdevs.eli"
+		fi
 
 		n=$(( $n + 1 ))
 	done # disks

From bb808254355827820e391e61dcc61e7085be11ab Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Sun, 18 Jan 2015 18:32:43 +0000
Subject: [PATCH 064/279] Refactor PowerPC (especially AIM) init sequence to be
 less baroque.

MFC after:	2 months
---
 sys/conf/ldscript.powerpc          |   1 +
 sys/conf/ldscript.powerpc64        |   3 +-
 sys/powerpc/aim/locore32.S         |  56 ++-----------
 sys/powerpc/aim/locore64.S         | 124 +++++++++--------------------
 sys/powerpc/aim/machdep.c          |  18 +++--
 sys/powerpc/aim/trap_subr64.S      |  15 ++--
 sys/powerpc/booke/locore.S         |  22 +----
 sys/powerpc/include/trap.h         |   3 +
 sys/powerpc/ofw/ofw_machdep.c      |  16 +++-
 sys/powerpc/powerpc/intr_machdep.c |   5 ++
 sys/powerpc/powerpc/vm_machdep.c   |   7 +-
 11 files changed, 92 insertions(+), 178 deletions(-)

diff --git a/sys/conf/ldscript.powerpc b/sys/conf/ldscript.powerpc
index efc0dfdf127..b190dc193b0 100644
--- a/sys/conf/ldscript.powerpc
+++ b/sys/conf/ldscript.powerpc
@@ -11,6 +11,7 @@ SECTIONS
   /* Read-only sections, merged into text segment: */
 
   . = kernbase + SIZEOF_HEADERS;
+  PROVIDE (begin = . - SIZEOF_HEADERS);
 
   .text      :
   {
diff --git a/sys/conf/ldscript.powerpc64 b/sys/conf/ldscript.powerpc64
index ce235516f0d..dba745538be 100644
--- a/sys/conf/ldscript.powerpc64
+++ b/sys/conf/ldscript.powerpc64
@@ -11,6 +11,7 @@ SECTIONS
   /* Read-only sections, merged into text segment: */
 
   . = kernbase + SIZEOF_HEADERS;
+  PROVIDE (begin = . - SIZEOF_HEADERS);
 
   .text      :
   {
@@ -68,7 +69,7 @@ SECTIONS
   .toc1      : ALIGN(8) { *(.toc1) }
   .opd       : ALIGN(8) { KEEP (*(.opd)) }
   .branch_lt : ALIGN(8) { *(.branch_lt) }
-  .got       : ALIGN(8) { *(.got .toc) }
+  .got       : ALIGN(8) { __tocbase = .; *(.got .toc) }
 
   .dynamic        : { *(.dynamic) }
   /* Put .ctors and .dtors next to the .got2 section, so that the pointers
diff --git a/sys/powerpc/aim/locore32.S b/sys/powerpc/aim/locore32.S
index 8cee6654b2e..6e462a05d33 100644
--- a/sys/powerpc/aim/locore32.S
+++ b/sys/powerpc/aim/locore32.S
@@ -76,29 +76,19 @@
         .globl  kernbase
         .set    kernbase, KERNBASE
 
-#define	TMPSTKSZ	8192		/* 8K temporary stack */
-
 /*
  * Globals
  */
 	.data
+	.align 3
+GLOBAL(__startkernel)
+	.long	begin
+GLOBAL(__endkernel)
+	.long	end
 	.align	4
+#define	TMPSTKSZ	8192		/* 8K temporary stack */
 GLOBAL(tmpstk)
 	.space	TMPSTKSZ
-GLOBAL(esym)
-	.long	0			/* end of symbol table */
-
-#define	INTRCNT_COUNT	256		/* max(HROWPIC_IRQMAX,OPENPIC_IRQMAX) */
-GLOBAL(intrnames)
-	.space	INTRCNT_COUNT * (MAXCOMLEN + 1) * 2
-GLOBAL(sintrnames)
-	.long	INTRCNT_COUNT * (MAXCOMLEN + 1) * 2
-
-	.align 4
-GLOBAL(intrcnt)
-	.space	INTRCNT_COUNT * 4 * 2
-GLOBAL(sintrcnt)
-	.long	INTRCNT_COUNT * 4 * 2
 
 	.text
 	.globl	btext
@@ -142,43 +132,9 @@ __start:
 	cmplw	8,9
 	blt	2b
 	
-	/* Save the argument pointer and length */
-	mr	20,6
-	mr	21,7
-
-	lis	8,openfirmware_entry@ha
-	stw	5,openfirmware_entry@l(8) /* save client interface handler */
-
 	lis	1,(tmpstk+TMPSTKSZ-16)@ha
 	addi	1,1,(tmpstk+TMPSTKSZ-16)@l
 
-	mfmsr	0
-	lis	9,ofmsr@ha
-	stwu	0,ofmsr@l(9)
-
-	mfsprg0	0			/* save SPRG0-3 */
-	stw	0,4(9)			/* ofmsr[1] = sprg0 */
-	mfsprg1 0
-	stw	0,8(9)			/* ofmsr[2] = sprg1 */
-	mfsprg2 0
-	stw	0,12(9)			/* ofmsr[3] = sprg2 */
-	mfsprg3 0
-	stw	0,16(9)			/* ofmsr[4] = sprg3 */
-	
-	bl	OF_initial_setup
-
-	lis	3,kernel_text@ha
-	addi	3,3,kernel_text@l
-
-	lis	4,end@ha
-	addi	4,4,end@l
-	add	4,4,3
-	mr	5,4
-
-	/* Restore the argument pointer and length */
-	mr	6,20
-	mr	7,21
-
 	bl	powerpc_init
 	mr	%r1, %r3
 	li	%r3, 0
diff --git a/sys/powerpc/aim/locore64.S b/sys/powerpc/aim/locore64.S
index 0c26e01125b..4865e75dc18 100644
--- a/sys/powerpc/aim/locore64.S
+++ b/sys/powerpc/aim/locore64.S
@@ -68,36 +68,28 @@
 /* Locate the per-CPU data structure */
 #define GET_CPUINFO(r)  \
         mfsprg0  r
+#define GET_TOCBASE(r)  \
+	li	r,TRAP_TOCBASE;	/* Magic address for TOC */ \
+	ld	r,0(r)
 
-/*
- * Compiled KERNBASE location and the kernel load address
- */
-        .globl  kernbase
-        .set    kernbase, KERNBASE
-
-#define	TMPSTKSZ	16384		/* 16K temporary stack */
+/* Glue for linker script */
+.globl  kernbase
+.set    kernbase, KERNBASE
 
 /*
  * Globals
  */
 	.data
+	.align 3
+GLOBAL(__startkernel)
+	.llong	begin
+GLOBAL(__endkernel)
+	.llong	end
+
 	.align	4
+#define	TMPSTKSZ	16384		/* 16K temporary stack */
 GLOBAL(tmpstk)
 	.space	TMPSTKSZ
-GLOBAL(esym)
-	.llong	0			/* end of symbol table */
-
-#define	INTRCNT_COUNT	256		/* max(HROWPIC_IRQMAX,OPENPIC_IRQMAX) */
-GLOBAL(intrnames)
-	.space	INTRCNT_COUNT * (MAXCOMLEN + 1) * 2
-GLOBAL(sintrnames)
-	.quad	INTRCNT_COUNT * (MAXCOMLEN + 1) * 2
-
-	.align 4
-GLOBAL(intrcnt)
-	.space	INTRCNT_COUNT * 4 * 2
-GLOBAL(sintrcnt)
-	.quad	INTRCNT_COUNT * 4 * 2
 
 	.text
 	.globl	btext
@@ -113,89 +105,51 @@ kernel_text:
 /*
  * Startup entry.  Note, this must be the first thing in the text
  * segment!
+ *
+ * Calling convention:
+ * r3: Flattened Device Tree pointer (or zero)
+ * r4: ignored
+ * r5: OF client interface pointer (or zero)
+ * r6: Loader metadata pointer (or zero)
  */
 	.text
 ASENTRY_NOPROF(__start)
-	li	8,0
-	li	9,0x100
-	mtctr	9
-1:
-	dcbf	0,8
-	icbi	0,8
-	addi	8,8,0x20
-	bdnz	1b
-	sync
-	isync
-
-	/* Save the argument pointer and length */
-	mr	20,6
-	mr	21,7
-
-	lis	8,openfirmware_entry@ha
-	std	5,openfirmware_entry@l(8) /* save client interface handler */
+	/* Set up the TOC pointer */
+	b	0f
+	.align 3
+0:	nop
+	bl	1f
+	.llong	__tocbase + 0x8000
+1:	mflr	%r2
+	ld	%r2,0(%r2)
 
 	/* Set up the stack pointer */
-	lis	1,(tmpstk+TMPSTKSZ-48)@ha
-	addi	1,1,(tmpstk+TMPSTKSZ-48)@l
-
-	/* Set up the TOC pointer */
-	lis	2,tocbase@ha
-	ld	2,tocbase@l(2)
-
-	mfmsr	0
-	lis	9,ofmsr@ha
-	stdu	0,ofmsr@l(9)
-
-	mfsprg0	0			/* save SPRG0-3 */
-	std	0,8(9)			/* ofmsr[1] = sprg0 */
-	mfsprg1 0
-	std	0,16(9)			/* ofmsr[2] = sprg1 */
-	mfsprg2 0
-	std	0,24(9)			/* ofmsr[3] = sprg2 */
-	mfsprg3 0
-	std	0,32(9)			/* ofmsr[4] = sprg3 */
+	lis	%r1,(tmpstk+TMPSTKSZ-48)@ha
+	addi	%r1,%r1,(tmpstk+TMPSTKSZ-48)@l
 
 	/* Switch to 64-bit mode */
-	mfmsr	9
-	li	8,1
-	insrdi	9,8,1,0
-	mtmsrd	9
+	mfmsr	%r9
+	li	%r8,1
+	insrdi	%r9,%r8,1,0
+	mtmsrd	%r9
 	isync
 	
-	bl	OF_initial_setup
-	nop
-
-	lis	3,kernbase@ha
-	addi	3,3,kernbase@l
-
-	lis	4,end@ha
-	addi	4,4,end@l
-	add	4,4,3
-	mr	5,4
-
-	/* Restore the argument pointer and length */
-	mr	6,20
-	mr	7,21
-
+	/* Begin CPU init */
+	mr	%r4,%r2 /* Replace ignored r4 with tocbase for trap handlers */
 	bl	powerpc_init
 	nop
+
+	/* Set stack pointer to new value and branch to mi_startup */
 	mr	%r1, %r3
 	li	%r3, 0
 	std	%r3, 0(%r1)
 	bl	mi_startup
 	nop
+
+	/* If this returns (it won't), go back to firmware */
 	b	OF_exit
 	nop
 
-/*
- * PPC64 ABI TOC base
- */
-
-        .align  3
-	.globl	tocbase
-tocbase:
-        .llong  .TOC.@tocbase
-
 /*
  * int setfault()
  *
diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c
index d1413a2c4ab..fdbe48b4b54 100644
--- a/sys/powerpc/aim/machdep.c
+++ b/sys/powerpc/aim/machdep.c
@@ -223,7 +223,7 @@ cpu_startup(void *dummy)
 	vm_pager_bufferinit();
 }
 
-extern char	kernel_text[], _end[];
+extern vm_offset_t	__startkernel, __endkernel;
 
 #ifndef __powerpc64__
 /* Bits for running on 64-bit systems in 32-bit mode. */
@@ -244,13 +244,12 @@ extern void	*dblow, *dbsize;
 extern void	*imisstrap, *imisssize;
 extern void	*dlmisstrap, *dlmisssize;
 extern void	*dsmisstrap, *dsmisssize;
-char 		save_trap_init[0x2f00];		/* EXC_LAST */
 
 uintptr_t
-powerpc_init(vm_offset_t startkernel, vm_offset_t endkernel,
-    vm_offset_t basekernel, void *mdp)
+powerpc_init(vm_offset_t fdt, vm_offset_t toc, vm_offset_t ofentry, void *mdp)
 {
 	struct		pcpu *pc;
+	vm_offset_t	startkernel, endkernel;
 	void		*generictrap;
 	size_t		trap_offset;
 	void		*kmdp;
@@ -273,8 +272,12 @@ powerpc_init(vm_offset_t startkernel, vm_offset_t endkernel,
 	trap_offset = 0;
 	cacheline_warn = 0;
 
-	/* Save trap vectors. */
-	ofw_save_trap_vec(save_trap_init);
+	/* Store boot environment state */
+	OF_initial_setup((void *)fdt, NULL, (int (*)(void *))ofentry);
+
+	/* First guess at start/end kernel positions */
+	startkernel = __startkernel;
+	endkernel = __endkernel;
 
 #ifdef WII
 	/*
@@ -490,6 +493,9 @@ powerpc_init(vm_offset_t startkernel, vm_offset_t endkernel,
 	#else /* powerpc64 */
 	cpu_features |= PPC_FEATURE_64;
 	generictrap = &trapcode;
+
+	/* Set TOC base so that the interrupt code can get at it */
+	*((register_t *)TRAP_TOCBASE) = toc;
 	#endif
 
 	bcopy(&rstcode, (void *)(EXC_RST + trap_offset),  (size_t)&rstsize);
diff --git a/sys/powerpc/aim/trap_subr64.S b/sys/powerpc/aim/trap_subr64.S
index 0dc46b34e86..ec3c9257911 100644
--- a/sys/powerpc/aim/trap_subr64.S
+++ b/sys/powerpc/aim/trap_subr64.S
@@ -310,8 +310,7 @@ cpu_reset:
 	lis	%r1,(tmpstk+TMPSTKSZ-48)@ha	/* get new SP */
 	addi	%r1,%r1,(tmpstk+TMPSTKSZ-48)@l
 
-	lis	%r3,tocbase@ha
-	ld	%r2,tocbase@l(%r3)
+	GET_TOCBASE(%r2)
 	bl	CNAME(cpudep_ap_early_bootstrap) /* Set PCPU */
 	nop
 	lis	%r3,1@l
@@ -445,8 +444,7 @@ kern_slbtrap:
 	addi	%r1,%r1,PC_SLBSTACK-48+1024
 	li	%r2,~15
 	and	%r1,%r1,%r2
-	lis	%r3,tocbase@ha
-	ld	%r2,tocbase@l(%r3)
+	GET_TOCBASE(%r2)
 	mflr	%r3
 	andi.	%r3,%r3,0xff80
 	mfdar	%r4
@@ -683,8 +681,7 @@ k_trap:
 	FRAME_SETUP(PC_TEMPSAVE)
 /* Call C interrupt dispatcher: */
 trapagain:
-	lis	%r3,tocbase@ha
-	ld	%r2,tocbase@l(%r3)
+	GET_TOCBASE(%r2)
 	addi	%r3,%r1,48
 	bl	CNAME(powerpc_interrupt)
 	nop
@@ -711,8 +708,7 @@ CNAME(trapexit):
 	ori	%r3,%r3,PSL_EE@l
 	mtmsr	%r3
 	isync
-	lis	%r3,tocbase@ha
-	ld	%r2,tocbase@l(%r3)
+	GET_TOCBASE(%r2)
 	addi	%r3,%r1,48
 	bl	CNAME(ast)
 	nop
@@ -760,8 +756,7 @@ dbtrap:
 
 	FRAME_SETUP(PC_DBSAVE)
 /* Call C trap code: */
-	lis	%r3,tocbase@ha
-	ld	%r2,tocbase@l(%r3)
+	GET_TOCBASE(%r2)
 	addi	%r3,%r1,48
 	bl	CNAME(db_trap_glue)
 	nop
diff --git a/sys/powerpc/booke/locore.S b/sys/powerpc/booke/locore.S
index 514afa66be7..0fcca91c36e 100644
--- a/sys/powerpc/booke/locore.S
+++ b/sys/powerpc/booke/locore.S
@@ -207,7 +207,7 @@ done_mapping:
  */
 	lis	%r1, tmpstack@ha
 	addi	%r1, %r1, tmpstack@l
-	addi	%r1, %r1, (TMPSTACKSZ - 8)
+	addi	%r1, %r1, (TMPSTACKSZ - 16)
 
 /*
  * Initialise exception vector offsets
@@ -367,7 +367,7 @@ bp_tlb1_end:
  */
 	lis	%r1, tmpstack@ha
 	addi	%r1, %r1, tmpstack@l
-	addi	%r1, %r1, (TMPSTACKSZ - 8)
+	addi	%r1, %r1, (TMPSTACKSZ - 16)
 
 /*
  * Initialise exception vector offsets
@@ -757,6 +757,8 @@ setfault:
 	.align	4
 tmpstack:
 	.space	TMPSTACKSZ
+tmpstackbound:
+	.space 10240	/* XXX: this really should not be necessary */
 
 /*
  * Compiled KERNBASE locations
@@ -764,20 +766,4 @@ tmpstack:
 	.globl	kernbase
 	.set	kernbase, KERNBASE
 
-/*
- * Globals
- */
-#define	INTRCNT_COUNT	256		/* max(HROWPIC_IRQMAX,OPENPIC_IRQMAX) */
-
-GLOBAL(intrnames)
-	.space	INTRCNT_COUNT * (MAXCOMLEN + 1) * 2
-GLOBAL(sintrnames)
-	.long	INTRCNT_COUNT * (MAXCOMLEN + 1) * 2
-
-	.align 4
-GLOBAL(intrcnt)
-	.space	INTRCNT_COUNT * 4 * 2
-GLOBAL(sintrcnt)
-	.long	INTRCNT_COUNT * 4 * 2
-
 #include <powerpc/booke/trap_subr.S>
diff --git a/sys/powerpc/include/trap.h b/sys/powerpc/include/trap.h
index 211afcf7822..3b1ade582e1 100644
--- a/sys/powerpc/include/trap.h
+++ b/sys/powerpc/include/trap.h
@@ -123,6 +123,9 @@
 /* DTrace trap opcode. */
 #define EXC_DTRACE	0x7c810808
 
+/* Magic pointer to store TOC base for trap handlers on ppc64 */
+#define TRAP_TOCBASE	0x1f8
+
 #ifndef LOCORE
 struct	trapframe;
 struct	pcb;
diff --git a/sys/powerpc/ofw/ofw_machdep.c b/sys/powerpc/ofw/ofw_machdep.c
index cc44847fa93..1c02fa3e90d 100644
--- a/sys/powerpc/ofw/ofw_machdep.c
+++ b/sys/powerpc/ofw/ofw_machdep.c
@@ -66,8 +66,8 @@ extern register_t ofmsr[5];
 extern void	*openfirmware_entry;
 static void	*fdt;
 int		ofw_real_mode;
-extern char     save_trap_init[0x2f00];          /* EXC_LAST */
-char            save_trap_of[0x2f00];            /* EXC_LAST */
+char		save_trap_init[0x2f00];          /* EXC_LAST */
+char		save_trap_of[0x2f00];            /* EXC_LAST */
 
 int		ofwcall(void *);
 static int	openfirmware(void *args);
@@ -257,18 +257,30 @@ ofw_mem_regions(struct mem_region *memp, int *memsz,
 void
 OF_initial_setup(void *fdt_ptr, void *junk, int (*openfirm)(void *))
 {
+	ofmsr[0] = mfmsr();
+	#ifdef __powerpc64__
+	ofmsr[0] &= ~PSL_SF;
+	#endif
+	__asm __volatile("mfsprg0 %0" : "=&r"(ofmsr[1]));
+	__asm __volatile("mfsprg1 %0" : "=&r"(ofmsr[2]));
+	__asm __volatile("mfsprg2 %0" : "=&r"(ofmsr[3]));
+	__asm __volatile("mfsprg3 %0" : "=&r"(ofmsr[4]));
+
 	if (ofmsr[0] & PSL_DR)
 		ofw_real_mode = 0;
 	else
 		ofw_real_mode = 1;
 
 	fdt = fdt_ptr;
+	openfirmware_entry = openfirm;
 
 	#ifdef FDT_DTB_STATIC
 	/* Check for a statically included blob */
 	if (fdt == NULL)
 		fdt = &fdt_static_dtb;
 	#endif
+
+	ofw_save_trap_vec(save_trap_init);
 }
 
 boolean_t
diff --git a/sys/powerpc/powerpc/intr_machdep.c b/sys/powerpc/powerpc/intr_machdep.c
index cc7624f61e4..b481279d657 100644
--- a/sys/powerpc/powerpc/intr_machdep.c
+++ b/sys/powerpc/powerpc/intr_machdep.c
@@ -127,6 +127,11 @@ static u_int nirqs = 0;		/* Allocated IRQs. */
 #endif
 static u_int stray_count;
 
+u_long intrcnt[INTR_VECTORS];
+char intrnames[INTR_VECTORS * MAXCOMLEN];
+size_t sintrcnt = sizeof(intrcnt);
+size_t sintrnames = sizeof(intrnames);
+
 device_t root_pic;
 
 #ifdef SMP
diff --git a/sys/powerpc/powerpc/vm_machdep.c b/sys/powerpc/powerpc/vm_machdep.c
index d6840947f0f..c816e32cc2b 100644
--- a/sys/powerpc/powerpc/vm_machdep.c
+++ b/sys/powerpc/powerpc/vm_machdep.c
@@ -99,11 +99,6 @@
 #include <vm/vm_map.h>
 #include <vm/vm_extern.h>
 
-#ifdef __powerpc64__
-extern uintptr_t tocbase;
-#endif
-
-
 /*
  * Finish a fork operation, with process p2 nearly set up.
  * Copy and update the pcb, set up the stack so that the child
@@ -149,7 +144,7 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
 	cf = (struct callframe *)tf - 1;
 	memset(cf, 0, sizeof(struct callframe));
 	#ifdef __powerpc64__
-	cf->cf_toc = tocbase;
+	cf->cf_toc = ((register_t *)fork_return)[1];
 	#endif
 	cf->cf_func = (register_t)fork_return;
 	cf->cf_arg0 = (register_t)td2;

From 9cecb88ce397aefd3733f3e46bbd128044ee7673 Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Sun, 18 Jan 2015 20:00:33 +0000
Subject: [PATCH 065/279] Use TOC to look up all kernel globals on powerpc64
 instead of doing the non-relocatable lis @ha, ori @l dance and hoping they
 are below 4 GB.

MFC after:	2 months
---
 sys/powerpc/aim/locore64.S    |  6 ++++--
 sys/powerpc/aim/trap_subr64.S | 13 ++++++++-----
 sys/powerpc/include/asm.h     |  6 ++++++
 sys/powerpc/ofw/ofwcall64.S   | 36 ++++++++++++++++++++++-------------
 sys/powerpc/powerpc/swtch64.S |  5 +++--
 5 files changed, 44 insertions(+), 22 deletions(-)

diff --git a/sys/powerpc/aim/locore64.S b/sys/powerpc/aim/locore64.S
index 4865e75dc18..30ee3a95dc9 100644
--- a/sys/powerpc/aim/locore64.S
+++ b/sys/powerpc/aim/locore64.S
@@ -91,6 +91,8 @@ GLOBAL(__endkernel)
 GLOBAL(tmpstk)
 	.space	TMPSTKSZ
 
+TOC_ENTRY(tmpstk)
+
 	.text
 	.globl	btext
 btext:
@@ -124,8 +126,8 @@ ASENTRY_NOPROF(__start)
 	ld	%r2,0(%r2)
 
 	/* Set up the stack pointer */
-	lis	%r1,(tmpstk+TMPSTKSZ-48)@ha
-	addi	%r1,%r1,(tmpstk+TMPSTKSZ-48)@l
+	ld	%r1,TOC_REF(tmpstk)(%r2)
+	addi	%r1,%r1,TMPSTKSZ-48
 
 	/* Switch to 64-bit mode */
 	mfmsr	%r9
diff --git a/sys/powerpc/aim/trap_subr64.S b/sys/powerpc/aim/trap_subr64.S
index ec3c9257911..63ca50a4bfe 100644
--- a/sys/powerpc/aim/trap_subr64.S
+++ b/sys/powerpc/aim/trap_subr64.S
@@ -307,10 +307,11 @@ CNAME(rstcode):
 CNAME(rstsize) = . - CNAME(rstcode)
 
 cpu_reset:
-	lis	%r1,(tmpstk+TMPSTKSZ-48)@ha	/* get new SP */
-	addi	%r1,%r1,(tmpstk+TMPSTKSZ-48)@l
-
 	GET_TOCBASE(%r2)
+
+	ld	%r1,TOC_REF(tmpstk)(%r2)	/* get new SP */
+	addi	%r1,%r1,(TMPSTKSZ-48)
+
 	bl	CNAME(cpudep_ap_early_bootstrap) /* Set PCPU */
 	nop
 	lis	%r3,1@l
@@ -751,8 +752,10 @@ dbtrap:
 	andi.	%r1,%r1,0xff00
 	mtsprg3	%r1
 
-	lis	%r1,(tmpstk+TMPSTKSZ-48)@ha	/* get new SP */
-	addi	%r1,%r1,(tmpstk+TMPSTKSZ-48)@l
+	li	%r1,TRAP_TOCBASE		/* get new SP */
+	ld	%r1,0(%r1)
+	ld	%r1,TOC_REF(tmpstk)(%r1)
+	addi	%r1,%r1,(TMPSTKSZ-48)
 
 	FRAME_SETUP(PC_DBSAVE)
 /* Call C trap code: */
diff --git a/sys/powerpc/include/asm.h b/sys/powerpc/include/asm.h
index e571316d15e..c04cbc2f78e 100644
--- a/sys/powerpc/include/asm.h
+++ b/sys/powerpc/include/asm.h
@@ -80,6 +80,12 @@
 	name:
 
 #ifdef __powerpc64__
+#define TOC_REF(name)	__CONCAT(.L,name)
+#define TOC_ENTRY(name) \
+	.section ".toc","aw"; \
+	TOC_REF(name): \
+        .tc name[TC],name
+
 #define	_ENTRY(name) \
 	.section ".text"; \
 	.p2align 2; \
diff --git a/sys/powerpc/ofw/ofwcall64.S b/sys/powerpc/ofw/ofwcall64.S
index beb6bdc7073..c1778cc8293 100644
--- a/sys/powerpc/ofw/ofwcall64.S
+++ b/sys/powerpc/ofw/ofwcall64.S
@@ -52,6 +52,13 @@ GLOBAL(openfirmware_entry)
 GLOBAL(rtas_entry)
 	.llong	0			/* RTAS entry point */
 
+TOC_ENTRY(ofmsr)
+TOC_ENTRY(ofwstk)
+TOC_ENTRY(rtasmsr)
+TOC_ENTRY(openfirmware_entry)
+TOC_ENTRY(rtas_entry)
+TOC_ENTRY(rtas_regsave)
+
 /*
  * Open Firmware Real-mode Entry Point. This is a huge pain.
  */
@@ -94,16 +101,20 @@ ASENTRY_NOPROF(ofwcall)
 	mfmsr	%r6
 
 	/* read client interface handler */
-	lis	%r4,openfirmware_entry@ha
-	ld	%r4,openfirmware_entry@l(%r4)
+	ld	%r4,TOC_REF(openfirmware_entry)(%r2)
+	ld	%r4,0(%r4)
+
+	/* Get OF stack pointer */
+	ld	%r7,TOC_REF(ofwstk)(%r2)
+	addi	%r7,%r7,OFWSTKSZ-32
 
 	/*
 	 * Set the MSR to the OF value. This has the side effect of disabling
 	 * exceptions, which is important for the next few steps.
 	 */
 
-	lis	%r5,ofmsr@ha
-	ld	%r5,ofmsr@l(%r5)
+	ld	%r5,TOC_REF(ofmsr)(%r2)
+	ld	%r5,0(%r5)
 	mtmsrd	%r5
 	isync
 
@@ -114,8 +125,7 @@ ASENTRY_NOPROF(ofwcall)
 	 * the old MSR so we can get them back later.
 	 */
 	mr	%r5,%r1
-	lis	%r1,(ofwstk+OFWSTKSZ-32)@ha
-	addi	%r1,%r1,(ofwstk+OFWSTKSZ-32)@l
+	mr	%r1,%r7
 	std	%r5,8(%r1)	/* Save real stack pointer */
 	std	%r2,16(%r1)	/* Save old TOC */
 	std	%r6,24(%r1)	/* Save old MSR */
@@ -212,17 +222,18 @@ ASENTRY_NOPROF(rtascall)
 	/* Record the old MSR */
 	mfmsr	%r6
 
-	/* read client interface handler */
-	lis	%r5,rtas_entry@ha
-	ld	%r5,rtas_entry@l(%r5)
+	/* Read RTAS entry and reg save area pointers */
+	ld	%r5,TOC_REF(rtas_entry)(%r2)
+	ld	%r5,0(%r5)
+	ld	%r8,TOC_REF(rtas_regsave)(%r2)
 
 	/*
 	 * Set the MSR to the RTAS value. This has the side effect of disabling
 	 * exceptions, which is important for the next few steps.
 	 */
 
-	lis	%r7,rtasmsr@ha
-	ld	%r7,rtasmsr@l(%r7)
+	ld	%r7,TOC_REF(rtasmsr)(%r2)
+	ld	%r7,0(%r7)
 	mtmsrd	%r7
 	isync
 
@@ -233,8 +244,7 @@ ASENTRY_NOPROF(rtascall)
 	 * are below 4 GB, so this is safe.
 	 */
 	mr	%r7,%r1
-	lis	%r1,rtas_regsave@ha
-	addi	%r1,%r1,rtas_regsave@l
+	mr	%r1,%r8
 	std	%r7,0(%r1)	/* Save 64-bit stack pointer */
 	std	%r2,8(%r1)	/* Save TOC */
 	std	%r6,16(%r1)	/* Save MSR */
diff --git a/sys/powerpc/powerpc/swtch64.S b/sys/powerpc/powerpc/swtch64.S
index ab6f5327895..6722bb64213 100644
--- a/sys/powerpc/powerpc/swtch64.S
+++ b/sys/powerpc/powerpc/swtch64.S
@@ -65,6 +65,8 @@
 #include <machine/param.h>
 #include <machine/asm.h>
 
+TOC_ENTRY(blocked_lock)
+
 /*
  * void cpu_throw(struct thread *old, struct thread *new)
  */
@@ -145,8 +147,7 @@ ENTRY(cpu_switch)
 cpu_switchin:
 #if defined(SMP) && defined(SCHED_ULE)
 	/* Wait for the new thread to become unblocked */
-	lis	%r6,blocked_lock@ha
-	addi	%r6,%r6,blocked_lock@l
+	ld	%r6,TOC_REF(blocked_lock)(%r2)
 blocked_loop:
 	ld	%r7,TD_LOCK(%r13)
 	cmpd	%r6,%r7 

From 61330de4b046c8ef69e7cdd060fd1d6b5f7c870d Mon Sep 17 00:00:00 2001
From: Michael Tuexen <tuexen@FreeBSD.org>
Date: Sun, 18 Jan 2015 20:20:27 +0000
Subject: [PATCH 066/279] Remove an unused variable.

Reported by:	Coverity
CID:		750999
MFC after:	1 week
---
 sys/netinet/sctp_usrreq.c | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/sys/netinet/sctp_usrreq.c b/sys/netinet/sctp_usrreq.c
index 95955dedcb9..b1d85f51eb3 100644
--- a/sys/netinet/sctp_usrreq.c
+++ b/sys/netinet/sctp_usrreq.c
@@ -4720,7 +4720,6 @@ sctp_setopt(struct socket *so, int optname, void *optval, size_t optsize,
 	case SCTP_CONNECT_X_COMPLETE:
 		{
 			struct sockaddr *sa;
-			struct sctp_nets *net;
 
 			/* FIXME MT: check correct? */
 			SCTP_CHECK_AND_CAST(sa, optval, struct sockaddr, optsize);
@@ -4731,7 +4730,6 @@ sctp_setopt(struct socket *so, int optname, void *optval, size_t optsize,
 				stcb = LIST_FIRST(&inp->sctp_asoc_list);
 				if (stcb) {
 					SCTP_TCB_LOCK(stcb);
-					net = sctp_findnet(stcb, sa);
 				}
 				SCTP_INP_RUNLOCK(inp);
 			} else {
@@ -4743,7 +4741,7 @@ sctp_setopt(struct socket *so, int optname, void *optval, size_t optsize,
 				 * TCB.. aka NULL.
 				 */
 				SCTP_INP_INCR_REF(inp);
-				stcb = sctp_findassociation_ep_addr(&inp, sa, &net, NULL, NULL);
+				stcb = sctp_findassociation_ep_addr(&inp, sa, NULL, NULL, NULL);
 				if (stcb == NULL) {
 					SCTP_INP_DECR_REF(inp);
 				}

From 955ba37baaf837592e5c1b532e0495cbc76e7a45 Mon Sep 17 00:00:00 2001
From: "Pedro F. Giffuni" <pfg@FreeBSD.org>
Date: Sun, 18 Jan 2015 20:26:27 +0000
Subject: [PATCH 067/279] Remove dead code.

After the ext2 variant of the "orlov allocator" was implemented,
the case for a negative or zero dirsize disappeared.

Drop the dead code and unsign dirsize given that it can't be
negative anyways.

CID:		1008669
MFC after:	1 week
---
 sys/fs/ext2fs/ext2_alloc.c | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/sys/fs/ext2fs/ext2_alloc.c b/sys/fs/ext2fs/ext2_alloc.c
index e2bb138793b..935cb0caeda 100644
--- a/sys/fs/ext2fs/ext2_alloc.c
+++ b/sys/fs/ext2fs/ext2_alloc.c
@@ -443,11 +443,11 @@ static u_long
 ext2_dirpref(struct inode *pip)
 {
 	struct m_ext2fs *fs;
-	int cg, prefcg, dirsize, cgsize;
+	int cg, prefcg, cgsize;
 	u_int avgifree, avgbfree, avgndir, curdirsize;
 	u_int minifree, minbfree, maxndir;
 	u_int mincg, minndir;
-	u_int maxcontigdirs;
+	u_int dirsize, maxcontigdirs;
 
 	mtx_assert(EXT2_MTX(pip->i_ump), MA_OWNED);
 	fs = pip->i_e2fs;
@@ -498,10 +498,7 @@ ext2_dirpref(struct inode *pip)
 	curdirsize = avgndir ? (cgsize - avgbfree * fs->e2fs_bsize) / avgndir : 0;
 	if (dirsize < curdirsize)
 		dirsize = curdirsize;
-	if (dirsize <= 0)
-		maxcontigdirs = 0;		/* dirsize overflowed */
-	else
-		maxcontigdirs = min((avgbfree * fs->e2fs_bsize) / dirsize, 255);
+	maxcontigdirs = min((avgbfree * fs->e2fs_bsize) / dirsize, 255);
 	maxcontigdirs = min(maxcontigdirs, fs->e2fs_ipg / AFPDIR);
 	if (maxcontigdirs == 0)
 		maxcontigdirs = 1;

From 86ee58d9929d8e687c7a59d25b1ea49b7507812d Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Sun, 18 Jan 2015 20:47:21 +0000
Subject: [PATCH 068/279] Save the command-and-flags value into the shadow
 register when it is written.

This doesn't actually change any behavior, because it just allows a 16-bit
read of the command register to return the correct value, and nothing
actually does a 16-bit read of that register.
---
 sys/arm/broadcom/bcm2835/bcm2835_sdhci.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c b/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c
index 7e1ad1988eb..5f31478266e 100644
--- a/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c
+++ b/sys/arm/broadcom/bcm2835/bcm2835_sdhci.c
@@ -399,8 +399,11 @@ bcm_sdhci_write_2(device_t dev, struct sdhci_slot *slot, bus_size_t off, uint16_
 	val32 |= (val << (off & 3)*8);
 	if (off == SDHCI_TRANSFER_MODE)
 		sc->cmd_and_mode = val32;
-	else
+	else {
 		WR4(sc, off & ~3, val32);
+		if (off == SDHCI_COMMAND_FLAGS)
+			sc->cmd_and_mode = val32;
+	}
 }
 
 static void

From 3ff78fbbd95efbce7db1d4f042a7c9f80cff76d2 Mon Sep 17 00:00:00 2001
From: Michael Tuexen <tuexen@FreeBSD.org>
Date: Sun, 18 Jan 2015 20:53:20 +0000
Subject: [PATCH 069/279] Add protection code to free memory in case of
 processing an address which is neither IPv4 or IPv6.

Reported by:	Coverity
CID:		749311
MFC after:	1 week
---
 sys/netinet/sctp_asconf.c | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/sys/netinet/sctp_asconf.c b/sys/netinet/sctp_asconf.c
index 069d4536a17..456990019c2 100644
--- a/sys/netinet/sctp_asconf.c
+++ b/sys/netinet/sctp_asconf.c
@@ -3346,6 +3346,11 @@ sctp_asconf_send_nat_state_update(struct sctp_tcb *stcb,
 		TAILQ_INSERT_TAIL(&stcb->asoc.asconf_queue, aa, next);
 		break;
 #endif
+	default:
+		SCTPDBG(SCTP_DEBUG_ASCONF1,
+		    "sctp_asconf_send_nat_state_update: unknown address family\n");
+		SCTP_FREE(aa, SCTP_M_ASC_ADDR);
+		return;
 	}
 	SCTP_MALLOC(aa, struct sctp_asconf_addr *, sizeof(*aa),
 	    SCTP_M_ASC_ADDR);
@@ -3379,6 +3384,11 @@ sctp_asconf_send_nat_state_update(struct sctp_tcb *stcb,
 		TAILQ_INSERT_TAIL(&stcb->asoc.asconf_queue, aa, next);
 		break;
 #endif
+	default:
+		SCTPDBG(SCTP_DEBUG_ASCONF1,
+		    "sctp_asconf_send_nat_state_update: unknown address family\n");
+		SCTP_FREE(aa, SCTP_M_ASC_ADDR);
+		return;
 	}
 	/* Now we must hunt the addresses and add all global addresses */
 	if (stcb->sctp_ep->sctp_flags & SCTP_PCB_FLAGS_BOUNDALL) {

From d6165c1fca36d01e3014777db2dd990a1c38563d Mon Sep 17 00:00:00 2001
From: Michael Tuexen <tuexen@FreeBSD.org>
Date: Sun, 18 Jan 2015 21:16:22 +0000
Subject: [PATCH 070/279] Remove an unnecessary check.

Reported by:	Coverity
CID:		749576
MFC after:	1 week
---
 sys/netinet/sctp_input.c | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/sys/netinet/sctp_input.c b/sys/netinet/sctp_input.c
index ab64435364f..feb7b892eca 100644
--- a/sys/netinet/sctp_input.c
+++ b/sys/netinet/sctp_input.c
@@ -3108,7 +3108,6 @@ sctp_handle_ecn_cwr(struct sctp_cwr_chunk *cp, struct sctp_tcb *stcb, struct sct
 	uint32_t cwr_tsn;
 
 	cwr_tsn = ntohl(cp->tsn);
-
 	override = cp->ch.chunk_flags & SCTP_CWR_REDUCE_OVERRIDE;
 	TAILQ_FOREACH(chk, &stcb->asoc.control_send_queue, sctp_next) {
 		if (chk->rec.chunk_id.id != SCTP_ECN_ECHO) {
@@ -3124,10 +3123,8 @@ sctp_handle_ecn_cwr(struct sctp_cwr_chunk *cp, struct sctp_tcb *stcb, struct sct
 			stcb->asoc.ecn_echo_cnt_onq--;
 			TAILQ_REMOVE(&stcb->asoc.control_send_queue, chk,
 			    sctp_next);
-			if (chk->data) {
-				sctp_m_freem(chk->data);
-				chk->data = NULL;
-			}
+			sctp_m_freem(chk->data);
+			chk->data = NULL;
 			stcb->asoc.ctrl_queue_cnt--;
 			sctp_free_a_chunk(stcb, chk, SCTP_SO_NOT_LOCKED);
 			if (override == 0) {

From 7075482d4ad90db5a501bbd1d1a87f1629bd6ab1 Mon Sep 17 00:00:00 2001
From: "Pedro F. Giffuni" <pfg@FreeBSD.org>
Date: Sun, 18 Jan 2015 21:18:28 +0000
Subject: [PATCH 071/279] ext2: fix for uninitialized pointer read.

path.ep_bp was being used uninitialized in ext4_ext_find_extent().

CID:		1062344
MFC after:	1 week
---
 sys/fs/ext2fs/ext2_bmap.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/fs/ext2fs/ext2_bmap.c b/sys/fs/ext2fs/ext2_bmap.c
index ce648586097..12f9560d93b 100644
--- a/sys/fs/ext2fs/ext2_bmap.c
+++ b/sys/fs/ext2fs/ext2_bmap.c
@@ -94,7 +94,7 @@ ext4_bmapext(struct vnode *vp, int32_t bn, int64_t *bnp, int *runp, int *runb)
 	struct inode *ip;
 	struct m_ext2fs *fs;
 	struct ext4_extent *ep;
-	struct ext4_extent_path path;
+	struct ext4_extent_path path = { .ep_bp = NULL };
 	daddr_t lbn;
 
 	ip = VTOI(vp);

From e1600e5058c4e733743f7e0a4610816d1c6bd3d5 Mon Sep 17 00:00:00 2001
From: Michael Tuexen <tuexen@FreeBSD.org>
Date: Sun, 18 Jan 2015 22:00:39 +0000
Subject: [PATCH 072/279] Fix a bug which only shows up when an mbuf allocation
 failed. Therefore chances are low that we hit this.

Reported by:	Coverity
CID:		1018886
MFC after:	1 week
---
 sys/netinet/sctp_output.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/netinet/sctp_output.c b/sys/netinet/sctp_output.c
index 86aa3afdeab..8e8157499d4 100644
--- a/sys/netinet/sctp_output.c
+++ b/sys/netinet/sctp_output.c
@@ -7420,7 +7420,7 @@ dont_do_it:
 				SCTP_TCB_SEND_LOCK(stcb);
 				send_lock_up = 1;
 			}
-			if (chk->data == NULL) {
+			if (sp->data == NULL) {
 				/* unsteal the data */
 				sp->data = chk->data;
 				sp->tail_mbuf = chk->last_mbuf;

From 5eab7e540605a77b6d141deee5fbaf56291be23a Mon Sep 17 00:00:00 2001
From: Steven Hartland <smh@FreeBSD.org>
Date: Sun, 18 Jan 2015 23:15:49 +0000
Subject: [PATCH 073/279] Clean ZFS spa config before syncing

A number of entries that can be present in the spa config shouldn't be saved
to disk so add a method to ensure this is case. Without this if the last
caller to vdev_config_generate requested stats then we can end up in the
cache file.

Also only skip a none writable pool in the cache file generation if its
active. This prevents unavailable pools incorrectly getting removed from
cache file.

Tested by:	delphij
MFC after:	2 weeks
Sponsored by:	Multiplay
---
 .../uts/common/fs/zfs/spa_config.c            | 27 ++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_config.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_config.c
index c4ee741e9a7..be1e528f7f4 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_config.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa_config.c
@@ -140,6 +140,26 @@ out:
 	kobj_close_file(file);
 }
 
+static void
+spa_config_clean(nvlist_t *nvl)
+{
+	nvlist_t **child;
+	nvlist_t *nvroot = NULL;
+	uint_t c, children;
+
+	if (nvlist_lookup_nvlist_array(nvl, ZPOOL_CONFIG_CHILDREN, &child,
+	    &children) == 0) {
+		for (c = 0; c < children; c++)
+			spa_config_clean(child[c]);
+	}
+
+	if (nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_VDEV_TREE, &nvroot) == 0)
+		spa_config_clean(nvroot);
+
+	nvlist_remove(nvl, ZPOOL_CONFIG_VDEV_STATS, DATA_TYPE_UINT64_ARRAY);
+	nvlist_remove(nvl, ZPOOL_CONFIG_SCAN_STATS, DATA_TYPE_UINT64_ARRAY);
+}
+
 static int
 spa_config_write(spa_config_dirent_t *dp, nvlist_t *nvl)
 {
@@ -233,6 +253,7 @@ spa_config_sync(spa_t *target, boolean_t removing, boolean_t postsysevent)
 		 */
 		nvl = NULL;
 		while ((spa = spa_next(spa)) != NULL) {
+			nvlist_t *nvroot = NULL;
 			/*
 			 * Skip over our own pool if we're about to remove
 			 * ourselves from the spa namespace or any pool that
@@ -241,7 +262,8 @@ spa_config_sync(spa_t *target, boolean_t removing, boolean_t postsysevent)
 			 * we don't allow them to be written to the cache file.
 			 */
 			if ((spa == target && removing) ||
-			    !spa_writeable(spa))
+			    (spa_state(spa) == POOL_STATE_ACTIVE &&
+			    !spa_writeable(spa)))
 				continue;
 
 			mutex_enter(&spa->spa_props_lock);
@@ -260,6 +282,9 @@ spa_config_sync(spa_t *target, boolean_t removing, boolean_t postsysevent)
 			VERIFY(nvlist_add_nvlist(nvl, spa->spa_name,
 			    spa->spa_config) == 0);
 			mutex_exit(&spa->spa_props_lock);
+
+			if (nvlist_lookup_nvlist(nvl, spa->spa_name, &nvroot) == 0)
+				spa_config_clean(nvroot);
 		}
 
 		error = spa_config_write(dp, nvl);

From 9745de4c2c78da747da5ddcb58b941cef9599132 Mon Sep 17 00:00:00 2001
From: Ryan Stone <rstone@FreeBSD.org>
Date: Mon, 19 Jan 2015 00:33:32 +0000
Subject: [PATCH 074/279] When mountd is creating sockets, it iterates over all
 addresses specified in the "hosts" array and eventually looks up the network
 address with getaddrinfo(). At one point it checks for a numeric address and
 if it sees one, it sets a hint parameter to force getaddrinfo to interpret
 the host as a numeric address. However that hint is not cleared for
 subsequent iterations of the loop and if any hosts seen after this point are
 host names, getaddrinfo will fail on the name.  The result of this bug is
 that you cannot pass a host name to the -h flag.

Unfortunately, the first iteration will either process ::1 or 127.0.0.1,
so the flag is set on the first iteration and all host names will fail
to be processed.

The same bug applies to rpc.lockd and rpc.statd, so fix them too.

Differential Revision:	https://reviews.freebsd.org/D1507
Reported by:	Dylan Martin
MFC after:	1 week
Sponsored by:	Sandvine Inc.
---
 usr.sbin/mountd/mountd.c   | 3 ++-
 usr.sbin/rpc.lockd/lockd.c | 2 +-
 usr.sbin/rpc.statd/statd.c | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/usr.sbin/mountd/mountd.c b/usr.sbin/mountd/mountd.c
index 6e4085c85aa..87794c3ce6d 100644
--- a/usr.sbin/mountd/mountd.c
+++ b/usr.sbin/mountd/mountd.c
@@ -627,7 +627,6 @@ create_service(struct netconfig *nconf)
 
 	/* Get mountd's address on this transport */
 	memset(&hints, 0, sizeof hints);
-	hints.ai_flags = AI_PASSIVE;
 	hints.ai_family = si.si_af;
 	hints.ai_socktype = si.si_socktype;
 	hints.ai_protocol = si.si_proto;
@@ -644,6 +643,8 @@ create_service(struct netconfig *nconf)
 		sock_fd[sock_fdcnt++] = -1;	/* Set invalid for now. */
 		mallocd_res = 0;
 
+		hints.ai_flags = AI_PASSIVE;
+
 		/*	
 		 * XXX - using RPC library internal functions.
 		 */
diff --git a/usr.sbin/rpc.lockd/lockd.c b/usr.sbin/rpc.lockd/lockd.c
index 2974acb6ca4..4f1347ef924 100644
--- a/usr.sbin/rpc.lockd/lockd.c
+++ b/usr.sbin/rpc.lockd/lockd.c
@@ -518,7 +518,6 @@ create_service(struct netconfig *nconf)
 
 	/* Get rpc.statd's address on this transport */
 	memset(&hints, 0, sizeof hints);
-	hints.ai_flags = AI_PASSIVE;
 	hints.ai_family = si.si_af;
 	hints.ai_socktype = si.si_socktype;
 	hints.ai_protocol = si.si_proto;
@@ -534,6 +533,7 @@ create_service(struct netconfig *nconf)
 			out_of_mem();
 		sock_fd[sock_fdcnt++] = -1;	/* Set invalid for now. */
 		mallocd_res = 0;
+		hints.ai_flags = AI_PASSIVE;
 
 		/*	
 		 * XXX - using RPC library internal functions.
diff --git a/usr.sbin/rpc.statd/statd.c b/usr.sbin/rpc.statd/statd.c
index ff537f84b53..faa8513ba59 100644
--- a/usr.sbin/rpc.statd/statd.c
+++ b/usr.sbin/rpc.statd/statd.c
@@ -343,7 +343,6 @@ create_service(struct netconfig *nconf)
 
 	/* Get rpc.statd's address on this transport */
 	memset(&hints, 0, sizeof hints);
-	hints.ai_flags = AI_PASSIVE;
 	hints.ai_family = si.si_af;
 	hints.ai_socktype = si.si_socktype;
 	hints.ai_protocol = si.si_proto;
@@ -359,6 +358,7 @@ create_service(struct netconfig *nconf)
 			out_of_mem();
 		sock_fd[sock_fdcnt++] = -1;	/* Set invalid for now. */
 		mallocd_res = 0;
+		hints.ai_flags = AI_PASSIVE;
 
 		/*	
 		 * XXX - using RPC library internal functions.

From 545ddfbe7d4fe8adfb862903b24eac1d5896c1ef Mon Sep 17 00:00:00 2001
From: Marcel Moolenaar <marcel@FreeBSD.org>
Date: Mon, 19 Jan 2015 02:22:03 +0000
Subject: [PATCH 075/279] Upgrade libxo to 0.2.0.

Obtained from:	https://github.com/Juniper/libxo
Requested by: Phil Shafer <phil@juniper.net>

Revisions 276253 & 276273 were incorporated into 0.2.0.
Revision 276260 has been merged-in.
---
 contrib/libxo/Makefile.am                     |   24 +-
 contrib/libxo/bin/Zaliases                    |    2 +-
 contrib/libxo/configure.ac                    |    6 +-
 contrib/libxo/doc/libxo.txt                   |  293 ++++-
 contrib/libxo/libxo/libxo.3                   |  257 +++-
 contrib/libxo/libxo/libxo.c                   | 1129 ++++++++++++++---
 contrib/libxo/libxo/xo.h                      |   81 +-
 contrib/libxo/libxo/xo_attr.3                 |    4 +
 contrib/libxo/libxo/xo_create.3               |   42 +-
 contrib/libxo/libxo/xo_emit.3                 |   15 +-
 contrib/libxo/libxo/xo_err.3                  |   23 +-
 contrib/libxo/libxo/xo_error.3                |   64 +
 contrib/libxo/libxo/xo_finish.3               |   20 +-
 contrib/libxo/libxo/xo_flush.3                |   23 +-
 contrib/libxo/libxo/xo_format.5               |  333 +++--
 contrib/libxo/libxo/xo_no_setlocale.3         |   37 +-
 contrib/libxo/libxo/xo_open_container.3       |   58 +-
 contrib/libxo/libxo/xo_open_list.3            |   55 +-
 contrib/libxo/libxo/xo_parse_args.3           |   88 +-
 contrib/libxo/libxo/xo_set_allocator.3        |   21 +-
 contrib/libxo/libxo/xo_set_flags.3            |  106 +-
 contrib/libxo/libxo/xo_set_info.3             |   51 +-
 contrib/libxo/libxo/xo_set_options.3          |   19 +-
 contrib/libxo/libxo/xo_set_style.3            |   21 +-
 contrib/libxo/libxo/xo_set_writer.3           |   30 +-
 contrib/libxo/libxo/xoconfig.h                |   12 +-
 contrib/libxo/libxo/xoconfig.h.in             |    6 +
 contrib/libxo/libxo/xoversion.h               |    6 +-
 contrib/libxo/packaging/libxo.rb.base.in      |   20 +
 contrib/libxo/tests/core/Makefile.am          |    6 +-
 contrib/libxo/tests/core/saved/test_01.H.out  |    2 +-
 .../libxo/tests/core/saved/test_01.HIPx.out   |   58 +
 contrib/libxo/tests/core/saved/test_01.HP.out |   58 +
 contrib/libxo/tests/core/saved/test_01.J.out  |    2 +-
 contrib/libxo/tests/core/saved/test_01.JP.out |    9 +-
 contrib/libxo/tests/core/saved/test_01.T.out  |    8 +
 contrib/libxo/tests/core/saved/test_01.X.out  |    2 +-
 contrib/libxo/tests/core/saved/test_01.XP.out |   23 +-
 contrib/libxo/tests/core/saved/test_02.J.out  |    2 +-
 contrib/libxo/tests/core/saved/test_02.JP.out |    5 +-
 contrib/libxo/tests/core/saved/test_07.J.out  |    2 +-
 contrib/libxo/tests/core/saved/test_07.JP.out |    2 +-
 contrib/libxo/tests/core/saved/test_08.H.err  |   18 +
 contrib/libxo/tests/core/saved/test_08.H.out  |    1 +
 .../libxo/tests/core/saved/test_08.HIPx.err   |   18 +
 .../libxo/tests/core/saved/test_08.HIPx.out   |  264 ++++
 contrib/libxo/tests/core/saved/test_08.HP.err |   18 +
 contrib/libxo/tests/core/saved/test_08.HP.out |  264 ++++
 contrib/libxo/tests/core/saved/test_08.J.err  |   18 +
 contrib/libxo/tests/core/saved/test_08.J.out  |    2 +
 contrib/libxo/tests/core/saved/test_08.JP.err |   18 +
 contrib/libxo/tests/core/saved/test_08.JP.out |  185 +++
 contrib/libxo/tests/core/saved/test_08.T.err  |   18 +
 contrib/libxo/tests/core/saved/test_08.T.out  |   52 +
 contrib/libxo/tests/core/saved/test_08.X.err  |   18 +
 contrib/libxo/tests/core/saved/test_08.X.out  |    1 +
 contrib/libxo/tests/core/saved/test_08.XP.err |   18 +
 contrib/libxo/tests/core/saved/test_08.XP.out |  165 +++
 contrib/libxo/tests/core/saved/test_09.H.err  |    0
 contrib/libxo/tests/core/saved/test_09.H.out  |    1 +
 .../libxo/tests/core/saved/test_09.HIPx.err   |    0
 .../libxo/tests/core/saved/test_09.HIPx.out   |   93 ++
 contrib/libxo/tests/core/saved/test_09.HP.err |    0
 contrib/libxo/tests/core/saved/test_09.HP.out |   93 ++
 contrib/libxo/tests/core/saved/test_09.J.err  |    0
 contrib/libxo/tests/core/saved/test_09.J.out  |    2 +
 contrib/libxo/tests/core/saved/test_09.JP.err |    0
 contrib/libxo/tests/core/saved/test_09.JP.out |   27 +
 contrib/libxo/tests/core/saved/test_09.T.err  |    0
 contrib/libxo/tests/core/saved/test_09.T.out  |   25 +
 contrib/libxo/tests/core/saved/test_09.X.err  |    0
 contrib/libxo/tests/core/saved/test_09.X.out  |    1 +
 contrib/libxo/tests/core/saved/test_09.XP.err |    0
 contrib/libxo/tests/core/saved/test_09.XP.out |   29 +
 contrib/libxo/tests/core/test_01.c            |   28 +
 contrib/libxo/tests/core/test_07.c            |    3 +-
 contrib/libxo/tests/core/test_08.c            |  157 +++
 contrib/libxo/tests/core/test_09.c            |  114 ++
 contrib/libxo/xo/xo.1                         |   52 +-
 contrib/libxo/xo/xo.c                         |    3 +-
 contrib/libxo/xolint/Makefile.am              |    2 +-
 contrib/libxo/xolint/xolint.1                 |   53 +-
 lib/libxo/Makefile                            |    1 +
 83 files changed, 4077 insertions(+), 710 deletions(-)
 create mode 100644 contrib/libxo/libxo/xo_error.3
 create mode 100644 contrib/libxo/packaging/libxo.rb.base.in
 create mode 100644 contrib/libxo/tests/core/saved/test_08.H.err
 create mode 100644 contrib/libxo/tests/core/saved/test_08.H.out
 create mode 100644 contrib/libxo/tests/core/saved/test_08.HIPx.err
 create mode 100644 contrib/libxo/tests/core/saved/test_08.HIPx.out
 create mode 100644 contrib/libxo/tests/core/saved/test_08.HP.err
 create mode 100644 contrib/libxo/tests/core/saved/test_08.HP.out
 create mode 100644 contrib/libxo/tests/core/saved/test_08.J.err
 create mode 100644 contrib/libxo/tests/core/saved/test_08.J.out
 create mode 100644 contrib/libxo/tests/core/saved/test_08.JP.err
 create mode 100644 contrib/libxo/tests/core/saved/test_08.JP.out
 create mode 100644 contrib/libxo/tests/core/saved/test_08.T.err
 create mode 100644 contrib/libxo/tests/core/saved/test_08.T.out
 create mode 100644 contrib/libxo/tests/core/saved/test_08.X.err
 create mode 100644 contrib/libxo/tests/core/saved/test_08.X.out
 create mode 100644 contrib/libxo/tests/core/saved/test_08.XP.err
 create mode 100644 contrib/libxo/tests/core/saved/test_08.XP.out
 create mode 100644 contrib/libxo/tests/core/saved/test_09.H.err
 create mode 100644 contrib/libxo/tests/core/saved/test_09.H.out
 create mode 100644 contrib/libxo/tests/core/saved/test_09.HIPx.err
 create mode 100644 contrib/libxo/tests/core/saved/test_09.HIPx.out
 create mode 100644 contrib/libxo/tests/core/saved/test_09.HP.err
 create mode 100644 contrib/libxo/tests/core/saved/test_09.HP.out
 create mode 100644 contrib/libxo/tests/core/saved/test_09.J.err
 create mode 100644 contrib/libxo/tests/core/saved/test_09.J.out
 create mode 100644 contrib/libxo/tests/core/saved/test_09.JP.err
 create mode 100644 contrib/libxo/tests/core/saved/test_09.JP.out
 create mode 100644 contrib/libxo/tests/core/saved/test_09.T.err
 create mode 100644 contrib/libxo/tests/core/saved/test_09.T.out
 create mode 100644 contrib/libxo/tests/core/saved/test_09.X.err
 create mode 100644 contrib/libxo/tests/core/saved/test_09.X.out
 create mode 100644 contrib/libxo/tests/core/saved/test_09.XP.err
 create mode 100644 contrib/libxo/tests/core/saved/test_09.XP.out
 create mode 100644 contrib/libxo/tests/core/test_08.c
 create mode 100644 contrib/libxo/tests/core/test_09.c

diff --git a/contrib/libxo/Makefile.am b/contrib/libxo/Makefile.am
index f1fc9997b8a..4ff2aad892d 100644
--- a/contrib/libxo/Makefile.am
+++ b/contrib/libxo/Makefile.am
@@ -35,6 +35,7 @@ docs:
 
 DIST_FILES_DIR = ~/Dropbox/dist-files/
 GH_PAGES_DIR = gh-pages/
+GH_PAGES_DIR_VER = gh-pages/${PACKAGE_VERSION}
 PACKAGE_FILE = ${PACKAGE_TARNAME}-${PACKAGE_VERSION}.tar.gz
 
 upload: dist upload-docs
@@ -45,10 +46,14 @@ upload-docs: docs
 	@echo "Uploading libxo-manual.html ... "
 	@-[ -d ${GH_PAGES_DIR} ] \
 		&& echo "Updating manual on gh-pages ..." \
+		&& mkdir -p ${GH_PAGES_DIR_VER} \
 		&& cp doc/libxo-manual.html ${GH_PAGES_DIR} \
+		&& cp doc/libxo-manual.html ${GH_PAGES_DIR_VER} \
 		&& (cd ${GH_PAGES_DIR} \
+			&& git add ${PACKAGE_VERSION} \
+			&& git add libxo-manual.html \
 			&& git commit -m 'new docs' \
-                               libxo-manual.html \
+                               libxo-manual.html ${PACKAGE_VERSION} \
 			&& git push origin gh-pages ) ; true
 
 pkgconfigdir=$(libdir)/pkgconfig
@@ -66,7 +71,7 @@ UPDATE_PACKAGE_FILE = \
     -e "s;__SHA256__;SHA256 (textproc/${PACKAGE_FILE}) = $$SHA256;" \
     -e "s;__SIZE__;SIZE (textproc/${PACKAGE_FILE}) = $$SIZE;"
 
-GH_PACKAGING_DIR = packaging/${PACKAGE_VERSION}
+GH_PACKAGING_DIR = ${PACKAGE_VERSION}/packaging
 GH_PAGES_PACKAGE_DIR = ${GH_PAGES_DIR}/${GH_PACKAGING_DIR}
 
 packages:
@@ -75,7 +80,6 @@ packages:
 		&& SHA1="`openssl sha1 ${PACKAGE_FILE} | awk '{print $$2}'`" \
 		&& SHA256="`openssl sha256 ${PACKAGE_FILE} | awk '{print $$2}'`" \
 		&& SIZE="`ls -l ${PACKAGE_FILE} | awk '{print $$5}'`" \
-		&& mkdir -p ${GH_PAGES_PACKAGE_DIR}/freebsd \
 		&& echo "... ${GH_PAGES_PACKAGE_DIR}/${PACKAGE_NAME}.rb ..." \
 		&& sed ${UPDATE_PACKAGE_FILE} \
 			packaging/${PACKAGE_NAME}.rb.base \
@@ -83,20 +87,10 @@ packages:
 		&& echo "... ${GH_PAGES_PACKAGE_DIR}/${PACKAGE_NAME}.spec ..." \
 		&& cp packaging/${PACKAGE_NAME}.spec \
 			${GH_PAGES_PACKAGE_DIR}/${PACKAGE_NAME}.spec \
-		&& echo "... ${GH_PAGES_PACKAGE_DIR}/freebsd ..." \
-		&& sed ${UPDATE_PACKAGE_FILE} \
-			${srcdir}/packaging/freebsd/distinfo.base \
-			> ${GH_PAGES_PACKAGE_DIR}/freebsd/distinfo \
-		&& cp ${srcdir}/packaging/freebsd/pkg-descr \
-			${GH_PAGES_PACKAGE_DIR}/freebsd/pkg-descr \
-		&& cp ${srcdir}/packaging/freebsd/pkg-plist \
-			${GH_PAGES_PACKAGE_DIR}/freebsd/pkg-plist \
-		&& cp ${srcdir}/packaging/freebsd/pkg-plist \
-			${GH_PAGES_PACKAGE_DIR}/freebsd/pkg-plist \
-		&& cp packaging/freebsd/port-Makefile \
-			${GH_PAGES_PACKAGE_DIR}/freebsd/Makefile \
 		&& (cd ${GH_PAGES_DIR} \
 			&& git add ${GH_PACKAGING_DIR} \
+			&& git add ${GH_PACKAGING_DIR}/libxo.rb \
+                                   ${GH_PACKAGING_DIR}/libxo.spec \
 			&& git commit -m 'new packaging data' \
                                ${GH_PACKAGING_DIR} \
 			&& git push origin gh-pages ) ; true
diff --git a/contrib/libxo/bin/Zaliases b/contrib/libxo/bin/Zaliases
index a24d33e7a52..b8fb5dbe522 100644
--- a/contrib/libxo/bin/Zaliases
+++ b/contrib/libxo/bin/Zaliases
@@ -1,5 +1,5 @@
 set top_src=`pwd`
-alias Zautoreconf "(cd $top_src ; autoreconf --install)"
+alias Zautoreconf "(cd $top_src ; autoreconf)"
 
 set opts=' \
 --with-libslax-prefix=/Users/phil/work/root \
diff --git a/contrib/libxo/configure.ac b/contrib/libxo/configure.ac
index 904af12bf68..b2553b89ac5 100644
--- a/contrib/libxo/configure.ac
+++ b/contrib/libxo/configure.ac
@@ -12,7 +12,7 @@
 #
 
 AC_PREREQ(2.2)
-AC_INIT([libxo], [0.1.6], [phil@juniper.net])
+AC_INIT([libxo], [0.2.0], [phil@juniper.net])
 AM_INIT_AUTOMAKE([-Wall -Werror foreign -Wno-portability])
 
 # Support silent build rules.  Requires at least automake-1.11.
@@ -57,8 +57,10 @@ AC_CHECK_FUNCS([getpass])
 AC_CHECK_FUNCS([sysctlbyname])
 AC_CHECK_FUNCS([flock])
 AC_CHECK_FUNCS([asprintf])
+AC_CHECK_FUNCS([__flbf])
 
 AC_CHECK_HEADERS([dlfcn.h])
+AC_CHECK_HEADERS([stdio_ext.h])
 AC_CHECK_HEADERS([tzfile.h])
 AC_CHECK_HEADERS([stdtime/tzfile.h])
 AC_CHECK_FUNCS([dlfunc])
@@ -164,7 +166,6 @@ AC_ARG_ENABLE([libxo-options],
 AC_MSG_RESULT([$LIBXO_OPTS])
 AM_CONDITIONAL([NO_LIBXO_OPTIONS], [test "$LIBXO_OPTS" != "yes"])
 
-
 case $host_os in
      darwin*)
         LIBTOOL=glibtool
@@ -238,6 +239,7 @@ AC_CONFIG_FILES([
   tests/core/Makefile
   tests/xo/Makefile
   packaging/libxo.spec
+  packaging/libxo.rb.base
 ])
 AC_OUTPUT
 
diff --git a/contrib/libxo/doc/libxo.txt b/contrib/libxo/doc/libxo.txt
index 5148de0f74a..31aec5333b5 100644
--- a/contrib/libxo/doc/libxo.txt
+++ b/contrib/libxo/doc/libxo.txt
@@ -12,14 +12,15 @@
 
 libxo - A Library for Generating Text, XML, JSON, and HTML Output
 
-You live in the present, but you want to live in the future.  You'd
-love a flying car, but need to get to work today.  You want to support
-features like XML, JSON, and HTML rendering to allow integration with
-NETCONF, REST, and web browsers, but you need to make text output for
-command line users.  And you don't want multiple code paths that can't
-help but get out of sync.  None of this "if (xml) {... } else {...}"
-logic.  And ifdefs are right out.  But you'd really, really like all
-the fancy features that modern encoding formats can provide.
+You want to prepare for the future, but you need to live in the
+present.  You'd love a flying car, but need to get to work today.  You
+want to support features like XML, JSON, and HTML rendering to allow
+integration with NETCONF, REST, and web browsers, but you need to make
+text output for command line users.  And you don't want multiple code
+paths that can't help but get out of sync.  None of this "if (xml)
+{... } else {...}"  logic.  And ifdefs are right out.  But you'd
+really, really like all the fancy features that modern encoding
+formats can provide.  libxo can help.
 
 The libxo library allows an application to generate text, XML, JSON,
 and HTML output using a common set of function calls.  The application
@@ -83,31 +84,37 @@ The latest release of libxo is available at:
 
   https://github.com/Juniper/libxo/releases
 
-We are following the branching scheme from 
-^http://nvie.com/posts/a-successful-git-branching-model/^
-which means we will do development under the "develop" branch, and
-release from the master.  To clone a developer tree, run the following
+We are following the branching scheme from
+^http://nvie.com/posts/a-successful-git-branching-model/^ which means
+we will do development under the "develop" branch, and release from
+the "master" branch.  To clone a developer tree, run the following
 command:
 
   git clone https://github.com/Juniper/libxo.git -b develop
 
-We're using semantic release numbering.
+We're using semantic release numbering, as defined in
+^http://semver.org/spec/v2.0.0.html^.
+
+libxo is open source, distributed under the BSD license.  It
+is shipped as part of FreeBSD 11.0.
 
 * Overview
 
 Most unix commands emit text output aimed at humans.  It is designed
-to be parsed and understood by a user.  Humans are gifted at extracted
-details and pattern matching.  Often programmers need to extract
-information from this human-oriented output.  Programmers use tools
-like grep, awk, and regular expressions to ferret out the pieces of
-information they need.  Such solutions are fragile and require
-updates when output contents change or evolve, requiring testing and
-validation.
+to be parsed and understood by a user.  Humans are gifted at
+extracting details and pattern matching in such output.  Often
+programmers need to extract information from this human-oriented
+output.  Programmers use tools like grep, awk, and regular expressions
+to ferret out the pieces of information they need.  Such solutions are
+fragile and require maintenance when output contents change or evolve,
+along with testing and validation.
 
-Modern tool developers favors encoding schemes like XML and JSON,
+Modern tool developers favor encoding schemes like XML and JSON,
 which allow trivial parsing and extraction of data.  Such formats are
 simple, well understood, hierarchical, easily parsed, and often
-integrate easier with common tools and environments.
+integrate easier with common tools and environments.  Changes to
+content can be done in ways that do not break existing users of the
+data, which can reduce maintenance costs and increase feature velocity.
 
 In addition, modern reality means that more output ends up in web
 browsers than in terminals, making HTML output valuable.
@@ -278,7 +285,7 @@ content.  The roles are listed below; only one role is permitted:
 |---+--------------+-------------------------------------------------|
 | M | Name         | Description                                     |
 |---+--------------+-------------------------------------------------|
-| D | decoration   | Field is non-text (e.g. colon, comma)           |
+| D | decoration   | Field is non-text (e.g., colon, comma)          |
 | E | error        | Field is an error message                       |
 | L | label        | Field is text that prefixes a value             |
 | N | note         | Field is text that follows a value              |
@@ -321,7 +328,7 @@ the field descriptor, or a printf-style format descriptor can be used,
 if preceded by a slash ("/"):
 
     xo_emit("{P:        }{Lwc:Cost}{:cost/%u}\n", cost);
-    xo_emit("{P:/30s}{Lwc:Cost}{:cost/%u}\n", "", cost);
+    xo_emit("{P:/%30s}{Lwc:Cost}{:cost/%u}\n", "", cost);
 
 **** The Title Role ({T:})
 
@@ -333,6 +340,16 @@ if preceded by a slash ("/"):
     xo_emit("{T:Interface Statistics}\n");
     xo_emit("{T:/%20.20s}{T:/%6.6s}\n", "Item Name", "Cost");
 
+Title fields have an extra convenience feature; if both content and
+format are specified, instead of looking to the argument list for a
+value, the content is used, allowing a mixture of format and content
+within the field descriptor:
+
+    xo_emit("{T:Name/%20s}{T:Count/%6s}\n");
+
+Since the incoming argument is a string, the format must be "%s" or
+something suitable.
+
 **** The Units Role ({U:})
 
 Units are the dimension by which values are measured, such as degrees,
@@ -412,6 +429,7 @@ content emitted for some output styles:
 | d | display      | Only emit field for display styles (text/HTML)  |
 | e | encoding     | Only emit for encoding styles (XML/JSON)        |
 | k | key          | Field is a key, suitable for XPath predicates   |
+| l | leaf-list    | Field is a leaf-list
 | n | no-quotes    | Do not quote the field when using JSON style    |
 | q | quotes       | Quote the field when using JSON style           |
 | w | white space  | A blank (" ") is appended after the label       |
@@ -433,7 +451,7 @@ The colon modifier appends a single colon to the data value:
       Name:phil
 
 The colon modifier is only used for the TEXT and HTML output
-styles. It is commonly combined with the space modifier ('{w:').
+styles. It is commonly combined with the space modifier ('{w:}').
 It is purely a convenience feature.
 
 **** The Display Modifier ({d:})
@@ -485,6 +503,24 @@ Currently the key modifier is only used when generating XPath value
 for the HTML output style when XOF_XPATH is set, but other uses are
 likely in the near future.
 
+**** The Leaf-List Modifier ({l:})
+
+The leaf-list modifier is used to distinguish lists where each
+instance consists of only a single value.  In XML, these are
+rendered as single elements, where JSON renders them as arrays.
+
+    EXAMPLE:
+        for (i = 0; i < num_users; i++) {
+            xo_emit("Member {l:user}\n", user[i].u_name);
+        }
+    XML:
+        <user>phil</user>
+        <user>pallavi</user>
+    JSON:
+        "user": [ "phil", "pallavi" ]
+
+The name of the field must match the name of the leaf list.
+
 **** The No-Quotes Modifier ({n:})
 
 The no-quotes modifier (and its twin, the 'quotes' modifier) affect
@@ -522,7 +558,7 @@ The white space modifier appends a single space to the data value:
       Name phil
 
 The white space modifier is only used for the TEXT and HTML output
-styles. It is commonly combined with the colon modifier ('{c:').
+styles. It is commonly combined with the colon modifier ('{c:}').
 It is purely a convenience feature.
 
 Note that the sense of the 'w' modifier is reversed for the units role
@@ -530,14 +566,15 @@ Note that the sense of the 'w' modifier is reversed for the units role
 
 *** Field Formatting
 
-The field format is similar to the format string for printf(3).  It's
-used varies based on the role of the field, but generally is used to
+The field format is similar to the format string for printf(3).  Its
+use varies based on the role of the field, but generally is used to
 format the field's contents.
 
-If not provided, the format string defaults to "%s".
+If the format string is not provided for a value field, it defaults to
+"%s".
 
 Note a field definition can contain zero or more printf-style
-'directives', which are sequences that start with a '%' and end with a
+'directives', which are sequences that start with a '%' and end with
 one of following characters: "diouxXDOUeEfFgGaAcCsSp".  Each directive
 is matched by one of more arguments to the xo_emit function.
 
@@ -557,7 +594,7 @@ argument.  If the width in columns of the output value is less that
 the minumum width, the value will be padded to reach the minimum.
 - a period followed by one or more digits indicating the maximum
 number of bytes which will be examined for a string argument, or the maximum
-width for a non-string argument.  When handling ASCII strings this is
+width for a non-string argument.  When handling ASCII strings this
 functions as the field width but for multi-byte characters, a single
 character may be composed of multiple bytes.
 xo_emit will never dereference memory beyond the given number of bytes.
@@ -630,8 +667,8 @@ ASCII data, a normal 7-bit ASCII string can be used.  '%ls' expects a
 Unicode values.  '%hs' expects a 'char *' pointer to a multi-byte
 string encoded with the current locale, as given by the LC_CTYPE,
 LANG, or LC_ALL environment varibles.  The first of this list of
-variables is used and if none of the variables, the locale defaults to
-"UTF-8".
+variables is used and if none of the variables are set, the locale
+defaults to "UTF-8".
 
 For example, a function is passed a locale-base name, a hat size,
 and a time value.  The hat size is formatted in a UTF-8 (ASCII)
@@ -676,10 +713,10 @@ columns.
 
 *** Characters Outside of Field Definitions
 
-Characters in the format string are not part of a field definition are
-copied to the output for the TEXT style, and are ignored for the JSON
-and XML styles.  For HTML, these characters are placed in a <div> with
-class "text".
+Characters in the format string that are not part of a field
+definition are copied to the output for the TEXT style, and are
+ignored for the JSON and XML styles.  For HTML, these characters are
+placed in a <div> with class "text".
 
   EXAMPLE:
       xo_emit("The hat is {:size/%s}.\n", size_val);
@@ -854,7 +891,7 @@ container, a warning will be generated.
 *** Lists and Instances
 
 A list is set of one or more instances that appear under the same
-parent.  The instances contains details about a specific object.  One
+parent.  The instances contain details about a specific object.  One
 can think of instances as objects or records.  A call is needed to
 open and close the list, while a distinct call is needed to open and
 close each instance of the list:
@@ -874,8 +911,8 @@ generation of XML and JSON data.
 
 *** DTRT Mode
 
-Some user may find tracking the names of open containers, lists, and
-instances inconvenient.  libxo offers "Do The Right Thing" mode, where
+Some users may find tracking the names of open containers, lists, and
+instances inconvenient.  libxo offers a "Do The Right Thing" mode, where
 libxo will track the names of open containers, lists, and instances so
 the close function can be called without a name.  To enable DTRT mode,
 turn on the XOF_DTRT flag prior to making any other libxo output.
@@ -889,10 +926,42 @@ will close the open container, list, or instance:
     ...
     xo_close_container_d();
 
+This also works for lists and instances:
+
+    xo_open_list("item");
+    for (...) {
+        xo_open_instance("item");
+        xo_emit(...);
+        xo_close_instance_d();
+    }
+    xo_close_list_d();
+
 Note that the XOF_WARN flag will also cause libxo to track open
-containers, lists, and instances.  A warning is generated with the
+containers, lists, and instances.  A warning is generated when the
 name given to the close function and the name recorded do not match.
 
+*** Markers
+
+Markers are used to protect and restore the state of open constructs.
+While a marker is open, no other open constructs can be closed.  When
+a marker is closed, all constructs open since the marker was opened
+will be closed.
+
+Markers use names which are not user-visible, allowing the caller to
+choose appropriate internal names.
+
+In this example, the code whiffles through a list of fish, calling a
+function to emit details about each fish.  The marker "fish-guts" is
+used to ensure that any constructs opened by the function are closed
+properly.
+
+    for (i = 0; fish[i]; i++) {
+        xo_open_instance("fish");
+        xo_open_marker("fish-guts");
+        dump_fish_details(i);
+        xo_close_marker("fish-guts");
+    }
+
 ** Handles
 
 libxo uses "handles" to control its rendering functionality.  The
@@ -952,7 +1021,7 @@ be passed NULL to access the default handle.
 
 For the typical command that is generating output on standard output,
 there is no need to create an explicit handle, but they are available
-when needed, e.g. for daemons that generate multiple streams of
+when needed, e.g., for daemons that generate multiple streams of
 output.
 
 *** xo_create
@@ -972,7 +1041,7 @@ See also ^styles^ and ^flags^.
 
 By default, libxo writes output to standard output.  A convenience
 function is provided for situations when output should be written to
-different file:
+a different file:
 
     xo_handle_t *xo_create_to_file (FILE *fp, unsigned style,
                                     unsigned flags);
@@ -987,10 +1056,13 @@ which can tailor how libxo writes data.  An opaque argument is
 recorded and passed back to the write function, allowing the function
 to acquire context information. The 'close' function can
 release this opaque data and any other resources as needed.
+The flush function can flush buffered data associated with the opaque
+object.
 
     void xo_set_writer (xo_handle_t *xop, void *opaque,
                         xo_write_func_t write_func,
                         xo_close_func_t close_func);
+                        xo_flush_func_t flush_func);
 
 *** xo_set_style
 
@@ -1068,7 +1140,7 @@ XML, JSON, and HTML output.  Text output is not affected.
 
 The XOF_WARN flag requests that warnings will trigger diagnostic
 output (on standard error) when the library notices errors during
-operations, or with arguments to functions.  Without warning enabled,
+operations, or with arguments to functions.  Without warnings enabled,
 such conditions are ignored.
 
 Warnings allow developers to debug their interaction with libxo.
@@ -1178,6 +1250,13 @@ parameter passed to xo_attr_hv().
     XML:
         <login-time seconds="1408336270">00:14</login-time>
 
+xo_attr is placed on the next container, instance, leaf, or leaf list
+that is emitted.
+
+Since attributes are only emitted in XML, their use should be limited
+to meta-data and additional or redundant representations of data
+already emitted in other form.
+
 *** Flushing Output (xo_flush)
 
 libxo buffers data, both for performance and consistency, but also to
@@ -1188,6 +1267,10 @@ xo_flush() call is used for this:
     void xo_flush (void);
     void xo_flush_h (xo_handle_t *xop);
 
+Calling xo_flush also triggers the flush function associated with the
+handle.  For the default handle, this is equivalent to
+"fflush(stdio);". 
+
 *** Finishing Output (xo_finish)
 
 When the program is ready to exit or close a handle, a call to
@@ -1250,7 +1333,7 @@ styles.  Calls must be made to open and close a list, and for each
 instance of data in that list, calls must be make to open and close
 that instance.
 
-The name given to all calls must be identical, and it is strong
+The name given to all calls must be identical, and it is strongly
 suggested that the name be singular, not plural, as a matter of
 style and usage expectations.
 
@@ -1314,6 +1397,16 @@ Following the call to xo_parse_args, the application can process the
 remaining arguments in a normal manner.  See ^command-line-arguments^
 for a description of valid arguments.
 
+*** xo_set_program
+
+The xo_set_program function sets name of the program as reported by
+functions like xo_failure, xo_warn, xo_err, etc.  The program name is
+initialized by xo_parse_args, but subsequent calls to xo_set_program
+can override this value.
+
+Note that the value is not copied, so the memory passed to
+xo_set_program (and xo_parse_args) must be maintained by the caller.
+
 *** Field Information (xo_info_t) @info@
 
 HTML data can include additional information in attributes that
@@ -1353,7 +1446,7 @@ known to the application:
     ...
     xo_set_info(NULL, info, info_count);
 
-Third, the emitting of info must be triggered with the XOF_INFO flag
+Third, the emission of info must be triggered with the XOF_INFO flag
 using either the xo_set_flags() function or the "--libxo=info" command
 line argument.
 
@@ -1405,6 +1498,10 @@ Complete HTML output can be generated with:
 
     % env LIBXO_OPTIONS=HXI my-app
 
+Since environment variables are inherited, child processes will have
+the same options, which may be undesirable, making the use of the
+"--libxo" option is preferable in most situations.
+
 *** Errors, Warnings, and Messages
 
 Many programs make use of the standard library functions err() and
@@ -1435,6 +1532,20 @@ message associated with either "errno" or the "code" parameter.
         if (open(filename, O_RDONLY) < 0)
             xo_err(1, "cannot open file '%s'", filename);
 
+*** xo_error
+
+The xo_error function can be used for generic errors that should be
+reported over the handle, rather than to stderr.  The xo_error
+function behaves like xo_err for TEXT and HTML output styles, but puts
+the error into XML or JSON elements:
+
+    EXAMPLE::
+        xo_error("Does not %s", "compute");
+    XML::
+        <error><message>Does not compute</message></error>
+    JSON::
+        "error": { "message": "Does not compute" }
+
 *** xo_no_setlocale
 
 libxo automatically initializes the locale based on setting of the
@@ -1580,7 +1691,7 @@ and errors, warning, or informational messages as needed.
 | -X         | Extract samples from xolint, suitable for testing |
 |------------+---------------------------------------------------|
 
-Output message contain the source filename and line number, the
+The output message will contain the source filename and line number, the
 class of the message, the message, and, if -p is given, the
 line that contains the error:
 
@@ -1627,6 +1738,88 @@ libxo is an effort to mix the best aspects of the JUNOS strategy into
 FreeBSD in a seemless way, allowing commands to make printf-like
 output calls without needing to care how the output is rendered.
 
+*** Did the complex semantics of format strings evolve over time?
+
+The history is both long and short: libxo's functionality is based
+on what JUNOS does in a data modeling language called ODL (output
+definition language).  In JUNOS, all subcomponents generate XML,
+which is feed to the CLI, where data from the ODL files tell is
+how to render that XML into text.  ODL might had a set of tags
+like:
+
+     tag docsis-state {
+         help "State of the DOCSIS interface";
+         type string;
+     }
+
+     tag docsis-mode {
+         help "DOCSIS mode (2.0/3.0) of the DOCSIS interface";
+         type string;
+     }
+
+     tag docsis-upstream-speed {
+         help "Operational upstream speed of the interface";
+         type string;
+     }
+
+     tag downstream-scanning {
+         help "Result of scanning in downstream direction";
+         type string;
+     }
+
+     tag ranging {
+         help "Result of ranging action";
+         type string;
+     }
+
+     tag signal-to-noise-ratio {
+         help "Signal to noise ratio for all channels";
+         type string;
+     }
+
+     tag power {
+         help "Operational power of the signal on all channels";
+         type string;
+     }
+
+     format docsis-status-format {
+         picture "
+   State   : @, Mode: @, Upstream speed: @
+   Downstream scanning: @, Ranging: @
+   Signal to noise ratio: @
+   Power: @
+";
+         line {
+             field docsis-state;
+             field docsis-mode;
+             field docsis-upstream-speed;
+             field downstream-scanning;
+             field ranging;
+             field signal-to-noise-ratio;
+             field power;
+         }
+     }
+
+These tag definitions are compiled into field definitions
+that are triggered when matching XML elements are seen.  ODL
+also supports other means of defining output.
+
+The roles and modifiers describe these details.
+
+In moving these ideas to bsd, two things had to happen: the
+formatting had to happen at the source since BSD won't have
+a JUNOS-like CLI to do the rendering, and we can't depend on
+external data models like ODL, which was seen as too hard a
+sell to the BSD community.
+
+The results were that the xo_emit strings are used to encode the
+roles, modifiers, names, and formats.  They are dense and a bit
+cryptic, but not so unlike printf format strings that developers will
+be lost.
+
+libxo is a new implementation of these ideas and is distinct from
+the previous implementation in JUNOS.
+
 *** What makes a good field name?
 
 To make useful, consistent field names, follow these guidelines:
@@ -1660,7 +1853,7 @@ Nothing's worse than writing expressions like:
     }
 
 Find someone else who is expressing similar data and follow their
-field's and hierarchy.  Remember the quote is not "Consistency is the
+fields and hierarchy.  Remember the quote is not "Consistency is the
 hobgoblin of little minds", but "A foolish consistency is the
 hobgoblin of little minds".
 = Think about your users
@@ -1670,7 +1863,7 @@ content with xo_attr() calls (^xo_attr^) or "{e:}" fields
 (^e-modifier^) to make the data useful.
 = Don't use an arbitrary number postfix
 What does "errors2" mean?  No one will know.  "errors-after-restart"
-would be a better choice.  Think of you users, and think of the
+would be a better choice.  Think of your users, and think of the
 future.  If you make "errors2", the next guy will happily make
 "errors3" and before you know it, someone will be asking what's the
 difference between errors37 and errors63.
@@ -1689,7 +1882,7 @@ After using "xolint" to find errors in your field descriptors, use
 "xolint -V" to spell check your field names and to detect different
 names for the same data.  "dropped-short" and "dropped-too-short" are
 both reasonable names, but using them both will lead users to ask the
-difference between the two fields.  If there isn't a difference,
+difference between the two fields.  If there is no difference,
 use only one of the field names.  If there is a difference, change the
 names to make that difference more obvious.
 
diff --git a/contrib/libxo/libxo/libxo.3 b/contrib/libxo/libxo/libxo.3
index fca0774eb13..f9b0e6f8b2a 100644
--- a/contrib/libxo/libxo/libxo.3
+++ b/contrib/libxo/libxo/libxo.3
@@ -7,7 +7,7 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 8, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
@@ -19,58 +19,82 @@
 .In libxo/xo.h
 .Sh DESCRIPTION
 The functions defined in
-.Lb libxo
+.Nm
 are used to generate a choice of
 .Em TEXT ,
 .Em XML ,
 .Em JSON ,
 or
 .Em HTML
-output.  A common set of functions are used, with
+output.
+A common set of functions are used, with
 command line switches passed to the library to control the details of
 the output.
 .Pp
-Most commands emit text output aimed at humans.  It is designed
-to be parsed and understood by a user.  Humans are gifted at extracted
-details and pattern matching.  Often programmers need to extract
-information from this human-oriented output.  Programmers use tools
-like grep, awk, and regular expressions to ferret out the pieces of
-information they need.  Such solutions are fragile and require
+Most commands emit text output aimed at humans.
+It is designed
+to be parsed and understood by a user.
+Humans are gifted at extracting
+details and pattern matching.
+Often programmers need to extract
+information from this human-oriented output.
+Programmers use tools
+like
+.Xr grep 1 ,
+.Xr awk 1 ,
+and regular expressions to ferret out the pieces of
+information they need.
+Such solutions are fragile and require
 updates when output contents change or evolve, requiring testing and
 validation.
 .Pp
-Modern tool developers favors encoding schemes like XML and JSON,
-which allow trivial parsing and extraction of data.  Such formats are
+Modern tool developers favor encoding schemes like XML and JSON,
+which allow trivial parsing and extraction of data.
+Such formats are
 simple, well understood, hierarchical, easily parsed, and often
 integrate easier with common tools and environments.
 .Pp
 In addition, modern reality means that more output ends up in web
 browsers than in terminals, making HTML output valuable.
 .Pp
-.Em libxo
+.Nm
 allows a single set of function calls in source code to generate
-traditional text output, as well as XML and JSON formatted data.  HTML
+traditional text output, as well as XML and JSON formatted data.
+HTML
 can also be generated; "<div>" elements surround the traditional text
 output, with attributes that detail how to render the data.
 .Pp
-There are four encoding styles supported by libxo: TEXT, HTML, JSON,
-and XML.  JSON and XML are suitable for encoding data, while TEXT and
-HTML are suited for display to the user.  TEXT output can be display
+There are four encoding styles supported by
+.Nm :
+TEXT, HTML, JSON,
+and XML.
+JSON and XML are suitable for encoding data, while TEXT and
+HTML are suited for display to the user.
+TEXT output can be display
 on a terminal session, allowing compatibility with traditional usage.
 HTML can be matched with a small CSS file to permit rendering in any
-HTML5 browser.  XML output is suitable for tools like XPath and
-protocols like NETCONF.  JSON output can be used for RESTful APIs.
+HTML5 browser.
+XML output is suitable for tools like XPath and
+protocols like NETCONF.
+JSON output can be used for RESTful APIs.
 .Pp
 The
-.Em libxo
+.Nm
 library allows an application to generate text, XML, JSON,
-and HTML output using a common set of function calls.  The application
-decides at run time which output style should be produced.  The
+and HTML output using a common set of function calls.
+The application
+decides at run time which output style should be produced.
+The
 application calls a function
-.Fn xo_emit
+.Xr xo_emit 3
 to product output that is
-described in a format string.  A "field descriptor" tells libxo what
-the field is and what it means.  Each field descriptor is placed in
+described in a format string.
+A
+.Dq field descriptor
+tells
+.Nm
+what the field is and what it means.
+Each field descriptor is placed in
 braces with a printf-like format string:
 .Bd -literal -offset indent
     xo_emit(" {:lines/%7ju} {:words/%7ju} "
@@ -79,55 +103,199 @@ braces with a printf-like format string:
 .Ed
 .Pp
 Each field can have a role, with the 'value' role being the default,
-and the role tells libxo how and when to render that field, as well as
+and the role tells
+.Nm
+how and when to render that field, as well as
 a
-.Xr printf 3 -like
+.Xr printf 3 Ns -like
 format string.
 .Pp
 Output
 can then be generated in various style, using the "--libxo" option.
 .Sh DEFAULT HANDLE
-Handles give an abstraction for libxo that encapsulates the state of a
-stream of output.  Handles have the data type "xo_handle_t" and are
+Handles give an abstraction for
+.Nm
+that encapsulates the state of a
+stream of output.
+Handles have the data type "xo_handle_t" and are
 opaque to the caller.
-
+.Pp
 The library has a default handle that is automatically initialized.
 By default, this handle will send text style output to standard output.
-The xo_set_style and xo_set_flags functions can be used to change this
+The
+.Xr xo_set_style 3
+and
+.Xr xo_set_flags 3
+functions can be used to change this
 behavior.
-
-Many libxo functions take a handle as their first parameter; most that
-do not use the default handle.  Any function taking a handle can
-be passed NULL to access the default handle.
-
+.Pp
+Many
+.Nm
+functions take a handle as their first parameter; most that
+do not use the default handle.
+Any function taking a handle can
+be passed
+.Dv NULL
+to access the default handle.
+.Pp
 For the typical command that is generating output on standard output,
 there is no need to create an explicit handle, but they are available
-when needed, e.g. for daemons that generate multiple streams of
+when needed, e.g., for daemons that generate multiple streams of
 output.
+.Sh FUNCTION OVERVIEW
+The
+.Nm
+library includes the following functions:
+.Bl -tag -width "xo_close_container_hd"
+.It Sy "Function               Description"
+.It Fn xo_attr
+.It Fn xo_attr_h
+.It Fn xo_attr_hv
+Allows the caller to emit XML attributes with the next open element.
+.It Fn xo_create
+.It Fn xo_create_to_file
+Allow the caller to create a new handle.
+Note that
+.Nm
+has a default handle that allows the caller to avoid use of an
+explicitly created handle.
+Only callers writing to files other than
+.Dv stdout
+would need to call
+.Fn xo_create .
+.It Fn xo_destroy
+Frees any resources associated with the handle, including the handle
+itself.
+.It Fn xo_emit
+.It Fn xo_emit_h
+.It Fn xo_emit_hv
+Emit formatted output.
+The
+.Fa fmt
+string controls the conversion of the remaining arguments into
+formatted output.
+See
+.Xr xo_format 5
+for details.
+.It Fn xo_warn
+.It Fn xo_warnx
+.It Fn xo_warn_c
+.It Fn xo_warn_hc
+.It Fn xo_err
+.It Fn xo_errc
+.It Fn xo_errx
+.It Fn xo_message
+.It Fn xo_message_c
+.It Fn xo_message_hc
+.It Fn xo_message_hcv
+These functions are meant to be compatible with their standard libc namesakes.
+.It Fn xo_finish
+.It Fn xo_finish_h
+Flush output, close open construct, and complete any pending
+operations.
+.It Fn xo_flush
+.It Fn xo_flush_h
+Allow the caller to flush any pending output for a handle.
+.It Fn xo_no_setlocale
+Direct
+.Nm
+to avoid initializing the locale.
+This function should be called before any other
+.Nm
+function is called.
+.It Fn xo_open_container
+.It Fn xo_open_container_h
+.It Fn xo_open_container_hd
+.It Fn xo_open_container_d
+.It Fn xo_close_container
+.It Fn xo_close_container_h
+.It Fn xo_close_container_hd
+.It Fn xo_close_container_d
+Containers a singleton levels of hierarchy, typically used to organize
+related content.
+.It Fn xo_open_list_h
+.It Fn xo_open_list
+.It Fn xo_open_list_hd
+.It Fn xo_open_list_d
+.It Fn xo_open_instance_h
+.It Fn xo_open_instance
+.It Fn xo_open_instance_hd
+.It Fn xo_open_instance_d
+.It Fn xo_close_instance_h
+.It Fn xo_close_instance
+.It Fn xo_close_instance_hd
+.It Fn xo_close_instance_d
+.It Fn xo_close_list_h
+.It Fn xo_close_list
+.It Fn xo_close_list_hd
+.It Fn xo_close_list_d
+Lists are levels of hierarchy that can appear multiple times within
+the same parent.
+Two calls are needed to encapsulate them, one for
+the list and one for each instance of that list.
+Typically
+.Fn xo_open_list
+and
+.Fn xo_close_list
+are called outside a
+for-loop, where
+.Fn xo_open_instance
+it called at the top of the loop, and
+.Fn xo_close_instance
+is called at the bottom of the loop.
+.It Fn xo_parse_args
+Inspects command line arguments for directions to
+.Nm .
+This function should be called before
+.Va argv
+is inspected by the application.
+.It Fn xo_set_allocator
+Instructs
+.Nm
+to use an alternative memory allocator and deallocator.
+.It Fn xo_set_flags
+.It Fn xo_clear_flags
+Change the flags set for a handle.
+.It Fn xo_set_info
+Provides additional information about elements for use with HTML
+rendering.
+.It Fn xo_set_options
+Changes formatting options used by handle.
+.It Fn xo_set_style
+.It Fn xo_set_style_name
+Changes the output style used by a handle.
+.It Fn xo_set_writer
+Instructs
+.Nm
+to use an alternative set of low-level output functions.
+.El
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
 .Sh SEE ALSO
+.Xr xo 1 ,
+.Xr xolint 1 ,
 .Xr xo_attr 3 ,
 .Xr xo_create 3 ,
 .Xr xo_emit 3 ,
 .Xr xo_err 3 ,
 .Xr xo_finish 3 ,
 .Xr xo_flush 3 ,
-.Xr xo_format 5 ,
 .Xr xo_no_setlocale 3 ,
 .Xr xo_open_container 3 ,
 .Xr xo_open_list 3 ,
@@ -138,12 +306,11 @@ https://github.com/Juniper/libxo/releases
 .Xr xo_set_options 3 ,
 .Xr xo_set_style 3 ,
 .Xr xo_set_writer 3 ,
-.Xr xo 1 ,
-and
-.Xr xolint 1 .
+.Xr xo_format 5
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/libxo.c b/contrib/libxo/libxo/libxo.c
index 2c973377337..e9d05ce0ceb 100644
--- a/contrib/libxo/libxo/libxo.c
+++ b/contrib/libxo/libxo/libxo.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, Juniper Networks, Inc.
+ * Copyright (c) 2014-2015, Juniper Networks, Inc.
  * All rights reserved.
  * This SOFTWARE is licensed under the LICENSE provided in the
  * ../Copyright file. By downloading, installing, copying, or otherwise
@@ -11,6 +11,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <stdint.h>
+#include <unistd.h>
 #include <stddef.h>
 #include <wchar.h>
 #include <locale.h>
@@ -27,6 +28,10 @@
 #include "xo.h"
 #include "xoversion.h"
 
+#ifdef HAVE_STDIO_EXT_H
+#include <stdio_ext.h>
+#endif /* HAVE_STDIO_EXT_H */
+
 const char xo_version[] = LIBXO_VERSION;
 const char xo_version_extra[] = LIBXO_VERSION_EXTRA;
 
@@ -58,6 +63,52 @@ typedef unsigned xo_xsf_flags_t; /* XSF_* flags */
 #define XSF_INSTANCE	(1<<2)	/* Frame is an instance */
 #define XSF_DTRT	(1<<3)	/* Save the name for DTRT mode */
 
+#define XSF_CONTENT	(1<<4)	/* Some content has been emitted */
+#define XSF_EMIT	(1<<5)	/* Some field has been emitted */
+#define XSF_EMIT_KEY	(1<<6)	/* A key has been emitted */
+#define XSF_EMIT_LEAF_LIST (1<<7) /* A leaf-list field has been emitted */
+
+/* These are the flags we propagate between markers and their parents */
+#define XSF_MARKER_FLAGS \
+ (XSF_NOT_FIRST | XSF_CONTENT | XSF_EMIT | XSF_EMIT_KEY | XSF_EMIT_LEAF_LIST )
+
+/*
+ * A word about states:  We're moving to a finite state machine (FMS)
+ * approach to help remove fragility from the caller's code.  Instead
+ * of requiring a specific order of calls, we'll allow the caller more
+ * flexibility and make the library responsible for recovering from
+ * missed steps.  The goal is that the library should not be capable of
+ * emitting invalid xml or json, but the developer shouldn't need
+ * to know or understand all the details about these encodings.
+ *
+ * You can think of states as either states or event, since they
+ * function rather like both.  None of the XO_CLOSE_* events will
+ * persist as states, since their stack frame will be popped.
+ * Same is true of XSS_EMIT, which is an event that asks us to
+ * prep for emitting output fields.
+ */
+
+/* Stack frame states */
+typedef unsigned xo_state_t;
+#define XSS_INIT		0      	/* Initial stack state */
+#define XSS_OPEN_CONTAINER	1
+#define XSS_CLOSE_CONTAINER	2
+#define XSS_OPEN_LIST		3
+#define XSS_CLOSE_LIST		4
+#define XSS_OPEN_INSTANCE	5
+#define XSS_CLOSE_INSTANCE	6
+#define XSS_OPEN_LEAF_LIST	7
+#define XSS_CLOSE_LEAF_LIST	8
+#define XSS_DISCARDING		9	/* Discarding data until recovered */
+#define XSS_MARKER		10	/* xo_open_marker's marker */
+#define XSS_EMIT		11	/* xo_emit has a leaf field */
+#define XSS_EMIT_LEAF_LIST	12	/* xo_emit has a leaf-list ({l:}) */
+#define XSS_FINISH		13	/* xo_finish was called */
+
+#define XSS_MAX			13
+
+#define XSS_TRANSITION(_old, _new) ((_old) << 8 | (_new))
+
 /*
  * xo_stack_t: As we open and close containers and levels, we
  * create a stack of frames to track them.  This is needed for
@@ -65,6 +116,7 @@ typedef unsigned xo_xsf_flags_t; /* XSF_* flags */
  */
 typedef struct xo_stack_s {
     xo_xsf_flags_t xs_flags;	/* Flags for this frame */
+    xo_state_t xs_state;	/* State for this stack frame */
     char *xs_name;		/* Name (for XPath value) */
     char *xs_keys;		/* XPath predicate for any key fields */
 } xo_stack_t;
@@ -74,12 +126,13 @@ typedef struct xo_stack_s {
  * It's used as a store for state, options, and content.
  */
 struct xo_handle_s {
-    unsigned long xo_flags;	/* Flags */
+    xo_xof_flags_t xo_flags;	/* Flags */
     unsigned short xo_style;	/* XO_STYLE_* value */
     unsigned short xo_indent;	/* Indent level (if pretty) */
     unsigned short xo_indent_by; /* Indent amount (tab stop) */
     xo_write_func_t xo_write;	/* Write callback */
     xo_close_func_t xo_close;	/* Close callback */
+    xo_flush_func_t xo_flush;	/* Flush callback */
     xo_formatter_t xo_formatter; /* Custom formating function */
     xo_checkpointer_t xo_checkpointer; /* Custom formating support function */
     void *xo_opaque;		/* Opaque data for write function */
@@ -197,7 +250,7 @@ typedef struct xo_format_s {
 static xo_handle_t xo_default_handle;
 static int xo_default_inited;
 static int xo_locale_inited;
-static char *xo_program;
+static const char *xo_program;
 
 /*
  * To allow libxo to be used in diverse environment, we allow the
@@ -210,6 +263,10 @@ static xo_free_func_t xo_free = free;
 static void
 xo_failure (xo_handle_t *xop, const char *fmt, ...);
 
+static int
+xo_transition (xo_handle_t *xop, xo_xsf_flags_t flags, const char *name,
+	       xo_state_t new_state);
+
 static void
 xo_buf_append_div (xo_handle_t *xop, const char *class, xo_xff_flags_t flags,
 		   const char *name, int nlen,
@@ -226,6 +283,7 @@ static int
 xo_write_to_file (void *opaque, const char *data)
 {
     FILE *fp = (FILE *) opaque;
+
     return fprintf(fp, "%s", data);
 }
 
@@ -236,9 +294,21 @@ static void
 xo_close_file (void *opaque)
 {
     FILE *fp = (FILE *) opaque;
+
     fclose(fp);
 }
 
+/*
+ * Callback to flush a FILE pointer
+ */
+static int
+xo_flush_file (void *opaque)
+{
+    FILE *fp = (FILE *) opaque;
+
+    return fflush(fp);
+}
+
 /*
  * Initialize the contents of an xo_buffer_t.
  */
@@ -290,6 +360,29 @@ xo_no_setlocale (void)
     xo_locale_inited = 1;	/* Skip initialization */
 }
 
+/*
+ * We need to decide if stdout is line buffered (_IOLBF).  Lacking a
+ * standard way to decide this (e.g. getlinebuf()), we have configure
+ * look to find __flbf, which glibc supported.  If not, we'll rely
+ * on isatty, with the assumption that terminals are the only thing
+ * that's line buffered.  We _could_ test for "steam._flags & _IOLBF",
+ * which is all __flbf does, but that's even tackier.  Like a
+ * bedazzled Elvis outfit on an ugly lap dog sort of tacky.  Not
+ * something we're willing to do.
+ */
+static int
+xo_is_line_buffered (FILE *stream)
+{
+#if HAVE___FLBF
+    if (__flbf(stream))
+	return 1;
+#else /* HAVE___FLBF */
+    if (isatty(fileno(stream)))
+	return 1;
+#endif /* HAVE___FLBF */
+    return 0;
+}
+
 /*
  * Initialize an xo_handle_t, using both static defaults and
  * the global settings from the LIBXO_OPTIONS environment
@@ -300,6 +393,10 @@ xo_init_handle (xo_handle_t *xop)
 {
     xop->xo_opaque = stdout;
     xop->xo_write = xo_write_to_file;
+    xop->xo_flush = xo_flush_file;
+
+    if (xo_is_line_buffered(stdout))
+	xop->xo_flags |= XOF_FLUSH_LINE;
 
     /*
      * We need to initialize the locale, which isn't really pretty.
@@ -415,7 +512,7 @@ xo_indent (xo_handle_t *xop)
 	    rc += xop->xo_indent_by;
     }
 
-    return rc;
+    return (rc > 0) ? rc : 0;
 }
 
 static void
@@ -500,9 +597,9 @@ xo_escape_json (xo_buffer_t *xbp, int len)
     char *cp, *ep, *ip;
 
     for (cp = xbp->xb_curp, ep = cp + len; cp < ep; cp++) {
-	if (*cp == '\\')
+	if (*cp == '\\' || *cp == '"')
 	    delta += 1;
-	else if (*cp == '"')
+	else if (*cp == '\n' || *cp == '\r')
 	    delta += 1;
     }
 
@@ -519,13 +616,18 @@ xo_escape_json (xo_buffer_t *xbp, int len)
 	cp -= 1;
 	ip -= 1;
 
-	if (*cp != '\\' && *cp != '"') {
+	if (*cp == '\\' || *cp == '"') {
+	    *ip-- = *cp;
+	    *ip = '\\';
+	} else if (*cp == '\n') {
+	    *ip-- = 'n';
+	    *ip = '\\';
+	} else if (*cp == '\r') {
+	    *ip-- = 'r';
+	    *ip = '\\';
+	} else {
 	    *ip = *cp;
-	    continue;
 	}
-
-	*ip-- = *cp;
-	*ip = '\\';
 	
     } while (cp > ep && cp != ip);
 
@@ -572,20 +674,23 @@ xo_buf_escape (xo_handle_t *xop, xo_buffer_t *xbp,
  * Write the current contents of the data buffer using the handle's
  * xo_write function.
  */
-static void
+static int
 xo_write (xo_handle_t *xop)
 {
+    int rc = 0;
     xo_buffer_t *xbp = &xop->xo_data;
 
     if (xbp->xb_curp != xbp->xb_bufp) {
 	xo_buf_append(xbp, "", 1); /* Append ending NUL */
 	xo_anchor_clear(xop);
-	xop->xo_write(xop->xo_opaque, xbp->xb_bufp);
+	rc = xop->xo_write(xop->xo_opaque, xbp->xb_bufp);
 	xbp->xb_curp = xbp->xb_bufp;
     }
 
     /* Turn off the flags that don't survive across writes */
     xop->xo_flags &= ~(XOF_UNITS_PENDING);
+
+    return rc;
 }
 
 /*
@@ -947,7 +1052,7 @@ xo_warn_hcv (xo_handle_t *xop, int code, int check_warn,
 
     int len = strlen(fmt);
     int plen = xo_program ? strlen(xo_program) : 0;
-    char *newfmt = alloca(len + 2 + plen + 2); /* newline, NUL, and ": " */
+    char *newfmt = alloca(len + 1 + plen + 2); /* NUL, and ": " */
 
     if (plen) {
 	memcpy(newfmt, xo_program, plen);
@@ -955,7 +1060,6 @@ xo_warn_hcv (xo_handle_t *xop, int code, int check_warn,
 	newfmt[plen++] = ' ';
     }
     memcpy(newfmt + plen, fmt, len);
-
     newfmt[len + plen] = '\0';
 
     if (xop->xo_flags & XOF_WARN_XML) {
@@ -994,7 +1098,7 @@ xo_warn_hcv (xo_handle_t *xop, int code, int check_warn,
 	xo_buf_append(xbp, msg_close, sizeof(msg_close) - 1);
 	xo_buf_append(xbp, err_close, sizeof(err_close) - 1);
 
-	if (code > 0) {
+	if (code >= 0) {
 	    const char *msg = strerror(code);
 	    if (msg) {
 		xo_buf_append(xbp, ": ", 2);
@@ -1003,11 +1107,16 @@ xo_warn_hcv (xo_handle_t *xop, int code, int check_warn,
 	}
 
 	xo_buf_append(xbp, "\n", 2); /* Append newline and NUL to string */
-	xo_write(xop);
+	(void) xo_write(xop);
 
     } else {
 	vfprintf(stderr, newfmt, vap);
-	fprintf(stderr, ": %s\n", strerror(code));
+	if (code >= 0) {
+	    const char *msg = strerror(code);
+	    if (msg)
+		fprintf(stderr, ": %s", msg);
+	}
+	fprintf(stderr, "\n");
     }
 }
 
@@ -1027,7 +1136,7 @@ xo_warn_c (int code, const char *fmt, ...)
     va_list vap;
 
     va_start(vap, fmt);
-    xo_warn_hcv(NULL, 0, code, fmt, vap);
+    xo_warn_hcv(NULL, code, 0, fmt, vap);
     va_end(vap);
 }
 
@@ -1149,7 +1258,7 @@ xo_message_hcv (xo_handle_t *xop, int code, const char *fmt, va_list vap)
 	xo_buf_append(xbp, msg_close, sizeof(msg_close) - 1);
 	if (need_nl)
 	    xo_buf_append(xbp, "\n", 2); /* Append newline and NUL to string */
-	xo_write(xop);
+	(void) xo_write(xop);
 	break;
 
     case XO_STYLE_HTML:
@@ -1211,7 +1320,7 @@ xo_message_hcv (xo_handle_t *xop, int code, const char *fmt, va_list vap)
 	break;
     }
 
-    xo_flush_h(xop);
+    (void) xo_flush_h(xop);
 }
 
 void
@@ -1299,6 +1408,7 @@ xo_create_to_file (FILE *fp, xo_style_t style, xo_xof_flags_t flags)
 	xop->xo_opaque = fp;
 	xop->xo_write = xo_write_to_file;
 	xop->xo_close = xo_close_file;
+	xop->xo_flush = xo_flush_file;
     }
 
     return xop;
@@ -1323,7 +1433,7 @@ xo_destroy (xo_handle_t *xop_arg)
     xo_buf_cleanup(&xop->xo_attrs);
 
     if (xop_arg == NULL) {
-	bzero(&xo_default_handle, sizeof(&xo_default_handle));
+	bzero(&xo_default_handle, sizeof(xo_default_handle));
 	xo_default_inited = 0;
     } else
 	xo_free(xop);
@@ -1451,6 +1561,10 @@ xo_set_options (xo_handle_t *xop, const char *input)
 		xop->xo_flags |= XOF_FLUSH;
 		break;
 
+	    case 'F':
+		xop->xo_flags |= XOF_FLUSH_LINE;
+		break;
+
 	    case 'H':
 		xop->xo_style = XO_STYLE_HTML;
 		break;
@@ -1651,6 +1765,33 @@ xo_clear_flags (xo_handle_t *xop, xo_xof_flags_t flags)
     xop->xo_flags &= ~flags;
 }
 
+static const char *
+xo_state_name (xo_state_t state)
+{
+    static const char *names[] = {
+	"init",
+	"open_container",
+	"close_container",
+	"open_list",
+	"close_list",
+	"open_instance",
+	"close_instance",
+	"open_leaf_list",
+	"close_leaf_list",
+	"discarding",
+	"marker",
+	"emit",
+	"emit_leaf_list",
+	"finish",
+	NULL
+    };
+
+    if (state < (sizeof(names) / sizeof(names[0])))
+	return names[state];
+
+    return "unknown";
+}
+
 static void
 xo_line_ensure_open (xo_handle_t *xop, xo_xff_flags_t flags UNUSED)
 {
@@ -1867,14 +2008,20 @@ xo_format_string_direct (xo_handle_t *xop, xo_buffer_t *xbp,
 		goto done_with_encoding; /* Need multi-level 'break' */
 
 	    case XO_STYLE_JSON:
-		if (wc != '\\' && wc != '"')
+		if (wc != '\\' && wc != '"' && wc != '\n' && wc != '\r')
 		    break;
 
 		if (!xo_buf_has_room(xbp, 2))
 		    return -1;
 
 		*xbp->xb_curp++ = '\\';
-		*xbp->xb_curp++ = wc & 0x7f;
+		if (wc == '\n')
+		    wc = 'n';
+		else if (wc == '\r')
+		    wc = 'r';
+		else wc = wc & 0x7f;
+
+		*xbp->xb_curp++ = wc;
 		goto done_with_encoding;
 	    }
 
@@ -2550,6 +2697,15 @@ xo_buf_append_div (xo_handle_t *xop, const char *class, xo_xff_flags_t flags,
 		if (xsp->xs_name == NULL)
 		    continue;
 
+		/*
+		 * XSS_OPEN_LIST and XSS_OPEN_LEAF_LIST stack frames
+		 * are directly under XSS_OPEN_INSTANCE frames so we
+		 * don't need to put these in our XPath expressions.
+		 */
+		if (xsp->xs_state == XSS_OPEN_LIST
+			|| xsp->xs_state == XSS_OPEN_LEAF_LIST)
+		    continue;
+
 		xo_data_append(xop, "/", 1);
 		xo_data_escape(xop, xsp->xs_name, strlen(xsp->xs_name));
 		if (xsp->xs_keys) {
@@ -2615,6 +2771,11 @@ xo_format_title (xo_handle_t *xop, const char *str, int len,
     static char div_open[] = "<div class=\"title\">";
     static char div_close[] = "</div>";
 
+    if (flen == 0) {
+	fmt = "%s";
+	flen = 2;
+    }
+
     switch (xop->xo_style) {
     case XO_STYLE_XML:
     case XO_STYLE_JSON:
@@ -2748,6 +2909,75 @@ xo_format_value (xo_handle_t *xop, const char *name, int nlen,
     int quote;
     xo_buffer_t *xbp;
 
+    /*
+     * Before we emit a value, we need to know that the frame is ready.
+     */
+    xo_stack_t *xsp = &xop->xo_stack[xop->xo_depth];
+
+    if (flags & XFF_LEAF_LIST) {
+	/*
+	 * Check if we've already started to emit normal leafs
+	 * or if we're not in a leaf list.
+	 */
+	if ((xsp->xs_flags & (XSF_EMIT | XSF_EMIT_KEY))
+	    || !(xsp->xs_flags & XSF_EMIT_LEAF_LIST)) {
+	    char nbuf[nlen + 1];
+	    memcpy(nbuf, name, nlen);
+	    nbuf[nlen] = '\0';
+
+	    int rc = xo_transition(xop, 0, nbuf, XSS_EMIT_LEAF_LIST);
+	    if (rc < 0)
+		flags |= XFF_DISPLAY_ONLY | XFF_ENCODE_ONLY;
+	    else
+		xop->xo_stack[xop->xo_depth].xs_flags |= XSF_EMIT_LEAF_LIST;
+	}
+
+	xsp = &xop->xo_stack[xop->xo_depth];
+	if (xsp->xs_name) {
+	    name = xsp->xs_name;
+	    nlen = strlen(name);
+	}
+
+    } else if (flags & XFF_KEY) {
+	/* Emitting a 'k' (key) field */
+	if ((xsp->xs_flags & XSF_EMIT) && !(flags & XFF_DISPLAY_ONLY)) {
+	    xo_failure(xop, "key field emitted after normal value field: '%.*s'",
+		       nlen, name);
+
+	} else if (!(xsp->xs_flags & XSF_EMIT_KEY)) {
+	    char nbuf[nlen + 1];
+	    memcpy(nbuf, name, nlen);
+	    nbuf[nlen] = '\0';
+
+	    int rc = xo_transition(xop, 0, nbuf, XSS_EMIT);
+	    if (rc < 0)
+		flags |= XFF_DISPLAY_ONLY | XFF_ENCODE_ONLY;
+	    else
+		xop->xo_stack[xop->xo_depth].xs_flags |= XSF_EMIT_KEY;
+
+	    xsp = &xop->xo_stack[xop->xo_depth];
+	    xsp->xs_flags |= XSF_EMIT_KEY;
+	}
+
+    } else {
+	/* Emitting a normal value field */
+	if ((xsp->xs_flags & XSF_EMIT_LEAF_LIST)
+	    || !(xsp->xs_flags & XSF_EMIT)) {
+	    char nbuf[nlen + 1];
+	    memcpy(nbuf, name, nlen);
+	    nbuf[nlen] = '\0';
+
+	    int rc = xo_transition(xop, 0, nbuf, XSS_EMIT);
+	    if (rc < 0)
+		flags |= XFF_DISPLAY_ONLY | XFF_ENCODE_ONLY;
+	    else
+		xop->xo_stack[xop->xo_depth].xs_flags |= XSF_EMIT;
+
+	    xsp = &xop->xo_stack[xop->xo_depth];
+	    xsp->xs_flags |= XSF_EMIT;
+	}
+    }
+
     switch (xop->xo_style) {
     case XO_STYLE_TEXT:
 	if (flags & XFF_ENCODE_ONLY)
@@ -3157,13 +3387,15 @@ xo_do_emit (xo_handle_t *xop, const char *fmt)
     const char *cp, *sp, *ep, *basep;
     char *newp = NULL;
     int flush = (xop->xo_flags & XOF_FLUSH) ? 1 : 0;
+    int flush_line = (xop->xo_flags & XOF_FLUSH_LINE) ? 1 : 0;
 
     xop->xo_columns = 0;	/* Always reset it */
 
     for (cp = fmt; *cp; ) {
 	if (*cp == '\n') {
 	    xo_line_close(xop);
-	    xo_flush_h(xop);
+	    if (flush_line && xo_flush_h(xop) < 0)
+		return -1;
 	    cp += 1;
 	    continue;
 
@@ -3375,42 +3607,53 @@ xo_do_emit (xo_handle_t *xop, const char *fmt)
 	    return -1;
 	}
 
-	if (format == NULL && ftype != '[' && ftype != ']' ) {
-	    format = "%s";
-	    flen = 2;
-	}
+	if (ftype == 0 || ftype == 'V') {
+	    if (format == NULL) {
+		/* Default format for value fields is '%s' */
+		format = "%s";
+		flen = 2;
+	    }
 
-	if (ftype == 0 || ftype == 'V')
 	    xo_format_value(xop, content, clen, format, flen,
 			    encoding, elen, flags);
-	else if (ftype == 'D')
-	    xo_format_content(xop, "decoration", NULL, 1,
-			      content, clen, format, flen);
-	else if (ftype == 'E')
-	    xo_format_content(xop, "error", "error", 0,
-			      content, clen, format, flen);
-	else if (ftype == 'L')
-	    xo_format_content(xop, "label", NULL, 1,
-			      content, clen, format, flen);
-	else if (ftype == 'N')
-	    xo_format_content(xop, "note", NULL, 1,
-			      content, clen, format, flen);
-	else if (ftype == 'P')
- 	    xo_format_content(xop, "padding", NULL, 1,
-			      content, clen, format, flen);
-	else if (ftype == 'T')
-	    xo_format_title(xop, content, clen, format, flen);
-	else if (ftype == 'U') {
-	    if (flags & XFF_WS)
-		xo_format_content(xop, "padding", NULL, 1, " ", 1, NULL, 0);
- 	    xo_format_units(xop, content, clen, format, flen);
-	} else if (ftype == 'W')
-	    xo_format_content(xop, "warning", "warning", 0,
-			      content, clen, format, flen);
-	else if (ftype == '[')
-	    xo_anchor_start(xop, content, clen, format, flen);
+
+	} else if (ftype == '[')
+		xo_anchor_start(xop, content, clen, format, flen);
 	else if (ftype == ']')
-	    xo_anchor_stop(xop, content, clen, format, flen);
+		xo_anchor_stop(xop, content, clen, format, flen);
+
+	else  if (clen || format) { /* Need either content or format */
+	    if (format == NULL) {
+		/* Default format for value fields is '%s' */
+		format = "%s";
+		flen = 2;
+	    }
+
+	    if (ftype == 'D')
+		xo_format_content(xop, "decoration", NULL, 1,
+				  content, clen, format, flen);
+	    else if (ftype == 'E')
+		xo_format_content(xop, "error", "error", 0,
+				  content, clen, format, flen);
+	    else if (ftype == 'L')
+		xo_format_content(xop, "label", NULL, 1,
+				  content, clen, format, flen);
+	    else if (ftype == 'N')
+		xo_format_content(xop, "note", NULL, 1,
+				  content, clen, format, flen);
+	    else if (ftype == 'P')
+		xo_format_content(xop, "padding", NULL, 1,
+				  content, clen, format, flen);
+	    else if (ftype == 'T')
+		xo_format_title(xop, content, clen, format, flen);
+	    else if (ftype == 'U') {
+		if (flags & XFF_WS)
+		    xo_format_content(xop, "padding", NULL, 1, " ", 1, NULL, 0);
+		xo_format_units(xop, content, clen, format, flen);
+	    } else if (ftype == 'W')
+		xo_format_content(xop, "warning", "warning", 0,
+				  content, clen, format, flen);
+	}
 
 	if (flags & XFF_COLON)
 	    xo_format_content(xop, "decoration", NULL, 1, ":", 1, NULL, 0);
@@ -3425,8 +3668,12 @@ xo_do_emit (xo_handle_t *xop, const char *fmt)
     }
 
     /* If we don't have an anchor, write the text out */
-    if (flush && !(xop->xo_flags & XOF_ANCHOR))
-	xo_write(xop);
+    if (flush && !(xop->xo_flags & XOF_ANCHOR)) {
+	if (xo_write(xop) < 0) 
+	    rc = -1;		/* Report failure */
+	else if (xop->xo_flush && xop->xo_flush(xop->xo_opaque) < 0)
+	    rc = -1;
+    }
 
     return (rc < 0) ? rc : (int) xop->xo_columns;
 }
@@ -3549,8 +3796,11 @@ xo_stack_set_flags (xo_handle_t *xop)
 
 static void
 xo_depth_change (xo_handle_t *xop, const char *name,
-		 int delta, int indent, xo_xsf_flags_t flags)
+		 int delta, int indent, xo_state_t state, xo_xsf_flags_t flags)
 {
+    if (xop->xo_style == XO_STYLE_HTML || xop->xo_style == XO_STYLE_TEXT)
+	indent = 0;
+
     if (xop->xo_flags & XOF_DTRT)
 	flags |= XSF_DTRT;
 
@@ -3560,18 +3810,17 @@ xo_depth_change (xo_handle_t *xop, const char *name,
 
 	xo_stack_t *xsp = &xop->xo_stack[xop->xo_depth + delta];
 	xsp->xs_flags = flags;
+	xsp->xs_state = state;
 	xo_stack_set_flags(xop);
 
-	unsigned save = (xop->xo_flags & (XOF_XPATH | XOF_WARN | XOF_DTRT));
-	save |= (flags & XSF_DTRT);
+	if (name == NULL)
+	    name = XO_FAILURE_NAME;
 
-	if (name && save) {
-	    int len = strlen(name) + 1;
-	    char *cp = xo_realloc(NULL, len);
-	    if (cp) {
-		memcpy(cp, name, len);
-		xsp->xs_name = cp;
-	    }
+	int len = strlen(name) + 1;
+	char *cp = xo_realloc(NULL, len);
+	if (cp) {
+	    memcpy(cp, name, len);
+	    xsp->xs_name = cp;
 	}
 
     } else {			/* Pop operation */
@@ -3636,10 +3885,8 @@ xo_stack_flags (unsigned xflags)
 }
 
 static int
-xo_open_container_hf (xo_handle_t *xop, xo_xof_flags_t flags, const char *name)
+xo_do_open_container (xo_handle_t *xop, xo_xof_flags_t flags, const char *name)
 {
-    xop = xo_default(xop);
-
     int rc = 0;
     const char *ppn = (xop->xo_flags & XOF_PRETTY) ? "\n" : "";
     const char *pre_nl = "";
@@ -3653,9 +3900,16 @@ xo_open_container_hf (xo_handle_t *xop, xo_xof_flags_t flags, const char *name)
 
     switch (xop->xo_style) {
     case XO_STYLE_XML:
-	rc = xo_printf(xop, "%*s<%s>%s", xo_indent(xop), "",
-		     name, ppn);
-	xo_depth_change(xop, name, 1, 1, xo_stack_flags(flags));
+	rc = xo_printf(xop, "%*s<%s", xo_indent(xop), "", name);
+
+	if (xop->xo_attrs.xb_curp != xop->xo_attrs.xb_bufp) {
+	    rc += xop->xo_attrs.xb_curp - xop->xo_attrs.xb_bufp;
+	    xo_data_append(xop, xop->xo_attrs.xb_bufp,
+			   xop->xo_attrs.xb_curp - xop->xo_attrs.xb_bufp);
+	    xop->xo_attrs.xb_curp = xop->xo_attrs.xb_bufp;
+	}
+
+	rc += xo_printf(xop, ">%s", ppn);
 	break;
 
     case XO_STYLE_JSON:
@@ -3674,18 +3928,21 @@ xo_open_container_hf (xo_handle_t *xop, xo_xof_flags_t flags, const char *name)
 
 	rc = xo_printf(xop, "%s%*s\"%s\": {%s",
 		       pre_nl, xo_indent(xop), "", name, ppn);
-	xo_depth_change(xop, name, 1, 1, xo_stack_flags(flags));
-	break;
-
-    case XO_STYLE_HTML:
-    case XO_STYLE_TEXT:
-	xo_depth_change(xop, name, 1, 0, xo_stack_flags(flags));
 	break;
     }
 
+    xo_depth_change(xop, name, 1, 1, XSS_OPEN_CONTAINER,
+		    xo_stack_flags(flags));
+
     return rc;
 }
 
+static int
+xo_open_container_hf (xo_handle_t *xop, xo_xof_flags_t flags, const char *name)
+{
+    return xo_transition(xop, flags, name, XSS_OPEN_CONTAINER);
+}
+
 int
 xo_open_container_h (xo_handle_t *xop, const char *name)
 {
@@ -3710,8 +3967,8 @@ xo_open_container_d (const char *name)
     return xo_open_container_hf(NULL, XOF_DTRT, name);
 }
 
-int
-xo_close_container_h (xo_handle_t *xop, const char *name)
+static int
+xo_do_close_container (xo_handle_t *xop, const char *name)
 {
     xop = xo_default(xop);
 
@@ -3721,8 +3978,6 @@ xo_close_container_h (xo_handle_t *xop, const char *name)
 
     if (name == NULL) {
 	xo_stack_t *xsp = &xop->xo_stack[xop->xo_depth];
-	if (!(xsp->xs_flags & XSF_DTRT))
-	    xo_failure(xop, "missing name without 'dtrt' mode");
 
 	name = xsp->xs_name;
 	if (name) {
@@ -3731,13 +3986,15 @@ xo_close_container_h (xo_handle_t *xop, const char *name)
 	    char *cp = alloca(len);
 	    memcpy(cp, name, len);
 	    name = cp;
-	} else
+	} else if (!(xsp->xs_flags & XSF_DTRT)) {
+	    xo_failure(xop, "missing name without 'dtrt' mode");
 	    name = XO_FAILURE_NAME;
+	}
     }
 
     switch (xop->xo_style) {
     case XO_STYLE_XML:
-	xo_depth_change(xop, name, -1, -1, 0);
+	xo_depth_change(xop, name, -1, -1, XSS_CLOSE_CONTAINER, 0);
 	rc = xo_printf(xop, "%*s</%s>%s", xo_indent(xop), "", name, ppn);
 	break;
 
@@ -3745,20 +4002,26 @@ xo_close_container_h (xo_handle_t *xop, const char *name)
 	pre_nl = (xop->xo_flags & XOF_PRETTY) ? "\n" : "";
 	ppn = (xop->xo_depth <= 1) ? "\n" : "";
 
-	xo_depth_change(xop, name, -1, -1, 0);
+	xo_depth_change(xop, name, -1, -1, XSS_CLOSE_CONTAINER, 0);
 	rc = xo_printf(xop, "%s%*s}%s", pre_nl, xo_indent(xop), "", ppn);
 	xop->xo_stack[xop->xo_depth].xs_flags |= XSF_NOT_FIRST;
 	break;
 
     case XO_STYLE_HTML:
     case XO_STYLE_TEXT:
-	xo_depth_change(xop, name, -1, 0, 0);
+	xo_depth_change(xop, name, -1, 0, XSS_CLOSE_CONTAINER, 0);
 	break;
     }
 
     return rc;
 }
 
+int
+xo_close_container_h (xo_handle_t *xop, const char *name)
+{
+    return xo_transition(xop, 0, name, XSS_CLOSE_CONTAINER);
+}
+
 int
 xo_close_container (const char *name)
 {
@@ -3778,42 +4041,52 @@ xo_close_container_d (void)
 }
 
 static int
-xo_open_list_hf (xo_handle_t *xop, xo_xsf_flags_t flags, const char *name)
+xo_do_open_list (xo_handle_t *xop, xo_xsf_flags_t flags, const char *name)
 {
+    int rc = 0;
+    int indent = 0;
+
     xop = xo_default(xop);
 
-    if (xop->xo_style != XO_STYLE_JSON)
-	return 0;
+    if (xop->xo_style == XO_STYLE_JSON) {
+	const char *ppn = (xop->xo_flags & XOF_PRETTY) ? "\n" : "";
+	const char *pre_nl = "";
 
-    int rc = 0;
-    const char *ppn = (xop->xo_flags & XOF_PRETTY) ? "\n" : "";
-    const char *pre_nl = "";
-
-    if (!(xop->xo_flags & XOF_NO_TOP)) {
-	if (!(xop->xo_flags & XOF_TOP_EMITTED)) {
-	    xo_printf(xop, "%*s{%s", xo_indent(xop), "", ppn);
-	    xop->xo_flags |= XOF_TOP_EMITTED;
+	indent = 1;
+	if (!(xop->xo_flags & XOF_NO_TOP)) {
+	    if (!(xop->xo_flags & XOF_TOP_EMITTED)) {
+		xo_printf(xop, "%*s{%s", xo_indent(xop), "", ppn);
+		xop->xo_flags |= XOF_TOP_EMITTED;
+	    }
 	}
+
+	if (name == NULL) {
+	    xo_failure(xop, "NULL passed for list name");
+	    name = XO_FAILURE_NAME;
+	}
+
+	xo_stack_set_flags(xop);
+
+	if (xop->xo_stack[xop->xo_depth].xs_flags & XSF_NOT_FIRST)
+	    pre_nl = (xop->xo_flags & XOF_PRETTY) ? ",\n" : ", ";
+	xop->xo_stack[xop->xo_depth].xs_flags |= XSF_NOT_FIRST;
+
+	rc = xo_printf(xop, "%s%*s\"%s\": [%s",
+		       pre_nl, xo_indent(xop), "", name, ppn);
     }
 
-    if (name == NULL) {
-	xo_failure(xop, "NULL passed for list name");
-	name = XO_FAILURE_NAME;
-    }
-
-    xo_stack_set_flags(xop);
-
-    if (xop->xo_stack[xop->xo_depth].xs_flags & XSF_NOT_FIRST)
-	pre_nl = (xop->xo_flags & XOF_PRETTY) ? ",\n" : ", ";
-    xop->xo_stack[xop->xo_depth].xs_flags |= XSF_NOT_FIRST;
-
-    rc = xo_printf(xop, "%s%*s\"%s\": [%s",
-		   pre_nl, xo_indent(xop), "", name, ppn);
-    xo_depth_change(xop, name, 1, 1, XSF_LIST | xo_stack_flags(flags));
+    xo_depth_change(xop, name, 1, indent, XSS_OPEN_LIST,
+		    XSF_LIST | xo_stack_flags(flags));
 
     return rc;
 }
 
+static int
+xo_open_list_hf (xo_handle_t *xop, xo_xsf_flags_t flags, const char *name)
+{
+    return xo_transition(xop, flags, name, XSS_OPEN_LIST);
+}
+
 int
 xo_open_list_h (xo_handle_t *xop, const char *name UNUSED)
 {
@@ -3838,21 +4111,14 @@ xo_open_list_d (const char *name)
     return xo_open_list_hf(NULL, XOF_DTRT, name);
 }
 
-int
-xo_close_list_h (xo_handle_t *xop, const char *name)
+static int
+xo_do_close_list (xo_handle_t *xop, const char *name)
 {
     int rc = 0;
     const char *pre_nl = "";
 
-    xop = xo_default(xop);
-
-    if (xop->xo_style != XO_STYLE_JSON)
-	return 0;
-
     if (name == NULL) {
 	xo_stack_t *xsp = &xop->xo_stack[xop->xo_depth];
-	if (!(xsp->xs_flags & XSF_DTRT))
-	    xo_failure(xop, "missing name without 'dtrt' mode");
 
 	name = xsp->xs_name;
 	if (name) {
@@ -3861,21 +4127,35 @@ xo_close_list_h (xo_handle_t *xop, const char *name)
 	    char *cp = alloca(len);
 	    memcpy(cp, name, len);
 	    name = cp;
-	} else
+	} else if (!(xsp->xs_flags & XSF_DTRT)) {
+	    xo_failure(xop, "missing name without 'dtrt' mode");
 	    name = XO_FAILURE_NAME;
+	}
     }
 
-    if (xop->xo_stack[xop->xo_depth].xs_flags & XSF_NOT_FIRST)
-	pre_nl = (xop->xo_flags & XOF_PRETTY) ? "\n" : "";
-    xop->xo_stack[xop->xo_depth].xs_flags |= XSF_NOT_FIRST;
+    if (xop->xo_style == XO_STYLE_JSON) {
+	if (xop->xo_stack[xop->xo_depth].xs_flags & XSF_NOT_FIRST)
+	    pre_nl = (xop->xo_flags & XOF_PRETTY) ? "\n" : "";
+	xop->xo_stack[xop->xo_depth].xs_flags |= XSF_NOT_FIRST;
 
-    xo_depth_change(xop, name, -1, -1, XSF_LIST);
-    rc = xo_printf(xop, "%s%*s]", pre_nl, xo_indent(xop), "");
-    xop->xo_stack[xop->xo_depth].xs_flags |= XSF_NOT_FIRST;
+	xo_depth_change(xop, name, -1, -1, XSS_CLOSE_LIST, XSF_LIST);
+	rc = xo_printf(xop, "%s%*s]", pre_nl, xo_indent(xop), "");
+	xop->xo_stack[xop->xo_depth].xs_flags |= XSF_NOT_FIRST;
+
+    } else {
+	xo_depth_change(xop, name, -1, 0, XSS_CLOSE_LIST, XSF_LIST);
+	xop->xo_stack[xop->xo_depth].xs_flags |= XSF_NOT_FIRST;
+    }
 
     return rc;
 }
 
+int
+xo_close_list_h (xo_handle_t *xop, const char *name)
+{
+    return xo_transition(xop, 0, name, XSS_CLOSE_LIST);
+}
+
 int
 xo_close_list (const char *name)
 {
@@ -3895,7 +4175,88 @@ xo_close_list_d (void)
 }
 
 static int
-xo_open_instance_hf (xo_handle_t *xop, xo_xsf_flags_t flags, const char *name)
+xo_do_open_leaf_list (xo_handle_t *xop, xo_xsf_flags_t flags, const char *name)
+{
+    int rc = 0;
+    int indent = 0;
+
+    xop = xo_default(xop);
+
+    if (xop->xo_style == XO_STYLE_JSON) {
+	const char *ppn = (xop->xo_flags & XOF_PRETTY) ? "\n" : "";
+	const char *pre_nl = "";
+
+	indent = 1;
+
+	if (!(xop->xo_flags & XOF_NO_TOP)) {
+	    if (!(xop->xo_flags & XOF_TOP_EMITTED)) {
+		xo_printf(xop, "%*s{%s", xo_indent(xop), "", ppn);
+		xop->xo_flags |= XOF_TOP_EMITTED;
+	    }
+	}
+
+	if (name == NULL) {
+	    xo_failure(xop, "NULL passed for list name");
+	    name = XO_FAILURE_NAME;
+	}
+
+	xo_stack_set_flags(xop);
+
+	if (xop->xo_stack[xop->xo_depth].xs_flags & XSF_NOT_FIRST)
+	    pre_nl = (xop->xo_flags & XOF_PRETTY) ? ",\n" : ", ";
+	xop->xo_stack[xop->xo_depth].xs_flags |= XSF_NOT_FIRST;
+
+	rc = xo_printf(xop, "%s%*s\"%s\": [%s",
+		       pre_nl, xo_indent(xop), "", name, ppn);
+    }
+
+    xo_depth_change(xop, name, 1, indent, XSS_OPEN_LEAF_LIST,
+		    XSF_LIST | xo_stack_flags(flags));
+
+    return rc;
+}
+
+static int
+xo_do_close_leaf_list (xo_handle_t *xop, const char *name)
+{
+    int rc = 0;
+    const char *pre_nl = "";
+
+    if (name == NULL) {
+	xo_stack_t *xsp = &xop->xo_stack[xop->xo_depth];
+
+	name = xsp->xs_name;
+	if (name) {
+	    int len = strlen(name) + 1;
+	    /* We need to make a local copy; xo_depth_change will free it */
+	    char *cp = alloca(len);
+	    memcpy(cp, name, len);
+	    name = cp;
+	} else if (!(xsp->xs_flags & XSF_DTRT)) {
+	    xo_failure(xop, "missing name without 'dtrt' mode");
+	    name = XO_FAILURE_NAME;
+	}
+    }
+
+    if (xop->xo_style == XO_STYLE_JSON) {
+	if (xop->xo_stack[xop->xo_depth].xs_flags & XSF_NOT_FIRST)
+	    pre_nl = (xop->xo_flags & XOF_PRETTY) ? "\n" : "";
+	xop->xo_stack[xop->xo_depth].xs_flags |= XSF_NOT_FIRST;
+
+	xo_depth_change(xop, name, -1, -1, XSS_CLOSE_LEAF_LIST, XSF_LIST);
+	rc = xo_printf(xop, "%s%*s]", pre_nl, xo_indent(xop), "");
+	xop->xo_stack[xop->xo_depth].xs_flags |= XSF_NOT_FIRST;
+
+    } else {
+	xo_depth_change(xop, name, -1, 0, XSS_CLOSE_LEAF_LIST, XSF_LIST);
+	xop->xo_stack[xop->xo_depth].xs_flags |= XSF_NOT_FIRST;
+    }
+
+    return rc;
+}
+
+static int
+xo_do_open_instance (xo_handle_t *xop, xo_xsf_flags_t flags, const char *name)
 {
     xop = xo_default(xop);
 
@@ -3912,8 +4273,16 @@ xo_open_instance_hf (xo_handle_t *xop, xo_xsf_flags_t flags, const char *name)
 
     switch (xop->xo_style) {
     case XO_STYLE_XML:
-	rc = xo_printf(xop, "%*s<%s>%s", xo_indent(xop), "", name, ppn);
-	xo_depth_change(xop, name, 1, 1, xo_stack_flags(flags));
+	rc = xo_printf(xop, "%*s<%s", xo_indent(xop), "", name);
+
+	if (xop->xo_attrs.xb_curp != xop->xo_attrs.xb_bufp) {
+	    rc += xop->xo_attrs.xb_curp - xop->xo_attrs.xb_bufp;
+	    xo_data_append(xop, xop->xo_attrs.xb_bufp,
+			   xop->xo_attrs.xb_curp - xop->xo_attrs.xb_bufp);
+	    xop->xo_attrs.xb_curp = xop->xo_attrs.xb_bufp;
+	}
+
+	rc += xo_printf(xop, ">%s", ppn);
 	break;
 
     case XO_STYLE_JSON:
@@ -3925,18 +4294,20 @@ xo_open_instance_hf (xo_handle_t *xop, xo_xsf_flags_t flags, const char *name)
 
 	rc = xo_printf(xop, "%s%*s{%s",
 		       pre_nl, xo_indent(xop), "", ppn);
-	xo_depth_change(xop, name, 1, 1, xo_stack_flags(flags));
-	break;
-
-    case XO_STYLE_HTML:
-    case XO_STYLE_TEXT:
-	xo_depth_change(xop, name, 1, 0, xo_stack_flags(flags));
 	break;
     }
 
+    xo_depth_change(xop, name, 1, 1, XSS_OPEN_INSTANCE, xo_stack_flags(flags));
+
     return rc;
 }
 
+static int
+xo_open_instance_hf (xo_handle_t *xop, xo_xsf_flags_t flags, const char *name)
+{
+    return xo_transition(xop, flags, name, XSS_OPEN_INSTANCE);
+}
+
 int
 xo_open_instance_h (xo_handle_t *xop, const char *name)
 {
@@ -3961,8 +4332,8 @@ xo_open_instance_d (const char *name)
     return xo_open_instance_hf(NULL, XOF_DTRT, name);
 }
 
-int
-xo_close_instance_h (xo_handle_t *xop, const char *name)
+static int
+xo_do_close_instance (xo_handle_t *xop, const char *name)
 {
     xop = xo_default(xop);
 
@@ -3972,8 +4343,6 @@ xo_close_instance_h (xo_handle_t *xop, const char *name)
 
     if (name == NULL) {
 	xo_stack_t *xsp = &xop->xo_stack[xop->xo_depth];
-	if (!(xsp->xs_flags & XSF_DTRT))
-	    xo_failure(xop, "missing name without 'dtrt' mode");
 
 	name = xsp->xs_name;
 	if (name) {
@@ -3982,33 +4351,41 @@ xo_close_instance_h (xo_handle_t *xop, const char *name)
 	    char *cp = alloca(len);
 	    memcpy(cp, name, len);
 	    name = cp;
-	} else
+	} else if (!(xsp->xs_flags & XSF_DTRT)) {
+	    xo_failure(xop, "missing name without 'dtrt' mode");
 	    name = XO_FAILURE_NAME;
+	}
     }
 
     switch (xop->xo_style) {
     case XO_STYLE_XML:
-	xo_depth_change(xop, name, -1, -1, 0);
+	xo_depth_change(xop, name, -1, -1, XSS_CLOSE_INSTANCE, 0);
 	rc = xo_printf(xop, "%*s</%s>%s", xo_indent(xop), "", name, ppn);
 	break;
 
     case XO_STYLE_JSON:
 	pre_nl = (xop->xo_flags & XOF_PRETTY) ? "\n" : "";
 
-	xo_depth_change(xop, name, -1, -1, 0);
+	xo_depth_change(xop, name, -1, -1, XSS_CLOSE_INSTANCE, 0);
 	rc = xo_printf(xop, "%s%*s}", pre_nl, xo_indent(xop), "");
 	xop->xo_stack[xop->xo_depth].xs_flags |= XSF_NOT_FIRST;
 	break;
 
     case XO_STYLE_HTML:
     case XO_STYLE_TEXT:
-	xo_depth_change(xop, name, -1, 0, 0);
+	xo_depth_change(xop, name, -1, 0, XSS_CLOSE_INSTANCE, 0);
 	break;
     }
 
     return rc;
 }
 
+int
+xo_close_instance_h (xo_handle_t *xop, const char *name)
+{
+    return xo_transition(xop, 0, name, XSS_CLOSE_INSTANCE);
+}
+
 int
 xo_close_instance (const char *name)
 {
@@ -4027,15 +4404,421 @@ xo_close_instance_d (void)
     return xo_close_instance_h(NULL, NULL);
 }
 
+static int
+xo_do_close_all (xo_handle_t *xop, xo_stack_t *limit)
+{
+    xo_stack_t *xsp;
+    int rc = 0;
+    xo_xsf_flags_t flags;
+
+    for (xsp = &xop->xo_stack[xop->xo_depth]; xsp >= limit; xsp--) {
+	switch (xsp->xs_state) {
+	case XSS_INIT:
+	    /* Nothing */
+	    rc = 0;
+	    break;
+
+	case XSS_OPEN_CONTAINER:
+	    rc = xo_do_close_container(xop, NULL);
+	    break;
+
+	case XSS_OPEN_LIST:
+	    rc = xo_do_close_list(xop, NULL);
+	    break;
+
+	case XSS_OPEN_INSTANCE:
+	    rc = xo_do_close_instance(xop, NULL);
+	    break;
+
+	case XSS_OPEN_LEAF_LIST:
+	    rc = xo_do_close_leaf_list(xop, NULL);
+	    break;
+
+	case XSS_MARKER:
+	    flags = xsp->xs_flags & XSF_MARKER_FLAGS;
+	    xo_depth_change(xop, xsp->xs_name, -1, 0, XSS_MARKER, 0);
+	    xop->xo_stack[xop->xo_depth].xs_flags |= flags;
+	    rc = 0;
+	    break;
+	}
+
+	if (rc < 0)
+	    xo_failure(xop, "close %d failed: %d", xsp->xs_state, rc);
+    }
+
+    return 0;
+}
+
+/*
+ * This function is responsible for clearing out whatever is needed
+ * to get to the desired state, if possible.
+ */
+static int
+xo_do_close (xo_handle_t *xop, const char *name, xo_state_t new_state)
+{
+    xo_stack_t *xsp, *limit = NULL;
+    int rc;
+    xo_state_t need_state = new_state;
+
+    if (new_state == XSS_CLOSE_CONTAINER)
+	need_state = XSS_OPEN_CONTAINER;
+    else if (new_state == XSS_CLOSE_LIST)
+	need_state = XSS_OPEN_LIST;
+    else if (new_state == XSS_CLOSE_INSTANCE)
+	need_state = XSS_OPEN_INSTANCE;
+    else if (new_state == XSS_CLOSE_LEAF_LIST)
+	need_state = XSS_OPEN_LEAF_LIST;
+    else if (new_state == XSS_MARKER)
+	need_state = XSS_MARKER;
+    else
+	return 0; /* Unknown or useless new states are ignored */
+
+    for (xsp = &xop->xo_stack[xop->xo_depth]; xsp > xop->xo_stack; xsp--) {
+	/*
+	 * Marker's normally stop us from going any further, unless
+	 * we are popping a marker (new_state == XSS_MARKER).
+	 */
+	if (xsp->xs_state == XSS_MARKER && need_state != XSS_MARKER) {
+	    if (name) {
+		xo_failure(xop, "close (xo_%s) fails at marker '%s'; "
+			   "not found '%s'",
+			   xo_state_name(new_state),
+			   xsp->xs_name, name);
+		return 0;
+
+	    } else {
+		limit = xsp;
+		xo_failure(xop, "close stops at marker '%s'", xsp->xs_name);
+	    }
+	    break;
+	}
+	
+	if (xsp->xs_state != need_state)
+	    continue;
+
+	if (name && xsp->xs_name && strcmp(name, xsp->xs_name) != 0)
+	    continue;
+
+	limit = xsp;
+	break;
+    }
+
+    if (limit == NULL) {
+	xo_failure(xop, "xo_%s can't find match for '%s'",
+		   xo_state_name(new_state), name);
+	return 0;
+    }
+
+    rc = xo_do_close_all(xop, limit);
+
+    return rc;
+}
+
+/*
+ * We are in a given state and need to transition to the new state.
+ */
+static int
+xo_transition (xo_handle_t *xop, xo_xsf_flags_t flags, const char *name,
+	       xo_state_t new_state)
+{
+    xo_stack_t *xsp;
+    int rc;
+    int old_state, on_marker;
+
+    xop = xo_default(xop);
+
+    rc = 0;
+    xsp = &xop->xo_stack[xop->xo_depth];
+    old_state = xsp->xs_state;
+    on_marker = (old_state == XSS_MARKER);
+
+    /* If there's a marker on top of the stack, we need to find a real state */
+    while (old_state == XSS_MARKER) {
+	if (xsp == xop->xo_stack)
+	    break;
+	xsp -= 1;
+	old_state = xsp->xs_state;
+    }
+
+    /*
+     * At this point, the list of possible states are:
+     *   XSS_INIT, XSS_OPEN_CONTAINER, XSS_OPEN_LIST,
+     *   XSS_OPEN_INSTANCE, XSS_OPEN_LEAF_LIST, XSS_DISCARDING
+     */
+    switch (XSS_TRANSITION(old_state, new_state)) {
+
+    open_container:
+    case XSS_TRANSITION(XSS_INIT, XSS_OPEN_CONTAINER):
+    case XSS_TRANSITION(XSS_OPEN_INSTANCE, XSS_OPEN_CONTAINER):
+    case XSS_TRANSITION(XSS_OPEN_CONTAINER, XSS_OPEN_CONTAINER):
+       rc = xo_do_open_container(xop, flags, name);
+       break;
+
+    case XSS_TRANSITION(XSS_OPEN_LIST, XSS_OPEN_CONTAINER):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close_list(xop, NULL);
+	if (rc >= 0)
+	    goto open_container;
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_LEAF_LIST, XSS_OPEN_CONTAINER):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close_leaf_list(xop, NULL);
+	if (rc >= 0)
+	    goto open_container;
+	break;
+
+    /*close_container:*/
+    case XSS_TRANSITION(XSS_OPEN_CONTAINER, XSS_CLOSE_CONTAINER):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close(xop, name, new_state);
+	break;
+
+    case XSS_TRANSITION(XSS_INIT, XSS_CLOSE_CONTAINER):
+	/* This is an exception for "xo --close" */
+	rc = xo_do_close_container(xop, name);
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_LIST, XSS_CLOSE_CONTAINER):
+    case XSS_TRANSITION(XSS_OPEN_INSTANCE, XSS_CLOSE_CONTAINER):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close(xop, name, new_state);
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_LEAF_LIST, XSS_CLOSE_CONTAINER):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close_leaf_list(xop, NULL);
+	if (rc >= 0)
+	    rc = xo_do_close(xop, name, new_state);
+	break;
+
+    open_list:
+    case XSS_TRANSITION(XSS_INIT, XSS_OPEN_LIST):
+    case XSS_TRANSITION(XSS_OPEN_CONTAINER, XSS_OPEN_LIST):
+    case XSS_TRANSITION(XSS_OPEN_INSTANCE, XSS_OPEN_LIST):
+	rc = xo_do_open_list(xop, flags, name);
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_LIST, XSS_OPEN_LIST):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close_list(xop, NULL);
+	if (rc >= 0)
+	    goto open_list;
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_LEAF_LIST, XSS_OPEN_LIST):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close_leaf_list(xop, NULL);
+	if (rc >= 0)
+	    goto open_list;
+	break;
+
+    /*close_list:*/
+    case XSS_TRANSITION(XSS_OPEN_LIST, XSS_CLOSE_LIST):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close(xop, name, new_state);
+	break;
+
+    case XSS_TRANSITION(XSS_INIT, XSS_CLOSE_LIST):
+    case XSS_TRANSITION(XSS_OPEN_CONTAINER, XSS_CLOSE_LIST):
+    case XSS_TRANSITION(XSS_OPEN_INSTANCE, XSS_CLOSE_LIST):
+    case XSS_TRANSITION(XSS_OPEN_LEAF_LIST, XSS_CLOSE_LIST):
+	rc = xo_do_close(xop, name, new_state);
+	break;
+
+    open_instance:
+    case XSS_TRANSITION(XSS_OPEN_LIST, XSS_OPEN_INSTANCE):
+	rc = xo_do_open_instance(xop, flags, name);
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_CONTAINER, XSS_OPEN_INSTANCE):
+    case XSS_TRANSITION(XSS_INIT, XSS_OPEN_INSTANCE):
+	rc = xo_do_open_list(xop, flags, name);
+	if (rc >= 0)
+	    goto open_instance;
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_INSTANCE, XSS_OPEN_INSTANCE):
+	if (on_marker) {
+	    rc = xo_do_open_list(xop, flags, name);
+	} else {
+	    rc = xo_do_close_instance(xop, NULL);
+	}
+	if (rc >= 0)
+	    goto open_instance;
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_LEAF_LIST, XSS_OPEN_INSTANCE):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close_leaf_list(xop, NULL);
+	if (rc >= 0)
+	    goto open_instance;
+	break;
+
+    /*close_instance:*/
+    case XSS_TRANSITION(XSS_OPEN_INSTANCE, XSS_CLOSE_INSTANCE):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close_instance(xop, name);
+	break;
+
+    case XSS_TRANSITION(XSS_INIT, XSS_CLOSE_INSTANCE):
+	/* This one makes no sense; ignore it */
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_CONTAINER, XSS_CLOSE_INSTANCE):
+    case XSS_TRANSITION(XSS_OPEN_LIST, XSS_CLOSE_INSTANCE):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close(xop, name, new_state);
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_LEAF_LIST, XSS_CLOSE_INSTANCE):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close_leaf_list(xop, NULL);
+	if (rc >= 0)
+	    rc = xo_do_close(xop, name, new_state);
+	break;
+
+    open_leaf_list:
+    case XSS_TRANSITION(XSS_OPEN_CONTAINER, XSS_OPEN_LEAF_LIST):
+    case XSS_TRANSITION(XSS_OPEN_INSTANCE, XSS_OPEN_LEAF_LIST):
+    case XSS_TRANSITION(XSS_INIT, XSS_OPEN_LEAF_LIST):
+	rc = xo_do_open_leaf_list(xop, flags, name);
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_LIST, XSS_OPEN_LEAF_LIST):
+    case XSS_TRANSITION(XSS_OPEN_LEAF_LIST, XSS_OPEN_LEAF_LIST):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close_list(xop, NULL);
+	if (rc >= 0)
+	    goto open_leaf_list;
+	break;
+
+    /*close_leaf_list:*/
+    case XSS_TRANSITION(XSS_OPEN_LEAF_LIST, XSS_CLOSE_LEAF_LIST):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close_leaf_list(xop, name);
+	break;
+
+    case XSS_TRANSITION(XSS_INIT, XSS_CLOSE_LEAF_LIST):
+	/* Makes no sense; ignore */
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_CONTAINER, XSS_CLOSE_LEAF_LIST):
+    case XSS_TRANSITION(XSS_OPEN_LIST, XSS_CLOSE_LEAF_LIST):
+    case XSS_TRANSITION(XSS_OPEN_INSTANCE, XSS_CLOSE_LEAF_LIST):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close(xop, name, new_state);
+	break;
+
+    /*emit:*/
+    case XSS_TRANSITION(XSS_OPEN_CONTAINER, XSS_EMIT):
+    case XSS_TRANSITION(XSS_OPEN_INSTANCE, XSS_EMIT):
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_LIST, XSS_EMIT):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close(xop, NULL, XSS_CLOSE_LIST);
+	break;
+
+    case XSS_TRANSITION(XSS_INIT, XSS_EMIT):
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_LEAF_LIST, XSS_EMIT):
+	if (on_marker)
+	    goto marker_prevents_close;
+	rc = xo_do_close_leaf_list(xop, NULL);
+	break;
+
+    /*emit_leaf_list:*/
+    case XSS_TRANSITION(XSS_INIT, XSS_EMIT_LEAF_LIST):
+    case XSS_TRANSITION(XSS_OPEN_CONTAINER, XSS_EMIT_LEAF_LIST):
+    case XSS_TRANSITION(XSS_OPEN_INSTANCE, XSS_EMIT_LEAF_LIST):
+	rc = xo_do_open_leaf_list(xop, flags, name);
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_LEAF_LIST, XSS_EMIT_LEAF_LIST):
+	break;
+
+    case XSS_TRANSITION(XSS_OPEN_LIST, XSS_EMIT_LEAF_LIST):
+	/*
+	 * We need to be backward compatible with the pre-xo_open_leaf_list
+	 * API, where both lists and leaf-lists were opened as lists.  So
+	 * if we find an open list that hasn't had anything written to it,
+	 * we'll accept it.
+	 */
+	break;
+
+    default:
+	xo_failure(xop, "unknown transition: (%u -> %u)",
+		   xsp->xs_state, new_state);
+    }
+
+    return rc;
+
+ marker_prevents_close:
+    xo_failure(xop, "marker '%s' prevents transition from %s to %s",
+	       xop->xo_stack[xop->xo_depth].xs_name,
+	       xo_state_name(old_state), xo_state_name(new_state));
+    return -1;
+}
+
+int
+xo_open_marker_h (xo_handle_t *xop, const char *name)
+{
+    xop = xo_default(xop);
+
+    xo_depth_change(xop, name, 1, 0, XSS_MARKER,
+		    xop->xo_stack[xop->xo_depth].xs_flags & XSF_MARKER_FLAGS);
+
+    return 0;
+}
+
+int
+xo_open_marker (const char *name)
+{
+    return xo_open_marker_h(NULL, name);
+}
+
+int
+xo_close_marker_h (xo_handle_t *xop, const char *name)
+{
+    xop = xo_default(xop);
+
+    return xo_do_close(xop, name, XSS_MARKER);
+}
+
+int
+xo_close_marker (const char *name)
+{
+    return xo_close_marker_h(NULL, name);
+}
+
 void
 xo_set_writer (xo_handle_t *xop, void *opaque, xo_write_func_t write_func,
-	       xo_close_func_t close_func)
+	       xo_close_func_t close_func, xo_flush_func_t flush_func)
 {
     xop = xo_default(xop);
 
     xop->xo_opaque = opaque;
     xop->xo_write = write_func;
     xop->xo_close = close_func;
+    xop->xo_flush = flush_func;
 }
 
 void
@@ -4045,10 +4828,11 @@ xo_set_allocator (xo_realloc_func_t realloc_func, xo_free_func_t free_func)
     xo_free = free_func;
 }
 
-void
+int
 xo_flush_h (xo_handle_t *xop)
 {
     static char div_close[] = "</div>";
+    int rc;
 
     xop = xo_default(xop);
 
@@ -4064,21 +4848,29 @@ xo_flush_h (xo_handle_t *xop)
 	break;
     }
 
-    xo_write(xop);
+    rc = xo_write(xop);
+    if (rc >= 0 && xop->xo_flush)
+	if (xop->xo_flush(xop->xo_opaque) < 0)
+	    return -1;
+
+    return rc;
 }
 
-void
+int
 xo_flush (void)
 {
-    xo_flush_h(NULL);
+    return xo_flush_h(NULL);
 }
 
-void
+int
 xo_finish_h (xo_handle_t *xop)
 {
     const char *cp = "";
     xop = xo_default(xop);
 
+    if (!(xop->xo_flags & XOF_NO_CLOSE))
+	xo_do_close_all(xop, xop->xo_stack);
+
     switch (xop->xo_style) {
     case XO_STYLE_JSON:
 	if (!(xop->xo_flags & XOF_NO_TOP)) {
@@ -4091,13 +4883,13 @@ xo_finish_h (xo_handle_t *xop)
 	break;
     }
 
-    xo_flush_h(xop);
+    return xo_flush_h(xop);
 }
 
-void
+int
 xo_finish (void)
 {
-    xo_finish_h(NULL);
+    return xo_finish_h(NULL);
 }
 
 /*
@@ -4141,6 +4933,7 @@ xo_error_hv (xo_handle_t *xop, const char *fmt, va_list vap)
 	break;
 
     case XO_STYLE_XML:
+    case XO_STYLE_JSON:
 	va_copy(xop->xo_vap, vap);
 
 	xo_open_container_h(xop, "error");
@@ -4235,6 +5028,30 @@ xo_parse_args (int argc, char **argv)
     return save;
 }
 
+void
+xo_dump_stack (xo_handle_t *xop)
+{
+    int i;
+    xo_stack_t *xsp;
+
+    xop = xo_default(xop);
+
+    fprintf(stderr, "Stack dump:\n");
+
+    xsp = xop->xo_stack;
+    for (i = 1, xsp++; i <= xop->xo_depth; i++, xsp++) {
+	fprintf(stderr, "   [%d] %s '%s' [%x]\n",
+		i, xo_state_name(xsp->xs_state),
+		xsp->xs_name ?: "--", xsp->xs_flags);
+    }
+}
+
+void
+xo_set_program (const char *name)
+{
+    xo_program = name;
+}
+
 #ifdef UNIT_TEST
 int
 main (int argc, char **argv)
diff --git a/contrib/libxo/libxo/xo.h b/contrib/libxo/libxo/xo.h
index 3a59e4c81ef..82b965a2920 100644
--- a/contrib/libxo/libxo/xo.h
+++ b/contrib/libxo/libxo/xo.h
@@ -25,35 +25,39 @@ typedef unsigned xo_style_t;
 #define XO_STYLE_HTML	3	/** Generate HTML output */
 
 /** Flags for libxo */
-typedef unsigned long xo_xof_flags_t;
-#define XOF_CLOSE_FP	(1<<0)	/** Close file pointer on xo_close() */
-#define XOF_PRETTY	(1<<1)	/** Make 'pretty printed' output */
-#define XOF_DIV_OPEN	(1<<2)	/** Internal use only: a <div> is open */
-#define XOF_LINE_OPEN	(1<<3)	/** Internal use only: a <div class="line"> */
+typedef unsigned long long xo_xof_flags_t;
+#define XOF_BIT(_n) ((xo_xof_flags_t) 1 << (_n))
+#define XOF_CLOSE_FP	XOF_BIT(0) /** Close file pointer on xo_close() */
+#define XOF_PRETTY	XOF_BIT(1) /** Make 'pretty printed' output */
+#define XOF_DIV_OPEN	XOF_BIT(2) /** Internal use only: a <div> is open */
+#define XOF_LINE_OPEN	XOF_BIT(3) /** Internal use only: <div class="line"> */
 
-#define XOF_WARN	(1<<4)	/** Generate warnings for broken calls */
-#define XOF_XPATH	(1<<5)	/** Emit XPath attributes in HTML  */
-#define XOF_INFO	(1<<6)	/** Emit additional info fields (HTML) */
-#define XOF_WARN_XML	(1<<7)	/** Emit warnings in XML (on stdout) */
+#define XOF_WARN	XOF_BIT(4) /** Generate warnings for broken calls */
+#define XOF_XPATH	XOF_BIT(5) /** Emit XPath attributes in HTML  */
+#define XOF_INFO	XOF_BIT(6) /** Emit additional info fields (HTML) */
+#define XOF_WARN_XML	XOF_BIT(7) /** Emit warnings in XML (on stdout) */
 
-#define XOF_NO_ENV	(1<<8)	/** Don't look at the LIBXO_OPTIONS env var */
-#define XOF_NO_VA_ARG	(1<<9)	/** Don't advance va_list w/ va_arg() */
-#define XOF_DTRT	(1<<10)	/** Enable "do the right thing" mode */
-#define XOF_KEYS	(1<<11)	/** Flag 'key' fields for xml and json */
+#define XOF_NO_ENV	XOF_BIT(8) /** Don't look at LIBXO_OPTIONS env var */
+#define XOF_NO_VA_ARG	XOF_BIT(9) /** Don't advance va_list w/ va_arg() */
+#define XOF_DTRT	XOF_BIT(10) /** Enable "do the right thing" mode */
+#define XOF_KEYS	XOF_BIT(11) /** Flag 'key' fields for xml and json */
 
-#define XOF_IGNORE_CLOSE (1<<12) /** Ignore errors on close tags */
-#define XOF_NOT_FIRST	(1<<13)	 /* Not the first item (JSON)  */
-#define XOF_NO_LOCALE	(1<<14)	 /** Don't bother with locale */
-#define XOF_TOP_EMITTED	(1<<15)	 /* The top JSON braces have been emitted  */
+#define XOF_IGNORE_CLOSE XOF_BIT(12) /** Ignore errors on close tags */
+#define XOF_NOT_FIRST	XOF_BIT(13) /* Not the first item (JSON)  */
+#define XOF_NO_LOCALE	XOF_BIT(14) /** Don't bother with locale */
+#define XOF_TOP_EMITTED	XOF_BIT(15) /* The top JSON braces have been emitted  */
 
-#define XOF_NO_TOP	(1<<16)	/** Don't emit the top braces in JSON */
-#define XOF_ANCHOR	(1<<17)	/** An anchor is in place  */
-#define XOF_UNITS	(1<<18)	/** Encode units in XML */
-#define XOF_UNITS_PENDING (1<<19) /** We have a units-insertion pending */
+#define XOF_NO_TOP	XOF_BIT(16) /** Don't emit the top braces in JSON */
+#define XOF_ANCHOR	XOF_BIT(17) /** An anchor is in place  */
+#define XOF_UNITS	XOF_BIT(18) /** Encode units in XML */
+#define XOF_UNITS_PENDING XOF_BIT(19) /** We have a units-insertion pending */
 
-#define XOF_UNDERSCORES	(1<<20)	/** Replace dashes with underscores (JSON)  */
-#define XOF_COLUMNS	(1<<21)	/** xo_emit should return a column count */
-#define XOF_FLUSH	(1<<22)	/** Flush after each xo_emit call */
+#define XOF_UNDERSCORES	XOF_BIT(20) /** Replace dashes with underscores (JSON)*/
+#define XOF_COLUMNS	XOF_BIT(21) /** xo_emit should return a column count */
+#define XOF_FLUSH	XOF_BIT(22) /** Flush after each xo_emit call */
+#define XOF_FLUSH_LINE	XOF_BIT(23) /** Flush after each newline */
+
+#define XOF_NO_CLOSE	XOF_BIT(24) /** xo_finish won't close open elements */
 
 /*
  * The xo_info_t structure provides a mapping between names and
@@ -70,6 +74,7 @@ typedef struct xo_handle_s xo_handle_t; /* Handle for XO output */
 
 typedef int (*xo_write_func_t)(void *, const char *);
 typedef void (*xo_close_func_t)(void *);
+typedef int (*xo_flush_func_t)(void *);
 typedef void *(*xo_realloc_func_t)(void *, size_t);
 typedef void (*xo_free_func_t)(void *);
 
@@ -93,7 +98,7 @@ xo_destroy (xo_handle_t *xop);
 
 void
 xo_set_writer (xo_handle_t *xop, void *opaque, xo_write_func_t write_func,
-	       xo_close_func_t close_func);
+	       xo_close_func_t close_func, xo_flush_func_t flush_func);
 
 void
 xo_set_allocator (xo_realloc_func_t realloc_func, xo_free_func_t free_func);
@@ -209,6 +214,18 @@ xo_close_instance_hd (xo_handle_t *xop);
 int
 xo_close_instance_d (void);
 
+int
+xo_open_marker_h (xo_handle_t *xop, const char *name);
+
+int
+xo_open_marker (const char *name);
+
+int
+xo_close_marker_h (xo_handle_t *xop, const char *name);
+
+int
+xo_close_marker (const char *name);
+
 int
 xo_attr_h (xo_handle_t *xop, const char *name, const char *fmt, ...);
 
@@ -227,16 +244,16 @@ xo_error_h (xo_handle_t *xop, const char *fmt, ...);
 void
 xo_error (const char *fmt, ...);
 
-void
+int
 xo_flush_h (xo_handle_t *xop);
 
-void
+int
 xo_flush (void);
 
-void
+int
 xo_finish_h (xo_handle_t *xop);
 
-void
+int
 xo_finish (void);
 
 void
@@ -297,4 +314,10 @@ xo_parse_args (int argc, char **argv);
 extern const char xo_version[];
 extern const char xo_version_extra[];
 
+void
+xo_dump_stack (xo_handle_t *xop);
+
+void
+xo_set_program (const char *name);
+
 #endif /* INCLUDE_XO_H */
diff --git a/contrib/libxo/libxo/xo_attr.3 b/contrib/libxo/libxo/xo_attr.3
index afd805f54f0..1c183605975 100644
--- a/contrib/libxo/libxo/xo_attr.3
+++ b/contrib/libxo/libxo/xo_attr.3
@@ -51,6 +51,10 @@ parameter as passed to
     XML:
         <login-time seconds="1408336270">00:14</login-time>
 .Ed
+.Pp
+Since attributes are only emitted in XML, their use should be limited
+to meta-data and additional or redundant representations of data
+already emitted in other form.
 .Sh ADDITIONAL DOCUMENTATION
 .Pp
 Complete documentation can be found on github:
diff --git a/contrib/libxo/libxo/xo_create.3 b/contrib/libxo/libxo/xo_create.3
index ec241ee1e0e..b0e89652270 100644
--- a/contrib/libxo/libxo/xo_create.3
+++ b/contrib/libxo/libxo/xo_create.3
@@ -7,7 +7,7 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
@@ -25,7 +25,7 @@
 .Fn xo_destroy "xo_handle_t *handle"
 .Sh DESCRIPTION
 A
-.Em libxo
+.Nm libxo
 handle can be allocated using the
 .Fn xo_create
 function.
@@ -37,49 +37,55 @@ function.
 .Ed
 .Pp
 By default,
-.Em libxo
-writes output to standard output.  A convenience
-function is provided for situations when output should be written to
+.Nm libxo
+writes output to standard output.
+A convenience
+function is provided for situations when output should be written to a
 different file.
 .Pp
 Use the
-.Em XOF_CLOSE_FP
+.Dv XOF_CLOSE_FP
 flag to trigger a call to
-.Em fclose 3
-for the FILE pointer when the handle is destroyed.
+.Xr fclose 3
+for the
+.Dv FILE
+pointer when the handle is destroyed.
 .Pp
 The
 .Fn xo_destroy
 function releases a handle and any resources it is
-using.  Calling
+using.
+Calling
 .Fn xo_destroy
 with a
-.Em NULL
+.Dv NULL
 handle will release any
 resources associated with the default handle.
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
 .Sh SEE ALSO
-.Xr xo_emit 3
-and
-.Xr xo_set_options 3 .
+.Xr xo_emit 3 ,
+.Xr xo_set_options 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_emit.3 b/contrib/libxo/libxo/xo_emit.3
index 9f76e13c632..70608245098 100644
--- a/contrib/libxo/libxo/xo_emit.3
+++ b/contrib/libxo/libxo/xo_emit.3
@@ -7,7 +7,7 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
@@ -44,13 +44,13 @@ accepts a
 .Fa va_list
 for additional flexibility.
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
@@ -61,11 +61,12 @@ https://github.com/Juniper/libxo/releases
 .Ed
 .Sh SEE ALSO
 .Xr xo_open_container 3 ,
-.Xr xo_open_list 3 , and
-.Xr xo_format 5 .
+.Xr xo_open_list 3 ,
+.Xr xo_format 5
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_err.3 b/contrib/libxo/libxo/xo_err.3
index 2445aa70f65..5584309f3bf 100644
--- a/contrib/libxo/libxo/xo_err.3
+++ b/contrib/libxo/libxo/xo_err.3
@@ -7,7 +7,7 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
@@ -45,32 +45,38 @@ Many programs make use of the standard library functions
 and
 .Xr warn 3
 to generate errors and warnings for the user.
-.Em libxo
+.Nm libxo
 wants to
 pass that information via the current output style, and provides
 compatible functions to allow this.
 .Pp
 These functions display the program name, a colon, a formatted message
 based on the arguments, and then optionally a colon and an error
-message associated with either "errno" or the "code" parameter.
+message associated with either
+.Fa errno
+or the
+.Fa code
+parameter.
 .Bd -literal -offset indent
     EXAMPLE:
         if (open(filename, O_RDONLY) < 0)
             xo_err(1, "cannot open file '%s'", filename);
 .Ed
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
@@ -78,7 +84,8 @@ https://github.com/Juniper/libxo/releases
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_error.3 b/contrib/libxo/libxo/xo_error.3
new file mode 100644
index 00000000000..da91785928d
--- /dev/null
+++ b/contrib/libxo/libxo/xo_error.3
@@ -0,0 +1,64 @@
+.\" #
+.\" # Copyright (c) 2014, Juniper Networks, Inc.
+.\" # All rights reserved.
+.\" # This SOFTWARE is licensed under the LICENSE provided in the
+.\" # ../Copyright file. By downloading, installing, copying, or 
+.\" # using the SOFTWARE, you agree to be bound by the terms of that
+.\" # LICENSE.
+.\" # Phil Shafer, July 2014
+.\" 
+.Dd December 4, 2014
+.Dt LIBXO 3
+.Os
+.Sh NAME
+.Nm xo_error
+.Nd generate error messages
+.Sh LIBRARY
+.Lb libxo
+.Sh SYNOPSIS
+.In libxo/xo.h
+.Ft void
+.Fn xo_error "const char *fmt" "..."
+.Sh DESCRIPTION
+Use the
+.Fn xo_error
+function to generate error messages to standard error.
+The 
+.Fa fmt
+argument is a string containing printf-style formatting
+instructions that describe the remaining arguments.
+.Pp
+When converting an application to libxo, one can replace
+.Em "fprintf(stderr,...)"
+calls with
+.Fn xo_error
+calls.
+.Pp
+.Sh ADDITIONAL DOCUMENTATION
+Complete documentation can be found on github:
+.Bd -literal -offset indent
+http://juniper.github.io/libxo/libxo-manual.html
+.Ed
+.Pp
+.Nm libxo
+lives on github as:
+.Bd -literal -offset indent
+https://github.com/Juniper/libxo
+.Ed
+.Pp
+The latest release of
+.Nm libxo
+is available at:
+.Bd -literal -offset indent
+https://github.com/Juniper/libxo/releases
+.Ed
+.Sh SEE ALSO
+.Xr printf 3
+.Xr xo_emit 3
+.Sh HISTORY
+The
+.Nm libxo
+library was added in
+.Fx 11.0 .
+.Sh AUTHOR
+Phil Shafer
diff --git a/contrib/libxo/libxo/xo_finish.3 b/contrib/libxo/libxo/xo_finish.3
index 3b25dc34cbf..421e94557bf 100644
--- a/contrib/libxo/libxo/xo_finish.3
+++ b/contrib/libxo/libxo/xo_finish.3
@@ -7,7 +7,7 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
@@ -24,9 +24,10 @@
 .Sh DESCRIPTION
 When the program is ready to exit or close a handle, a call to
 .Fn xo_finish
-is required.  This flushes any buffered data, closes
+is required.
+This flushes any buffered data, closes
 open
-.Em libxo
+.Nm libxo
 constructs, and completes any pending operations.
 .Pp
 Calling this function is
@@ -34,18 +35,20 @@ Calling this function is
 to the proper operation of libxo,
 especially for the non-TEXT output styles.
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
@@ -53,7 +56,8 @@ https://github.com/Juniper/libxo/releases
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_flush.3 b/contrib/libxo/libxo/xo_flush.3
index 160f634773d..b85e9f786d0 100644
--- a/contrib/libxo/libxo/xo_flush.3
+++ b/contrib/libxo/libxo/xo_flush.3
@@ -7,7 +7,7 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
@@ -22,25 +22,29 @@
 .Ft void
 .Fn xo_flush_h "xo_handle_t *handle"
 .Sh DESCRIPTION
-.Em libxo
+.Nm libxo
 buffers data, both for performance and consistency, but also to
-allow some advanced features to work properly.  At various times, the
-caller may wish to flush any data buffered within the library.  The
+allow some advanced features to work properly.
+At various times, the
+caller may wish to flush any data buffered within the library.
+The
 .Fn xo_flush
 function is used for this.
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
@@ -48,7 +52,8 @@ https://github.com/Juniper/libxo/releases
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_format.5 b/contrib/libxo/libxo/xo_format.5
index 62cfeb6b92f..b021b98fd55 100644
--- a/contrib/libxo/libxo/xo_format.5
+++ b/contrib/libxo/libxo/xo_format.5
@@ -7,7 +7,7 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
@@ -15,7 +15,7 @@
 .Nd content of format descriptors for xo_emit
 .Sh DESCRIPTION
 .Pp
-.Em libxo
+.Nm libxo
 uses format strings to control the rendering of data into
 various output styles, including
 .Em text ,
@@ -25,25 +25,28 @@ and
 .Em HTML .
 Each format string contains a set of zero or more
 .Dq field descriptions ,
-which describe independent data fields.  Each
-field description contains a set of
+which describe independent data fields.
+Each field description contains a set of
 .Dq modifiers ,
 a
 .Dq content string ,
 and zero, one, or two
 .Dq format descriptors .
 The modifiers tell
-.Em libxo
+.Nm libxo
 what the field is and how to treat it, while the format descriptors are
 formatting instructions using
-.Xr printf 3 -style
+.Xr printf 3 Ns -style
 format strings, telling
-libxo how to format the field.  The field description is placed inside
+.Nm libxo
+how to format the field.
+The field description is placed inside
 a set of braces, with a colon
 .Ql ( \&: )
 after the modifiers and a slash
 .Ql ( \&/ )
-before each format descriptors.  Text may be intermixed with
+before each format descriptors.
+Text may be intermixed with
 field descriptions within the format string.
 .Pp
 The field description is given as follows:
@@ -53,15 +56,19 @@ The field description is given as follows:
 .Ed
 .Pp
 The role describes the function of the field, while the modifiers
-enable optional behaviors.  The contents, field-format, and
-encoding-format are used in varying ways, based on the role.  These
-are described in the following sections.
+enable optional behaviors.
+The contents, field-format, and
+encoding-format are used in varying ways, based on the role.
+These are described in the following sections.
 .Pp
-In the following example, three field descriptors appear.  The first
+In the following example, three field descriptors appear.
+The first
 is a padding field containing three spaces of padding, the second is a
-label ("In stock"), and the third is a value field ("in-stock").  The
-in-stock field has a "%u" format that will parse the next argument
-passed to the xo_emit function as an unsigned integer.
+label ("In stock"), and the third is a value field ("in-stock").
+The in-stock field has a "%u" format that will parse the next argument
+passed to the
+.Xr xo_emit 3 ,
+function as an unsigned integer.
 .Bd -literal -offset indent
         xo_emit("{P:   }{Lwc:In stock}{:in-stock/%u}\\n", 65);
 .Ed
@@ -71,7 +78,8 @@ This single line of code can generate text ("In stock: 65\\n"), XML
 lengthy to be listed here).
 .Ss Modifier Roles
 Modifiers are optional, and indicate the role and formatting of the
-content.  The roles are listed below; only one role is permitted:
+content.
+The roles are listed below; only one role is permitted:
 .Pp
 .Bl -column "M" "Name12341234"
 .It Sy "M  Name           Description"
@@ -91,7 +99,8 @@ content.  The roles are listed below; only one role is permitted:
 .Ss The Decoration Role ({D:})
 Decorations are typically punctuation marks such as colons,
 semi-colons, and commas used to decorate the text and make it simpler
-for human readers.  By marking these distinctly, HTML usage scenarios
+for human readers.
+By marking these distinctly, HTML usage scenarios
 can use CSS to direct their display parameters.
 .Bd -literal -offset indent
     xo_emit("{D:((}{:name}{D:))}\\n", name);
@@ -116,8 +125,9 @@ if preceded by a slash ("/"):
     xo_emit("{P:/30s}{Lwc:Cost}{:cost/%u}\\n", "", cost);
 .Ed
 .Ss The Title Role ({T:})
-Title are heading or column headers that are meant to be displayed to
-the user.  The title can be either static, when placed directly within
+Titles are heading or column headers that are meant to be displayed to
+the user.
+The title can be either static, when placed directly within
 the field descriptor, or a printf-style format descriptor can be used,
 if preceded by a slash ("/"):
 .Bd -literal -offset indent
@@ -126,7 +136,8 @@ if preceded by a slash ("/"):
 .Ed
 .Ss The Units Role ({U:})
 Units are the dimension by which values are measured, such as degrees,
-miles, bytes, and decibels.  The units field carries this information
+miles, bytes, and decibels.
+The units field carries this information
 for the previous value field.
 .Bd -literal -offset indent
     xo_emit("{Lwc:Distance}{:distance/%u}{Uw:miles}\\n", miles);
@@ -136,7 +147,7 @@ Note that the sense of the 'w' modifier is reversed for units;
 a blank is added before the contents, rather than after it.
 .Pp
 When the
-.Em XOF_UNITS
+.Dv XOF_UNITS
 flag is set, units are rendered in XML as the
 .Dq units
 attribute:
@@ -151,14 +162,19 @@ Units can also be rendered in HTML as the "data-units" attribute:
 .Ed
 .Ss The Value Role ({V:} and {:})
 The value role is used to represent the a data value that is
-interesting for the non-display output styles (XML and JSON).  Value
+interesting for the non-display output styles (XML and JSON).
+Value
 is the default role; if no other role designation is given, the field
-is a value.  The field name must appear within the field descriptor,
-followed by one or two format descriptors.  The first format
+is a value.
+The field name must appear within the field descriptor,
+followed by one or two format descriptors.
+The first format
 descriptor is used for display styles (TEXT and HTML), while the
-second one is used for encoding styles (XML and JSON).  If no second
+second one is used for encoding styles (XML and JSON).
+If no second
 format is given, the encoding format defaults to the first format,
-with any minimum width removed.  If no first format is given, both
+with any minimum width removed.
+If no first format is given, both
 format descriptors default to "%s".
 .Bd -literal -offset indent
     xo_emit("{:length/%02u}x{:width/%02u}x{:height/%02u}\\n",
@@ -168,9 +184,13 @@ format descriptors default to "%s".
 .Ed
 .Ss The Anchor Modifiers ({[:} and {]:})
 The anchor roles allow a set of strings by be padded as a group,
-but still be visible to xo_emit as distinct fields.  Either the start
+but still be visible to
+.Xr xo_emit 3
+as distinct fields.
+Either the start
 or stop anchor can give a field width and it can be either directly in
-the descriptor or passed as an argument.  Any fields between the start
+the descriptor or passed as an argument.
+Any fields between the start
 and stop anchor are padded to meet the minimum width given.
 .Pp
 To give a width directly, encode it as the content of the anchor tag:
@@ -179,21 +199,26 @@ To give a width directly, encode it as the content of the anchor tag:
 .Ed
 .Pp
 To pass a width as an argument, use "%d" as the format, which must
-appear after the "/".  Note that only "%d" is supported for widths.
+appear after the "/".
+Note that only "%d" is supported for widths.
 Using any other value could ruin your day.
 .Bd -literal -offset indent
     xo_emit("({[:/%d}{:min/%d}/{:max/%d}{]:})\\n", width, min, max);
 .Ed
 .Pp
 If the width is negative, padding will be added on the right, suitable
-for left justification.  Otherwise the padding will be added to the
+for left justification.
+Otherwise the padding will be added to the
 left of the fields between the start and stop anchors, suitable for
-right justification.  If the width is zero, nothing happens.  If the
+right justification.
+If the width is zero, nothing happens.
+If the
 number of columns of output between the start and stop anchors is less
 than the absolute value of the given width, nothing happens.
 .Pp
-Widths over 8k are considered probable errors and not supported.  If
-.Em XOF_WARN
+Widths over 8k are considered probable errors and not supported.
+If
+.Dv XOF_WARN
 is set, a warning will be generated.
 .Ss Modifier Flags
 The modifiers can also include the following flags, which modify the
@@ -201,19 +226,20 @@ content emitted for some output styles:
 .Pp
 .Bl -column M "Name12341234"
 .It Sy M "Name        Description"
-.It c "colon       " "A colon (":") is appended after the label"
+.It c "colon       " "A colon ("":"") is appended after the label"
 .It d "display     " "Only emit field for display styles (text/HTML)"
 .It e "encoding    " "Only emit for encoding styles (XML/JSON)"
 .It k "key         " "Field is a key, suitable for XPath predicates"
 .It n "no-quotes   " "Do not quote the field when using JSON style"
 .It q "quotes      " "Quote the field when using JSON style"
-.It w "white space " "A blank (" ") is appended after the label"
+.It w "white space " "A blank ("" "") is appended after the label"
 .El
 .Pp
 For example, the modifier string "Lwc" means the field has a label
 role (text that describes the next field) and should be followed by a
-colon ('c') and a space ('w').  The modifier string "Vkq" means the
-field is has value role, that it is a key for the current instance, and
+colon ('c') and a space ('w').
+The modifier string "Vkq" means the
+field has a value role, that it is a key for the current instance, and
 that the value should be quoted when encoded for JSON.
 .Ss The Colon Modifier ({c:})
 The colon modifier appends a single colon to the data value:
@@ -225,7 +251,8 @@ The colon modifier appends a single colon to the data value:
 .Ed
 .Pp
 The colon modifier is only used for the TEXT and HTML output
-styles. It is commonly combined with the space modifier ('{w:').
+styles.
+It is commonly combined with the space modifier ('{w:}').
 It is purely a convenience feature.
 .Ss The Display Modifier ({d:})
 The display modifier indicated the field should only be generated for
@@ -270,15 +297,34 @@ uniquely identify an instance of list data.
         xo_close_list("user");
 .Ed
 .Pp
-Currently the key modifier is only used when generating XPath value
+Currently the key modifier is only used when generating XPath values
 for the HTML output style when
-.Em XOF_XPATH
+.Dv XOF_XPATH
 is set, but other uses are likely in the near future.
+.Ss The Leaf-List Modifier ({l:})
+The leaf-list modifier is used to distinguish lists where each
+instance consists of only a single value.  In XML, these are
+rendered as single elements, where JSON renders them as arrays.
+.Bd -literal -offset indent
+    EXAMPLE:
+        xo_open_list("user");
+        for (i = 0; i < num_users; i++) {
+            xo_emit("Member {l:name}\n", user[i].u_name);
+        }
+        xo_close_list("user");
+    XML:
+        <user>phil</user>
+        <user>pallavi</user>
+    JSON:
+        "user": [ "phil", "pallavi" ]
+.Ed
 .Ss The No-Quotes Modifier ({n:})
 The no-quotes modifier (and its twin, the 'quotes' modifier) affect
-the quoting of values in the JSON output style.  JSON uses quotes for
-string value, but no quotes for numeric, boolean, and null data.
-xo_emit applies a simple heuristic to determine whether quotes are
+the quoting of values in the JSON output style.
+JSON uses quotes for
+string values, but no quotes for numeric, boolean, and null data.
+.Xr xo_emit 3
+applies a simple heuristic to determine whether quotes are
 needed, but often this needs to be controlled by the caller.
 .Bd -literal -offset indent
     EXAMPLE:
@@ -289,9 +335,11 @@ needed, but often this needs to be controlled by the caller.
 .Ed
 .Ss The Quotes Modifier ({q:})
 The quotes modifier (and its twin, the 'no-quotes' modifier) affect
-the quoting of values in the JSON output style.  JSON uses quotes for
-string value, but no quotes for numeric, boolean, and null data.
-xo_emit applies a simple heuristic to determine whether quotes are
+the quoting of values in the JSON output style.
+JSON uses quotes for
+string values, but no quotes for numeric, boolean, and null data.
+.Xr xo_emit 3
+applies a simple heuristic to determine whether quotes are
 needed, but often this needs to be controlled by the caller.
 .Bd -literal -offset indent
     EXAMPLE:
@@ -309,7 +357,8 @@ The white space modifier appends a single space to the data value:
 .Ed
 .Pp
 The white space modifier is only used for the TEXT and HTML output
-styles. It is commonly combined with the colon modifier ('{c:').
+styles.
+It is commonly combined with the colon modifier ('{c:}').
 It is purely a convenience feature.
 .Pp
 Note that the sense of the 'w' modifier is reversed for the units role
@@ -317,16 +366,20 @@ Note that the sense of the 'w' modifier is reversed for the units role
 .Ss Field Formatting
 The field format is similar to the format string for
 .Xr printf 3 .
-It's used varies based on the role of the field, but generally is used to
+Its use varies based on the role of the field, but generally is used to
 format the field's contents.
 .Pp
-If not provided, the format string defaults to "%s".
+If the format string is not provided for a value field, it defaults
+to "%s".
 .Pp
 Note a field definition can contain zero or more printf-style
 .Dq directives ,
-which are sequences that start with a '%' and end with a
-one of following characters: "diouxXDOUeEfFgGaAcCsSp".  Each directive
-is matched by one of more arguments to the xo_emit function.
+which are sequences that start with a '%' and end with
+one of following characters: "diouxXDOUeEfFgGaAcCsSp".
+Each directive
+is matched by one of more arguments to the
+.Xr xo_emit 3
+function.
 .Pp
 The format string has the form:
 .Bd -literal -offset indent
@@ -346,19 +399,22 @@ a leading zero ('0') indicating the output value should be padded on the
 left with zeroes instead of spaces (' ').
 .It
 one or more digits ('0' - '9') indicating the minimum width of the
-argument.  If the width in columns of the output value is less that
-the minumum width, the value will be padded to reach the minimum.
+argument.
+If the width in columns of the output value is less than
+the minimum width, the value will be padded to reach the minimum.
 .It
 a period followed by one or more digits indicating the maximum
 number of bytes which will be examined for a string argument, or the maximum
-width for a non-string argument.  When handling ASCII strings this is
+width for a non-string argument.
+When handling ASCII strings this
 functions as the field width but for multi-byte characters, a single
 character may be composed of multiple bytes.
-xo_emit will never dereference memory beyond the given number of bytes.
+.Xr xo_emit 3
+will never dereference memory beyond the given number of bytes.
 .It
 a second period followed by one or more digits indicating the maximum
-width for a string argument.  This modifier cannot be given for non-string
-arguments. 
+width for a string argument.
+This modifier cannot be given for non-string arguments. 
 .It
 one or more 'h' characters, indicating shorter input data.
 .It
@@ -420,39 +476,50 @@ argument:
 .El
 .Pp
 .Ss UTF-8 and Locale Strings
-All strings for libxo must be UTF-8.  libxo will handle turning them
+All strings for
+.Nm libxo
+must be UTF-8.
+.Nm libxo
+will handle turning them
 into locale-based strings for display to the user.
 .Pp
 For strings, the 'h' and 'l' modifiers affect the interpretation of
-the bytes pointed to argument.  The default '%s' string is a 'char *'
-pointer to a string encoded as UTF-8.  Since UTF-8 is compatible with
+the bytes pointed to argument.
+The default '%s' string is a 'char *'
+pointer to a string encoded as UTF-8.
+Since UTF-8 is compatible with
 .Em ASCII
 data, a normal 7-bit
 .Em ASCII
- string can be used.  '%ls' expects a
-'wchar_t *' pointer to a wide-character string, encoded as a 32-bit
-Unicode values.  '%hs' expects a 'char *' pointer to a multi-byte
+string can be used.
+'%ls' expects a
+'wchar_t *' pointer to a wide-character string, encoded as 32-bit
+Unicode values.
+'%hs' expects a 'char *' pointer to a multi-byte
 string encoded with the current locale, as given by the
-.Em LC_CTYPE ,
-.Em LANG ,
+.Ev LC_CTYPE ,
+.Ev LANG ,
 or
-.Em LC_ALL
-environment varibles.  The first of this list of
-variables is used and if none of the variables, the locale defaults to
-.Em UTF-8.
+.Ev LC_ALL
+environment variables.
+The first of this list of
+variables is used and if none of the variables are set, the locale defaults to
+.Em UTF-8 .
 .Pp
-libxo will
+.Nm libxo
+will
 convert these arguments as needed to either UTF-8 (for XML, JSON, and
 HTML styles) or locale-based strings for display in text style.
 .Bd -literal -offset indent
-   xo_emit("Alll strings are utf-8 content {:tag/%ls}",
+   xo_emit("All strings are utf-8 content {:tag/%ls}",
            L"except for wide strings");
 .Ed
 .Pp
 "%S" is equivalent to "%ls".
 .Pp
 For example, a function is passed a locale-base name, a hat size,
-and a time value.  The hat size is formatted in a UTF-8 (ASCII)
+and a time value.
+The hat size is formatted in a UTF-8 (ASCII)
 string, and the time value is formatted into a wchar_t string.
 .Bd -literal -offset indent
     void print_order (const char *name, int size,
@@ -475,29 +542,35 @@ string, and the time value is formatted into a wchar_t string.
     }
 .Ed
 .Pp
-It is important to note that xo_emit will perform the conversion
-required to make appropriate output.  Text style output uses the
+It is important to note that
+.Xr xo_emit 3
+will perform the conversion
+required to make appropriate output.
+Text style output uses the
 current locale (as described above), while XML, JSON, and HTML use
 UTF-8.
 .Pp
 UTF-8 and locale-encoded strings can use multiple bytes to encode one
-column of data.  The traditional "precision'" (aka "max-width") value
+column of data.
+The traditional "precision'" (aka "max-width") value
 for "%s" printf formatting becomes overloaded since it specifies both
 the number of bytes that can be safely referenced and the maximum
-number of columns to emit.  xo_emit uses the precision as the former,
+number of columns to emit.
+.Xr xo_emit 3
+uses the precision as the former,
 and adds a third value for specifying the maximum number of columns.
 .Pp
 In this example, the name field is printed with a minimum of 3 columns
-and a maximum of 6.  Up to ten bytes are in used in filling those
-columns. 
+and a maximum of 6.
+Up to ten bytes are in used in filling those columns. 
 .Bd -literal -offset indent
     xo_emit("{:name/%3.10.6s}", name);
 .Ed
 .Ss Characters Outside of Field Definitions
-Characters in the format string are not part of a field definition are
+Characters in the format string that are not part of a field definition are
 copied to the output for the TEXT style, and are ignored for the JSON
-and XML styles.  For HTML, these characters are placed in a <div> with
-class "text".
+and XML styles.
+For HTML, these characters are placed in a <div> with class "text".
 .Bd -literal -offset indent
   EXAMPLE:
       xo_emit("The hat is {:size/%s}.\\n", size_val);
@@ -513,13 +586,16 @@ class "text".
       <div class="text">.</div>
 .Ed
 .Ss "%n" is Not Supported
-libxo does not support the '%n' directive.  It's a bad idea and we
-just don't do it.
+.Nm libxo
+does not support the '%n' directive.
+It is a bad idea and we
+just do not do it.
 .Ss The Encoding Format (eformat)
 The "eformat" string is the format string used when encoding the field
-for JSON and XML.  If not provided, it defaults to the primary format
-with any minimum width removed.  If the primary is not given, both
-default to "%s".
+for JSON and XML.
+If not provided, it defaults to the primary format
+with any minimum width removed.
+If the primary is not given, both default to "%s".
 .Sh EXAMPLE
 In this example, the value for the number of items in stock is emitted:
 .Bd -literal -offset indent
@@ -547,9 +623,9 @@ This call will generate the following output:
 .Pp
 Clearly HTML wins the verbosity award, and this output does
 not include
-.Em XOF_XPATH
+.Dv XOF_XPATH
 or
-.Em XOF_INFO
+.Dv XOF_INFO
 data, which would expand the penultimate line to:
 .Bd -literal -offset indent
        <div class="data" data-tag="in-stock"
@@ -561,28 +637,33 @@ data, which would expand the penultimate line to:
 To make useful, consistent field names, follow these guidelines:
 .Pp
 .Ss Use lower case, even for TLAs
-Lower case is more civilized.  Even TLAs should be lower case
+Lower case is more civilized.
+Even TLAs should be lower case
 to avoid scenarios where the differences between "XPath" and
-"Xpath" drive your users crazy.  Using "xpath" is simpler and better.
+"Xpath" drive your users crazy.
+Using "xpath" is simpler and better.
 .Ss Use hyphens, not underscores
 Use of hyphens is traditional in XML, and the
-.Em XOF_UNDERSCORES
+.Dv XOF_UNDERSCORES
 flag can be used to generate underscores in JSON, if desired.
 But the raw field name should use hyphens.
-.Se Use full words
-Don't abbreviate especially when the abbreviation is not obvious or
-not widely used.  Use "data-size", not "dsz" or "dsize".  Use
+.Ss Use full words
+Do not abbreviate especially when the abbreviation is not obvious or
+not widely used.
+Use "data-size", not "dsz" or "dsize".
+Use
 "interface" instead of "ifname", "if-name", "iface", "if", or "intf".
-.Se Use <verb>-<units>
+.Ss Use <verb>-<units>
 Using the form <verb>-<units> or <verb>-<classifier>-<units> helps in
 making consistent, useful names, avoiding the situation where one app
 uses "sent-packet" and another "packets-sent" and another
-"packets-we-have-sent".  The <units> can be dropped when it is
+"packets-we-have-sent".
+The <units> can be dropped when it is
 obvious, as can obvious words in the classification.
 Use "receive-after-window-packets" instead of
 "received-packets-of-data-after-window".
-.Se Reuse existing field names
-Nothing's worse than writing expressions like:
+.Ss Reuse existing field names
+Nothing is worse than writing expressions like:
 .Bd -literal -offset indent
     if ($src1/process[pid == $pid]/name == 
         $src2/proc-table/proc/p[process-id == $pid]/proc-name) {
@@ -591,32 +672,38 @@ Nothing's worse than writing expressions like:
 .Ed
 .Pp
 Find someone else who is expressing similar data and follow their
-field's and hierarchy.  Remember the quote is not
+fields and hierarchy.
+Remember the quote is not
 .Dq Consistency is the hobgoblin of little minds
 but
-.Dq A foolish consistency is the hobgoblin of little minds.
+.Dq A foolish consistency is the hobgoblin of little minds .
 .Ss Think about your users
 Have empathy for your users, choosing clear and useful fields that
-contain clear and useful data.  You may need to augment the display
-content with
+contain clear and useful data.
+You may need to augment the display content with
 .Xr xo_attr 3
 calls or "{e:}" fields to make the data useful.
-.Ss Don't use an arbitrary number postfix
-What does "errors2" mean?  No one will know.  "errors-after-restart"
-would be a better choice.  Think of you users, and think of the
-future.  If you make "errors2", the next guy will happily make
-"errors3" and before you know it, someone will be asking what's the
+.Ss Do not use an arbitrary number postfix
+What does "errors2" mean?
+No one will know.
+"errors-after-restart" would be a better choice.
+Think of your users, and think of the future.
+If you make "errors2", the next guy will happily make
+"errors3" and before you know it, someone will be asking what is the
 difference between errors37 and errors63.
 .Ss Be consistent, uniform, unsurprising, and predictable
-Think of your field vocabulary as an API.  You want it useful,
-expressive, meaningful, direct, and obvious.  You want the client
+Think of your field vocabulary as an API.
+You want it useful,
+expressive, meaningful, direct, and obvious.
+You want the client
 application's programmer to move between without the need to
-understand a variety of opinions on how fields are named.  They should
+understand a variety of opinions on how fields are named.
+They should
 see the system as a single cohesive whole, not a sack of cats.
 .Pp
 Field names constitute the means by which client programmers interact
-with our system.  By choosing wise names now, you are making their
-lives better.
+with our system.
+By choosing wise names now, you are making their lives better.
 .Pp
 After using
 .Xr xolint 1
@@ -628,30 +715,36 @@ names for the same data.
 and
 .Dq dropped-too-short
 are both reasonable names, but using them both will lead users to ask the
-difference between the two fields.  If there isn't a difference,
-use only one of the field names.  If there is a difference, change the
+difference between the two fields.
+If there is no difference,
+use only one of the field names.
+If there is a difference, change the
 names to make that difference more obvious.
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
 .Sh SEE ALSO
+.Xr xolint 1 ,
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_no_setlocale.3 b/contrib/libxo/libxo/xo_no_setlocale.3
index 94a126440f8..c3f32a403d0 100644
--- a/contrib/libxo/libxo/xo_no_setlocale.3
+++ b/contrib/libxo/libxo/xo_no_setlocale.3
@@ -7,12 +7,13 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
 .Nm xo_no_setlocale
-.Nd prevent implicit call to setlocale()
+.Nd prevent implicit call to
+.Fn setlocale
 .Sh LIBRARY
 .Lb libxo
 .Sh SYNOPSIS
@@ -20,44 +21,48 @@
 .Ft void
 .Fn xo_no_setlocale "void"
 .Sh DESCRIPTION
-.Em libxo
-automatically initializes the locale based on setting of the
+.Nm libxo
+automatically initializes the locale based on the setting of the
 environment variables
-.Em LC_CTYPE ,
-.Em LANG ,
+.Ev LC_CTYPE ,
+.Ev LANG ,
 and
-.Em LC_ALL .
+.Ev LC_ALL .
 The first of this
-list of variables is used and if none of the variables, the locale
+list of variables is used and if none of the variables are set, the locale
 defaults to
-.Em UTF-8.  The caller may wish to avoid this behavior, and
+.Em UTF-8 .
+The caller may wish to avoid this behavior, and
 can do so by calling the
 .Fn xo_no_setlocale
 function.
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
 .Sh SEE ALSO
 .Xr xo_emit 3 ,
 .Xr xo_open_container 3 ,
-.Xr xo_open_list 3 , and
-.Xr xo_format 5 .
+.Xr xo_open_list 3 ,
+.Xr xo_format 5
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_open_container.3 b/contrib/libxo/libxo/xo_open_container.3
index 9ba32c98f6f..86412ca91da 100644
--- a/contrib/libxo/libxo/xo_open_container.3
+++ b/contrib/libxo/libxo/xo_open_container.3
@@ -7,7 +7,7 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
@@ -47,13 +47,14 @@
 .Ft int
 .Fn xo_close_container_d "void"
 .Sh DESCRIPTION
-.Fa libxo
+.Nm libxo
 represents to types of hierarchy:
 .Dq containers
 and
 .Dq lists .
 A container appears once under a given parent where a list contains
-instances that can appear multiple times.  A container is used to hold
+instances that can appear multiple times.
+A container is used to hold
 related fields and to give the data organization and scope.
 The container has no value, but serves to
 contain other nodes.
@@ -71,8 +72,9 @@ or
 .Fn xo_close_container_h
 functions.
 .Pp
-Each open call must have a matching close call.  If the
-.Fa XOF_WARN
+Each open call must have a matching close call.
+If the
+.Dv XOF_WARN
 flag is set and the name given does not match the name of 
 the currently open
 container, a warning will be generated.
@@ -115,8 +117,8 @@ The
 .Fa handle
 parameter contains a handle such as returned by
 .Xr xo_create 3
-or a
-.Em NULL
+or
+.Dv NULL
 to use the default handle.
 The
 .Fa name
@@ -134,18 +136,18 @@ suffix are used in
 .Dq \&Do The Right Thing
 mode, where the name of the open containers, lists, and
 instances are maintained internally by
-.Em libxo
+.Nm libxo
 to allow the caller to
 avoid keeping track of the open container name.
 .Pp
 Use the
-.Em XOF_WARN
+.Dv XOF_WARN
 flag to generate a warning if the name given on the
 close does not match the current open container.
 .Pp
 For TEXT and HTML output, containers are not rendered into output
 text, though for HTML they are used when the
-.Em XOF_XPATH
+.Dv XOF_XPATH
 flag is set.
 .Pp
 .Bd -literal -offset indent -compact
@@ -157,21 +159,22 @@ flag is set.
        <system><host-name>foo</host-name></system>
 .Ed
 .Sh DTRT MODE
-Some user may find tracking the names of open containers, lists, and
+Some users may find tracking the names of open containers, lists, and
 instances inconvenient.
-.Em libxo
-offers
+.Nm libxo
+offers a
 .Dq \&Do The Right Thing
 mode, where
-.Em libxo
+.Nm libxo
 will track the names of open containers, lists, and instances so
-the close function can be called without a name.  To enable
+the close function can be called without a name.
+To enable
 .Em DTRT
 mode,
 turn on the
-.Em XOF_DTRT
+.Dv XOF_DTRT
 flag prior to making any other
-.Em libxo
+.Nm libxo
 output.
 .Bd -literal -offset indent -compact
     xo_set_flags(NULL, XOF_DTRT);
@@ -185,24 +188,28 @@ which will close the open container, list, or instance:
     xo_close_container_d();
 .Ed
 Note that the
-.Em XOF_WARN
-flag will also cause libxo to track open
+.Dv XOF_WARN
+flag will also cause
+.Nm libxo
+to track open
 containers, lists, and instances.
-A warning is generated with the name given to the close function
+A warning is generated when the name given to the close function
 and the name recorded do not match.
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
@@ -210,7 +217,8 @@ https://github.com/Juniper/libxo/releases
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_open_list.3 b/contrib/libxo/libxo/xo_open_list.3
index 9a6a21504c7..047af87596f 100644
--- a/contrib/libxo/libxo/xo_open_list.3
+++ b/contrib/libxo/libxo/xo_open_list.3
@@ -7,7 +7,7 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
@@ -71,19 +71,23 @@
 .Ft int
 .Fn xo_close_list_d "void"
 .Sh DESCRIPTION
-Lists are sequences of instances of homogeneous data objects.  Two
+Lists are sequences of instances of homogeneous data objects.
+Two
 distinct levels of calls are needed to represent them in our output
-styles.  Calls must be made to open and close a list, and for each
+styles.
+Calls must be made to open and close a list, and for each
 instance of data in that list, calls must be make to open and close
 that instance.
 .Pp
-The name given to all calls must be identical, and it is strong
+The name given to all calls must be identical, and it is strongly
 suggested that the name be singular, not plural, as a matter of
 style and usage expectations.
 .Pp
-A list is set of one or more instances that appear under the same
-parent.  The instances contains details about a specific object.  One
-can think of instances as objects or records.  A call is needed to
+A list is a set of one or more instances that appear under the same
+parent.
+The instances contain details about a specific object.
+One can think of instances as objects or records.
+A call is needed to
 open and close the list, while a distinct call is needed to open and
 close each instance of the list:
 .Bd -literal -offset indent -compact
@@ -143,19 +147,45 @@ generation of XML and JSON data.
             }
         ]
 .Ed
-.Sh ADDITIONAL DOCUMENTATION
 .Pp
+.Sh LEAF LISTS
+In contrast to a list of instances, a "leaf list" is list of simple
+values.
+To emit a leaf list, call the
+.Fn xo_emit
+function using the ""l"" modifier:
+.Bd -literal -offset indent -compact
+    for (ip = list; ip->i_title; ip++) {
+	xo_emit("{Lwc:Item}{l:item}\n", ip->i_title);
+    }
+.Ed
+.Pp
+The name of the field must match the name of the leaf list.
+.Pp
+In JSON, leaf lists are rendered as arrays of values.  In XML, they
+are rendered as multiple leaf elements.
+.Bd -literal -offset indent -compact
+    JSON:
+        "item": "hammer", "nail"
+    XML:
+        <item>hammer</item>
+        <item>nail</item>
+.Ed
+.Sh ADDITIONAL DOCUMENTATION
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
@@ -163,7 +193,8 @@ https://github.com/Juniper/libxo/releases
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_parse_args.3 b/contrib/libxo/libxo/xo_parse_args.3
index a9b4cec29b3..f66546bd3c9 100644
--- a/contrib/libxo/libxo/xo_parse_args.3
+++ b/contrib/libxo/libxo/xo_parse_args.3
@@ -7,27 +7,31 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
 .Nm xo_parse_args
 .Nd detect, parse, and remove arguments for libxo
 .Sh LIBRARY
-.Nm libxo
+.Lb libxo
 .Sh SYNOPSIS
 .In libxo/xo.h
 .Ft int
 .Fn xo_parse_args "int argc" "char **argv"
+.Ft int
+.Fn xo_set_program "const char *name"
 .Sh DESCRIPTION
 The
 .Fn xo_parse_args
 function is used to process command-line arguments.
-.Em libxo -specific
+.Nm libxo
+specific
 options are processed and removed
 from the argument list so the calling application does not
-need to process them.  If successful, a new value for argc
-is returned.  On failure, a message it emitted and -1 is returned.
+need to process them.
+If successful, a new value for argc is returned.
+On failure, a message it emitted and -1 is returned.
 .Bd -literal -offset indent
     argc = xo_parse_args(argc, argv);
     if (argc < 0)
@@ -38,9 +42,9 @@ Following the call to
 .Fn xo_parse_args ,
 the application can process the remaining arguments in a normal manner.
 .Pp
-.Em libxo
-uses command line options to trigger rendering behavior.  The
-following options are recognised:
+.Nm libxo
+uses command line options to trigger rendering behavior.
+The following options are recognised:
 .Pp
 .Bl -tag -width "--libxo"
 .It
@@ -56,44 +60,45 @@ styles, flags, or features:
 .Pp
 .Bl -tag -width "12345678"
 .It Sy "Token   Action"
-.It dtrt
+.It Dv dtrt
 Enable "Do The Right Thing" mode
-.It html
+.It Dv html
 Emit HTML output
-.It indent=xx
+.It Dv indent=xx
 Set the indentation level
-.It info
+.It Dv info
 Add info attributes (HTML)
-.It json
+.It Dv json
 Emit JSON output
-.It keys
+.It Dv keys
 Emit the key attribute for keys (XML)
-.It no-locale
+.It Dv no-locale
 Do not initialize the locale setting
-.It no-top
+.It Dv no-top
 Do not emit a top set of braces (JSON)
-.It not-first
+.It Dv not-first
 Pretend the 1st output item was not 1st (JSON)
-.It pretty
+.It Dv pretty
 Emit pretty-printed output
-.It text
+.It Dv text
 Emit TEXT output
-.It units
+.It Dv units
 Add the 'units' (XML) or 'data-units (HTML) attribute
-.It warn
+.It Dv warn
 Emit warnings when libxo detects bad calls
-.It warn-xml
+.It Dv warn-xml
 Emit warnings in XML
-.It xml
+.It Dv xml
 Emit XML output
-.It xpath
+.It Dv xpath
 Add XPath expressions (HTML)
 .El
 .Pp
 The
 .Dq brief-options
 are single letter commands, designed for those with
-too little patience to use real tokens.  No comma separator is used.
+too little patience to use real tokens.
+No comma separator is used.
 .Bl -column "i<num>"
 .It Sy "Token   Action"
 .It "H      " "Enable HTML output (XO_STYLE_HTML)"
@@ -107,19 +112,41 @@ too little patience to use real tokens.  No comma separator is used.
 .It "x      " "Enable XPath data (XOF_XPATH)"
 .El
 .Pp
-.Sh ADDITIONAL DOCUMENTATION
+The
+.Fn xo_set_program
+function sets name of the program as reported by
+functions like
+.Fn xo_failure ,
+.Fn xo_warn ,
+.Fn xo_err ,
+etc.
+The program name is initialized by
+.Fn xo_parse_args ,
+but subsequent calls to
+.Fn xo_set_program
+can override this value.
 .Pp
+Note that the value is not copied, so the memory passed to
+.Fn xo_set_program
+(and
+.Fn xo_parse_args )
+must be maintained by the caller.
+.Pp
+.Sh ADDITIONAL DOCUMENTATION
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
@@ -127,7 +154,8 @@ https://github.com/Juniper/libxo/releases
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_set_allocator.3 b/contrib/libxo/libxo/xo_set_allocator.3
index 508cc8e9af5..70bfdd59a45 100644
--- a/contrib/libxo/libxo/xo_set_allocator.3
+++ b/contrib/libxo/libxo/xo_set_allocator.3
@@ -7,7 +7,7 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
@@ -25,7 +25,9 @@
 .Sh DESCRIPTION
 The
 .Fn xo_set_allocator
-function allows libxo to be used in environments
+function allows
+.Nm libxo
+to be used in environments
 where the standard
 .Xr realloc 3
 and
@@ -40,7 +42,7 @@ a pointer to memory following the same convention.
 .Fa free_func
 will receive the same argument as
 .Xr free 3
-and should release it, asappropriate for the environment.
+and should release it, as appropriate for the environment.
 .Pp
 By default, the standard
 .Xr realloc 3
@@ -48,18 +50,20 @@ and
 .Xr free 3
 functions are used.
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
@@ -67,7 +71,8 @@ https://github.com/Juniper/libxo/releases
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_set_flags.3 b/contrib/libxo/libxo/xo_set_flags.3
index a23de7bfdd7..ca6655360d2 100644
--- a/contrib/libxo/libxo/xo_set_flags.3
+++ b/contrib/libxo/libxo/xo_set_flags.3
@@ -7,7 +7,7 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
@@ -25,96 +25,121 @@
 Use the
 .Fn xo_set_flags
 function to set the flags for a
-.Em libxo
-handle. To use the default handle, pass a NULL handle.
+.Nm libxo
+handle.
+To use the default handle, pass a
+.Dv NULL
+handle.
 .Pp
 The set of valid flags include:
 .Bl -tag -width "XOF_UNDERSCORES"
 .It Sy "Flag            Description"
-.It XOF_CLOSE_FP
-Close file pointer on xo_destroy().  This
-flag will trigger the call of the close_func
+.It Dv XOF_CLOSE_FP
+Close file pointer on
+.Xr xo_destroy 3 .
+This flag will trigger the call of the
+.Fn close_func
 (provided via
-.Fn xo_set_writer 3 )
+.Xr xo_set_writer 3 )
 when the handle is destroyed.
-.It XOF_DTRT
+.It Dv XOF_DTRT
 Enable "do the right thing" mode
-.It XOF_INFO
+.It Dv XOF_INFO
 Display info data attributes (HTML)
-.It XOF_KEYS
+.It Dv XOF_KEYS
 Emit the key attribute (XML)
-.It XOF_NO_ENV
-Do not use the LIBXO_OPTIONS env var
-.It XOF_PRETTY
+.It Dv XOF_NO_ENV
+Do not use the
+.Ev LIBXO_OPTIONS
+environment variable.
+.It Dv XOF_PRETTY
 Make 'pretty printed' output, with the
 addition of indentation and newlines to enhance the readability of
-XML, JSON, and HTML output.  Text output is not affected.
-.It XOF_UNDERSCORES
+XML, JSON, and HTML output.
+Text output is not affected.
+.It Dv XOF_UNDERSCORES
 Replaces hyphens with underscores
-.It XOF_UNITS
+.It Dv XOF_UNITS
 Display units (XML and HMTL)
-.It XOF_WARN
+.It Dv XOF_WARN
 Generate warnings for broken calls, triggering diagnostic
 output (on standard error) when the library notices errors during
-operations, or with arguments to functions.  Without warning enabled,
-such conditions are ignored.
-Warnings allow developers to debug their interaction with libxo.
-The function "xo_failure" can used as a breakpoint for a debugger,
+operations, or with arguments to functions.
+Without warnings enabled, such conditions are ignored.
+Warnings allow developers to debug their interaction with
+.Nm libxo .
+The function
+.Fn xo_failure
+can be used as a breakpoint for a debugger,
 regardless of whether warnings are enabled.
-.It XOF_WARN_XML
+.It Dv XOF_WARN_XML
 Generate warnings in XML on stdout
-.It XOF_XPATH
+.It Dv XOF_XPATH
 Emit XPath expressions (HTML)
-.It XOF_COLUMNS
-Force xo_emit to return columns used
-.It XOF_FLUSH
-Flush output after each xo_emit call
+.It Dv XOF_COLUMNS
+Force
+.Xr xo_emit 3
+to return columns used
+.It Dv XOF_FLUSH
+Flush output after each
+.Xr xo_emit 3
+call
 .El
 .Pp
-If the style is XO_STYLE_HTML, the following additional flags can be
+If the style is
+.Dv XO_STYLE_HTML ,
+the following additional flags can be
 used:
 .Bl -tag -width "XOF_UNDERSCORES"
 .It Sy "Flag            Description"
-.It XOF_XPATH
+.It Dv XOF_XPATH
 Emit "data-xpath" attributes
-.It XOF_INFO
+.It Dv XOF_INFO
 Emit additional informational fields for HTML
-output.  See
+output.
+See
 .Xr xo_set_info 3
 for details.
 .El
 .Pp
 The
-.Em XOF_XPATH
+.Dv XOF_XPATH
 flag enables the emission of XPath expressions detailing
 the hierarchy of XML elements used to encode the data field, if the
 XPATH style of output were requested.
 .Pp
-If the style is XO_STYLE_XML, the following additional flags can be
+If the style is
+.Dv XO_STYLE_XML ,
+the following additional flags can be
 used:
 .Bl -tag -width "XOF_UNDERSCORES"
 .It Sy "Flag            Description"
 .It XOF_KEYS
 Add 'key' attribute to the XML encoding for
-field definitions that use the 'k' modifier.  The key attribute has
+field definitions that use the 'k' modifier.
+The key attribute has
 the value "key".
 .El
 .Pp
-The xo_clear_flags() function turns off the given flags in a specific
+The
+.Fn xo_clear_flags
+function turns off the given flags in a specific
 handle. 
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
@@ -122,7 +147,8 @@ https://github.com/Juniper/libxo/releases
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_set_info.3 b/contrib/libxo/libxo/xo_set_info.3
index dbb3c9c88c9..4f8c5877c72 100644
--- a/contrib/libxo/libxo/xo_set_info.3
+++ b/contrib/libxo/libxo/xo_set_info.3
@@ -7,7 +7,7 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
@@ -21,19 +21,23 @@
 .Fn xo_set_info "xo_handle_t *handle" "xo_info_t *info" "int count"
 .Sh DESCRIPTION
 HTML data can include additional information in attributes that
-begin with "data-".  To enable this, three things must occur:
+begin with "data-".
+To enable this, three things must occur:
 .Pp
-First the application must build an array of xo_info_t structures,
-one per tag.  The array must be sorted by name, since
-.Em libxo
+First the application must build an array of
+.Dv xo_info_t
+structures,
+one per tag.
+The array must be sorted by name, since
+.Nm libxo
 uses a
 binary search to find the entry that matches names from format
 instructions.
 .Pp
 The
-.Em xo_info_t
+.Dv xo_info_t
 structure is defined in
-.Em <libxo/xo.h> :
+.In libxo/xo.h :
 .Bd -literal -offset indent
     typedef struct xo_info_s {
         const char *xi_name;    /* Name of the element */
@@ -43,21 +47,27 @@ structure is defined in
 .Ed
 .Pp
 Second, the application must inform
-.Em libxo
+.Nm libxo
 about this information using the
 .Fn xo_set_info
-call.  Like other libxo calls, passing NULL for the handle tells
-.Em libxo
+call.
+Like other
+.Nm libxo
+calls, passing
+.Dv NULL
+for the handle tells
+.Nm libxo
 to use the default handle.
 .Pp
 If the
 .Fa count
 is -1,
-.Em libxo
+.Nm libxo
 will count the elements of
 .Fa info ,
 but there
-must be an empty element at the end.  More typically, the number is
+must be an empty element at the end.
+More typically, the number is
 known to the application:
 .Bd -literal -offset indent
     xo_info_t info[] = {
@@ -72,8 +82,8 @@ known to the application:
     xo_set_info(NULL, info, info_count);
 .Ed
 .Pp
-Third, the emitting of info must be triggered with the
-.Em XOF_INFO
+Third, the emission of info must be triggered with the
+.Dv XOF_INFO
 flag
 using either the
 .Fn xo_set_flags
@@ -88,18 +98,20 @@ and "data-help" attributes:
        data-help="Stock Keeping Unit">GRO-000-533</div>
 .Ed
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
@@ -107,7 +119,8 @@ https://github.com/Juniper/libxo/releases
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_set_options.3 b/contrib/libxo/libxo/xo_set_options.3
index af7e95cdcce..bedbf912da1 100644
--- a/contrib/libxo/libxo/xo_set_options.3
+++ b/contrib/libxo/libxo/xo_set_options.3
@@ -7,12 +7,12 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
-.Nm xo_emit
-.Nd emit formatted output based on format string and arguments
+.Nm xo_set_options
+.Nd change options used by a handle
 .Sh LIBRARY
 .Lb libxo
 .Sh SYNOPSIS
@@ -27,18 +27,20 @@ and flags and enables them for a specific handle.
 The options are identical to those listed in
 .Xr xo_parse_args 3 .
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
@@ -46,7 +48,8 @@ https://github.com/Juniper/libxo/releases
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_set_style.3 b/contrib/libxo/libxo/xo_set_style.3
index 83371ac71b1..f11f190bf13 100644
--- a/contrib/libxo/libxo/xo_set_style.3
+++ b/contrib/libxo/libxo/xo_set_style.3
@@ -7,7 +7,7 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
@@ -25,9 +25,11 @@
 Use the
 .Fn xo_set_style
 function to set the output style for a handle.
-To use the default handle, pass a NULL handle.
+To use the default handle, pass a
+.Dv NULL
+handle.
 The set of output styles used by
-.Em libxo
+.Nm libxo
 is:
 .Bl -column "XO_STYLE_TEXT12"
 .It Sy "Flag          Description"
@@ -47,18 +49,20 @@ The name can be any of the styles: "text", "xml", "json", or "html".
         xo_set_style_name(NULL, "html");
 .Ed
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
@@ -66,7 +70,8 @@ https://github.com/Juniper/libxo/releases
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xo_set_writer.3 b/contrib/libxo/libxo/xo_set_writer.3
index 9185f10dd60..942bcc2866b 100644
--- a/contrib/libxo/libxo/xo_set_writer.3
+++ b/contrib/libxo/libxo/xo_set_writer.3
@@ -7,7 +7,7 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
+.Dd December 4, 2014
 .Dt LIBXO 3
 .Os
 .Sh NAME
@@ -21,9 +21,12 @@
 .Sy typedef int (*xo_write_func_t)(void *, const char *);
 .Pp
 .Sy typedef void (*xo_close_func_t)(void *);
+.Pp
+.Sy typedef int (*xo_flush_func_t)(void *);
 .Fn xo_set_writer "xo_handle_t *handle" "void *opaque"
                         "xo_write_func_t write_func"
                         "xo_close_func_t close_func"
+                        "xo_flush_func_t flush_func"
 .Sh DESCRIPTION
 The
 .Fn xo_set_writer
@@ -31,30 +34,38 @@ function allows custom
 .Dq write
 functions
 which can tailor how
-.Em libxo
-writes data.  An
+.Nm libxo
+writes data.
+An
 .Fa opaque
 argument is
 recorded and passed back to the
 .Fa write_func
 function, allowing the function
-to acquire context information. The
+to acquire context information.
+The
 .Fa close_func
 function can
 release this opaque data and any other resources as needed.
+The
+.Fa flush_func
+function should
+flush any pending data associated with the opaque pointer.
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
@@ -62,7 +73,8 @@ https://github.com/Juniper/libxo/releases
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/libxo/xoconfig.h b/contrib/libxo/libxo/xoconfig.h
index 0870e358720..4c596b4adf6 100644
--- a/contrib/libxo/libxo/xoconfig.h
+++ b/contrib/libxo/libxo/xoconfig.h
@@ -87,6 +87,9 @@
 /* Define to 1 if you have the <stdint.h> header file. */
 #define HAVE_STDINT_H 1
 
+/* Define to 1 if you have the <stdio_ext.h> header file. */
+/* #undef HAVE_STDIO_EXT_H */
+
 /* Define to 1 if you have the <stdio.h> header file. */
 #define HAVE_STDIO_H 1
 
@@ -141,6 +144,9 @@
 /* Define to 1 if you have the <unistd.h> header file. */
 #define HAVE_UNISTD_H 1
 
+/* Define to 1 if you have the `__flbf' function. */
+/* #undef HAVE___FLBF */
+
 /* Enable debugging */
 /* #undef LIBXO_DEBUG */
 
@@ -158,7 +164,7 @@
 #define PACKAGE_NAME "libxo"
 
 /* Define to the full name and version of this package. */
-#define PACKAGE_STRING "libxo 0.1.6"
+#define PACKAGE_STRING "libxo 0.2.0"
 
 /* Define to the one symbol short name of this package. */
 #define PACKAGE_TARNAME "libxo"
@@ -167,7 +173,7 @@
 #define PACKAGE_URL ""
 
 /* Define to the version of this package. */
-#define PACKAGE_VERSION "0.1.6"
+#define PACKAGE_VERSION "0.2.0"
 
 /* If using the C implementation of alloca, define if you know the
    direction of stack growth for your system; otherwise it will be
@@ -181,7 +187,7 @@
 #define STDC_HEADERS 1
 
 /* Version number of package */
-#define VERSION "0.1.6"
+#define VERSION "0.2.0"
 
 /* Define to `__inline__' or `__inline' if that's what the C compiler
    calls it, or to nothing if 'inline' is not supported under any name.  */
diff --git a/contrib/libxo/libxo/xoconfig.h.in b/contrib/libxo/libxo/xoconfig.h.in
index 3fe736571d0..467f5644d6b 100644
--- a/contrib/libxo/libxo/xoconfig.h.in
+++ b/contrib/libxo/libxo/xoconfig.h.in
@@ -86,6 +86,9 @@
 /* Define to 1 if you have the <stdint.h> header file. */
 #undef HAVE_STDINT_H
 
+/* Define to 1 if you have the <stdio_ext.h> header file. */
+#undef HAVE_STDIO_EXT_H
+
 /* Define to 1 if you have the <stdio.h> header file. */
 #undef HAVE_STDIO_H
 
@@ -140,6 +143,9 @@
 /* Define to 1 if you have the <unistd.h> header file. */
 #undef HAVE_UNISTD_H
 
+/* Define to 1 if you have the `__flbf' function. */
+#undef HAVE___FLBF
+
 /* Enable debugging */
 #undef LIBXO_DEBUG
 
diff --git a/contrib/libxo/libxo/xoversion.h b/contrib/libxo/libxo/xoversion.h
index 60c6118be93..4c29e82963e 100644
--- a/contrib/libxo/libxo/xoversion.h
+++ b/contrib/libxo/libxo/xoversion.h
@@ -18,17 +18,17 @@
 /**
  * The version string
  */
-#define LIBXO_VERSION		"0.1.6"
+#define LIBXO_VERSION		"0.2.0"
 
 /**
  * The version number
  */
-#define LIBXO_VERSION_NUMBER	1006
+#define LIBXO_VERSION_NUMBER	2000
 
 /**
  * The version number as a string
  */
-#define LIBXO_VERSION_STRING	"1006"
+#define LIBXO_VERSION_STRING	"2000"
 
 /**
  * The version number extra info as a string
diff --git a/contrib/libxo/packaging/libxo.rb.base.in b/contrib/libxo/packaging/libxo.rb.base.in
new file mode 100644
index 00000000000..70b712da2ac
--- /dev/null
+++ b/contrib/libxo/packaging/libxo.rb.base.in
@@ -0,0 +1,20 @@
+#
+# Homebrew formula file for libxo
+# https://github.com/mxcl/homebrew
+#
+
+require 'formula'
+
+class Libxo < Formula
+  homepage 'https://github.com/Juniper/@PACKAGE-NAME@'
+  url 'https://github.com/Juniper/@PACKAGE_NAME@/releases/@PACKAGE_VERSION@/@PACKAGE_NAME@-@PACKAGE_VERSION@.tar.gz'
+  sha1 '__SHA1__'
+
+  depends_on 'libtool' => :build
+
+  def install
+    system "./configure", "--disable-dependency-tracking",
+                          "--prefix=#{prefix}"
+    system "make install"
+  end
+end
diff --git a/contrib/libxo/tests/core/Makefile.am b/contrib/libxo/tests/core/Makefile.am
index a5470f37547..f145d183d81 100644
--- a/contrib/libxo/tests/core/Makefile.am
+++ b/contrib/libxo/tests/core/Makefile.am
@@ -18,7 +18,9 @@ test_03.c \
 test_04.c \
 test_05.c \
 test_06.c \
-test_07.c
+test_07.c \
+test_08.c \
+test_09.c
 
 test_01_test_SOURCES = test_01.c
 test_02_test_SOURCES = test_02.c
@@ -27,6 +29,8 @@ test_04_test_SOURCES = test_04.c
 test_05_test_SOURCES = test_05.c
 test_06_test_SOURCES = test_06.c
 test_07_test_SOURCES = test_07.c
+test_08_test_SOURCES = test_08.c
+test_09_test_SOURCES = test_09.c
 
 # TEST_CASES := $(shell cd ${srcdir} ; echo *.c )
 
diff --git a/contrib/libxo/tests/core/saved/test_01.H.out b/contrib/libxo/tests/core/saved/test_01.H.out
index e61eecc0aa2..4d4f2f171b6 100644
--- a/contrib/libxo/tests/core/saved/test_01.H.out
+++ b/contrib/libxo/tests/core/saved/test_01.H.out
@@ -1 +1 @@
-<div class="line"><div class="title">Item      </div><div class="title">  Total Sold</div><div class="title">    In Stock</div><div class="title">    On Order</div><div class="title">  SKU</div></div><div class="line"><div class="data" data-tag="name" data-key="key">gum       </div><div class="data" data-tag="sold">        1412</div><div class="data" data-tag="in-stock">          54</div><div class="data" data-tag="on-order">          10</div><div class="data" data-tag="sku" data-key="key">  GRO-000-415</div></div><div class="line"><div class="data" data-tag="name" data-key="key">rope      </div><div class="data" data-tag="sold">          85</div><div class="data" data-tag="in-stock">           4</div><div class="data" data-tag="on-order">           2</div><div class="data" data-tag="sku" data-key="key">  HRD-000-212</div></div><div class="line"><div class="data" data-tag="name" data-key="key">ladder    </div><div class="data" data-tag="sold">           0</div><div class="data" data-tag="in-stock">           2</div><div class="data" data-tag="on-order">           1</div><div class="data" data-tag="sku" data-key="key">  HRD-000-517</div></div><div class="line"><div class="data" data-tag="name" data-key="key">bolt      </div><div class="data" data-tag="sold">        4123</div><div class="data" data-tag="in-stock">         144</div><div class="data" data-tag="on-order">          42</div><div class="data" data-tag="sku" data-key="key">  HRD-000-632</div></div><div class="line"><div class="data" data-tag="name" data-key="key">water     </div><div class="data" data-tag="sold">          17</div><div class="data" data-tag="in-stock">          14</div><div class="data" data-tag="on-order">           2</div><div class="data" data-tag="sku" data-key="key">  GRO-000-2331</div></div><div class="line"></div><div class="line"></div><div class="line"><div class="label">Item</div><div class="text"> '</div><div class="data" data-tag="name" data-key="key">gum</div><div class="text">':</div></div><div class="line"><div class="padding">   </div><div class="label">Total sold</div><div class="text">: </div><div class="data" data-tag="sold">1412.0</div></div><div class="line"><div class="padding">   </div><div class="label">In stock</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="in-stock">54</div></div><div class="line"><div class="padding">   </div><div class="label">On order</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="on-order">10</div></div><div class="line"><div class="padding">   </div><div class="label">SKU</div><div class="text">: </div><div class="data" data-tag="sku" data-key="key">GRO-000-415</div></div><div class="line"><div class="label">Item</div><div class="text"> '</div><div class="data" data-tag="name" data-key="key">rope</div><div class="text">':</div></div><div class="line"><div class="padding">   </div><div class="label">Total sold</div><div class="text">: </div><div class="data" data-tag="sold">85.0</div></div><div class="line"><div class="padding">   </div><div class="label">In stock</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="in-stock">4</div></div><div class="line"><div class="padding">   </div><div class="label">On order</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="on-order">2</div></div><div class="line"><div class="padding">   </div><div class="label">SKU</div><div class="text">: </div><div class="data" data-tag="sku" data-key="key">HRD-000-212</div></div><div class="line"><div class="label">Item</div><div class="text"> '</div><div class="data" data-tag="name" data-key="key">ladder</div><div class="text">':</div></div><div class="line"><div class="padding">   </div><div class="label">Total sold</div><div class="text">: </div><div class="data" data-tag="sold">0</div></div><div class="line"><div class="padding">   </div><div class="label">In stock</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="in-stock">2</div></div><div class="line"><div class="padding">   </div><div class="label">On order</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="on-order">1</div></div><div class="line"><div class="padding">   </div><div class="label">SKU</div><div class="text">: </div><div class="data" data-tag="sku" data-key="key">HRD-000-517</div></div><div class="line"><div class="label">Item</div><div class="text"> '</div><div class="data" data-tag="name" data-key="key">bolt</div><div class="text">':</div></div><div class="line"><div class="padding">   </div><div class="label">Total sold</div><div class="text">: </div><div class="data" data-tag="sold">4123.0</div></div><div class="line"><div class="padding">   </div><div class="label">In stock</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="in-stock">144</div></div><div class="line"><div class="padding">   </div><div class="label">On order</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="on-order">42</div></div><div class="line"><div class="padding">   </div><div class="label">SKU</div><div class="text">: </div><div class="data" data-tag="sku" data-key="key">HRD-000-632</div></div><div class="line"><div class="label">Item</div><div class="text"> '</div><div class="data" data-tag="name" data-key="key">water</div><div class="text">':</div></div><div class="line"><div class="padding">   </div><div class="label">Total sold</div><div class="text">: </div><div class="data" data-tag="sold">17.0</div></div><div class="line"><div class="padding">   </div><div class="label">In stock</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="in-stock">14</div></div><div class="line"><div class="padding">   </div><div class="label">On order</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="on-order">2</div></div><div class="line"><div class="padding">   </div><div class="label">SKU</div><div class="text">: </div><div class="data" data-tag="sku" data-key="key">GRO-000-2331</div></div><div class="line"><div class="label">Item</div><div class="text"> '</div><div class="data" data-tag="name" data-key="key">fish</div><div class="text">':</div></div><div class="line"><div class="padding">   </div><div class="label">Total sold</div><div class="text">: </div><div class="data" data-tag="sold">1321.0</div></div><div class="line"><div class="padding">   </div><div class="label">In stock</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="in-stock">45</div></div><div class="line"><div class="padding">   </div><div class="label">On order</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="on-order">1</div></div><div class="line"><div class="padding">   </div><div class="label">SKU</div><div class="text">: </div><div class="data" data-tag="sku" data-key="key">GRO-000-533</div></div>
\ No newline at end of file
+<div class="line"><div class="title">Item      </div><div class="title">  Total Sold</div><div class="title">    In Stock</div><div class="title">    On Order</div><div class="title">  SKU</div></div><div class="line"><div class="data" data-tag="name" data-key="key">gum       </div><div class="data" data-tag="sold">        1412</div><div class="data" data-tag="in-stock">          54</div><div class="data" data-tag="on-order">          10</div><div class="data" data-tag="sku" data-key="key">  GRO-000-415</div></div><div class="line"><div class="data" data-tag="name" data-key="key">rope      </div><div class="data" data-tag="sold">          85</div><div class="data" data-tag="in-stock">           4</div><div class="data" data-tag="on-order">           2</div><div class="data" data-tag="sku" data-key="key">  HRD-000-212</div></div><div class="line"><div class="data" data-tag="name" data-key="key">ladder    </div><div class="data" data-tag="sold">           0</div><div class="data" data-tag="in-stock">           2</div><div class="data" data-tag="on-order">           1</div><div class="data" data-tag="sku" data-key="key">  HRD-000-517</div></div><div class="line"><div class="data" data-tag="name" data-key="key">bolt      </div><div class="data" data-tag="sold">        4123</div><div class="data" data-tag="in-stock">         144</div><div class="data" data-tag="on-order">          42</div><div class="data" data-tag="sku" data-key="key">  HRD-000-632</div></div><div class="line"><div class="data" data-tag="name" data-key="key">water     </div><div class="data" data-tag="sold">          17</div><div class="data" data-tag="in-stock">          14</div><div class="data" data-tag="on-order">           2</div><div class="data" data-tag="sku" data-key="key">  GRO-000-2331</div></div><div class="line"></div><div class="line"></div><div class="line"><div class="label">Item</div><div class="text"> '</div><div class="data" data-tag="name" data-key="key">gum</div><div class="text">':</div></div><div class="line"><div class="padding">   </div><div class="label">Total sold</div><div class="text">: </div><div class="data" data-tag="sold">1412.0</div></div><div class="line"><div class="padding">   </div><div class="label">In stock</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="in-stock">54</div></div><div class="line"><div class="padding">   </div><div class="label">On order</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="on-order">10</div></div><div class="line"><div class="padding">   </div><div class="label">SKU</div><div class="text">: </div><div class="data" data-tag="sku" data-key="key">GRO-000-415</div></div><div class="line"><div class="label">Item</div><div class="text"> '</div><div class="data" data-tag="name" data-key="key">rope</div><div class="text">':</div></div><div class="line"><div class="padding">   </div><div class="label">Total sold</div><div class="text">: </div><div class="data" data-tag="sold">85.0</div></div><div class="line"><div class="padding">   </div><div class="label">In stock</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="in-stock">4</div></div><div class="line"><div class="padding">   </div><div class="label">On order</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="on-order">2</div></div><div class="line"><div class="padding">   </div><div class="label">SKU</div><div class="text">: </div><div class="data" data-tag="sku" data-key="key">HRD-000-212</div></div><div class="line"><div class="label">Item</div><div class="text"> '</div><div class="data" data-tag="name" data-key="key">ladder</div><div class="text">':</div></div><div class="line"><div class="padding">   </div><div class="label">Total sold</div><div class="text">: </div><div class="data" data-tag="sold">0</div></div><div class="line"><div class="padding">   </div><div class="label">In stock</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="in-stock">2</div></div><div class="line"><div class="padding">   </div><div class="label">On order</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="on-order">1</div></div><div class="line"><div class="padding">   </div><div class="label">SKU</div><div class="text">: </div><div class="data" data-tag="sku" data-key="key">HRD-000-517</div></div><div class="line"><div class="label">Item</div><div class="text"> '</div><div class="data" data-tag="name" data-key="key">bolt</div><div class="text">':</div></div><div class="line"><div class="padding">   </div><div class="label">Total sold</div><div class="text">: </div><div class="data" data-tag="sold">4123.0</div></div><div class="line"><div class="padding">   </div><div class="label">In stock</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="in-stock">144</div></div><div class="line"><div class="padding">   </div><div class="label">On order</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="on-order">42</div></div><div class="line"><div class="padding">   </div><div class="label">SKU</div><div class="text">: </div><div class="data" data-tag="sku" data-key="key">HRD-000-632</div></div><div class="line"><div class="label">Item</div><div class="text"> '</div><div class="data" data-tag="name" data-key="key">water</div><div class="text">':</div></div><div class="line"><div class="padding">   </div><div class="label">Total sold</div><div class="text">: </div><div class="data" data-tag="sold">17.0</div></div><div class="line"><div class="padding">   </div><div class="label">In stock</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="in-stock">14</div></div><div class="line"><div class="padding">   </div><div class="label">On order</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="on-order">2</div></div><div class="line"><div class="padding">   </div><div class="label">SKU</div><div class="text">: </div><div class="data" data-tag="sku" data-key="key">GRO-000-2331</div></div><div class="line"><div class="label">Item</div><div class="text"> '</div><div class="data" data-tag="name" data-key="key">fish</div><div class="text">':</div></div><div class="line"><div class="padding">   </div><div class="label">Total sold</div><div class="text">: </div><div class="data" data-tag="sold">1321.0</div></div><div class="line"><div class="padding">   </div><div class="label">In stock</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="in-stock">45</div></div><div class="line"><div class="padding">   </div><div class="label">On order</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="on-order">1</div></div><div class="line"><div class="padding">   </div><div class="label">SKU</div><div class="text">: </div><div class="data" data-tag="sku" data-key="key">GRO-000-533</div></div><div class="line"><div class="label">Item</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="item">gum</div></div><div class="line"><div class="label">Item</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="item">rope</div></div><div class="line"><div class="label">Item</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="item">ladder</div></div><div class="line"><div class="label">Item</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="item">bolt</div></div><div class="line"><div class="label">Item</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="item">water</div></div><div class="line"><div class="text">X</div><div class="text">X</div><div class="text">X</div><div class="text">X</div><div class="text">X</div><div class="text">X</div><div class="text">X</div><div class="text">X</div></div><div class="line"><div class="text">X</div><div class="padding">        </div><div class="text">X</div><div class="label">Cost</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="cost">425</div></div><div class="line"><div class="text">X</div><div class="padding">                              </div><div class="text">X</div><div class="label">Cost</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="cost">455</div></div>
\ No newline at end of file
diff --git a/contrib/libxo/tests/core/saved/test_01.HIPx.out b/contrib/libxo/tests/core/saved/test_01.HIPx.out
index c38eb04d3d2..2bafff91c45 100644
--- a/contrib/libxo/tests/core/saved/test_01.HIPx.out
+++ b/contrib/libxo/tests/core/saved/test_01.HIPx.out
@@ -236,3 +236,61 @@
   <div class="text">: </div>
   <div class="data" data-tag="sku" data-xpath="/top/data/item/sku" data-type="string" data-help="Stock Keeping Unit" data-key="key">GRO-000-533</div>
 </div>
+<div class="line">
+  <div class="label">Item</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/item">gum</div>
+</div>
+<div class="line">
+  <div class="label">Item</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/item">rope</div>
+</div>
+<div class="line">
+  <div class="label">Item</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/item">ladder</div>
+</div>
+<div class="line">
+  <div class="label">Item</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/item">bolt</div>
+</div>
+<div class="line">
+  <div class="label">Item</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/item">water</div>
+</div>
+<div class="line">
+  <div class="text">X</div>
+  <div class="text">X</div>
+  <div class="text">X</div>
+  <div class="text">X</div>
+  <div class="text">X</div>
+  <div class="text">X</div>
+  <div class="text">X</div>
+  <div class="text">X</div>
+</div>
+<div class="line">
+  <div class="text">X</div>
+  <div class="padding">        </div>
+  <div class="text">X</div>
+  <div class="label">Cost</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="cost" data-xpath="/top/cost">425</div>
+</div>
+<div class="line">
+  <div class="text">X</div>
+  <div class="padding">                              </div>
+  <div class="text">X</div>
+  <div class="label">Cost</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="cost" data-xpath="/top/cost">455</div>
+</div>
diff --git a/contrib/libxo/tests/core/saved/test_01.HP.out b/contrib/libxo/tests/core/saved/test_01.HP.out
index a8874769d88..a007778c39a 100644
--- a/contrib/libxo/tests/core/saved/test_01.HP.out
+++ b/contrib/libxo/tests/core/saved/test_01.HP.out
@@ -236,3 +236,61 @@
   <div class="text">: </div>
   <div class="data" data-tag="sku" data-key="key">GRO-000-533</div>
 </div>
+<div class="line">
+  <div class="label">Item</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="item">gum</div>
+</div>
+<div class="line">
+  <div class="label">Item</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="item">rope</div>
+</div>
+<div class="line">
+  <div class="label">Item</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="item">ladder</div>
+</div>
+<div class="line">
+  <div class="label">Item</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="item">bolt</div>
+</div>
+<div class="line">
+  <div class="label">Item</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="item">water</div>
+</div>
+<div class="line">
+  <div class="text">X</div>
+  <div class="text">X</div>
+  <div class="text">X</div>
+  <div class="text">X</div>
+  <div class="text">X</div>
+  <div class="text">X</div>
+  <div class="text">X</div>
+  <div class="text">X</div>
+</div>
+<div class="line">
+  <div class="text">X</div>
+  <div class="padding">        </div>
+  <div class="text">X</div>
+  <div class="label">Cost</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="cost">425</div>
+</div>
+<div class="line">
+  <div class="text">X</div>
+  <div class="padding">                              </div>
+  <div class="text">X</div>
+  <div class="label">Cost</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="cost">455</div>
+</div>
diff --git a/contrib/libxo/tests/core/saved/test_01.J.out b/contrib/libxo/tests/core/saved/test_01.J.out
index 289a9521053..6fcdbd41630 100644
--- a/contrib/libxo/tests/core/saved/test_01.J.out
+++ b/contrib/libxo/tests/core/saved/test_01.J.out
@@ -1,2 +1,2 @@
-{"top": {"data": {"item": [{"sku":"GRO-000-415","name":"gum","sold":1412,"in-stock":54,"on-order":10}, {"sku":"HRD-000-212","name":"rope","sold":85,"in-stock":4,"on-order":2}, {"sku":"HRD-000-517","name":"ladder","sold":0,"in-stock":2,"on-order":1}, {"sku":"HRD-000-632","name":"bolt","sold":4123,"in-stock":144,"on-order":42}, {"sku":"GRO-000-2331","name":"water","sold":17,"in-stock":14,"on-order":2}]}, "data": {"item": [{"sku":"GRO-000-415","name":"gum","sold":1412.0,"in-stock":54,"on-order":10}, {"sku":"HRD-000-212","name":"rope","sold":85.0,"in-stock":4,"on-order":2}, {"sku":"HRD-000-517","name":"ladder","sold":0,"in-stock":2,"on-order":1}, {"sku":"HRD-000-632","name":"bolt","sold":4123.0,"in-stock":144,"on-order":42}, {"sku":"GRO-000-2331","name":"water","sold":17.0,"in-stock":14,"on-order":2}]}, "data": {"item": [{"sku":"GRO-000-533","name":"fish","sold":1321.0,"in-stock":45,"on-order":1}]}}
+{"top": {"data": {"item": [{"sku":"GRO-000-415","name":"gum","sold":1412,"in-stock":54,"on-order":10}, {"sku":"HRD-000-212","name":"rope","sold":85,"in-stock":4,"on-order":2}, {"sku":"HRD-000-517","name":"ladder","sold":0,"in-stock":2,"on-order":1}, {"sku":"HRD-000-632","name":"bolt","sold":4123,"in-stock":144,"on-order":42}, {"sku":"GRO-000-2331","name":"water","sold":17,"in-stock":14,"on-order":2}]}, "data": {"item": [{"sku":"GRO-000-415","name":"gum","sold":1412.0,"in-stock":54,"on-order":10}, {"sku":"HRD-000-212","name":"rope","sold":85.0,"in-stock":4,"on-order":2}, {"sku":"HRD-000-517","name":"ladder","sold":0,"in-stock":2,"on-order":1}, {"sku":"HRD-000-632","name":"bolt","sold":4123.0,"in-stock":144,"on-order":42}, {"sku":"GRO-000-2331","name":"water","sold":17.0,"in-stock":14,"on-order":2}]}, "data": {"item": [{"sku":"GRO-000-533","name":"fish","sold":1321.0,"in-stock":45,"on-order":1}]}, "data": {"item": ["gum","rope","ladder","bolt","water"]},"cost":425,"cost":455}
 }
diff --git a/contrib/libxo/tests/core/saved/test_01.JP.out b/contrib/libxo/tests/core/saved/test_01.JP.out
index 56758457f36..e1fd2318d67 100644
--- a/contrib/libxo/tests/core/saved/test_01.JP.out
+++ b/contrib/libxo/tests/core/saved/test_01.JP.out
@@ -88,6 +88,13 @@
           "on-order": 1
         }
       ]
-    }
+    },
+    "data": {
+      "item": [
+         "gum", "rope", "ladder", "bolt", "water"
+      ]
+    },
+    "cost": 425,
+    "cost": 455
   }
 }
diff --git a/contrib/libxo/tests/core/saved/test_01.T.out b/contrib/libxo/tests/core/saved/test_01.T.out
index c2ad7a00527..c45b13001a1 100644
--- a/contrib/libxo/tests/core/saved/test_01.T.out
+++ b/contrib/libxo/tests/core/saved/test_01.T.out
@@ -36,3 +36,11 @@ Item 'fish':
    In stock: 45
    On order: 1
    SKU: GRO-000-533
+Item: gum
+Item: rope
+Item: ladder
+Item: bolt
+Item: water
+XXXXXXXX
+X        XCost: 425
+X                              XCost: 455
diff --git a/contrib/libxo/tests/core/saved/test_01.X.out b/contrib/libxo/tests/core/saved/test_01.X.out
index c3e07c85ee6..ce2719268aa 100644
--- a/contrib/libxo/tests/core/saved/test_01.X.out
+++ b/contrib/libxo/tests/core/saved/test_01.X.out
@@ -1 +1 @@
-<top><data><item><sku key="key">GRO-000-415</sku><name key="key">gum</name><sold>1412</sold><in-stock>54</in-stock><on-order>10</on-order></item><item><sku key="key">HRD-000-212</sku><name key="key">rope</name><sold>85</sold><in-stock>4</in-stock><on-order>2</on-order></item><item><sku key="key">HRD-000-517</sku><name key="key">ladder</name><sold>0</sold><in-stock>2</in-stock><on-order>1</on-order></item><item><sku key="key">HRD-000-632</sku><name key="key">bolt</name><sold>4123</sold><in-stock>144</in-stock><on-order>42</on-order></item><item><sku key="key">GRO-000-2331</sku><name key="key">water</name><sold>17</sold><in-stock>14</in-stock><on-order>2</on-order></item></data><data><item><sku key="key">GRO-000-415</sku><name key="key">gum</name><sold>1412.0</sold><in-stock>54</in-stock><on-order>10</on-order></item><item><sku key="key">HRD-000-212</sku><name key="key">rope</name><sold>85.0</sold><in-stock>4</in-stock><on-order>2</on-order></item><item><sku key="key">HRD-000-517</sku><name key="key">ladder</name><sold>0</sold><in-stock>2</in-stock><on-order>1</on-order></item><item><sku key="key">HRD-000-632</sku><name key="key">bolt</name><sold>4123.0</sold><in-stock>144</in-stock><on-order>42</on-order></item><item><sku key="key">GRO-000-2331</sku><name key="key">water</name><sold>17.0</sold><in-stock>14</in-stock><on-order>2</on-order></item></data><data><item><sku key="key">GRO-000-533</sku><name key="key">fish</name><sold>1321.0</sold><in-stock>45</in-stock><on-order>1</on-order></item></data></top>
\ No newline at end of file
+<top><data test="value"><item test2="value2"><sku test3="value3" key="key">GRO-000-415</sku><name key="key">gum</name><sold>1412</sold><in-stock>54</in-stock><on-order>10</on-order></item><item><sku test3="value3" key="key">HRD-000-212</sku><name key="key">rope</name><sold>85</sold><in-stock>4</in-stock><on-order>2</on-order></item><item><sku test3="value3" key="key">HRD-000-517</sku><name key="key">ladder</name><sold>0</sold><in-stock>2</in-stock><on-order>1</on-order></item><item><sku test3="value3" key="key">HRD-000-632</sku><name key="key">bolt</name><sold>4123</sold><in-stock>144</in-stock><on-order>42</on-order></item><item><sku test3="value3" key="key">GRO-000-2331</sku><name key="key">water</name><sold>17</sold><in-stock>14</in-stock><on-order>2</on-order></item></data><data><item><sku key="key">GRO-000-415</sku><name key="key">gum</name><sold>1412.0</sold><in-stock>54</in-stock><on-order>10</on-order></item><item><sku key="key">HRD-000-212</sku><name key="key">rope</name><sold>85.0</sold><in-stock>4</in-stock><on-order>2</on-order></item><item><sku key="key">HRD-000-517</sku><name key="key">ladder</name><sold>0</sold><in-stock>2</in-stock><on-order>1</on-order></item><item><sku key="key">HRD-000-632</sku><name key="key">bolt</name><sold>4123.0</sold><in-stock>144</in-stock><on-order>42</on-order></item><item><sku key="key">GRO-000-2331</sku><name key="key">water</name><sold>17.0</sold><in-stock>14</in-stock><on-order>2</on-order></item></data><data><item><sku key="key">GRO-000-533</sku><name key="key">fish</name><sold>1321.0</sold><in-stock>45</in-stock><on-order>1</on-order></item></data><data><item test4="value4">gum</item><item test4="value4">rope</item><item test4="value4">ladder</item><item test4="value4">bolt</item><item test4="value4">water</item></data><cost>425</cost><cost>455</cost></top>
\ No newline at end of file
diff --git a/contrib/libxo/tests/core/saved/test_01.XP.out b/contrib/libxo/tests/core/saved/test_01.XP.out
index 49fc6dab91f..e5ea3e08897 100644
--- a/contrib/libxo/tests/core/saved/test_01.XP.out
+++ b/contrib/libxo/tests/core/saved/test_01.XP.out
@@ -1,35 +1,35 @@
 <top>
-  <data>
-    <item>
-      <sku key="key">GRO-000-415</sku>
+  <data test="value">
+    <item test2="value2">
+      <sku test3="value3" key="key">GRO-000-415</sku>
       <name key="key">gum</name>
       <sold>1412</sold>
       <in-stock>54</in-stock>
       <on-order>10</on-order>
     </item>
     <item>
-      <sku key="key">HRD-000-212</sku>
+      <sku test3="value3" key="key">HRD-000-212</sku>
       <name key="key">rope</name>
       <sold>85</sold>
       <in-stock>4</in-stock>
       <on-order>2</on-order>
     </item>
     <item>
-      <sku key="key">HRD-000-517</sku>
+      <sku test3="value3" key="key">HRD-000-517</sku>
       <name key="key">ladder</name>
       <sold>0</sold>
       <in-stock>2</in-stock>
       <on-order>1</on-order>
     </item>
     <item>
-      <sku key="key">HRD-000-632</sku>
+      <sku test3="value3" key="key">HRD-000-632</sku>
       <name key="key">bolt</name>
       <sold>4123</sold>
       <in-stock>144</in-stock>
       <on-order>42</on-order>
     </item>
     <item>
-      <sku key="key">GRO-000-2331</sku>
+      <sku test3="value3" key="key">GRO-000-2331</sku>
       <name key="key">water</name>
       <sold>17</sold>
       <in-stock>14</in-stock>
@@ -82,4 +82,13 @@
       <on-order>1</on-order>
     </item>
   </data>
+  <data>
+    <item test4="value4">gum</item>
+    <item test4="value4">rope</item>
+    <item test4="value4">ladder</item>
+    <item test4="value4">bolt</item>
+    <item test4="value4">water</item>
+  </data>
+  <cost>425</cost>
+  <cost>455</cost>
 </top>
diff --git a/contrib/libxo/tests/core/saved/test_02.J.out b/contrib/libxo/tests/core/saved/test_02.J.out
index 621e0619090..5b4502a1d15 100644
--- a/contrib/libxo/tests/core/saved/test_02.J.out
+++ b/contrib/libxo/tests/core/saved/test_02.J.out
@@ -1,2 +1,2 @@
-{"top": {"data": {"mbuf-current":10,"mbuf-cache":20,"mbuf-total":30,"distance":50,"location":"Boston","memory":64,"total":640,"memory":64,"total":640,"ten":10,"eleven":11,"unknown":1010,"min":15,"cur":20,"max":30,"min":15,"cur":20,"max":125,"min":15,"cur":20,"max":125,"min":15,"cur":20,"max":125, "flag": ["one","two","three"],"empty-tag":true,"t1":"1000","t2":"test5000","t3":"ten-longx","t4":"xtest","count":10,"test":4}}
+{"top": {"data": {"mbuf-current":10,"mbuf-cache":20,"mbuf-total":30,"distance":50,"location":"Boston","memory":64,"total":640,"memory":64,"total":640,"ten":10,"eleven":11,"unknown":1010,"min":15,"cur":20,"max":30,"min":15,"cur":20,"max":125,"min":15,"cur":20,"max":125,"min":15,"cur":20,"max":125, "flag": ["one","two","three"],"empty-tag":true,"t1":"1000","t2":"test5000","t3":"ten-longx","t4":"xtest","count":10,"test":4, "error": {"message":"Shut 'er down, Clancey!  She's a-pumpin' mud!  <>!,\"!<>\n"}}}
 }
diff --git a/contrib/libxo/tests/core/saved/test_02.JP.out b/contrib/libxo/tests/core/saved/test_02.JP.out
index 9479817b91a..21b168bdad7 100644
--- a/contrib/libxo/tests/core/saved/test_02.JP.out
+++ b/contrib/libxo/tests/core/saved/test_02.JP.out
@@ -34,7 +34,10 @@
       "t3": "ten-longx",
       "t4": "xtest",
       "count": 10,
-      "test": 4
+      "test": 4,
+      "error": {
+        "message": "Shut 'er down, Clancey!  She's a-pumpin' mud!  <>!,\"!<>\n"
+      }
     }
   }
 }
diff --git a/contrib/libxo/tests/core/saved/test_07.J.out b/contrib/libxo/tests/core/saved/test_07.J.out
index 9285ff5a6c5..8e9efaec800 100644
--- a/contrib/libxo/tests/core/saved/test_07.J.out
+++ b/contrib/libxo/tests/core/saved/test_07.J.out
@@ -1,2 +1,2 @@
-{"employees": {"test": [{"filename":"(null)"}],"v1":"γιγνώσκειν","v2":"ὦ ἄνδρες ᾿Αθηναῖοι","columns":28,"columns":2,"v1":"ახლავე გაიაროთ რეგისტრაცია","v2":"Unicode-ის მეათე საერთაშორისო","columns":55, "employee": ["columns":0, {"first-name":"Jim","nic-name":"\"რეგტ\"","last-name":"გთხოვთ ახ","department":431,"percent-time":90,"columns":23,"benefits":"full"}, {"first-name":"Terry","nic-name":"\"<one\"","last-name":"Οὐχὶ ταὐτὰ παρίσταταί μοι Jones","department":660,"percent-time":90,"columns":47,"benefits":"full"}, {"first-name":"Leslie","nic-name":"\"Les\"","last-name":"Patterson","department":341,"percent-time":60,"columns":25,"benefits":"full"}, {"first-name":"Ashley","nic-name":"\"Ash\"","last-name":"Meter & Smith","department":1440,"percent-time":40,"columns":30}, {"first-name":"0123456789","nic-name":"\"0123456789\"","last-name":"012345678901234567890","department":1440,"percent-time":40,"columns":49}, {"first-name":"ახლა","nic-name":"\"გაიარო\"","last-name":"საერთაშორისო","department":123,"percent-time":90,"columns":29,"benefits":"full"}]}
+{"employees": {"test": [{"filename":"(null)"}],"v1":"γιγνώσκειν","v2":"ὦ ἄνδρες ᾿Αθηναῖοι","columns":28,"columns":2,"v1":"ახლავე გაიაროთ რეგისტრაცია","v2":"Unicode-ის მეათე საერთაშორისო","columns":55,"columns":0, "employee": [{"first-name":"Jim","nic-name":"\"რეგტ\"","last-name":"გთხოვთ ახ","department":431,"percent-time":90,"columns":23,"benefits":"full"}, {"first-name":"Terry","nic-name":"\"<one\"","last-name":"Οὐχὶ ταὐτὰ παρίσταταί μοι Jones","department":660,"percent-time":90,"columns":47,"benefits":"full"}, {"first-name":"Leslie","nic-name":"\"Les\"","last-name":"Patterson","department":341,"percent-time":60,"columns":25,"benefits":"full"}, {"first-name":"Ashley","nic-name":"\"Ash\"","last-name":"Meter & Smith","department":1440,"percent-time":40,"columns":30}, {"first-name":"0123456789","nic-name":"\"0123456789\"","last-name":"012345678901234567890","department":1440,"percent-time":40,"columns":49}, {"first-name":"ახლა","nic-name":"\"გაიარო\"","last-name":"საერთაშორისო","department":123,"percent-time":90,"columns":29,"benefits":"full"}]}
 }
diff --git a/contrib/libxo/tests/core/saved/test_07.JP.out b/contrib/libxo/tests/core/saved/test_07.JP.out
index 206a14658e3..9c969f97689 100644
--- a/contrib/libxo/tests/core/saved/test_07.JP.out
+++ b/contrib/libxo/tests/core/saved/test_07.JP.out
@@ -12,8 +12,8 @@
     "v1": "ახლავე გაიაროთ რეგისტრაცია",
     "v2": "Unicode-ის მეათე საერთაშორისო",
     "columns": 55,
+    "columns": 0,
     "employee": [
-      "columns": 0,
       {
         "first-name": "Jim",
         "nic-name": "\"რეგტ\"",
diff --git a/contrib/libxo/tests/core/saved/test_08.H.err b/contrib/libxo/tests/core/saved/test_08.H.err
new file mode 100644
index 00000000000..445bfb7172f
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.H.err
@@ -0,0 +1,18 @@
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'top'
diff --git a/contrib/libxo/tests/core/saved/test_08.H.out b/contrib/libxo/tests/core/saved/test_08.H.out
new file mode 100644
index 00000000000..81f12d8412f
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.H.out
@@ -0,0 +1 @@
+<div class="line"><div class="title">Item      </div><div class="title">       Count</div></div><div class="line"><div class="data" data-tag="name" data-key="key">gum       </div><div class="data" data-tag="count">        1412</div></div><div class="line"><div class="data" data-tag="name" data-key="key">rope      </div><div class="data" data-tag="count">          85</div></div><div class="line"><div class="data" data-tag="name" data-key="key">ladder    </div><div class="data" data-tag="count">           0</div></div><div class="line"><div class="data" data-tag="name" data-key="key">bolt      </div><div class="data" data-tag="count">        4123</div></div><div class="line"><div class="data" data-tag="name" data-key="key">water     </div><div class="data" data-tag="count">          17</div></div><div class="line"></div><div class="line"></div><div class="line"><div class="title">Item      </div><div class="title">       Count</div></div><div class="line"><div class="data" data-tag="name" data-key="key">gum       </div><div class="data" data-tag="count">        1412</div></div><div class="line"><div class="data" data-tag="name" data-key="key">rope      </div><div class="data" data-tag="count">          85</div></div><div class="line"><div class="data" data-tag="name" data-key="key">ladder    </div><div class="data" data-tag="count">           0</div></div><div class="line"><div class="data" data-tag="name" data-key="key">bolt      </div><div class="data" data-tag="count">        4123</div></div><div class="line"><div class="data" data-tag="name" data-key="key">water     </div><div class="data" data-tag="count">          17</div></div><div class="line"></div><div class="line"></div><div class="line"><div class="title">Item      </div><div class="title">       Count</div></div><div class="line"><div class="data" data-tag="name" data-key="key">gum       </div><div class="data" data-tag="count">        1412</div></div><div class="line"><div class="data" data-tag="name" data-key="key">rope      </div><div class="data" data-tag="count">          85</div></div><div class="line"><div class="data" data-tag="name" data-key="key">ladder    </div><div class="data" data-tag="count">           0</div></div><div class="line"><div class="data" data-tag="name" data-key="key">bolt      </div><div class="data" data-tag="count">        4123</div></div><div class="line"><div class="data" data-tag="name" data-key="key">water     </div><div class="data" data-tag="count">          17</div></div><div class="line"><div class="data" data-tag="test">one</div></div><div class="line"></div><div class="line"><div class="title">Item      </div><div class="title">       Count</div></div><div class="line"><div class="data" data-tag="name" data-key="key">gum       </div><div class="data" data-tag="count">        1412</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">0</div><div class="text"> + 1 = </div><div class="data" data-tag="next">1</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">1</div><div class="text"> + 1 = </div><div class="data" data-tag="next">2</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">2</div><div class="text"> + 1 = </div><div class="data" data-tag="next">3</div></div><div class="line"><div class="label">Last</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="last">3</div></div><div class="line"><div class="data" data-tag="name" data-key="key">rope      </div><div class="data" data-tag="count">          85</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">0</div><div class="text"> + 1 = </div><div class="data" data-tag="next">1</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">1</div><div class="text"> + 1 = </div><div class="data" data-tag="next">2</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">2</div><div class="text"> + 1 = </div><div class="data" data-tag="next">3</div></div><div class="line"><div class="label">Last</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="last">3</div></div><div class="line"><div class="data" data-tag="name" data-key="key">ladder    </div><div class="data" data-tag="count">           0</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">0</div><div class="text"> + 1 = </div><div class="data" data-tag="next">1</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">1</div><div class="text"> + 1 = </div><div class="data" data-tag="next">2</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">2</div><div class="text"> + 1 = </div><div class="data" data-tag="next">3</div></div><div class="line"><div class="label">Last</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="last">3</div></div><div class="line"><div class="data" data-tag="name" data-key="key">bolt      </div><div class="data" data-tag="count">        4123</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">0</div><div class="text"> + 1 = </div><div class="data" data-tag="next">1</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">1</div><div class="text"> + 1 = </div><div class="data" data-tag="next">2</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">2</div><div class="text"> + 1 = </div><div class="data" data-tag="next">3</div></div><div class="line"><div class="label">Last</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="last">3</div></div><div class="line"><div class="data" data-tag="name" data-key="key">water     </div><div class="data" data-tag="count">          17</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">0</div><div class="text"> + 1 = </div><div class="data" data-tag="next">1</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">1</div><div class="text"> + 1 = </div><div class="data" data-tag="next">2</div></div><div class="line"><div class="label">Name</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="name">2</div><div class="text"> + 1 = </div><div class="data" data-tag="next">3</div></div><div class="line"><div class="label">Last</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="last">3</div></div><div class="line"><div class="data" data-tag="test">one</div></div><div class="line"></div>
\ No newline at end of file
diff --git a/contrib/libxo/tests/core/saved/test_08.HIPx.err b/contrib/libxo/tests/core/saved/test_08.HIPx.err
new file mode 100644
index 00000000000..445bfb7172f
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.HIPx.err
@@ -0,0 +1,18 @@
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'top'
diff --git a/contrib/libxo/tests/core/saved/test_08.HIPx.out b/contrib/libxo/tests/core/saved/test_08.HIPx.out
new file mode 100644
index 00000000000..87bfbed33d1
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.HIPx.out
@@ -0,0 +1,264 @@
+<div class="line">
+  <div class="title">Item      </div>
+  <div class="title">       Count</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/item/name" data-key="key">gum       </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/contents/item[name = 'gum']/count">        1412</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/item/name" data-key="key">rope      </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/contents/item[name = 'rope']/count">          85</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/item/name" data-key="key">ladder    </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/contents/item[name = 'ladder']/count">           0</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/item/name" data-key="key">bolt      </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/contents/item[name = 'bolt']/count">        4123</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/item/name" data-key="key">water     </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/contents/item[name = 'water']/count">          17</div>
+</div>
+<div class="line">
+</div>
+<div class="line">
+</div>
+<div class="line">
+  <div class="title">Item      </div>
+  <div class="title">       Count</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/item/name" data-key="key">gum       </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/contents/item[name = 'gum']/count">        1412</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/item/name" data-key="key">rope      </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/contents/item[name = 'rope']/count">          85</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/item/name" data-key="key">ladder    </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/contents/item[name = 'ladder']/count">           0</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/item/name" data-key="key">bolt      </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/contents/item[name = 'bolt']/count">        4123</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/item/name" data-key="key">water     </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/contents/item[name = 'water']/count">          17</div>
+</div>
+<div class="line">
+</div>
+<div class="line">
+</div>
+<div class="line">
+  <div class="title">Item      </div>
+  <div class="title">       Count</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item/name" data-key="key">gum       </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/m1/contents/item[name = 'gum']/count">        1412</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item/name" data-key="key">rope      </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/m1/contents/item[name = 'rope']/count">          85</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item/name" data-key="key">ladder    </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/m1/contents/item[name = 'ladder']/count">           0</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item/name" data-key="key">bolt      </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/m1/contents/item[name = 'bolt']/count">        4123</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item/name" data-key="key">water     </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/m1/contents/item[name = 'water']/count">          17</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="test" data-xpath="/top/data/m1/contents/item[name = 'water']/test">one</div>
+</div>
+<div class="line">
+</div>
+<div class="line">
+  <div class="title">Item      </div>
+  <div class="title">       Count</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item/name" data-key="key">gum       </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/m1/contents/item[name = 'gum']/count">        1412</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'gum']/m2/sub/name">0</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'gum']/m2/sub/next">1</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'gum']/m2/sub/name">1</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'gum']/m2/sub/next">2</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'gum']/m2/sub/name">2</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'gum']/m2/sub/next">3</div>
+</div>
+<div class="line">
+  <div class="label">Last</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="last" data-xpath="/top/data/m1/contents/item[name = 'gum']/last">3</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item/name" data-key="key">rope      </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/m1/contents/item[name = 'rope']/count">          85</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'rope']/m2/sub/name">0</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'rope']/m2/sub/next">1</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'rope']/m2/sub/name">1</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'rope']/m2/sub/next">2</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'rope']/m2/sub/name">2</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'rope']/m2/sub/next">3</div>
+</div>
+<div class="line">
+  <div class="label">Last</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="last" data-xpath="/top/data/m1/contents/item[name = 'rope']/last">3</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item/name" data-key="key">ladder    </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/m1/contents/item[name = 'ladder']/count">           0</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'ladder']/m2/sub/name">0</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'ladder']/m2/sub/next">1</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'ladder']/m2/sub/name">1</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'ladder']/m2/sub/next">2</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'ladder']/m2/sub/name">2</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'ladder']/m2/sub/next">3</div>
+</div>
+<div class="line">
+  <div class="label">Last</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="last" data-xpath="/top/data/m1/contents/item[name = 'ladder']/last">3</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item/name" data-key="key">bolt      </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/m1/contents/item[name = 'bolt']/count">        4123</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'bolt']/m2/sub/name">0</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'bolt']/m2/sub/next">1</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'bolt']/m2/sub/name">1</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'bolt']/m2/sub/next">2</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'bolt']/m2/sub/name">2</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'bolt']/m2/sub/next">3</div>
+</div>
+<div class="line">
+  <div class="label">Last</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="last" data-xpath="/top/data/m1/contents/item[name = 'bolt']/last">3</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item/name" data-key="key">water     </div>
+  <div class="data" data-tag="count" data-xpath="/top/data/m1/contents/item[name = 'water']/count">          17</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'water']/m2/sub/name">0</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'water']/m2/sub/next">1</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'water']/m2/sub/name">1</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'water']/m2/sub/next">2</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/m1/contents/item[name = 'water']/m2/sub/name">2</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next" data-xpath="/top/data/m1/contents/item[name = 'water']/m2/sub/next">3</div>
+</div>
+<div class="line">
+  <div class="label">Last</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="last" data-xpath="/top/data/m1/contents/item[name = 'water']/last">3</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="test" data-xpath="/top/data/m1/contents/item[name = 'water']/test">one</div>
+</div>
+<div class="line">
+</div>
diff --git a/contrib/libxo/tests/core/saved/test_08.HP.err b/contrib/libxo/tests/core/saved/test_08.HP.err
new file mode 100644
index 00000000000..445bfb7172f
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.HP.err
@@ -0,0 +1,18 @@
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'top'
diff --git a/contrib/libxo/tests/core/saved/test_08.HP.out b/contrib/libxo/tests/core/saved/test_08.HP.out
new file mode 100644
index 00000000000..3524f5e9fac
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.HP.out
@@ -0,0 +1,264 @@
+<div class="line">
+  <div class="title">Item      </div>
+  <div class="title">       Count</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">gum       </div>
+  <div class="data" data-tag="count">        1412</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">rope      </div>
+  <div class="data" data-tag="count">          85</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">ladder    </div>
+  <div class="data" data-tag="count">           0</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">bolt      </div>
+  <div class="data" data-tag="count">        4123</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">water     </div>
+  <div class="data" data-tag="count">          17</div>
+</div>
+<div class="line">
+</div>
+<div class="line">
+</div>
+<div class="line">
+  <div class="title">Item      </div>
+  <div class="title">       Count</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">gum       </div>
+  <div class="data" data-tag="count">        1412</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">rope      </div>
+  <div class="data" data-tag="count">          85</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">ladder    </div>
+  <div class="data" data-tag="count">           0</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">bolt      </div>
+  <div class="data" data-tag="count">        4123</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">water     </div>
+  <div class="data" data-tag="count">          17</div>
+</div>
+<div class="line">
+</div>
+<div class="line">
+</div>
+<div class="line">
+  <div class="title">Item      </div>
+  <div class="title">       Count</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">gum       </div>
+  <div class="data" data-tag="count">        1412</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">rope      </div>
+  <div class="data" data-tag="count">          85</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">ladder    </div>
+  <div class="data" data-tag="count">           0</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">bolt      </div>
+  <div class="data" data-tag="count">        4123</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">water     </div>
+  <div class="data" data-tag="count">          17</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="test">one</div>
+</div>
+<div class="line">
+</div>
+<div class="line">
+  <div class="title">Item      </div>
+  <div class="title">       Count</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">gum       </div>
+  <div class="data" data-tag="count">        1412</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">0</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">1</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">1</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">2</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">2</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">3</div>
+</div>
+<div class="line">
+  <div class="label">Last</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="last">3</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">rope      </div>
+  <div class="data" data-tag="count">          85</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">0</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">1</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">1</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">2</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">2</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">3</div>
+</div>
+<div class="line">
+  <div class="label">Last</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="last">3</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">ladder    </div>
+  <div class="data" data-tag="count">           0</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">0</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">1</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">1</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">2</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">2</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">3</div>
+</div>
+<div class="line">
+  <div class="label">Last</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="last">3</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">bolt      </div>
+  <div class="data" data-tag="count">        4123</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">0</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">1</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">1</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">2</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">2</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">3</div>
+</div>
+<div class="line">
+  <div class="label">Last</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="last">3</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="name" data-key="key">water     </div>
+  <div class="data" data-tag="count">          17</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">0</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">1</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">1</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">2</div>
+</div>
+<div class="line">
+  <div class="label">Name</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="name">2</div>
+  <div class="text"> + 1 = </div>
+  <div class="data" data-tag="next">3</div>
+</div>
+<div class="line">
+  <div class="label">Last</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="last">3</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="test">one</div>
+</div>
+<div class="line">
+</div>
diff --git a/contrib/libxo/tests/core/saved/test_08.J.err b/contrib/libxo/tests/core/saved/test_08.J.err
new file mode 100644
index 00000000000..445bfb7172f
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.J.err
@@ -0,0 +1,18 @@
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'top'
diff --git a/contrib/libxo/tests/core/saved/test_08.J.out b/contrib/libxo/tests/core/saved/test_08.J.out
new file mode 100644
index 00000000000..cbce0910b6c
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.J.out
@@ -0,0 +1,2 @@
+{"top": {"data": {"contents": {"item": [{"name":"gum","count":1412}, {"name":"rope","count":85}, {"name":"ladder","count":0}, {"name":"bolt","count":4123}, {"name":"water","count":17}]}}, "data": {"contents": {"item": [{"name":"gum","count":1412}, {"name":"rope","count":85}, {"name":"ladder","count":0}, {"name":"bolt","count":4123}, {"name":"water","count":17}]}}, "data": {"contents": {"item": [{"name":"gum","count":1412}, {"name":"rope","count":85}, {"name":"ladder","count":0}, {"name":"bolt","count":4123}, {"name":"water","count":17,"test":"one"}]}}, "data": {"contents": {"item": [{"name":"gum","count":1412, "sub": [{"name":0,"next":1}, {"name":1,"next":2}, {"name":2,"next":3}],"last":3}, {"name":"rope","count":85, "sub": [{"name":0,"next":1}, {"name":1,"next":2}, {"name":2,"next":3}],"last":3}, {"name":"ladder","count":0, "sub": [{"name":0,"next":1}, {"name":1,"next":2}, {"name":2,"next":3}],"last":3}, {"name":"bolt","count":4123, "sub": [{"name":0,"next":1}, {"name":1,"next":2}, {"name":2,"next":3}],"last":3}, {"name":"water","count":17, "sub": [{"name":0,"next":1}, {"name":1,"next":2}, {"name":2,"next":3}],"last":3,"test":"one"}]}}}
+}
diff --git a/contrib/libxo/tests/core/saved/test_08.JP.err b/contrib/libxo/tests/core/saved/test_08.JP.err
new file mode 100644
index 00000000000..445bfb7172f
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.JP.err
@@ -0,0 +1,18 @@
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'top'
diff --git a/contrib/libxo/tests/core/saved/test_08.JP.out b/contrib/libxo/tests/core/saved/test_08.JP.out
new file mode 100644
index 00000000000..932d6a12d94
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.JP.out
@@ -0,0 +1,185 @@
+{
+  "top": {
+    "data": {
+      "contents": {
+        "item": [
+          {
+            "name": "gum",
+            "count": 1412
+          },
+          {
+            "name": "rope",
+            "count": 85
+          },
+          {
+            "name": "ladder",
+            "count": 0
+          },
+          {
+            "name": "bolt",
+            "count": 4123
+          },
+          {
+            "name": "water",
+            "count": 17
+          }
+        ]
+      }
+    },
+    "data": {
+      "contents": {
+        "item": [
+          {
+            "name": "gum",
+            "count": 1412
+          },
+          {
+            "name": "rope",
+            "count": 85
+          },
+          {
+            "name": "ladder",
+            "count": 0
+          },
+          {
+            "name": "bolt",
+            "count": 4123
+          },
+          {
+            "name": "water",
+            "count": 17
+          }
+        ]
+      }
+    },
+    "data": {
+      "contents": {
+        "item": [
+          {
+            "name": "gum",
+            "count": 1412
+          },
+          {
+            "name": "rope",
+            "count": 85
+          },
+          {
+            "name": "ladder",
+            "count": 0
+          },
+          {
+            "name": "bolt",
+            "count": 4123
+          },
+          {
+            "name": "water",
+            "count": 17,
+            "test": "one"
+          }
+        ]
+      }
+    },
+    "data": {
+      "contents": {
+        "item": [
+          {
+            "name": "gum",
+            "count": 1412,
+            "sub": [
+              {
+                "name": 0,
+                "next": 1
+              },
+              {
+                "name": 1,
+                "next": 2
+              },
+              {
+                "name": 2,
+                "next": 3
+              }
+            ],
+            "last": 3
+          },
+          {
+            "name": "rope",
+            "count": 85,
+            "sub": [
+              {
+                "name": 0,
+                "next": 1
+              },
+              {
+                "name": 1,
+                "next": 2
+              },
+              {
+                "name": 2,
+                "next": 3
+              }
+            ],
+            "last": 3
+          },
+          {
+            "name": "ladder",
+            "count": 0,
+            "sub": [
+              {
+                "name": 0,
+                "next": 1
+              },
+              {
+                "name": 1,
+                "next": 2
+              },
+              {
+                "name": 2,
+                "next": 3
+              }
+            ],
+            "last": 3
+          },
+          {
+            "name": "bolt",
+            "count": 4123,
+            "sub": [
+              {
+                "name": 0,
+                "next": 1
+              },
+              {
+                "name": 1,
+                "next": 2
+              },
+              {
+                "name": 2,
+                "next": 3
+              }
+            ],
+            "last": 3
+          },
+          {
+            "name": "water",
+            "count": 17,
+            "sub": [
+              {
+                "name": 0,
+                "next": 1
+              },
+              {
+                "name": 1,
+                "next": 2
+              },
+              {
+                "name": 2,
+                "next": 3
+              }
+            ],
+            "last": 3,
+            "test": "one"
+          }
+        ]
+      }
+    }
+  }
+}
diff --git a/contrib/libxo/tests/core/saved/test_08.T.err b/contrib/libxo/tests/core/saved/test_08.T.err
new file mode 100644
index 00000000000..445bfb7172f
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.T.err
@@ -0,0 +1,18 @@
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'top'
diff --git a/contrib/libxo/tests/core/saved/test_08.T.out b/contrib/libxo/tests/core/saved/test_08.T.out
new file mode 100644
index 00000000000..8923b93642a
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.T.out
@@ -0,0 +1,52 @@
+Item             Count
+gum               1412
+rope                85
+ladder               0
+bolt              4123
+water               17
+
+
+Item             Count
+gum               1412
+rope                85
+ladder               0
+bolt              4123
+water               17
+
+
+Item             Count
+gum               1412
+rope                85
+ladder               0
+bolt              4123
+water               17
+one
+
+Item             Count
+gum               1412
+Name: 0 + 1 = 1
+Name: 1 + 1 = 2
+Name: 2 + 1 = 3
+Last: 3
+rope                85
+Name: 0 + 1 = 1
+Name: 1 + 1 = 2
+Name: 2 + 1 = 3
+Last: 3
+ladder               0
+Name: 0 + 1 = 1
+Name: 1 + 1 = 2
+Name: 2 + 1 = 3
+Last: 3
+bolt              4123
+Name: 0 + 1 = 1
+Name: 1 + 1 = 2
+Name: 2 + 1 = 3
+Last: 3
+water               17
+Name: 0 + 1 = 1
+Name: 1 + 1 = 2
+Name: 2 + 1 = 3
+Last: 3
+one
+
diff --git a/contrib/libxo/tests/core/saved/test_08.X.err b/contrib/libxo/tests/core/saved/test_08.X.err
new file mode 100644
index 00000000000..445bfb7172f
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.X.err
@@ -0,0 +1,18 @@
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'top'
diff --git a/contrib/libxo/tests/core/saved/test_08.X.out b/contrib/libxo/tests/core/saved/test_08.X.out
new file mode 100644
index 00000000000..5eb72b209ad
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.X.out
@@ -0,0 +1 @@
+<top><data><contents><item><name key="key">gum</name><count>1412</count></item><item><name key="key">rope</name><count>85</count></item><item><name key="key">ladder</name><count>0</count></item><item><name key="key">bolt</name><count>4123</count></item><item><name key="key">water</name><count>17</count></item></contents></data><data><contents><item><name key="key">gum</name><count>1412</count></item><item><name key="key">rope</name><count>85</count></item><item><name key="key">ladder</name><count>0</count></item><item><name key="key">bolt</name><count>4123</count></item><item><name key="key">water</name><count>17</count></item></contents></data><data><contents><item><name key="key">gum</name><count>1412</count></item><item><name key="key">rope</name><count>85</count></item><item><name key="key">ladder</name><count>0</count></item><item><name key="key">bolt</name><count>4123</count></item><item><name key="key">water</name><count>17</count><test>one</test></item></contents></data><data><contents><item><name key="key">gum</name><count>1412</count><sub><name>0</name><next>1</next></sub><sub><name>1</name><next>2</next></sub><sub><name>2</name><next>3</next></sub><last>3</last></item><item><name key="key">rope</name><count>85</count><sub><name>0</name><next>1</next></sub><sub><name>1</name><next>2</next></sub><sub><name>2</name><next>3</next></sub><last>3</last></item><item><name key="key">ladder</name><count>0</count><sub><name>0</name><next>1</next></sub><sub><name>1</name><next>2</next></sub><sub><name>2</name><next>3</next></sub><last>3</last></item><item><name key="key">bolt</name><count>4123</count><sub><name>0</name><next>1</next></sub><sub><name>1</name><next>2</next></sub><sub><name>2</name><next>3</next></sub><last>3</last></item><item><name key="key">water</name><count>17</count><sub><name>0</name><next>1</next></sub><sub><name>1</name><next>2</next></sub><sub><name>2</name><next>3</next></sub><last>3</last><test>one</test></item></contents></data></top>
\ No newline at end of file
diff --git a/contrib/libxo/tests/core/saved/test_08.XP.err b/contrib/libxo/tests/core/saved/test_08.XP.err
new file mode 100644
index 00000000000..445bfb7172f
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.XP.err
@@ -0,0 +1,18 @@
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm2'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'data'
+test: close (xo_close_container) fails at marker 'm1'; not found 'top'
diff --git a/contrib/libxo/tests/core/saved/test_08.XP.out b/contrib/libxo/tests/core/saved/test_08.XP.out
new file mode 100644
index 00000000000..99520c074a9
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_08.XP.out
@@ -0,0 +1,165 @@
+<top>
+  <data>
+    <contents>
+      <item>
+        <name key="key">gum</name>
+        <count>1412</count>
+      </item>
+      <item>
+        <name key="key">rope</name>
+        <count>85</count>
+      </item>
+      <item>
+        <name key="key">ladder</name>
+        <count>0</count>
+      </item>
+      <item>
+        <name key="key">bolt</name>
+        <count>4123</count>
+      </item>
+      <item>
+        <name key="key">water</name>
+        <count>17</count>
+      </item>
+    </contents>
+  </data>
+  <data>
+    <contents>
+      <item>
+        <name key="key">gum</name>
+        <count>1412</count>
+      </item>
+      <item>
+        <name key="key">rope</name>
+        <count>85</count>
+      </item>
+      <item>
+        <name key="key">ladder</name>
+        <count>0</count>
+      </item>
+      <item>
+        <name key="key">bolt</name>
+        <count>4123</count>
+      </item>
+      <item>
+        <name key="key">water</name>
+        <count>17</count>
+      </item>
+    </contents>
+  </data>
+  <data>
+    <contents>
+      <item>
+        <name key="key">gum</name>
+        <count>1412</count>
+      </item>
+      <item>
+        <name key="key">rope</name>
+        <count>85</count>
+      </item>
+      <item>
+        <name key="key">ladder</name>
+        <count>0</count>
+      </item>
+      <item>
+        <name key="key">bolt</name>
+        <count>4123</count>
+      </item>
+      <item>
+        <name key="key">water</name>
+        <count>17</count>
+        <test>one</test>
+      </item>
+    </contents>
+  </data>
+  <data>
+    <contents>
+      <item>
+        <name key="key">gum</name>
+        <count>1412</count>
+        <sub>
+          <name>0</name>
+          <next>1</next>
+        </sub>
+        <sub>
+          <name>1</name>
+          <next>2</next>
+        </sub>
+        <sub>
+          <name>2</name>
+          <next>3</next>
+        </sub>
+        <last>3</last>
+      </item>
+      <item>
+        <name key="key">rope</name>
+        <count>85</count>
+        <sub>
+          <name>0</name>
+          <next>1</next>
+        </sub>
+        <sub>
+          <name>1</name>
+          <next>2</next>
+        </sub>
+        <sub>
+          <name>2</name>
+          <next>3</next>
+        </sub>
+        <last>3</last>
+      </item>
+      <item>
+        <name key="key">ladder</name>
+        <count>0</count>
+        <sub>
+          <name>0</name>
+          <next>1</next>
+        </sub>
+        <sub>
+          <name>1</name>
+          <next>2</next>
+        </sub>
+        <sub>
+          <name>2</name>
+          <next>3</next>
+        </sub>
+        <last>3</last>
+      </item>
+      <item>
+        <name key="key">bolt</name>
+        <count>4123</count>
+        <sub>
+          <name>0</name>
+          <next>1</next>
+        </sub>
+        <sub>
+          <name>1</name>
+          <next>2</next>
+        </sub>
+        <sub>
+          <name>2</name>
+          <next>3</next>
+        </sub>
+        <last>3</last>
+      </item>
+      <item>
+        <name key="key">water</name>
+        <count>17</count>
+        <sub>
+          <name>0</name>
+          <next>1</next>
+        </sub>
+        <sub>
+          <name>1</name>
+          <next>2</next>
+        </sub>
+        <sub>
+          <name>2</name>
+          <next>3</next>
+        </sub>
+        <last>3</last>
+        <test>one</test>
+      </item>
+    </contents>
+  </data>
+</top>
diff --git a/contrib/libxo/tests/core/saved/test_09.H.err b/contrib/libxo/tests/core/saved/test_09.H.err
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/libxo/tests/core/saved/test_09.H.out b/contrib/libxo/tests/core/saved/test_09.H.out
new file mode 100644
index 00000000000..899cd2f4045
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_09.H.out
@@ -0,0 +1 @@
+<div class="line"><div class="title">Item      </div><div class="title">       Count</div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="name">gum       </div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="name">rope      </div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="name">ladder    </div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="name">bolt      </div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="name">water     </div></div><div class="line"></div><div class="line"></div><div class="line"><div class="title">Item      </div><div class="title">       Count</div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="item">gum       </div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="item">rope      </div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="item">ladder    </div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="item">bolt      </div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="item">water     </div></div><div class="line"></div><div class="line"></div><div class="line"><div class="title">Test      </div><div class="title">       Three</div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="item">gum       </div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="item">rope      </div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="item">ladder    </div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="item">bolt      </div></div><div class="line"><div class="text">Name: </div><div class="data" data-tag="item">water     </div></div><div class="line"><div class="label">Total:</div><div class="decoration">:</div><div class="padding"> </div><div class="data" data-tag="total">six</div></div><div class="line"><div class="data" data-tag="one">one</div><div class="data" data-tag="two">two</div><div class="data" data-tag="three">three</div></div><div class="line"></div>
\ No newline at end of file
diff --git a/contrib/libxo/tests/core/saved/test_09.HIPx.err b/contrib/libxo/tests/core/saved/test_09.HIPx.err
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/libxo/tests/core/saved/test_09.HIPx.out b/contrib/libxo/tests/core/saved/test_09.HIPx.out
new file mode 100644
index 00000000000..a63f29264df
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_09.HIPx.out
@@ -0,0 +1,93 @@
+<div class="line">
+  <div class="title">Item      </div>
+  <div class="title">       Count</div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/name">gum       </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/name">rope      </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/name">ladder    </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/name">bolt      </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="name" data-xpath="/top/data/contents/name">water     </div>
+</div>
+<div class="line">
+</div>
+<div class="line">
+</div>
+<div class="line">
+  <div class="title">Item      </div>
+  <div class="title">       Count</div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/contents/item">gum       </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/contents/item">rope      </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/contents/item">ladder    </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/contents/item">bolt      </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/contents/item">water     </div>
+</div>
+<div class="line">
+</div>
+<div class="line">
+</div>
+<div class="line">
+  <div class="title">Test      </div>
+  <div class="title">       Three</div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/contents/item">gum       </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/contents/item">rope      </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/contents/item">ladder    </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/contents/item">bolt      </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item" data-xpath="/top/data/contents/item">water     </div>
+</div>
+<div class="line">
+  <div class="label">Total:</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="total" data-xpath="/top/data/contents/total">six</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="one" data-xpath="/top/data/contents/one">one</div>
+  <div class="data" data-tag="two" data-xpath="/top/data/contents/two">two</div>
+  <div class="data" data-tag="three" data-xpath="/top/data/contents/three">three</div>
+</div>
+<div class="line">
+</div>
diff --git a/contrib/libxo/tests/core/saved/test_09.HP.err b/contrib/libxo/tests/core/saved/test_09.HP.err
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/libxo/tests/core/saved/test_09.HP.out b/contrib/libxo/tests/core/saved/test_09.HP.out
new file mode 100644
index 00000000000..8a8f700338e
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_09.HP.out
@@ -0,0 +1,93 @@
+<div class="line">
+  <div class="title">Item      </div>
+  <div class="title">       Count</div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="name">gum       </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="name">rope      </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="name">ladder    </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="name">bolt      </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="name">water     </div>
+</div>
+<div class="line">
+</div>
+<div class="line">
+</div>
+<div class="line">
+  <div class="title">Item      </div>
+  <div class="title">       Count</div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item">gum       </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item">rope      </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item">ladder    </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item">bolt      </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item">water     </div>
+</div>
+<div class="line">
+</div>
+<div class="line">
+</div>
+<div class="line">
+  <div class="title">Test      </div>
+  <div class="title">       Three</div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item">gum       </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item">rope      </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item">ladder    </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item">bolt      </div>
+</div>
+<div class="line">
+  <div class="text">Name: </div>
+  <div class="data" data-tag="item">water     </div>
+</div>
+<div class="line">
+  <div class="label">Total:</div>
+  <div class="decoration">:</div>
+  <div class="padding"> </div>
+  <div class="data" data-tag="total">six</div>
+</div>
+<div class="line">
+  <div class="data" data-tag="one">one</div>
+  <div class="data" data-tag="two">two</div>
+  <div class="data" data-tag="three">three</div>
+</div>
+<div class="line">
+</div>
diff --git a/contrib/libxo/tests/core/saved/test_09.J.err b/contrib/libxo/tests/core/saved/test_09.J.err
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/libxo/tests/core/saved/test_09.J.out b/contrib/libxo/tests/core/saved/test_09.J.out
new file mode 100644
index 00000000000..e43ad3150c3
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_09.J.out
@@ -0,0 +1,2 @@
+{"top": {"data": {"contents": {"name": ["gum","rope","ladder","bolt","water"]}, "contents": {"item": ["gum","rope","ladder","bolt","water"]}, "contents": {"item": ["gum","rope","ladder","bolt","water"],"total":"six","one":"one", "two": ["two"],"three":"three"}}}
+}
diff --git a/contrib/libxo/tests/core/saved/test_09.JP.err b/contrib/libxo/tests/core/saved/test_09.JP.err
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/libxo/tests/core/saved/test_09.JP.out b/contrib/libxo/tests/core/saved/test_09.JP.out
new file mode 100644
index 00000000000..8340b275606
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_09.JP.out
@@ -0,0 +1,27 @@
+{
+  "top": {
+    "data": {
+      "contents": {
+        "name": [
+           "gum", "rope", "ladder", "bolt", "water"
+        ]
+      },
+      "contents": {
+        "item": [
+           "gum", "rope", "ladder", "bolt", "water"
+        ]
+      },
+      "contents": {
+        "item": [
+           "gum", "rope", "ladder", "bolt", "water"
+        ],
+        "total": "six",
+        "one": "one",
+        "two": [
+           "two"
+        ],
+        "three": "three"
+      }
+    }
+  }
+}
diff --git a/contrib/libxo/tests/core/saved/test_09.T.err b/contrib/libxo/tests/core/saved/test_09.T.err
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/libxo/tests/core/saved/test_09.T.out b/contrib/libxo/tests/core/saved/test_09.T.out
new file mode 100644
index 00000000000..5bb163ed866
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_09.T.out
@@ -0,0 +1,25 @@
+Item             Count
+Name: gum       
+Name: rope      
+Name: ladder    
+Name: bolt      
+Name: water     
+
+
+Item             Count
+Name: gum       
+Name: rope      
+Name: ladder    
+Name: bolt      
+Name: water     
+
+
+Test             Three
+Name: gum       
+Name: rope      
+Name: ladder    
+Name: bolt      
+Name: water     
+Total:: six
+onetwothree
+
diff --git a/contrib/libxo/tests/core/saved/test_09.X.err b/contrib/libxo/tests/core/saved/test_09.X.err
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/libxo/tests/core/saved/test_09.X.out b/contrib/libxo/tests/core/saved/test_09.X.out
new file mode 100644
index 00000000000..21ce1ce6956
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_09.X.out
@@ -0,0 +1 @@
+<top><data><contents><name>gum</name><name>rope</name><name>ladder</name><name>bolt</name><name>water</name></contents><contents><item>gum</item><item>rope</item><item>ladder</item><item>bolt</item><item>water</item></contents><contents><item>gum</item><item>rope</item><item>ladder</item><item>bolt</item><item>water</item><total>six</total><one>one</one><two>two</two><three>three</three></contents></data></top>
\ No newline at end of file
diff --git a/contrib/libxo/tests/core/saved/test_09.XP.err b/contrib/libxo/tests/core/saved/test_09.XP.err
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/contrib/libxo/tests/core/saved/test_09.XP.out b/contrib/libxo/tests/core/saved/test_09.XP.out
new file mode 100644
index 00000000000..9476126e372
--- /dev/null
+++ b/contrib/libxo/tests/core/saved/test_09.XP.out
@@ -0,0 +1,29 @@
+<top>
+  <data>
+    <contents>
+      <name>gum</name>
+      <name>rope</name>
+      <name>ladder</name>
+      <name>bolt</name>
+      <name>water</name>
+    </contents>
+    <contents>
+      <item>gum</item>
+      <item>rope</item>
+      <item>ladder</item>
+      <item>bolt</item>
+      <item>water</item>
+    </contents>
+    <contents>
+      <item>gum</item>
+      <item>rope</item>
+      <item>ladder</item>
+      <item>bolt</item>
+      <item>water</item>
+      <total>six</total>
+      <one>one</one>
+      <two>two</two>
+      <three>three</three>
+    </contents>
+  </data>
+</top>
diff --git a/contrib/libxo/tests/core/test_01.c b/contrib/libxo/tests/core/test_01.c
index 164a38b9943..9a9ed2c07fe 100644
--- a/contrib/libxo/tests/core/test_01.c
+++ b/contrib/libxo/tests/core/test_01.c
@@ -11,6 +11,8 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <unistd.h>
+#include <errno.h>
 
 #include "xo.h"
 
@@ -69,6 +71,10 @@ main (int argc, char **argv)
 	    xo_set_flags(NULL, XOF_XPATH);
 	else if (strcmp(argv[argc], "info") == 0)
 	    xo_set_flags(NULL, XOF_INFO);
+        else if (strcmp(argv[argc], "error") == 0) {
+            close(-1);
+            xo_err(1, "error detected");
+        }
     }
 
     xo_set_info(NULL, info, info_count);
@@ -76,14 +82,17 @@ main (int argc, char **argv)
 
     xo_open_container_h(NULL, "top");
 
+    xo_attr("test", "value");
     xo_open_container("data");
     xo_open_list("item");
+    xo_attr("test2", "value2");
 
     xo_emit("{T:Item/%-10s}{T:Total Sold/%12s}{T:In Stock/%12s}"
 	    "{T:On Order/%12s}{T:SKU/%5s}\n");
 
     for (ip = list; ip->i_title; ip++) {
 	xo_open_instance("item");
+	xo_attr("test3", "value3");
 
 	xo_emit("{keq:sku/%s-%u/%s-000-%u}"
 		"{k:name/%-10s/%s}{n:sold/%12u/%u}{:in-stock/%12u/%u}"
@@ -142,6 +151,25 @@ main (int argc, char **argv)
     xo_close_list("item");
     xo_close_container("data");
 
+    xo_open_container("data");
+    xo_open_list("item");
+
+    for (ip = list; ip->i_title; ip++) {
+	xo_attr("test4", "value4");
+	xo_emit("{Lwc:Item}{l:item}\n", ip->i_title);
+    }
+
+    xo_close_list("item");
+    xo_close_container("data");
+
+    xo_emit("X{P:}X", "epic fail");
+    xo_emit("X{T:}X", "epic fail");
+    xo_emit("X{N:}X", "epic fail");
+    xo_emit("X{L:}X\n", "epic fail");
+
+    xo_emit("X{P:        }X{Lwc:Cost}{:cost/%u}\n", 425);
+    xo_emit("X{P:/%30s}X{Lwc:Cost}{:cost/%u}\n", "", 455);
+
     xo_close_container_h(NULL, "top");
 
     xo_finish();
diff --git a/contrib/libxo/tests/core/test_07.c b/contrib/libxo/tests/core/test_07.c
index 18b7baa146b..5b1ed57292a 100644
--- a/contrib/libxo/tests/core/test_07.c
+++ b/contrib/libxo/tests/core/test_07.c
@@ -68,11 +68,12 @@ main (int argc, char **argv)
 	    "Unicode-ის მეათე საერთაშორისო");
     xo_emit("{:columns/%d}\n", rc);
 
-    xo_open_list("employee");
 
     rc = xo_emit("{T:First Name/%-25s}{T:Last Name/%-14s}"
 	    "{T:/%-12s}{T:Time (%)}\n", "Department");
     xo_emit("{:columns/%d}\n", rc);
+
+    xo_open_list("employee");
     for ( ; ep->e_first; ep++) {
 	xo_open_instance("employee");
 	rc = xo_emit("{[:-25}{:first-name/%s} ({:nic-name/\"%s\"}){]:}"
diff --git a/contrib/libxo/tests/core/test_08.c b/contrib/libxo/tests/core/test_08.c
new file mode 100644
index 00000000000..eb3776dcfbb
--- /dev/null
+++ b/contrib/libxo/tests/core/test_08.c
@@ -0,0 +1,157 @@
+/*
+ * Copyright (c) 2015, Juniper Networks, Inc.
+ * All rights reserved.
+ * This SOFTWARE is licensed under the LICENSE provided in the
+ * ../Copyright file. By downloading, installing, copying, or otherwise
+ * using the SOFTWARE, you agree to be bound by the terms of that
+ * LICENSE.
+ * Phil Shafer, July 2015
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+#include "xo.h"
+
+int
+main (int argc, char **argv)
+{
+    struct item {
+	const char *i_title;
+	int i_count;
+    };
+    struct item list[] = {
+	{ "gum", 1412 },
+	{ "rope", 85 },
+	{ "ladder", 0 },
+	{ "bolt", 4123 },
+	{ "water", 17 },
+	{ NULL, 0 }
+    };
+    struct item *ip;
+    int i;
+    
+    argc = xo_parse_args(argc, argv);
+    if (argc < 0)
+	return 1;
+
+    for (argc = 1; argv[argc]; argc++) {
+	if (strcmp(argv[argc], "xml") == 0)
+	    xo_set_style(NULL, XO_STYLE_XML);
+	else if (strcmp(argv[argc], "json") == 0)
+	    xo_set_style(NULL, XO_STYLE_JSON);
+	else if (strcmp(argv[argc], "text") == 0)
+	    xo_set_style(NULL, XO_STYLE_TEXT);
+	else if (strcmp(argv[argc], "html") == 0)
+	    xo_set_style(NULL, XO_STYLE_HTML);
+	else if (strcmp(argv[argc], "pretty") == 0)
+	    xo_set_flags(NULL, XOF_PRETTY);
+	else if (strcmp(argv[argc], "xpath") == 0)
+	    xo_set_flags(NULL, XOF_XPATH);
+	else if (strcmp(argv[argc], "info") == 0)
+	    xo_set_flags(NULL, XOF_INFO);
+        else if (strcmp(argv[argc], "error") == 0) {
+            close(-1);
+            xo_err(1, "error detected");
+        }
+    }
+
+    xo_set_flags(NULL, XOF_KEYS);
+    xo_set_program("test");
+
+    xo_open_container_h(NULL, "top");
+
+    xo_open_container("data");
+    xo_open_container("contents");
+    xo_open_list("item");
+
+    xo_emit("{T:Item/%-10s}{T:Count/%12s}\n");
+
+    for (ip = list; ip->i_title; ip++) {
+	xo_open_instance("item");
+
+	xo_emit("{k:name/%-10s/%s}{n:count/%12u/%u}\n",
+		ip->i_title, ip->i_count);
+
+	xo_close_instance("item");
+    }
+
+    xo_close_list("item");
+    xo_close_container("contents");
+    xo_close_container("data");
+
+    xo_emit("\n\n");
+
+    xo_open_container("data");
+    xo_open_container("contents");
+
+    xo_emit("{T:Item/%-10s}{T:Count/%12s}\n");
+
+    for (ip = list; ip->i_title; ip++) {
+	xo_open_instance("item");
+
+	xo_emit("{k:name/%-10s/%s}{n:count/%12u/%u}\n",
+		ip->i_title, ip->i_count);
+    }
+
+    xo_close_container("data");
+
+    xo_emit("\n\n");
+
+    xo_open_container("data");
+    xo_open_marker("m1");
+    xo_open_container("contents");
+
+    xo_emit("{T:Item/%-10s}{T:Count/%12s}\n");
+
+    for (ip = list; ip->i_title; ip++) {
+	xo_open_instance("item");
+
+	xo_emit("{k:name/%-10s/%s}{n:count/%12u/%u}\n",
+		ip->i_title, ip->i_count);
+    }
+
+    xo_close_container("data");	/* Should be a noop */
+    xo_emit("{:test}", "one");
+
+    xo_close_marker("m1");
+    xo_close_container("data");	/* Should be a noop */
+
+    xo_emit("\n\n");
+
+    xo_open_container("data");
+    xo_open_marker("m1");
+    xo_open_container("contents");
+
+    xo_emit("{T:Item/%-10s}{T:Count/%12s}\n");
+
+    for (ip = list; ip->i_title; ip++) {
+	xo_open_instance("item");
+
+	xo_emit("{k:name/%-10s/%s}{n:count/%12u/%u}\n",
+		ip->i_title, ip->i_count);
+	
+	xo_open_marker("m2");
+	for (i = 0; i < 3; i++) {
+	    xo_open_instance("sub");
+	    xo_emit("{Lwc:/Name}{:name/%d} + 1 = {:next/%d}\n", i, i + 1);
+	    xo_close_container("data");
+	}
+	xo_close_marker("m2");
+	xo_emit("{Lwc:/Last}{:last/%d}\n", i);
+    }
+
+    xo_close_container("data");	/* Should be a noop */
+    xo_emit("{:test}", "one");
+
+    xo_emit("\n\n");
+
+    xo_close_container_h(NULL, "top");
+
+    xo_finish();
+
+    return 0;
+}
diff --git a/contrib/libxo/tests/core/test_09.c b/contrib/libxo/tests/core/test_09.c
new file mode 100644
index 00000000000..a612a647d7f
--- /dev/null
+++ b/contrib/libxo/tests/core/test_09.c
@@ -0,0 +1,114 @@
+/*
+ * Copyright (c) 2015, Juniper Networks, Inc.
+ * All rights reserved.
+ * This SOFTWARE is licensed under the LICENSE provided in the
+ * ../Copyright file. By downloading, installing, copying, or otherwise
+ * using the SOFTWARE, you agree to be bound by the terms of that
+ * LICENSE.
+ * Phil Shafer, July 2015
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+#include <errno.h>
+
+#include "xo.h"
+
+int
+main (int argc, char **argv)
+{
+    struct item {
+	const char *i_title;
+	int i_count;
+    };
+    struct item list[] = {
+	{ "gum", 1412 },
+	{ "rope", 85 },
+	{ "ladder", 0 },
+	{ "bolt", 4123 },
+	{ "water", 17 },
+	{ NULL, 0 }
+    };
+    struct item *ip;
+    int i;
+    
+    argc = xo_parse_args(argc, argv);
+    if (argc < 0)
+	return 1;
+
+    for (argc = 1; argv[argc]; argc++) {
+	if (strcmp(argv[argc], "xml") == 0)
+	    xo_set_style(NULL, XO_STYLE_XML);
+	else if (strcmp(argv[argc], "json") == 0)
+	    xo_set_style(NULL, XO_STYLE_JSON);
+	else if (strcmp(argv[argc], "text") == 0)
+	    xo_set_style(NULL, XO_STYLE_TEXT);
+	else if (strcmp(argv[argc], "html") == 0)
+	    xo_set_style(NULL, XO_STYLE_HTML);
+	else if (strcmp(argv[argc], "pretty") == 0)
+	    xo_set_flags(NULL, XOF_PRETTY);
+	else if (strcmp(argv[argc], "xpath") == 0)
+	    xo_set_flags(NULL, XOF_XPATH);
+	else if (strcmp(argv[argc], "info") == 0)
+	    xo_set_flags(NULL, XOF_INFO);
+        else if (strcmp(argv[argc], "error") == 0) {
+            close(-1);
+            xo_err(1, "error detected");
+        }
+    }
+
+    xo_set_flags(NULL, XOF_KEYS);
+    xo_set_program("test");
+
+    xo_open_container_h(NULL, "top");
+
+    xo_open_container("data");
+    xo_open_container("contents");
+
+    xo_emit("{T:Item/%-10s}{T:Count/%12s}\n");
+
+    for (ip = list; ip->i_title; ip++) {
+	xo_emit("Name: {l:name/%-10s/%s}\n", ip->i_title);
+    }
+
+    xo_close_container("contents");
+
+    xo_emit("\n\n");
+    xo_open_container("contents");
+
+    xo_emit("{T:Item/%-10s}{T:Count/%12s}\n");
+
+    for (ip = list; ip->i_title; ip++) {
+	xo_emit("Name: {l:item/%-10s/%s}\n", ip->i_title);
+    }
+
+    xo_close_container("contents");
+
+    xo_emit("\n\n");
+
+    xo_open_container("contents");
+    xo_emit("{T:Test/%-10s}{T:Three/%12s}\n");
+
+    xo_open_list("item");
+    for (ip = list; ip->i_title; ip++) {
+	xo_emit("Name: {l:item/%-10s/%s}\n", ip->i_title);
+    }
+    xo_emit("{Lwc:/Total:}{:total}\n", "six");
+
+    xo_emit("{:one}", "one");
+    xo_emit("{l:two}", "two");
+    xo_emit("{:three}", "three");
+
+
+    xo_close_container("contents");
+
+    xo_emit("\n\n");
+
+    xo_close_container_h(NULL, "top");
+
+    xo_finish();
+
+    return 0;
+}
diff --git a/contrib/libxo/xo/xo.1 b/contrib/libxo/xo/xo.1
index 1833b0a4559..12fc959481c 100644
--- a/contrib/libxo/xo/xo.1
+++ b/contrib/libxo/xo/xo.1
@@ -7,27 +7,29 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
-.Dt LIBXO 3
+.Dd December 4, 2014
+.Dt XO 1
 .Os
 .Sh NAME
 .Nm xo
 .Nd emit formatted output based on format string and arguments
 .Sh SYNOPSIS
-.Nm xo
+.Nm
 .Op Fl options
 .Op Ar argument...
 .Sh DESCRIPTION
 The
-.Nm xo
+.Nm
 utility allows command line access to the functionality of
 the
-.Em libxo
-library.  Using
-.Nm xo ,
+.Nm libxo
+library.
+Using
+.Nm ,
 shell scripts can emit
 .Em XML ,
-.Em JSON , or
+.Em JSON ,
+or
 .Em HTML
 using the same commands that emit text output.
 .Pp
@@ -63,11 +65,11 @@ Wrap output in a set of containers
 .It Fl "-xml OR -X"
 Generate XML output
 .It Fl "-xpath"
-Add XPath data to HTML output);
+Add XPath data to HTML output
 .El
 .Pp
 The
-.Nm xo
+.Nm
 utility accepts a format string suitable for
 .Xr xo_emit 3
 and a set of zero or more arguments used to supply data for that string.
@@ -95,7 +97,8 @@ and a set of zero or more arguments used to supply data for that string.
 The
 .Fl "-wrap <path>"
 option can be used to wrap emitted content in a
-specific hierarchy.  The path is a set of hierarchical names separated
+specific hierarchy.
+The path is a set of hierarchical names separated
 by the '/' character.
 .Bd -literal -offset indent
     xo --wrap top/a/b/c '{:tag}' value
@@ -128,11 +131,14 @@ and
 .Fl "\-close <path>"
 can be used to emit
 hierarchical information without the matching close and open
-tag.  This allows a shell script to emit open tags, data, and
-then close tags.  The
+tag.
+This allows a shell script to emit open tags, data, and
+then close tags.
+The
 .Fl \-depth
 option may be used to set the
-depth for indentation.  The
+depth for indentation.
+The
 .Fl "\-leading-xpath"
 may be used to
 prepend data to the XPath values used for HTML output style.
@@ -154,37 +160,39 @@ prepend data to the XPath values used for HTML output style.
       }
     }
 .Ed
-.Pp
 .Sh EXAMPLE
-.Bd -literal -offset indent
+.Bd -literal
   % xo 'The {:product} is {:status}\n' stereo "in route"
   The stereo is in route
   % xo -p -X 'The {:product} is {:status}\n' stereo "in route"
   <product>stereo</product>
   <status>in route</status>
 .Ed
-.Pp
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
 .Sh SEE ALSO
+.Xr libxo 3 ,
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 11.0.
+.Nm libxo
+library was added in
+.Fx 11.0 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/contrib/libxo/xo/xo.c b/contrib/libxo/xo/xo.c
index 698d9d65309..c364539091a 100644
--- a/contrib/libxo/xo/xo.c
+++ b/contrib/libxo/xo/xo.c
@@ -362,8 +362,7 @@ main (int argc UNUSED, char **argv)
     }
 
     xo_set_formatter(NULL, formatter, checkpoint);
-    xo_set_flags(NULL, XOF_NO_VA_ARG);
-    xo_set_flags(NULL, XOF_NO_TOP);
+    xo_set_flags(NULL, XOF_NO_VA_ARG | XOF_NO_TOP | XOF_NO_CLOSE);
 
     fmt = *argv++;
     if (opt_opener == NULL && opt_closer == NULL && fmt == NULL) {
diff --git a/contrib/libxo/xolint/Makefile.am b/contrib/libxo/xolint/Makefile.am
index 0f8ef62613a..a847e72c784 100644
--- a/contrib/libxo/xolint/Makefile.am
+++ b/contrib/libxo/xolint/Makefile.am
@@ -11,4 +11,4 @@ man_MANS = xolint.1
 EXTRA_DIST = xolint.1 xolint.pl
 
 install-exec-hook:
-	install ${srcdir}/xolint.pl ${bindir}/xolint
+	install ${srcdir}/xolint.pl ${DESTDIR}${bindir}/xolint
diff --git a/contrib/libxo/xolint/xolint.1 b/contrib/libxo/xolint/xolint.1
index b7ed130a4fe..fcf7bcdeec7 100644
--- a/contrib/libxo/xolint/xolint.1
+++ b/contrib/libxo/xolint/xolint.1
@@ -7,12 +7,13 @@
 .\" # LICENSE.
 .\" # Phil Shafer, July 2014
 .\" 
-.Dd July, 2014
-.Dt LIBXO 3
+.Dd December 4, 2014
+.Dt XOLINT 1
 .Os
 .Sh NAME
 .Nm xolint
-.Nd detect errors in programs using xo_emit
+.Nd detect errors in programs using
+.Xr xo_emit 3
 .Sh SYNOPSIS
 .Nm xolint
 .Op Fl c
@@ -25,34 +26,45 @@
 .Op Fl "X"
 .Op Ar files...
 .Sh DESCRIPTION
-xolint is a tool for reporting common mistakes in format strings
-in source code that invokes xo_emit().  It allows these errors
+.Nm
+is a tool for reporting common mistakes in format strings
+in source code that invokes
+.Xr xo_emit 3 .
+It allows these errors
 to be diagnosed at build time, rather than waiting until runtime.
 .Pp
-xolint takes the one or more C files as arguments, and reports
-and errors, warning, or informational messages as needed.
+.Nm
+takes one or more C files as arguments, and reports
+error, warning, or informational messages as needed.
 .Bl -tag -width "C <flags>"
 .It Fl c
 Invoke 'cpp' against the input file
 .It Fl "C <flags>"
-Flags that are passed to 'cpp
+Flags that are passed to 'cpp'
 .It Fl "d"
 Enable debug output
 .It Fl "D"
-Generate documentation for all xolint messages
+Generate documentation for all
+.Nm
+messages
 .It Fl "I"
-Generate a table of xo_info_t structures.
+Generate a table of
+.Dv xo_info_t
+structures.
 .It Fl "p"
 Print the offending lines after the error message is displayed
 .It Fl "V"
 Do not report errors, but instead print a complete list of
-all field names, sorted alphabetically.  The output can help spot
+all field names, sorted alphabetically.
+The output can help spot
 inconsistencies and spelling errors.
 .It Fl "X"
-Extract samples from xolint, suitable for internal testing.
+Extract samples from
+.Nm ,
+suitable for internal testing.
 .El
 .Pp
-Output message contain the source filename and line number, the
+The output message contains the source filename and line number, the
 class of the message, the message, and, if
 .Fl p
 is given, the
@@ -62,28 +74,31 @@ line that contains the error:
     xolint.c: 16: error: anchor format should be "%d"
     16         xo_emit("{[:/%s}");
 .Ed
-.Pp
 .Sh ADDITIONAL DOCUMENTATION
-.Pp
 Complete documentation can be found on github:
 .Bd -literal -offset indent
 http://juniper.github.io/libxo/libxo-manual.html
 .Ed
 .Pp
-libxo lives on github as:
+.Nm libxo
+lives on github as:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo
 .Ed
 .Pp
-The latest release of libxo is available at:
+The latest release of
+.Nm libxo
+is available at:
 .Bd -literal -offset indent
 https://github.com/Juniper/libxo/releases
 .Ed
 .Sh SEE ALSO
+.Xr libxo 3 ,
 .Xr xo_emit 3
 .Sh HISTORY
 The
-.Fa libxo
-library was added in FreeBSD 10.1.
+.Nm libxo
+library was added in
+.Fx 10.1 .
 .Sh AUTHOR
 Phil Shafer
diff --git a/lib/libxo/Makefile b/lib/libxo/Makefile
index 770db20c125..cc0e72aa4a9 100644
--- a/lib/libxo/Makefile
+++ b/lib/libxo/Makefile
@@ -21,6 +21,7 @@ MAN+=	xo_attr.3 \
 	xo_create.3 \
 	xo_emit.3 \
 	xo_err.3 \
+	xo_error.3 \
 	xo_finish.3 \
 	xo_flush.3 \
 	xo_no_setlocale.3 \

From 9a53618ab27141fe3d74b4bc7faf765162de0e31 Mon Sep 17 00:00:00 2001
From: "Pedro F. Giffuni" <pfg@FreeBSD.org>
Date: Mon, 19 Jan 2015 03:30:45 +0000
Subject: [PATCH 076/279] ext2: Garbage-collect some unused variables

Reported by:	clang static analysis
MFC after:	2 weeks
---
 sys/fs/ext2fs/ext2_bmap.c    | 2 --
 sys/fs/ext2fs/ext2_extents.c | 2 --
 sys/fs/ext2fs/ext2_htree.c   | 4 ++--
 sys/fs/ext2fs/ext2_inode.c   | 2 --
 sys/fs/ext2fs/ext2_lookup.c  | 1 -
 sys/fs/ext2fs/ext2_vfsops.c  | 2 --
 sys/fs/ext2fs/ext2_vnops.c   | 3 ---
 7 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/sys/fs/ext2fs/ext2_bmap.c b/sys/fs/ext2fs/ext2_bmap.c
index 12f9560d93b..d144e926f0b 100644
--- a/sys/fs/ext2fs/ext2_bmap.c
+++ b/sys/fs/ext2fs/ext2_bmap.c
@@ -145,7 +145,6 @@ ext2_bmaparray(struct vnode *vp, daddr_t bn, daddr_t *bnp, int *runp, int *runb)
 	struct buf *bp;
 	struct ext2mount *ump;
 	struct mount *mp;
-	struct vnode *devvp;
 	struct indir a[NIADDR+1], *ap;
 	daddr_t daddr;
 	e2fs_lbn_t metalbn;
@@ -156,7 +155,6 @@ ext2_bmaparray(struct vnode *vp, daddr_t bn, daddr_t *bnp, int *runp, int *runb)
 	ip = VTOI(vp);
 	mp = vp->v_mount;
 	ump = VFSTOEXT2(mp);
-	devvp = ump->um_devvp;
 
 	bsize = EXT2_BLOCK_SIZE(ump->um_e2fs);
 
diff --git a/sys/fs/ext2fs/ext2_extents.c b/sys/fs/ext2fs/ext2_extents.c
index 26e6a222cc9..68704bb8850 100644
--- a/sys/fs/ext2fs/ext2_extents.c
+++ b/sys/fs/ext2fs/ext2_extents.c
@@ -131,13 +131,11 @@ struct ext4_extent_path *
 ext4_ext_find_extent(struct m_ext2fs *fs, struct inode *ip,
 		     daddr_t lbn, struct ext4_extent_path *path)
 {
-	struct vnode *vp;
 	struct ext4_extent_header *ehp;
 	uint16_t i;
 	int error, size;
 	daddr_t nblk;
 
-	vp = ITOV(ip);
 	ehp = (struct ext4_extent_header *)(char *)ip->i_db;
 
 	if (ehp->eh_magic != EXT4_EXT_MAGIC)
diff --git a/sys/fs/ext2fs/ext2_htree.c b/sys/fs/ext2fs/ext2_htree.c
index 33e4c0f7d42..70a2f4786e9 100644
--- a/sys/fs/ext2fs/ext2_htree.c
+++ b/sys/fs/ext2fs/ext2_htree.c
@@ -395,7 +395,7 @@ ext2_htree_append_block(struct vnode *vp, char *data,
 	int error;
 
 	cursize = roundup(dp->i_size, blksize);
-	newsize = roundup(dp->i_size, blksize) + blksize;
+	newsize = cursize + blksize;
 
 	auio.uio_offset = cursize;
 	auio.uio_resid = blksize;
@@ -771,7 +771,7 @@ ext2_htree_add_entry(struct vnode *dvp, struct ext2fs_direct_2 *entry,
 		dst_node->h_fake_dirent.e2d_reclen = blksize;
 
 		cursize = roundup(ip->i_size, blksize);
-		dirsize = roundup(ip->i_size, blksize) + blksize;
+		dirsize = cursize + blksize;
 		blknum = dirsize / blksize - 1;
 
 		error = ext2_htree_append_block(dvp, newidxblock,
diff --git a/sys/fs/ext2fs/ext2_inode.c b/sys/fs/ext2fs/ext2_inode.c
index 43b0fc9d144..2fc2192d05b 100644
--- a/sys/fs/ext2fs/ext2_inode.c
+++ b/sys/fs/ext2fs/ext2_inode.c
@@ -115,7 +115,6 @@ ext2_truncate(struct vnode *vp, off_t length, int flags, struct ucred *cred,
 	struct inode *oip;
 	int32_t bn, lbn, lastiblock[NIADDR], indir_lbn[NIADDR];
 	uint32_t oldblks[NDADDR + NIADDR], newblks[NDADDR + NIADDR];
-	struct bufobj *bo;
 	struct m_ext2fs *fs;
 	struct buf *bp;
 	int offset, size, level;
@@ -124,7 +123,6 @@ ext2_truncate(struct vnode *vp, off_t length, int flags, struct ucred *cred,
 	off_t osize;
 
 	oip = VTOI(ovp);
-	bo = &ovp->v_bufobj;
 
 	ASSERT_VOP_LOCKED(vp, "ext2_truncate");	
 
diff --git a/sys/fs/ext2fs/ext2_lookup.c b/sys/fs/ext2fs/ext2_lookup.c
index e59b60614c4..f9845bd3e65 100644
--- a/sys/fs/ext2fs/ext2_lookup.c
+++ b/sys/fs/ext2fs/ext2_lookup.c
@@ -343,7 +343,6 @@ restart:
 	 * we watch for a place to put the new file in
 	 * case it doesn't already exist.
 	 */
-	ino = 0;
 	i_diroff = dp->i_diroff;
 	ss.slotstatus = FOUND;
 	ss.slotfreespace = ss.slotsize = ss.slotneeded = 0;
diff --git a/sys/fs/ext2fs/ext2_vfsops.c b/sys/fs/ext2fs/ext2_vfsops.c
index b97356d072a..6879e42ac95 100644
--- a/sys/fs/ext2fs/ext2_vfsops.c
+++ b/sys/fs/ext2fs/ext2_vfsops.c
@@ -902,7 +902,6 @@ ext2_vget(struct mount *mp, ino_t ino, int flags, struct vnode **vpp)
 	struct ext2mount *ump;
 	struct buf *bp;
 	struct vnode *vp;
-	struct cdev *dev;
 	struct thread *td;
 	int i, error;
 	int used_blocks;
@@ -913,7 +912,6 @@ ext2_vget(struct mount *mp, ino_t ino, int flags, struct vnode **vpp)
 		return (error);
 
 	ump = VFSTOEXT2(mp);
-	dev = ump->um_dev;
 	ip = malloc(sizeof(struct inode), M_EXT2NODE, M_WAITOK | M_ZERO);
 
 	/* Allocate a new vnode/inode. */
diff --git a/sys/fs/ext2fs/ext2_vnops.c b/sys/fs/ext2fs/ext2_vnops.c
index 5baca5aacba..dd30aea279c 100644
--- a/sys/fs/ext2fs/ext2_vnops.c
+++ b/sys/fs/ext2fs/ext2_vnops.c
@@ -1223,7 +1223,6 @@ ext2_rmdir(struct vop_rmdir_args *ap)
 	 *  the current directory and thus be
 	 *  non-empty.)
 	 */
-	error = 0;
 	if (ip->i_nlink != 2 || !ext2_dirempty(ip, dp->i_number, cnp->cn_cred)) {
 		error = ENOTEMPTY;
 		goto out;
@@ -1329,12 +1328,10 @@ ext2_strategy(struct vop_strategy_args *ap)
 {
 	struct buf *bp = ap->a_bp;
 	struct vnode *vp = ap->a_vp;
-	struct inode *ip;
 	struct bufobj *bo;
 	daddr_t blkno;
 	int error;
 
-	ip = VTOI(vp);
 	if (vp->v_type == VBLK || vp->v_type == VCHR)
 		panic("ext2_strategy: spec");
 	if (bp->b_blkno == bp->b_lblkno) {

From 1d2c1366bfbed3599a134c76ed5d3cbdd7f47811 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Mon, 19 Jan 2015 04:56:17 +0000
Subject: [PATCH 077/279] For armv6 builds, add -mfloat-abi=softfp.  This tells
 the compiler it can use floating point hardware instructions (because all
 armv6/7 systems we support have fp hardware), but it passes args using a
 soft-float compatible ABI.  This should give noticible performance
 improvement (but not as much as using the armv6hf arch).

---
 share/mk/bsd.cpu.mk | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/share/mk/bsd.cpu.mk b/share/mk/bsd.cpu.mk
index a3b9c1fd016..da5f5f2d38e 100644
--- a/share/mk/bsd.cpu.mk
+++ b/share/mk/bsd.cpu.mk
@@ -251,6 +251,10 @@ MACHINE_CPU = v9 ultrasparc ultrasparc3
 CFLAGS += -G0
 .endif
 
+.if ${MACHINE_ARCH} == "armv6"
+_CPUCFLAGS += -mfloat-abi=softfp
+.endif
+
 # NB: COPTFLAGS is handled in /usr/src/sys/conf/kern.pre.mk
 
 .if !defined(NO_CPU_CFLAGS)

From c5e8bb4f2ecb5972f60bb586cf56f062bb98ec1d Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Mon, 19 Jan 2015 05:14:07 +0000
Subject: [PATCH 078/279] Provide a tunable (machdep.moea64_bpvo_pool_size) to
 set the bootstrap PVO pool size. The default errs on the exceedingly large
 side, so absent any intelligent automatic tuning, at least let the user set
 it to save RAM on memory-constrained systems.

MFC after:	2 weeks
---
 sys/powerpc/aim/mmu_oea64.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c
index 1eb4b83cdf7..84ba8bbc1f9 100644
--- a/sys/powerpc/aim/mmu_oea64.c
+++ b/sys/powerpc/aim/mmu_oea64.c
@@ -221,9 +221,10 @@ struct	pvo_head *moea64_pvo_table;		/* pvo entries by pteg index */
 uma_zone_t	moea64_upvo_zone; /* zone for pvo entries for unmanaged pages */
 uma_zone_t	moea64_mpvo_zone; /* zone for pvo entries for managed pages */
 
-#define	BPVO_POOL_SIZE	327680
 static struct	pvo_entry *moea64_bpvo_pool;
 static int	moea64_bpvo_pool_index = 0;
+static int	moea64_bpvo_pool_size = 327680;
+TUNABLE_INT("machdep.moea64_bpvo_pool_size", &moea64_bpvo_pool_size);
 SYSCTL_INT(_machdep, OID_AUTO, moea64_allocated_bpvo_entries, CTLFLAG_RD, 
     &moea64_bpvo_pool_index, 0, "");
 
@@ -647,7 +648,7 @@ moea64_setup_direct_map(mmu_t mmup, vm_offset_t kernelstart,
 		off = (vm_offset_t)(moea64_pvo_table);
 		for (pa = off; pa < off + size; pa += PAGE_SIZE) 
 			moea64_kenter(mmup, pa, pa);
-		size = BPVO_POOL_SIZE*sizeof(struct pvo_entry);
+		size = moea64_bpvo_pool_size*sizeof(struct pvo_entry);
 		off = (vm_offset_t)(moea64_bpvo_pool);
 		for (pa = off; pa < off + size; pa += PAGE_SIZE) 
 		moea64_kenter(mmup, pa, pa);
@@ -815,7 +816,7 @@ moea64_mid_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
 	 * Initialise the unmanaged pvo pool.
 	 */
 	moea64_bpvo_pool = (struct pvo_entry *)moea64_bootstrap_alloc(
-		BPVO_POOL_SIZE*sizeof(struct pvo_entry), 0);
+		moea64_bpvo_pool_size*sizeof(struct pvo_entry), 0);
 	moea64_bpvo_pool_index = 0;
 
 	/*
@@ -2282,10 +2283,10 @@ moea64_pvo_enter(mmu_t mmu, pmap_t pm, uma_zone_t zone,
 	 * If we aren't overwriting a mapping, try to allocate.
 	 */
 	if (bootstrap) {
-		if (moea64_bpvo_pool_index >= BPVO_POOL_SIZE) {
+		if (moea64_bpvo_pool_index >= moea64_bpvo_pool_size) {
 			panic("moea64_enter: bpvo pool exhausted, %d, %d, %zd",
-			      moea64_bpvo_pool_index, BPVO_POOL_SIZE, 
-			      BPVO_POOL_SIZE * sizeof(struct pvo_entry));
+			      moea64_bpvo_pool_index, moea64_bpvo_pool_size, 
+			      moea64_bpvo_pool_size * sizeof(struct pvo_entry));
 		}
 		pvo = &moea64_bpvo_pool[moea64_bpvo_pool_index];
 		moea64_bpvo_pool_index++;

From 2e5b60079b7d8c3ca68f1390cd90f305e651f8d3 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Mon, 19 Jan 2015 06:10:01 +0000
Subject: [PATCH 079/279] Expect :overflow to fail with FreeBSD's expr as it
 doesn't have stringent overflow checks like NetBSD's expr does

MFC after: 3 days
PR: 196867
---
 contrib/netbsd-tests/bin/expr/t_expr.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/contrib/netbsd-tests/bin/expr/t_expr.sh b/contrib/netbsd-tests/bin/expr/t_expr.sh
index 97a559b1d97..ef99b021cd3 100755
--- a/contrib/netbsd-tests/bin/expr/t_expr.sh
+++ b/contrib/netbsd-tests/bin/expr/t_expr.sh
@@ -54,6 +54,9 @@ overflow_head() {
 	atf_set "descr" "Test overflow cases"
 }
 overflow_body() {
+	# Begin FreeBSD
+	atf_expect_fail "FreeBSD's expr does not check overflow to the same degree NetBSD's expr does; see bug 196867 for more details"
+	# End FreeBSD
 	test_expr '4611686018427387904 + 4611686018427387903' \
 	          '9223372036854775807'
 	test_expr '4611686018427387904 + 4611686018427387904' \

From e15138e5740afa089b01612329b30b82e236737a Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Mon, 19 Jan 2015 06:13:07 +0000
Subject: [PATCH 080/279] Integrate contrib/netbsd-tests/bin/expr into the
 build/kyua as bin/expr/tests

MFC after: 1 week
Sponsored by: EMC / Isilon Storage Division
---
 bin/expr/Makefile        |  6 ++++++
 bin/expr/tests/Makefile  | 16 ++++++++++++++++
 etc/mtree/BSD.tests.dist |  2 ++
 3 files changed, 24 insertions(+)
 create mode 100644 bin/expr/tests/Makefile

diff --git a/bin/expr/Makefile b/bin/expr/Makefile
index b86cf6686ab..a1f6cbf2109 100644
--- a/bin/expr/Makefile
+++ b/bin/expr/Makefile
@@ -1,9 +1,15 @@
 # $FreeBSD$
 
+.include <src.opts.mk>
+
 PROG=	expr
 SRCS=	expr.y
 YFLAGS=
 
 NO_WMISSING_VARIABLE_DECLARATIONS=
 
+.if ${MK_TESTS} != "no"
+SUBDIR+=	tests
+.endif
+
 .include <bsd.prog.mk>
diff --git a/bin/expr/tests/Makefile b/bin/expr/tests/Makefile
new file mode 100644
index 00000000000..80c130c3df2
--- /dev/null
+++ b/bin/expr/tests/Makefile
@@ -0,0 +1,16 @@
+# $FreeBSD$
+
+OBJTOP=		${.OBJDIR}/../../..
+SRCTOP=		${.CURDIR}/../../..
+TESTSRC=	${SRCTOP}/contrib/netbsd-tests/bin/expr
+
+TESTSDIR=	${TESTSBASE}/bin/expr
+
+NETBSD_ATF_TESTS_SH=	expr_test
+
+ATF_TESTS_SH_SED_expr_test+=	-e 's/eval expr/eval expr --/g'
+ATF_TESTS_SH_SED_expr_test+=	-e 's/"expr: integer overflow or underflow occurred for operation.*"/"expr: overflow"/g'
+
+.include <netbsd-tests.test.mk>
+
+.include <bsd.test.mk>
diff --git a/etc/mtree/BSD.tests.dist b/etc/mtree/BSD.tests.dist
index cbd65dc8a6a..c43f1aea48d 100644
--- a/etc/mtree/BSD.tests.dist
+++ b/etc/mtree/BSD.tests.dist
@@ -12,6 +12,8 @@
         ..
         date
         ..
+        expr
+        ..
         mv
         ..
         pax

From 009e2acba6c14814b896cc0527bda97697d9c8f5 Mon Sep 17 00:00:00 2001
From: Neel Natu <neel@FreeBSD.org>
Date: Mon, 19 Jan 2015 06:51:04 +0000
Subject: [PATCH 081/279] Fix a bug in libvmmapi 'vm_copy_setup()' where it
 would return success even if the 'gpa' was in the guest MMIO region. This
 would manifest as a segmentation fault in 'vm_map_copyin()' or
 'vm_map_copyout()' because 'vm_map_gpa()' would return NULL for this 'gpa'.

Fix this by calling 'vm_map_gpa()' in 'vm_copy_setup' and returning a failure
if the 'gpa' cannot be mapped. This matches the behavior of 'vm_copy_setup()'
in vmm.ko.

MFC after:	1 week
---
 lib/libvmmapi/vmmapi.c | 22 +++++++++++++++-------
 lib/libvmmapi/vmmapi.h |  2 ++
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/lib/libvmmapi/vmmapi.c b/lib/libvmmapi/vmmapi.c
index 5660f3b7bc5..98288768866 100644
--- a/lib/libvmmapi/vmmapi.c
+++ b/lib/libvmmapi/vmmapi.c
@@ -987,6 +987,7 @@ int
 vm_copy_setup(struct vmctx *ctx, int vcpu, struct vm_guest_paging *paging,
     uint64_t gla, size_t len, int prot, struct iovec *iov, int iovcnt)
 {
+	void *va;
 	uint64_t gpa;
 	int error, fault, i, n, off;
 
@@ -1006,7 +1007,11 @@ vm_copy_setup(struct vmctx *ctx, int vcpu, struct vm_guest_paging *paging,
 		off = gpa & PAGE_MASK;
 		n = min(len, PAGE_SIZE - off);
 
-		iov->iov_base = (void *)gpa;
+		va = vm_map_gpa(ctx, gpa, n);
+		if (va == NULL)
+			return (-1);
+
+		iov->iov_base = va;
 		iov->iov_len = n;
 		iov++;
 		iovcnt--;
@@ -1017,20 +1022,25 @@ vm_copy_setup(struct vmctx *ctx, int vcpu, struct vm_guest_paging *paging,
 	return (0);
 }
 
+void
+vm_copy_teardown(struct vmctx *ctx, int vcpu, struct iovec *iov, int iovcnt)
+{
+
+	return;
+}
+
 void
 vm_copyin(struct vmctx *ctx, int vcpu, struct iovec *iov, void *vp, size_t len)
 {
 	const char *src;
 	char *dst;
-	uint64_t gpa;
 	size_t n;
 
 	dst = vp;
 	while (len) {
 		assert(iov->iov_len);
-		gpa = (uint64_t)iov->iov_base;
 		n = min(len, iov->iov_len);
-		src = vm_map_gpa(ctx, gpa, n);
+		src = iov->iov_base;
 		bcopy(src, dst, n);
 
 		iov++;
@@ -1045,15 +1055,13 @@ vm_copyout(struct vmctx *ctx, int vcpu, const void *vp, struct iovec *iov,
 {
 	const char *src;
 	char *dst;
-	uint64_t gpa;
 	size_t n;
 
 	src = vp;
 	while (len) {
 		assert(iov->iov_len);
-		gpa = (uint64_t)iov->iov_base;
 		n = min(len, iov->iov_len);
-		dst = vm_map_gpa(ctx, gpa, n);
+		dst = iov->iov_base;
 		bcopy(src, dst, n);
 
 		iov++;
diff --git a/lib/libvmmapi/vmmapi.h b/lib/libvmmapi/vmmapi.h
index 8634762d006..06b29302640 100644
--- a/lib/libvmmapi/vmmapi.h
+++ b/lib/libvmmapi/vmmapi.h
@@ -137,6 +137,8 @@ void	vm_copyin(struct vmctx *ctx, int vcpu, struct iovec *guest_iov,
 	    void *host_dst, size_t len);
 void	vm_copyout(struct vmctx *ctx, int vcpu, const void *host_src,
 	    struct iovec *guest_iov, size_t len);
+void	vm_copy_teardown(struct vmctx *ctx, int vcpu, struct iovec *iov,
+	    int iovcnt);
 
 /* RTC */
 int	vm_rtc_write(struct vmctx *ctx, int offset, uint8_t value);

From 753463535984e9aba0b3d92f088b384ac28dce31 Mon Sep 17 00:00:00 2001
From: Neel Natu <neel@FreeBSD.org>
Date: Mon, 19 Jan 2015 06:53:31 +0000
Subject: [PATCH 082/279] MOVS instruction emulation.

These instructions are emitted by 'bus_space_read_region()' when accessing
MMIO regions.

Since MOVS can be used with a repeat prefix start decoding the REPZ and
REPNZ prefixes. Also start decoding the segment override prefix since MOVS
allows overriding the source operand segment register.

Tested by:	tychon
MFC after:	1 week
---
 sys/amd64/include/vmm.h              |   6 +-
 sys/amd64/vmm/vmm_instruction_emul.c | 271 ++++++++++++++++++++++++++-
 2 files changed, 272 insertions(+), 5 deletions(-)

diff --git a/sys/amd64/include/vmm.h b/sys/amd64/include/vmm.h
index 253654d4375..cf7f5bcae0c 100644
--- a/sys/amd64/include/vmm.h
+++ b/sys/amd64/include/vmm.h
@@ -446,8 +446,11 @@ struct vie {
 			rex_x:1,
 			rex_b:1,
 			rex_present:1,
+			repz_present:1,		/* REP/REPE/REPZ prefix */
+			repnz_present:1,	/* REPNE/REPNZ prefix */
 			opsize_override:1,	/* Operand size override */
-			addrsize_override:1;	/* Address size override */
+			addrsize_override:1,	/* Address size override */
+			segment_override:1;	/* Segment override */
 
 	uint8_t		mod:2,			/* ModRM byte */
 			reg:4,
@@ -463,6 +466,7 @@ struct vie {
 	uint8_t		scale;
 	int		base_register;		/* VM_REG_GUEST_xyz */
 	int		index_register;		/* VM_REG_GUEST_xyz */
+	int		segment_register;	/* VM_REG_GUEST_xyz */
 
 	int64_t		displacement;		/* optional addr displacement */
 	int64_t		immediate;		/* optional immediate operand */
diff --git a/sys/amd64/vmm/vmm_instruction_emul.c b/sys/amd64/vmm/vmm_instruction_emul.c
index d1d7173447d..3db890e9559 100644
--- a/sys/amd64/vmm/vmm_instruction_emul.c
+++ b/sys/amd64/vmm/vmm_instruction_emul.c
@@ -70,6 +70,7 @@ enum {
 	VIE_OP_TYPE_PUSH,
 	VIE_OP_TYPE_CMP,
 	VIE_OP_TYPE_POP,
+	VIE_OP_TYPE_MOVS,
 	VIE_OP_TYPE_LAST
 };
 
@@ -78,6 +79,7 @@ enum {
 #define	VIE_OP_F_IMM8		(1 << 1)  /* 8-bit immediate operand */
 #define	VIE_OP_F_MOFFSET	(1 << 2)  /* 16/32/64-bit immediate moffset */
 #define	VIE_OP_F_NO_MODRM	(1 << 3)
+#define	VIE_OP_F_NO_GLA_VERIFICATION (1 << 4)
 
 static const struct vie_op two_byte_opcodes[256] = {
 	[0xB6] = {
@@ -133,6 +135,16 @@ static const struct vie_op one_byte_opcodes[256] = {
 		.op_type = VIE_OP_TYPE_MOV,
 		.op_flags = VIE_OP_F_MOFFSET | VIE_OP_F_NO_MODRM,
 	},
+	[0xA4] = {
+		.op_byte = 0xA4,
+		.op_type = VIE_OP_TYPE_MOVS,
+		.op_flags = VIE_OP_F_NO_MODRM | VIE_OP_F_NO_GLA_VERIFICATION
+	},
+	[0xA5] = {
+		.op_byte = 0xA5,
+		.op_type = VIE_OP_TYPE_MOVS,
+		.op_flags = VIE_OP_F_NO_MODRM | VIE_OP_F_NO_GLA_VERIFICATION
+	},
 	[0xC6] = {
 		/* XXX Group 11 extended opcode - not just MOV */
 		.op_byte = 0xC6,
@@ -559,6 +571,217 @@ emulate_movx(void *vm, int vcpuid, uint64_t gpa, struct vie *vie,
 	return (error);
 }
 
+/*
+ * Helper function to calculate and validate a linear address.
+ *
+ * Returns 0 on success and 1 if an exception was injected into the guest.
+ */
+static int
+get_gla(void *vm, int vcpuid, struct vie *vie, struct vm_guest_paging *paging,
+    int opsize, int addrsize, int prot, enum vm_reg_name seg,
+    enum vm_reg_name gpr, uint64_t *gla)
+{
+	struct seg_desc desc;
+	uint64_t cr0, val, rflags;
+	int error;
+
+	error = vie_read_register(vm, vcpuid, VM_REG_GUEST_CR0, &cr0);
+	KASSERT(error == 0, ("%s: error %d getting cr0", __func__, error));
+
+	error = vie_read_register(vm, vcpuid, VM_REG_GUEST_RFLAGS, &rflags);
+	KASSERT(error == 0, ("%s: error %d getting rflags", __func__, error));
+
+	error = vm_get_seg_desc(vm, vcpuid, seg, &desc);
+	KASSERT(error == 0, ("%s: error %d getting segment descriptor %d",
+	    __func__, error, seg));
+
+	error = vie_read_register(vm, vcpuid, gpr, &val);
+	KASSERT(error == 0, ("%s: error %d getting register %d", __func__,
+	    error, gpr));
+
+	if (vie_calculate_gla(paging->cpu_mode, seg, &desc, val, opsize,
+	    addrsize, prot, gla)) {
+		if (seg == VM_REG_GUEST_SS)
+			vm_inject_ss(vm, vcpuid, 0);
+		else
+			vm_inject_gp(vm, vcpuid);
+		return (1);
+	}
+
+	if (vie_canonical_check(paging->cpu_mode, *gla)) {
+		if (seg == VM_REG_GUEST_SS)
+			vm_inject_ss(vm, vcpuid, 0);
+		else
+			vm_inject_gp(vm, vcpuid);
+		return (1);
+	}
+
+	if (vie_alignment_check(paging->cpl, opsize, cr0, rflags, *gla)) {
+		vm_inject_ac(vm, vcpuid, 0);
+		return (1);
+	}
+
+	return (0);
+}
+
+static int
+emulate_movs(void *vm, int vcpuid, uint64_t gpa, struct vie *vie,
+    struct vm_guest_paging *paging, mem_region_read_t memread,
+    mem_region_write_t memwrite, void *arg)
+{
+#ifdef _KERNEL
+	struct vm_copyinfo copyinfo[2];
+#else
+	struct iovec copyinfo[2];
+#endif
+	uint64_t dstaddr, srcaddr, val;
+	uint64_t rcx, rdi, rsi, rflags;
+	int error, opsize, seg, repeat;
+
+	opsize = (vie->op.op_byte == 0xA4) ? 1 : vie->opsize;
+	val = 0;
+	error = 0;
+
+	/*
+	 * XXX although the MOVS instruction is only supposed to be used with
+	 * the "rep" prefix some guests like FreeBSD will use "repnz" instead.
+	 *
+	 * Empirically the "repnz" prefix has identical behavior to "rep"
+	 * and the zero flag does not make a difference.
+	 */
+	repeat = vie->repz_present | vie->repnz_present;
+
+	if (repeat) {
+		error = vie_read_register(vm, vcpuid, VM_REG_GUEST_RCX, &rcx);
+		KASSERT(!error, ("%s: error %d getting rcx", __func__, error));
+
+		/*
+		 * The count register is %rcx, %ecx or %cx depending on the
+		 * address size of the instruction.
+		 */
+		if ((rcx & vie_size2mask(vie->addrsize)) == 0)
+			return (0);
+	}
+
+	/*
+	 *	Source		Destination	Comments
+	 *	--------------------------------------------
+	 * (1)  memory		memory		n/a
+	 * (2)  memory		mmio		emulated
+	 * (3)  mmio		memory		emulated
+	 * (4)  mmio		mmio		not emulated
+	 *
+	 * At this point we don't have sufficient information to distinguish
+	 * between (2), (3) and (4). We use 'vm_copy_setup()' to tease this
+	 * out because it will succeed only when operating on regular memory.
+	 *
+	 * XXX the emulation doesn't properly handle the case where 'gpa'
+	 * is straddling the boundary between the normal memory and MMIO.
+	 */
+
+	seg = vie->segment_override ? vie->segment_register : VM_REG_GUEST_DS;
+	error = get_gla(vm, vcpuid, vie, paging, opsize, vie->addrsize,
+	    PROT_READ, seg, VM_REG_GUEST_RSI, &srcaddr);
+	if (error)
+		goto done;
+
+	error = vm_copy_setup(vm, vcpuid, paging, srcaddr, opsize, PROT_READ,
+	    copyinfo, nitems(copyinfo));
+	if (error == 0) {
+		/*
+		 * case (2): read from system memory and write to mmio.
+		 */
+		vm_copyin(vm, vcpuid, copyinfo, &val, opsize);
+		vm_copy_teardown(vm, vcpuid, copyinfo, nitems(copyinfo));
+		error = memwrite(vm, vcpuid, gpa, val, opsize, arg);
+		goto done;
+	} else if (error > 0) {
+		/*
+		 * Resume guest execution to handle fault.
+		 */
+		goto done;
+	} else {
+		/*
+		 * 'vm_copy_setup()' is expected to fail for cases (3) and (4)
+		 * if 'srcaddr' is in the mmio space.
+		 */
+	}
+
+	error = get_gla(vm, vcpuid, vie, paging, opsize, vie->addrsize,
+	    PROT_WRITE, VM_REG_GUEST_ES, VM_REG_GUEST_RDI, &dstaddr);
+	if (error)
+		goto done;
+
+	error = vm_copy_setup(vm, vcpuid, paging, dstaddr, opsize,
+	    PROT_WRITE, copyinfo, nitems(copyinfo));
+	if (error == 0) {
+		/*
+		 * case (3): read from MMIO and write to system memory.
+		 *
+		 * A MMIO read can have side-effects so we commit to it
+		 * only after vm_copy_setup() is successful. If a page-fault
+		 * needs to be injected into the guest then it will happen
+		 * before the MMIO read is attempted.
+		 */
+		error = memread(vm, vcpuid, gpa, &val, opsize, arg);
+		if (error)
+			goto done;
+
+		vm_copyout(vm, vcpuid, &val, copyinfo, opsize);
+		vm_copy_teardown(vm, vcpuid, copyinfo, nitems(copyinfo));
+	} else if (error > 0) {
+		/*
+		 * Resume guest execution to handle fault.
+		 */
+		goto done;
+	} else {
+		goto done;
+	}
+
+	error = vie_read_register(vm, vcpuid, VM_REG_GUEST_RSI, &rsi);
+	KASSERT(error == 0, ("%s: error %d getting rsi", __func__, error));
+
+	error = vie_read_register(vm, vcpuid, VM_REG_GUEST_RDI, &rdi);
+	KASSERT(error == 0, ("%s: error %d getting rdi", __func__, error));
+
+	error = vie_read_register(vm, vcpuid, VM_REG_GUEST_RFLAGS, &rflags);
+	KASSERT(error == 0, ("%s: error %d getting rflags", __func__, error));
+
+	if (rflags & PSL_D) {
+		rsi -= opsize;
+		rdi -= opsize;
+	} else {
+		rsi += opsize;
+		rdi += opsize;
+	}
+
+	error = vie_update_register(vm, vcpuid, VM_REG_GUEST_RSI, rsi,
+	    vie->addrsize);
+	KASSERT(error == 0, ("%s: error %d updating rsi", __func__, error));
+
+	error = vie_update_register(vm, vcpuid, VM_REG_GUEST_RDI, rdi,
+	    vie->addrsize);
+	KASSERT(error == 0, ("%s: error %d updating rdi", __func__, error));
+
+	if (repeat) {
+		rcx = rcx - 1;
+		error = vie_update_register(vm, vcpuid, VM_REG_GUEST_RCX,
+		    rcx, vie->addrsize);
+		KASSERT(!error, ("%s: error %d updating rcx", __func__, error));
+
+		/*
+		 * Repeat the instruction if the count register is not zero.
+		 */
+		if ((rcx & vie_size2mask(vie->addrsize)) != 0)
+			vm_restart_instruction(vm, vcpuid);
+	}
+done:
+	if (error < 0)
+		return (EFAULT);
+	else
+		return (0);
+}
+
 static int
 emulate_and(void *vm, int vcpuid, uint64_t gpa, struct vie *vie,
 	    mem_region_read_t memread, mem_region_write_t memwrite, void *arg)
@@ -926,9 +1149,7 @@ emulate_stack_op(void *vm, int vcpuid, uint64_t mmio_gpa, struct vie *vie,
 		error = memwrite(vm, vcpuid, mmio_gpa, val, size, arg);
 		rsp += size;
 	}
-#ifdef _KERNEL
 	vm_copy_teardown(vm, vcpuid, copyinfo, nitems(copyinfo));
-#endif
 
 	if (error == 0) {
 		error = vie_update_register(vm, vcpuid, VM_REG_GUEST_RSP, rsp,
@@ -1012,6 +1233,10 @@ vmm_emulate_instruction(void *vm, int vcpuid, uint64_t gpa, struct vie *vie,
 		error = emulate_movx(vm, vcpuid, gpa, vie,
 				     memread, memwrite, memarg);
 		break;
+	case VIE_OP_TYPE_MOVS:
+		error = emulate_movs(vm, vcpuid, gpa, vie, paging, memread,
+		    memwrite, memarg);
+		break;
 	case VIE_OP_TYPE_AND:
 		error = emulate_and(vm, vcpuid, gpa, vie,
 				    memread, memwrite, memarg);
@@ -1193,6 +1418,7 @@ vie_init(struct vie *vie, const char *inst_bytes, int inst_length)
 
 	vie->base_register = VM_REG_LAST;
 	vie->index_register = VM_REG_LAST;
+	vie->segment_register = VM_REG_LAST;
 
 	if (inst_length) {
 		bcopy(inst_bytes, vie->inst, inst_length);
@@ -1458,6 +1684,35 @@ vie_advance(struct vie *vie)
 	vie->num_processed++;
 }
 
+static bool
+segment_override(uint8_t x, int *seg)
+{
+
+	switch (x) {
+	case 0x2E:
+		*seg = VM_REG_GUEST_CS;
+		break;
+	case 0x36:
+		*seg = VM_REG_GUEST_SS;
+		break;
+	case 0x3E:
+		*seg = VM_REG_GUEST_DS;
+		break;
+	case 0x26:
+		*seg = VM_REG_GUEST_ES;
+		break;
+	case 0x64:
+		*seg = VM_REG_GUEST_FS;
+		break;
+	case 0x65:
+		*seg = VM_REG_GUEST_GS;
+		break;
+	default:
+		return (false);
+	}
+	return (true);
+}
+
 static int
 decode_prefixes(struct vie *vie, enum vm_cpu_mode cpu_mode, int cs_d)
 {
@@ -1471,6 +1726,12 @@ decode_prefixes(struct vie *vie, enum vm_cpu_mode cpu_mode, int cs_d)
 			vie->opsize_override = 1;
 		else if (x == 0x67)
 			vie->addrsize_override = 1;
+		else if (x == 0xF3)
+			vie->repz_present = 1;
+		else if (x == 0xF2)
+			vie->repnz_present = 1;
+		else if (segment_override(x, &vie->segment_register))
+			vie->segment_override = 1;
 		else
 			break;
 
@@ -1923,8 +2184,10 @@ vmm_decode_instruction(struct vm *vm, int cpuid, uint64_t gla,
 	if (verify_inst_length(vie))
 		return (-1);
 
-	if (verify_gla(vm, cpuid, gla, vie))
-		return (-1);
+	if ((vie->op.op_flags & VIE_OP_F_NO_GLA_VERIFICATION) == 0) {
+		if (verify_gla(vm, cpuid, gla, vie))
+			return (-1);
+	}
 
 	vie->decoded = 1;	/* success */
 

From ae266f893fd7b82680066c53b6d24ebf4353178b Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Mon, 19 Jan 2015 07:10:08 +0000
Subject: [PATCH 083/279] Fix the build when INVARIANTS is defined by restoring
 `bo`'s definition in ext2_truncate(..) and by putting it under INVARIANTS
 ifdefs

X-MFC with: r277354
MFC after: 2 weeks
---
 sys/fs/ext2fs/ext2_inode.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/sys/fs/ext2fs/ext2_inode.c b/sys/fs/ext2fs/ext2_inode.c
index 2fc2192d05b..36768fb1c41 100644
--- a/sys/fs/ext2fs/ext2_inode.c
+++ b/sys/fs/ext2fs/ext2_inode.c
@@ -121,8 +121,14 @@ ext2_truncate(struct vnode *vp, off_t length, int flags, struct ucred *cred,
 	e4fs_daddr_t count, nblocks, blocksreleased = 0;
 	int error, i, allerror;
 	off_t osize;
+#ifdef INVARIANTS
+	struct bufobj *bo;
+#endif
 
 	oip = VTOI(ovp);
+#ifdef INVARIANTS
+	bo = &ovp->v_bufobj;
+#endif
 
 	ASSERT_VOP_LOCKED(vp, "ext2_truncate");	
 

From 79592d52d5d6ca5cc7909f265899b958835e05a4 Mon Sep 17 00:00:00 2001
From: Hans Petter Selasky <hselasky@FreeBSD.org>
Date: Mon, 19 Jan 2015 07:29:07 +0000
Subject: [PATCH 084/279] Minor refactoring of code block.

MFC after:		1 day
---
 sys/cam/scsi/scsi_da.c | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/sys/cam/scsi/scsi_da.c b/sys/cam/scsi/scsi_da.c
index 6865a61b9a7..2d09d571826 100644
--- a/sys/cam/scsi/scsi_da.c
+++ b/sys/cam/scsi/scsi_da.c
@@ -3101,11 +3101,10 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 			 * give them an 'illegal' value we'll avoid that
 			 * here.
 			 */
-			if (block_size == 0 && maxsector == 0) {
-				block_size = 512;
-				maxsector = -1;
-			} else if (block_size == 0) {
+			if (block_size == 0) {
 				block_size = 512;
+				if (maxsector == 0)
+					maxsector = -1;
 			}
 			if (block_size >= MAXPHYS) {
 				xpt_print(periph->path,

From 09eb425a04adf663823864a3960d0753a5a10a45 Mon Sep 17 00:00:00 2001
From: Andrew Turner <andrew@FreeBSD.org>
Date: Mon, 19 Jan 2015 11:06:56 +0000
Subject: [PATCH 085/279] Make the clock-frequency property optional as it may
 not be present on FDT systems.

Sponsored by:	The FreeBSD Foundation
---
 sys/dev/ofw/ofw_cpu.c | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/sys/dev/ofw/ofw_cpu.c b/sys/dev/ofw/ofw_cpu.c
index 1d01b63993e..02dc2b50055 100644
--- a/sys/dev/ofw/ofw_cpu.c
+++ b/sys/dev/ofw/ofw_cpu.c
@@ -1,7 +1,11 @@
 /*-
  * Copyright (C) 2009 Nathan Whitehorn
+ * Copyright (C) 2015 The FreeBSD Foundation
  * All rights reserved.
  *
+ * Portions of this software were developed by Andrew Turner
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -193,10 +197,11 @@ ofw_cpu_attach(device_t dev)
 	}
 	sc->sc_cpu_pcpu = pcpu_find(cell);
 	if (OF_getencprop(node, "clock-frequency", &cell, sizeof(cell)) < 0) {
-		device_printf(dev, "missing 'clock-frequency' property\n");
-		return (ENXIO);
-	}
-	sc->sc_nominal_mhz = cell / 1000000; /* convert to MHz */
+		if (bootverbose)
+			device_printf(dev,
+			    "missing 'clock-frequency' property\n");
+	} else
+		sc->sc_nominal_mhz = cell / 1000000; /* convert to MHz */
 
 	bus_generic_probe(dev);
 	return (bus_generic_attach(dev));
@@ -214,8 +219,11 @@ ofw_cpu_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
 		*result = (uintptr_t)sc->sc_cpu_pcpu;
 		return (0);
 	case CPU_IVAR_NOMINAL_MHZ:
-		*result = (uintptr_t)sc->sc_nominal_mhz;
-		return (0);
+		if (sc->sc_nominal_mhz > 0) {
+			*result = (uintptr_t)sc->sc_nominal_mhz;
+			return (0);
+		}
+		break;
 	}
 
 	return (ENOENT);

From 2010054d917b6fb68bdeb139590ef53b598761b4 Mon Sep 17 00:00:00 2001
From: Michael Tuexen <tuexen@FreeBSD.org>
Date: Mon, 19 Jan 2015 11:52:08 +0000
Subject: [PATCH 086/279] Code cleanup.

Reported by:	Coverity
CID:		749578
MFC after:	1 week
---
 sys/netinet/sctp_timer.c | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/sys/netinet/sctp_timer.c b/sys/netinet/sctp_timer.c
index 7d861acbeff..d7a3f0c405b 100644
--- a/sys/netinet/sctp_timer.c
+++ b/sys/netinet/sctp_timer.c
@@ -337,7 +337,7 @@ sctp_find_alternate_net(struct sctp_tcb *stcb,
 			return (NULL);
 		}
 	}
-	do {
+	for (;;) {
 		alt = TAILQ_NEXT(mnet, sctp_next);
 		if (alt == NULL) {
 			once++;
@@ -356,7 +356,6 @@ sctp_find_alternate_net(struct sctp_tcb *stcb,
 			}
 			alt->src_addr_selected = 0;
 		}
-		/* sa_ignore NO_NULL_CHK */
 		if (((alt->dest_state & SCTP_ADDR_REACHABLE) == SCTP_ADDR_REACHABLE) &&
 		    (alt->ro.ro_rt != NULL) &&
 		    (!(alt->dest_state & SCTP_ADDR_UNCONFIRMED))) {
@@ -364,14 +363,14 @@ sctp_find_alternate_net(struct sctp_tcb *stcb,
 			break;
 		}
 		mnet = alt;
-	} while (alt != NULL);
+	}
 
 	if (alt == NULL) {
 		/* Case where NO insv network exists (dormant state) */
 		/* we rotate destinations */
 		once = 0;
 		mnet = net;
-		do {
+		for (;;) {
 			if (mnet == NULL) {
 				return (TAILQ_FIRST(&stcb->asoc.nets));
 			}
@@ -382,15 +381,17 @@ sctp_find_alternate_net(struct sctp_tcb *stcb,
 					break;
 				}
 				alt = TAILQ_FIRST(&stcb->asoc.nets);
+				if (alt == NULL) {
+					break;
+				}
 			}
-			/* sa_ignore NO_NULL_CHK */
 			if ((!(alt->dest_state & SCTP_ADDR_UNCONFIRMED)) &&
 			    (alt != net)) {
 				/* Found an alternate address */
 				break;
 			}
 			mnet = alt;
-		} while (alt != NULL);
+		}
 	}
 	if (alt == NULL) {
 		return (net);

From 438e86ce0cdc17a89f7cbc5d433f1266b03d4670 Mon Sep 17 00:00:00 2001
From: Alexander Motin <mav@FreeBSD.org>
Date: Mon, 19 Jan 2015 15:52:32 +0000
Subject: [PATCH 087/279] Remove extra mtx_unlock().

Submitted by:	Dmitry Luhtionov <dmitryluhtionov@gmail.com>
MFC after:	1 week
---
 sys/cam/cam_xpt.c | 1 -
 1 file changed, 1 deletion(-)

diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c
index 36f22145146..f37bc0a890c 100644
--- a/sys/cam/cam_xpt.c
+++ b/sys/cam/cam_xpt.c
@@ -893,7 +893,6 @@ xpt_init(void *dummy)
 	if ((status = xpt_create_path(&path, NULL, CAM_XPT_PATH_ID,
 				      CAM_TARGET_WILDCARD,
 				      CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
-		mtx_unlock(&xsoftc.xpt_lock);
 		printf("xpt_init: xpt_create_path failed with status %#x,"
 		       " failing attach\n", status);
 		return (EINVAL);

From a57a934a38c8c6ca37c1b1e20659a8fb0251dbac Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Mon, 19 Jan 2015 17:24:52 +0000
Subject: [PATCH 088/279] Ignore devfs directory entries for devices either
 being destroyed or delisted.  The check is racy.

Tested by:	pho
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
---
 sys/fs/devfs/devfs_devs.c  | 10 ++++++++++
 sys/fs/devfs/devfs_vnops.c |  3 +++
 2 files changed, 13 insertions(+)

diff --git a/sys/fs/devfs/devfs_devs.c b/sys/fs/devfs/devfs_devs.c
index 294bd62be7b..986d06692bb 100644
--- a/sys/fs/devfs/devfs_devs.c
+++ b/sys/fs/devfs/devfs_devs.c
@@ -192,6 +192,16 @@ devfs_find(struct devfs_dirent *dd, const char *name, int namelen, int type)
 			continue;
 		if (type != 0 && type != de->de_dirent->d_type)
 			continue;
+
+		/*
+		 * The race with finding non-active name is not
+		 * completely closed by the check, but it is similar
+		 * to the devfs_allocv() in making it unlikely enough.
+		 */
+		if (de->de_dirent->d_type == DT_CHR &&
+		    (de->de_cdp->cdp_flags & CDP_ACTIVE) == 0)
+			continue;
+
 		if (bcmp(name, de->de_dirent->d_name, namelen) != 0)
 			continue;
 		break;
diff --git a/sys/fs/devfs/devfs_vnops.c b/sys/fs/devfs/devfs_vnops.c
index 91535881952..570f710e649 100644
--- a/sys/fs/devfs/devfs_vnops.c
+++ b/sys/fs/devfs/devfs_vnops.c
@@ -1045,6 +1045,9 @@ devfs_mknod(struct vop_mknod_args *ap)
 	TAILQ_FOREACH(de, &dd->de_dlist, de_list) {
 		if (cnp->cn_namelen != de->de_dirent->d_namlen)
 			continue;
+		if (de->de_dirent->d_type == DT_CHR &&
+		    (de->de_cdp->cdp_flags & CDP_ACTIVE) == 0)
+			continue;
 		if (bcmp(cnp->cn_nameptr, de->de_dirent->d_name,
 		    de->de_dirent->d_namlen) != 0)
 			continue;

From 3b50dff506e494f7aab6dee427f331a6c9133a37 Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Mon, 19 Jan 2015 17:36:52 +0000
Subject: [PATCH 089/279] Stop enforcing additional reference on all cdevs,
 which was introduced in r277199.  Acquire the neccessary reference in
 delist_dev_locked() and inform destroy_devl() about it using CDP_UNREF_DTR
 flag.

Fix some style nits, add asserts.

Discussed with:	hselasky
Tested by:	pho
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
---
 sys/fs/devfs/devfs_devs.c |  6 ------
 sys/fs/devfs/devfs_int.h  |  1 +
 sys/kern/kern_conf.c      | 33 ++++++++++++++++++++++++++++-----
 3 files changed, 29 insertions(+), 11 deletions(-)

diff --git a/sys/fs/devfs/devfs_devs.c b/sys/fs/devfs/devfs_devs.c
index 986d06692bb..6620aefc4c4 100644
--- a/sys/fs/devfs/devfs_devs.c
+++ b/sys/fs/devfs/devfs_devs.c
@@ -137,12 +137,6 @@ devfs_alloc(int flags)
 	vfs_timestamp(&ts);
 	cdev->si_atime = cdev->si_mtime = cdev->si_ctime = ts;
 	cdev->si_cred = NULL;
-	/*
-	 * Avoid race with dev_rel() by setting the initial
-	 * reference count to 1. This last reference is taken
-	 * by the destroy_dev() function.
-	 */
-	cdev->si_refcount = 1;
 
 	return (cdev);
 }
diff --git a/sys/fs/devfs/devfs_int.h b/sys/fs/devfs/devfs_int.h
index ce55416f6c0..6c571092877 100644
--- a/sys/fs/devfs/devfs_int.h
+++ b/sys/fs/devfs/devfs_int.h
@@ -56,6 +56,7 @@ struct cdev_priv {
 	u_int			cdp_flags;
 #define CDP_ACTIVE		(1 << 0)
 #define CDP_SCHED_DTR		(1 << 1)
+#define	CDP_UNREF_DTR		(1 << 2)
 
 	u_int			cdp_inuse;
 	u_int			cdp_maxdirent;
diff --git a/sys/kern/kern_conf.c b/sys/kern/kern_conf.c
index bcd6fb926fe..79c8fea69e0 100644
--- a/sys/kern/kern_conf.c
+++ b/sys/kern/kern_conf.c
@@ -116,6 +116,8 @@ dev_free_devlocked(struct cdev *cdev)
 
 	mtx_assert(&devmtx, MA_OWNED);
 	cdp = cdev2priv(cdev);
+	KASSERT((cdp->cdp_flags & CDP_UNREF_DTR) == 0,
+	    ("destroy_dev() was not called after delist_dev(%p)", cdev));
 	TAILQ_INSERT_HEAD(&cdevp_free_list, cdp, cdp_list);
 }
 
@@ -1035,6 +1037,7 @@ destroy_devl(struct cdev *dev)
 {
 	struct cdevsw *csw;
 	struct cdev_privdata *p;
+	struct cdev_priv *cdp;
 
 	mtx_assert(&devmtx, MA_OWNED);
 	KASSERT(dev->si_flags & SI_NAMED,
@@ -1043,7 +1046,18 @@ destroy_devl(struct cdev *dev)
 	    ("WARNING: Driver mistake: destroy_dev on eternal %d\n",
 	     dev2unit(dev)));
 
-	devfs_destroy(dev);
+	cdp = cdev2priv(dev);
+	if ((cdp->cdp_flags & CDP_UNREF_DTR) == 0) {
+		/*
+		 * Avoid race with dev_rel(), e.g. from the populate
+		 * loop.  If CDP_UNREF_DTR flag is set, the reference
+		 * to be dropped at the end of destroy_devl() was
+		 * already taken by delist_dev_locked().
+		 */
+		dev_refl(dev);
+
+		devfs_destroy(dev);
+	}
 
 	/* Remove name marking */
 	dev->si_flags &= ~SI_NAMED;
@@ -1103,19 +1117,27 @@ destroy_devl(struct cdev *dev)
 		}
 	}
 	dev->si_flags &= ~SI_ALIAS;
-	dev->si_refcount--;	/* Avoid race with dev_rel() */
+	cdp->cdp_flags &= ~CDP_UNREF_DTR;
+	dev->si_refcount--;
 
-	if (dev->si_refcount > 0) {
+	if (dev->si_refcount > 0)
 		LIST_INSERT_HEAD(&dead_cdevsw.d_devs, dev, si_list);
-	} else {
+	else
 		dev_free_devlocked(dev);
-	}
 }
 
 static void
 delist_dev_locked(struct cdev *dev)
 {
+	struct cdev_priv *cdp;
 	struct cdev *child;
+
+	mtx_assert(&devmtx, MA_OWNED);
+	cdp = cdev2priv(dev);
+	if ((cdp->cdp_flags & CDP_UNREF_DTR) != 0)
+		return;
+	cdp->cdp_flags |= CDP_UNREF_DTR;
+	dev_refl(dev);
 	devfs_destroy(dev);
 	LIST_FOREACH(child, &dev->si_children, si_siblings)
 		delist_dev_locked(child);
@@ -1124,6 +1146,7 @@ delist_dev_locked(struct cdev *dev)
 void
 delist_dev(struct cdev *dev)
 {
+
 	dev_lock();
 	delist_dev_locked(dev);
 	dev_unlock();

From 98cd7a66555d803a842c0155df736bbfc0e758c7 Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Mon, 19 Jan 2015 17:58:01 +0000
Subject: [PATCH 090/279] Add some initial infrastructure for relocating the
 kernel in place.

MFC after:	2 months
Differential revision:	D1554
---
 sys/powerpc/aim/locore64.S          | 26 ++++++++++++++++++++---
 sys/powerpc/powerpc/elf64_machdep.c | 33 +++++++++++++++++++++++++++++
 2 files changed, 56 insertions(+), 3 deletions(-)

diff --git a/sys/powerpc/aim/locore64.S b/sys/powerpc/aim/locore64.S
index 30ee3a95dc9..abce9d83e5b 100644
--- a/sys/powerpc/aim/locore64.S
+++ b/sys/powerpc/aim/locore64.S
@@ -121,13 +121,33 @@ ASENTRY_NOPROF(__start)
 	.align 3
 0:	nop
 	bl	1f
-	.llong	__tocbase + 0x8000
+	.llong	__tocbase + 0x8000 - .
 1:	mflr	%r2
-	ld	%r2,0(%r2)
+	ld	%r1,0(%r2)
+	add	%r2,%r1,%r2
 
 	/* Set up the stack pointer */
 	ld	%r1,TOC_REF(tmpstk)(%r2)
-	addi	%r1,%r1,TMPSTKSZ-48
+	addi	%r1,%r1,TMPSTKSZ-96
+
+	/* Relocate kernel */
+	std	%r3,48(%r1)
+	std	%r4,56(%r1)
+	std	%r5,64(%r1)
+	std	%r6,72(%r1)
+	bl	1f
+	.llong _DYNAMIC-.
+1:	mflr	%r3
+	ld	%r4,0(%r3)
+	add	%r3,%r4,%r3
+	ld	%r4,-0x8000(%r2) /* First TOC entry is TOC base */
+	subf    %r4,%r4,%r2	/* Subtract from real TOC base to get base */
+	bl	elf_reloc_self
+	nop
+	ld	%r3,48(%r1)
+	ld	%r4,56(%r1)
+	ld	%r5,64(%r1)
+	ld	%r6,72(%r1)
 
 	/* Switch to 64-bit mode */
 	mfmsr	%r9
diff --git a/sys/powerpc/powerpc/elf64_machdep.c b/sys/powerpc/powerpc/elf64_machdep.c
index a0fc7b75bb9..d23f7ffaa2f 100644
--- a/sys/powerpc/powerpc/elf64_machdep.c
+++ b/sys/powerpc/powerpc/elf64_machdep.c
@@ -119,6 +119,8 @@ SYSINIT(oelf64, SI_SUB_EXEC, SI_ORDER_ANY,
 	(sysinit_cfunc_t) elf64_insert_brand_entry,
 	&freebsd_brand_oinfo);
 
+void elf_reloc_self(Elf_Dyn *dynp, Elf_Addr relocbase);
+
 void
 elf64_dump_thread(struct thread *td, void *dst, size_t *off)
 {
@@ -198,6 +200,37 @@ elf_reloc_internal(linker_file_t lf, Elf_Addr relocbase, const void *data,
 	return(0);
 }
 
+void
+elf_reloc_self(Elf_Dyn *dynp, Elf_Addr relocbase)
+{
+	Elf_Rela *rela = 0, *relalim;
+	Elf_Addr relasz = 0;
+	Elf_Addr *where;
+
+	/*
+	 * Extract the rela/relasz values from the dynamic section
+	 */
+	for (; dynp->d_tag != DT_NULL; dynp++) {
+		switch (dynp->d_tag) {
+		case DT_RELA:
+			rela = (Elf_Rela *)(relocbase+dynp->d_un.d_ptr);
+			break;
+		case DT_RELASZ:
+			relasz = dynp->d_un.d_val;
+			break;
+		}
+	}
+
+	/*
+	 * Relocate these values
+	 */
+	relalim = (Elf_Rela *)((caddr_t)rela + relasz);
+	for (; rela < relalim; rela++) {
+		where = (Elf_Addr *)(relocbase + rela->r_offset);
+		*where = (Elf_Addr)(relocbase + rela->r_addend);
+	}
+}
+
 int
 elf_reloc(linker_file_t lf, Elf_Addr relocbase, const void *data, int type,
     elf_lookup_fn lookup)

From 7c3892fc82a34e48c23b2b732392d3c6cfa23c2d Mon Sep 17 00:00:00 2001
From: Hans Petter Selasky <hselasky@FreeBSD.org>
Date: Mon, 19 Jan 2015 20:39:48 +0000
Subject: [PATCH 091/279] Add more functions to the Linux kernel compatibility
 layer. Add some missing includes which are needed when the header files are
 not included in a particular order.

MFC after:	1 month
Sponsored by:	Mellanox Technologies
---
 sys/ofed/include/linux/bitops.h      |  6 ++++++
 sys/ofed/include/linux/cache.h       |  3 +--
 sys/ofed/include/linux/dma-mapping.h |  8 ++++++++
 sys/ofed/include/linux/etherdevice.h |  5 ++++-
 sys/ofed/include/linux/gfp.h         |  2 ++
 sys/ofed/include/linux/io.h          | 15 +++++++++++++++
 sys/ofed/include/linux/kernel.h      |  5 +++++
 sys/ofed/include/linux/ktime.h       |  9 +++++++++
 sys/ofed/include/linux/slab.h        |  2 ++
 9 files changed, 52 insertions(+), 3 deletions(-)

diff --git a/sys/ofed/include/linux/bitops.h b/sys/ofed/include/linux/bitops.h
index 93a3aa93a15..f225fdc6d9d 100644
--- a/sys/ofed/include/linux/bitops.h
+++ b/sys/ofed/include/linux/bitops.h
@@ -288,9 +288,15 @@ bitmap_empty(unsigned long *addr, int size)
 
 #define	NBLONG	(NBBY * sizeof(long))
 
+#define	__set_bit(i, a)							\
+    atomic_set_long(&((volatile long *)(a))[(i)/NBLONG], 1UL << ((i) % NBLONG))
+
 #define	set_bit(i, a)							\
     atomic_set_long(&((volatile long *)(a))[(i)/NBLONG], 1UL << ((i) % NBLONG))
 
+#define	__clear_bit(i, a)						\
+    atomic_clear_long(&((volatile long *)(a))[(i)/NBLONG], 1UL << ((i) % NBLONG))
+
 #define	clear_bit(i, a)							\
     atomic_clear_long(&((volatile long *)(a))[(i)/NBLONG], 1UL << ((i) % NBLONG))
 
diff --git a/sys/ofed/include/linux/cache.h b/sys/ofed/include/linux/cache.h
index e4a9d092463..921a507ea66 100644
--- a/sys/ofed/include/linux/cache.h
+++ b/sys/ofed/include/linux/cache.h
@@ -30,8 +30,7 @@
 #ifndef	_LINUX_CACHE_H_
 #define _LINUX_CACHE_H_
 
-
 #define	cache_line_size()	CACHE_LINE_SIZE
-
+#define	L1_CACHE_BYTES		CACHE_LINE_SIZE
 
 #endif	/* _LINUX_CACHE_H_ */
diff --git a/sys/ofed/include/linux/dma-mapping.h b/sys/ofed/include/linux/dma-mapping.h
index 2f0762b430b..f9fc3cb095b 100644
--- a/sys/ofed/include/linux/dma-mapping.h
+++ b/sys/ofed/include/linux/dma-mapping.h
@@ -139,6 +139,14 @@ dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
 		*dma_handle = 0;
 	return (mem);
 }
+
+static inline void *
+dma_zalloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_handle,
+    gfp_t flag)
+{
+
+	return (dma_alloc_coherent(dev, size, dma_handle, flag | __GFP_ZERO));
+}
                        
 static inline void
 dma_free_coherent(struct device *dev, size_t size, void *cpu_addr,
diff --git a/sys/ofed/include/linux/etherdevice.h b/sys/ofed/include/linux/etherdevice.h
index 7d1114501ce..d863651894a 100644
--- a/sys/ofed/include/linux/etherdevice.h
+++ b/sys/ofed/include/linux/etherdevice.h
@@ -89,6 +89,9 @@ static inline bool is_valid_ether_addr(const u8 *addr)
         return !is_multicast_ether_addr(addr) && !is_zero_ether_addr(addr);
 }
 
-
+static inline void ether_addr_copy(u8 *dst, const u8 *src)
+{
+	memcpy(dst, src, 6);
+}
 
 #endif /* _LINUX_ETHERDEVICE */
diff --git a/sys/ofed/include/linux/gfp.h b/sys/ofed/include/linux/gfp.h
index af30faacab6..3e773028dee 100644
--- a/sys/ofed/include/linux/gfp.h
+++ b/sys/ofed/include/linux/gfp.h
@@ -30,6 +30,8 @@
 #ifndef	_LINUX_GFP_H_
 #define	_LINUX_GFP_H_
 
+#include <sys/cdefs.h>
+#include <sys/types.h>
 #include <sys/systm.h>
 #include <sys/malloc.h>
 
diff --git a/sys/ofed/include/linux/io.h b/sys/ofed/include/linux/io.h
index 2fc25b567eb..4c8c9f524b2 100644
--- a/sys/ofed/include/linux/io.h
+++ b/sys/ofed/include/linux/io.h
@@ -31,6 +31,7 @@
 #define	_LINUX_IO_H_
 
 #include <machine/vm.h>
+#include <sys/endian.h>
 
 static inline uint32_t
 __raw_readl(const volatile void *addr)
@@ -89,6 +90,20 @@ writew(uint16_t b, void *addr)
         *(volatile uint16_t *)addr = b;
 }
 
+#undef ioread32be
+static inline uint32_t
+ioread32be(const volatile void *addr)
+{
+	return be32toh(*(const volatile uint32_t *)addr);
+}
+
+#undef iowrite32be
+static inline void
+iowrite32be(uint32_t v, volatile void *addr)
+{
+	*(volatile uint32_t *)addr = htobe32(v);
+}
+
 void *_ioremap_attr(vm_paddr_t phys_addr, unsigned long size, int attr);
 #define	ioremap_nocache(addr, size)					\
     _ioremap_attr((addr), (size), VM_MEMATTR_UNCACHEABLE)
diff --git a/sys/ofed/include/linux/kernel.h b/sys/ofed/include/linux/kernel.h
index d9d1ab6613b..5d4e50bdab3 100644
--- a/sys/ofed/include/linux/kernel.h
+++ b/sys/ofed/include/linux/kernel.h
@@ -29,6 +29,8 @@
 #ifndef	_LINUX_KERNEL_H_
 #define	_LINUX_KERNEL_H_
 
+#include <sys/cdefs.h>
+#include <sys/types.h>
 #include <sys/systm.h>
 #include <sys/param.h>
 #include <sys/libkern.h>
@@ -57,6 +59,8 @@
 #define	KERN_INFO	"<6>"
 #define	KERN_DEBUG	"<7>"
 
+#define	BUILD_BUG_ON(x)		CTASSERT(x)
+
 #define BUG()			panic("BUG")
 #define BUG_ON(condition)	do { if (condition) BUG(); } while(0)
 #define	WARN_ON			BUG_ON
@@ -84,6 +88,7 @@
 #endif
 
 #define udelay(t)       	DELAY(t)
+#define usleep_range(min,max)	DELAY(min)
 
 #ifndef pr_fmt
 #define pr_fmt(fmt) fmt
diff --git a/sys/ofed/include/linux/ktime.h b/sys/ofed/include/linux/ktime.h
index c59c7b9dacd..7524afcf1ab 100644
--- a/sys/ofed/include/linux/ktime.h
+++ b/sys/ofed/include/linux/ktime.h
@@ -288,4 +288,13 @@ static inline s64 ktime_to_ns(const ktime_t kt)
 
 #endif	/* !((BITS_PER_LONG == 64) || defined(CONFIG_KTIME_SCALAR)) */
 
+static inline s64 ktime_get_ns(void)
+{
+	struct timespec ts;
+	ktime_t kt;
+	ktime_get_ts(&ts);
+	kt = timespec_to_ktime(ts);
+	return (ktime_to_ns(kt));
+}
+
 #endif	/* _LINUX_KTIME_H */
diff --git a/sys/ofed/include/linux/slab.h b/sys/ofed/include/linux/slab.h
index 1d373ce0322..0d01a3659c2 100644
--- a/sys/ofed/include/linux/slab.h
+++ b/sys/ofed/include/linux/slab.h
@@ -40,6 +40,7 @@
 MALLOC_DECLARE(M_KMALLOC);
 
 #define	kmalloc(size, flags)		malloc((size), M_KMALLOC, (flags))
+#define	kvmalloc(size)			kmalloc((size), 0)
 #define	kzalloc(size, flags)		kmalloc((size), (flags) | M_ZERO)
 #define	kzalloc_node(size, flags, node)	kzalloc(size, flags)
 #define	kfree(ptr)			free(__DECONST(void *, (ptr)), M_KMALLOC)
@@ -47,6 +48,7 @@ MALLOC_DECLARE(M_KMALLOC);
 #define	kcalloc(n, size, flags)	        kmalloc((n) * (size), flags | M_ZERO)
 #define	vzalloc(size)			kzalloc(size, GFP_KERNEL | __GFP_NOWARN)
 #define	vfree(arg)			kfree(arg)
+#define	kvfree(arg)			kfree(arg)
 #define	vmalloc(size)                   kmalloc(size, GFP_KERNEL)
 #define	vmalloc_node(size, node)        kmalloc(size, GFP_KERNEL)
 

From d39d7c8636e4eb71d383a57db0e01e406ebe6528 Mon Sep 17 00:00:00 2001
From: Hans Petter Selasky <hselasky@FreeBSD.org>
Date: Mon, 19 Jan 2015 21:53:00 +0000
Subject: [PATCH 092/279] Add missing linuxapi module dependencies and always
 use the FreeBSD "MODULE_VERSION" macro definition. Remove the redefinition of
 the "MODULE_VERSION" macro from the Linux kernel compatibility API.

MFC after:	1 month
Reported by:	np@
Sponsored by:	Mellanox Technologies
---
 sys/contrib/rdma/krping/krping.c                   | 2 ++
 sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb.c                 | 5 ++---
 sys/dev/cxgbe/iw_cxgbe/device.c                    | 3 +--
 sys/ofed/drivers/infiniband/core/device.c          | 2 --
 sys/ofed/drivers/infiniband/hw/mlx4/main.c         | 4 ++--
 sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c  | 4 +++-
 sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c | 2 --
 sys/ofed/drivers/net/mlx4/en_main.c                | 4 ++--
 sys/ofed/drivers/net/mlx4/main.c                   | 5 -----
 sys/ofed/include/linux/module.h                    | 8 ++++----
 10 files changed, 16 insertions(+), 23 deletions(-)

diff --git a/sys/contrib/rdma/krping/krping.c b/sys/contrib/rdma/krping/krping.c
index a770423cfde..c89339edf31 100644
--- a/sys/contrib/rdma/krping/krping.c
+++ b/sys/contrib/rdma/krping/krping.c
@@ -60,6 +60,8 @@ extern int krping_debug;
 MODULE_AUTHOR("Steve Wise");
 MODULE_DESCRIPTION("RDMA ping client/server");
 MODULE_LICENSE("Dual BSD/GPL");
+MODULE_VERSION(krping, 1);
+MODULE_DEPEND(krping, linuxapi, 1, 1, 1);
 
 static __inline uint64_t
 get_cycles(void)
diff --git a/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb.c b/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb.c
index 1825a430d04..18876fe4245 100644
--- a/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb.c
+++ b/sys/dev/cxgb/ulp/iw_cxgb/iw_cxgb.c
@@ -251,9 +251,6 @@ iwch_mod_unload(void)
 }
 #endif	/* TCP_OFFLOAD */
 
-#undef MODULE_VERSION
-#include <sys/module.h>
-
 static int
 iwch_modevent(module_t mod, int cmd, void *arg)
 {
@@ -299,3 +296,5 @@ MODULE_DEPEND(t3_tom, cxgbc, 1, 1, 1);
 MODULE_DEPEND(iw_cxgb, toecore, 1, 1, 1);
 MODULE_DEPEND(iw_cxgb, t3_tom, 1, 1, 1);
 MODULE_DEPEND(iw_cxgb, ibcore, 1, 1, 1);
+MODULE_DEPEND(iw_cxgb, linuxapi, 1, 1, 1);
+
diff --git a/sys/dev/cxgbe/iw_cxgbe/device.c b/sys/dev/cxgbe/iw_cxgbe/device.c
index adb283d8f71..92a574215b2 100644
--- a/sys/dev/cxgbe/iw_cxgbe/device.c
+++ b/sys/dev/cxgbe/iw_cxgbe/device.c
@@ -321,8 +321,6 @@ c4iw_mod_unload(void)
 }
 
 #endif
-#undef MODULE_VERSION
-#include <sys/module.h>
 
 /*
  * t4_tom won't load on kernels without TCP_OFFLOAD and this module's dependency
@@ -366,4 +364,5 @@ MODULE_VERSION(iw_cxgbe, 1);
 MODULE_DEPEND(iw_cxgbe, t4nex, 1, 1, 1);
 MODULE_DEPEND(iw_cxgbe, t4_tom, 1, 1, 1);
 MODULE_DEPEND(iw_cxgbe, ibcore, 1, 1, 1);
+MODULE_DEPEND(iw_cxgbe, linuxapi, 1, 1, 1);
 DECLARE_MODULE(iw_cxgbe, c4iw_mod_data, SI_SUB_EXEC, SI_ORDER_ANY);
diff --git a/sys/ofed/drivers/infiniband/core/device.c b/sys/ofed/drivers/infiniband/core/device.c
index e2e6bafdf69..98adf489b81 100644
--- a/sys/ofed/drivers/infiniband/core/device.c
+++ b/sys/ofed/drivers/infiniband/core/device.c
@@ -754,8 +754,6 @@ static void __exit ib_core_cleanup(void)
 module_init(ib_core_init);
 module_exit(ib_core_cleanup);
 
-#undef MODULE_VERSION
-#include <sys/module.h>
 static int
 ibcore_evhand(module_t mod, int event, void *arg)
 {
diff --git a/sys/ofed/drivers/infiniband/hw/mlx4/main.c b/sys/ofed/drivers/infiniband/hw/mlx4/main.c
index 441ee739bb2..fd0b7235873 100644
--- a/sys/ofed/drivers/infiniband/hw/mlx4/main.c
+++ b/sys/ofed/drivers/infiniband/hw/mlx4/main.c
@@ -67,7 +67,9 @@
 MODULE_AUTHOR("Roland Dreier");
 MODULE_DESCRIPTION("Mellanox ConnectX HCA InfiniBand driver");
 MODULE_LICENSE("Dual BSD/GPL");
+#ifdef __linux__
 MODULE_VERSION(DRV_VERSION);
+#endif
 
 int mlx4_ib_sm_guid_assign = 1;
 
@@ -2404,8 +2406,6 @@ static void __exit mlx4_ib_cleanup(void)
 module_init_order(mlx4_ib_init, SI_ORDER_MIDDLE);
 module_exit(mlx4_ib_cleanup);
 
-#undef MODULE_VERSION
-#include <sys/module.h>
 static int
 mlx4ib_evhand(module_t mod, int event, void *arg)
 {
diff --git a/sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c b/sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c
index d1da694f62a..8596bdd3d7e 100644
--- a/sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c
+++ b/sys/ofed/drivers/infiniband/hw/mthca/mthca_main.c
@@ -47,7 +47,9 @@
 MODULE_AUTHOR("Roland Dreier");
 MODULE_DESCRIPTION("Mellanox InfiniBand HCA low-level driver");
 MODULE_LICENSE("Dual BSD/GPL");
-MODULE_VERSION(DRV_VERSION);
+MODULE_VERSION(mthca, 1);
+MODULE_DEPEND(mthca, linuxapi, 1, 1, 1);
+MODULE_DEPEND(mthca, ibcore, 1, 1, 1);
 
 #ifdef CONFIG_INFINIBAND_MTHCA_DEBUG
 
diff --git a/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c b/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c
index fe517389b65..695621f16a8 100644
--- a/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/sys/ofed/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -1527,8 +1527,6 @@ ipoib_resolvemulti(struct ifnet *ifp, struct sockaddr **llsa,
 module_init(ipoib_init_module);
 module_exit(ipoib_cleanup_module);
 
-#undef MODULE_VERSION
-#include <sys/module.h>
 static int
 ipoib_evhand(module_t mod, int event, void *arg)
 {
diff --git a/sys/ofed/drivers/net/mlx4/en_main.c b/sys/ofed/drivers/net/mlx4/en_main.c
index 983f99e382c..1249d62a4f9 100644
--- a/sys/ofed/drivers/net/mlx4/en_main.c
+++ b/sys/ofed/drivers/net/mlx4/en_main.c
@@ -45,7 +45,9 @@
 MODULE_AUTHOR("Liran Liss, Yevgeny Petrilin");
 MODULE_DESCRIPTION("Mellanox ConnectX HCA Ethernet driver");
 MODULE_LICENSE("Dual BSD/GPL");
+#ifdef __linux__
 MODULE_VERSION(DRV_VERSION " ("DRV_RELDATE")");
+#endif
 
 static const char mlx4_en_version[] =
 	DRV_NAME ": Mellanox ConnectX HCA Ethernet driver v"
@@ -336,8 +338,6 @@ static void __exit mlx4_en_cleanup(void)
 module_init(mlx4_en_init);
 module_exit(mlx4_en_cleanup);
 
-#undef MODULE_VERSION
-#include <sys/module.h>
 static int
 mlxen_evhand(module_t mod, int event, void *arg)
 {
diff --git a/sys/ofed/drivers/net/mlx4/main.c b/sys/ofed/drivers/net/mlx4/main.c
index 1b8f85fc2dc..84c40eddf24 100644
--- a/sys/ofed/drivers/net/mlx4/main.c
+++ b/sys/ofed/drivers/net/mlx4/main.c
@@ -34,10 +34,6 @@
  */
 
 #include <linux/kmod.h> 
-/* 
- * kmod.h must be included before module.h since it includes (indirectly) sys/module.h
- * To use the FBSD macro sys/module.h should define MODULE_VERSION before linux/module does.
-*/
 #include <linux/module.h>
 #include <linux/errno.h>
 #include <linux/pci.h>
@@ -3783,7 +3779,6 @@ static void __exit mlx4_cleanup(void)
 module_init_order(mlx4_init, SI_ORDER_MIDDLE);
 module_exit(mlx4_cleanup);
 
-#include <sys/module.h>
 static int
 mlx4_evhand(module_t mod, int event, void *arg)
 {
diff --git a/sys/ofed/include/linux/module.h b/sys/ofed/include/linux/module.h
index fbbc45219f8..cd9f4d55607 100644
--- a/sys/ofed/include/linux/module.h
+++ b/sys/ofed/include/linux/module.h
@@ -29,6 +29,10 @@
 #ifndef	_LINUX_MODULE_H_
 #define	_LINUX_MODULE_H_
 
+#include <sys/cdefs.h>
+#include <sys/types.h>
+#include <sys/module.h>
+
 #include <linux/list.h>
 #include <linux/compiler.h>
 #include <linux/kobject.h>
@@ -39,10 +43,6 @@
 #define MODULE_DESCRIPTION(name)
 #define MODULE_LICENSE(name)
 
-#ifndef MODULE_VERSION
-#define MODULE_VERSION(name)
-#endif
-
 #define	THIS_MODULE	((struct module *)0)
 
 #define	EXPORT_SYMBOL(name)

From eaf6610dd78266f251e251ff5ce05e12be8ab8a3 Mon Sep 17 00:00:00 2001
From: Oleksandr Tymoshenko <gonzo@FreeBSD.org>
Date: Tue, 20 Jan 2015 02:24:08 +0000
Subject: [PATCH 093/279] Properly clear IRQ status in order to fix "Spurious
 IRQ" message from AINT.

This register is not documented in TRM but that's what linux driver does
---
 sys/arm/ti/am335x/am335x_lcd.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sys/arm/ti/am335x/am335x_lcd.c b/sys/arm/ti/am335x/am335x_lcd.c
index 09f81f0d517..a4c56b91492 100644
--- a/sys/arm/ti/am335x/am335x_lcd.c
+++ b/sys/arm/ti/am335x/am335x_lcd.c
@@ -150,6 +150,7 @@ __FBSDID("$FreeBSD$");
 #define		IRQ_SYNC_LOST		(1 << 2)
 #define		IRQ_RASTER_DONE		(1 << 1)
 #define		IRQ_FRAME_DONE		(1 << 0)
+#define	LCD_END_OF_INT_IND	0x68
 #define	LCD_CLKC_ENABLE		0x6C
 #define		CLKC_ENABLE_DMA		(1 << 2)
 #define		CLKC_ENABLE_LDID	(1 << 1)
@@ -397,6 +398,8 @@ am335x_lcd_intr(void *arg)
 	if (reg & IRQ_ACB) {
 		/* TODO: Handle ACB */
 	}
+
+	LCD_WRITE4(sc, LCD_END_OF_INT_IND, 0);
 }
 
 static int

From d1b1b60065e8b8f547cc44874509050a8c8d8b7b Mon Sep 17 00:00:00 2001
From: Neel Natu <neel@FreeBSD.org>
Date: Tue, 20 Jan 2015 03:54:30 +0000
Subject: [PATCH 094/279] Update the vdso timehands only via tc_windup().

Prior to this change CLOCK_MONOTONIC could go backwards when the timecounter
hardware was changed via 'sysctl kern.timecounter.hardware'. This happened
because the vdso timehands update was missing the special treatment in
tc_windup() when changing timecounters.

Reviewed by:	kib
---
 sys/kern/kern_tc.c            | 15 +++++++++++----
 sys/kern/subr_dummy_vdso_tc.c |  5 +++--
 sys/sys/vdso.h                |  8 ++++++--
 sys/x86/x86/tsc.c             | 13 +++++++------
 4 files changed, 27 insertions(+), 14 deletions(-)

diff --git a/sys/kern/kern_tc.c b/sys/kern/kern_tc.c
index 7a6b4e6d959..1c29041b1f2 100644
--- a/sys/kern/kern_tc.c
+++ b/sys/kern/kern_tc.c
@@ -1424,7 +1424,15 @@ sysctl_kern_timecounter_hardware(SYSCTL_HANDLER_ARGS)
 		(void)newtc->tc_get_timecount(newtc);
 
 		timecounter = newtc;
-		timekeep_push_vdso();
+
+		/*
+		 * The vdso timehands update is deferred until the next
+		 * 'tc_windup()'.
+		 *
+		 * This is prudent given that 'timekeep_push_vdso()' does not
+		 * use any locking and that it can be called in hard interrupt
+		 * context via 'tc_windup()'.
+		 */
 		return (0);
 	}
 	return (EINVAL);
@@ -1982,7 +1990,6 @@ sysctl_fast_gettime(SYSCTL_HANDLER_ARGS)
 	if (error != 0)
 		return (error);
 	vdso_th_enable = old_vdso_th_enable;
-	timekeep_push_vdso();
 	return (0);
 }
 SYSCTL_PROC(_kern_timecounter, OID_AUTO, fast_gettime,
@@ -2002,7 +2009,7 @@ tc_fill_vdso_timehands(struct vdso_timehands *vdso_th)
 	vdso_th->th_counter_mask = th->th_counter->tc_counter_mask;
 	vdso_th->th_offset = th->th_offset;
 	vdso_th->th_boottime = boottimebin;
-	enabled = cpu_fill_vdso_timehands(vdso_th);
+	enabled = cpu_fill_vdso_timehands(vdso_th, th->th_counter);
 	if (!vdso_th_enable)
 		enabled = 0;
 	return (enabled);
@@ -2024,7 +2031,7 @@ tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32)
 	*(uint64_t *)&vdso_th32->th_offset.frac[0] = th->th_offset.frac;
 	vdso_th32->th_boottime.sec = boottimebin.sec;
 	*(uint64_t *)&vdso_th32->th_boottime.frac[0] = boottimebin.frac;
-	enabled = cpu_fill_vdso_timehands32(vdso_th32);
+	enabled = cpu_fill_vdso_timehands32(vdso_th32, th->th_counter);
 	if (!vdso_th_enable)
 		enabled = 0;
 	return (enabled);
diff --git a/sys/kern/subr_dummy_vdso_tc.c b/sys/kern/subr_dummy_vdso_tc.c
index 9c845014f50..d1c36b74ead 100644
--- a/sys/kern/subr_dummy_vdso_tc.c
+++ b/sys/kern/subr_dummy_vdso_tc.c
@@ -33,7 +33,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/vdso.h>
 
 uint32_t
-cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th)
+cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc)
 {
 
 	return (0);
@@ -41,7 +41,8 @@ cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th)
 
 #ifdef COMPAT_FREEBSD32
 uint32_t
-cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32)
+cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32,
+    struct timecounter *tc)
 {
 
 	return (0);
diff --git a/sys/sys/vdso.h b/sys/sys/vdso.h
index f584315b956..d905304ac68 100644
--- a/sys/sys/vdso.h
+++ b/sys/sys/vdso.h
@@ -69,6 +69,8 @@ int __vdso_gettimekeep(struct vdso_timekeep **tk);
 
 #ifdef _KERNEL
 
+struct timecounter;
+
 void timekeep_push_vdso(void);
 
 uint32_t tc_fill_vdso_timehands(struct vdso_timehands *vdso_th);
@@ -81,7 +83,8 @@ uint32_t tc_fill_vdso_timehands(struct vdso_timehands *vdso_th);
  * global sysctl enable override is handled by machine-independed code
  * after cpu_fill_vdso_timehands() call is made.
  */
-uint32_t cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th);
+uint32_t cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th,
+    struct timecounter *tc);
 
 #define	VDSO_TH_NUM	4
 
@@ -110,7 +113,8 @@ struct vdso_timekeep32 {
 };
 
 uint32_t tc_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32);
-uint32_t cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32);
+uint32_t cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32,
+    struct timecounter *tc);
 
 #endif
 #endif
diff --git a/sys/x86/x86/tsc.c b/sys/x86/x86/tsc.c
index c84f28a3e60..e2ae73517fa 100644
--- a/sys/x86/x86/tsc.c
+++ b/sys/x86/x86/tsc.c
@@ -720,21 +720,22 @@ tsc_get_timecount_low_mfence(struct timecounter *tc)
 }
 
 uint32_t
-cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th)
+cpu_fill_vdso_timehands(struct vdso_timehands *vdso_th, struct timecounter *tc)
 {
 
-	vdso_th->th_x86_shift = (int)(intptr_t)timecounter->tc_priv;
+	vdso_th->th_x86_shift = (int)(intptr_t)tc->tc_priv;
 	bzero(vdso_th->th_res, sizeof(vdso_th->th_res));
-	return (timecounter == &tsc_timecounter);
+	return (tc == &tsc_timecounter);
 }
 
 #ifdef COMPAT_FREEBSD32
 uint32_t
-cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32)
+cpu_fill_vdso_timehands32(struct vdso_timehands32 *vdso_th32,
+    struct timecounter *tc)
 {
 
-	vdso_th32->th_x86_shift = (int)(intptr_t)timecounter->tc_priv;
+	vdso_th32->th_x86_shift = (int)(intptr_t)tc->tc_priv;
 	bzero(vdso_th32->th_res, sizeof(vdso_th32->th_res));
-	return (timecounter == &tsc_timecounter);
+	return (tc == &tsc_timecounter);
 }
 #endif

From 3dcd1c9585c8dc53bfdbba81ed38c7091ece99eb Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Tue, 20 Jan 2015 05:28:03 +0000
Subject: [PATCH 095/279] Zero BSS explicitly if not started by loader(8). Add
 a check for the magic values that ePAPR-compliant loaders (like skiboot) put
 in the register loader uses for the metadata pointer to avoid confusing them.

---
 sys/powerpc/aim/machdep.c | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c
index fdbe48b4b54..d0aac4c7f46 100644
--- a/sys/powerpc/aim/machdep.c
+++ b/sys/powerpc/aim/machdep.c
@@ -224,6 +224,10 @@ cpu_startup(void *dummy)
 }
 
 extern vm_offset_t	__startkernel, __endkernel;
+extern unsigned char	__bss_start[];
+extern unsigned char	__sbss_start[];
+extern unsigned char	__sbss_end[];
+extern unsigned char	_end[];
 
 #ifndef __powerpc64__
 /* Bits for running on 64-bit systems in 32-bit mode. */
@@ -272,9 +276,6 @@ powerpc_init(vm_offset_t fdt, vm_offset_t toc, vm_offset_t ofentry, void *mdp)
 	trap_offset = 0;
 	cacheline_warn = 0;
 
-	/* Store boot environment state */
-	OF_initial_setup((void *)fdt, NULL, (int (*)(void *))ofentry);
-
 	/* First guess at start/end kernel positions */
 	startkernel = __startkernel;
 	endkernel = __endkernel;
@@ -289,6 +290,10 @@ powerpc_init(vm_offset_t fdt, vm_offset_t toc, vm_offset_t ofentry, void *mdp)
 		mdp = NULL;
 #endif
 
+	/* Check for ePAPR loader, which puts a magic value into r6 */
+	if (mdp == (void *)0x65504150)
+		mdp = NULL;
+
 	/*
 	 * Parse metadata if present and fetch parameters.  Must be done
 	 * before console is inited so cninit gets the right value of
@@ -308,8 +313,14 @@ powerpc_init(vm_offset_t fdt, vm_offset_t toc, vm_offset_t ofentry, void *mdp)
 			db_fetch_ksymtab(ksym_start, ksym_end);
 #endif
 		}
+	} else {
+		bzero(__sbss_start, __sbss_end - __sbss_start);
+		bzero(__bss_start, _end - __bss_start);
 	}
 
+	/* Store boot environment state */
+	OF_initial_setup((void *)fdt, NULL, (int (*)(void *))ofentry);
+
 	/*
 	 * Init params/tunables that can be overridden by the loader
 	 */

From b9d056f3d641ac14069d53ef4de2d52d6a8fa78e Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Tue, 20 Jan 2015 05:44:21 +0000
Subject: [PATCH 096/279] Remove space in the FDT reservation map from the
 available memory regions in ofw_mem_regions(). This function is actually MI
 and should move to dev/ofw at some point in the near future so that ARM and
 MIPS can use the same code.

---
 sys/powerpc/ofw/ofw_machdep.c | 91 +++++++++++++++++++++++++++++++++--
 1 file changed, 87 insertions(+), 4 deletions(-)

diff --git a/sys/powerpc/ofw/ofw_machdep.c b/sys/powerpc/ofw/ofw_machdep.c
index 1c02fa3e90d..50f9d9e8653 100644
--- a/sys/powerpc/ofw/ofw_machdep.c
+++ b/sys/powerpc/ofw/ofw_machdep.c
@@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/smp.h>
 #include <sys/stat.h>
+#include <sys/endian.h>
 
 #include <net/ethernet.h>
 
@@ -211,12 +212,89 @@ parse_ofw_memory(phandle_t node, const char *prop, struct mem_region *output)
 	return (sz);
 }
 
+static int
+excise_fdt_reserved(struct mem_region *avail, int asz)
+{
+	struct {
+		uint64_t address;
+		uint64_t size;
+	} fdtmap[16];
+	ssize_t fdtmapsize;
+	phandle_t chosen;
+	int i, j, k;
+
+	chosen = OF_finddevice("/chosen");
+	fdtmapsize = OF_getprop(chosen, "fdtmemreserv", fdtmap, sizeof(fdtmap));
+
+	for (j = 0; j < fdtmapsize/sizeof(fdtmap[0]); j++) {
+		fdtmap[j].address = be64toh(fdtmap[j].address);
+		fdtmap[j].size = be64toh(fdtmap[j].size);
+	}
+
+	for (i = 0; i < asz; i++) {
+		for (j = 0; j < fdtmapsize/sizeof(fdtmap[0]); j++) {
+			/*
+			 * Case 1: Exclusion region encloses complete
+			 * available entry. Drop it and move on.
+			 */
+			if (fdtmap[j].address <= avail[i].mr_start &&
+			    fdtmap[j].address + fdtmap[j].size >=
+			    avail[i].mr_start + avail[i].mr_size) {
+				for (k = i+1; k < asz; k++)
+					avail[k-1] = avail[k];
+				asz--;
+				i--; /* Repeat some entries */
+				continue;
+			}
+
+			/*
+			 * Case 2: Exclusion region starts in available entry.
+			 * Trim it to where the entry begins and append
+			 * a new available entry with the region after
+			 * the excluded region, if any.
+			 */
+			if (fdtmap[j].address >= avail[i].mr_start &&
+			    fdtmap[j].address < avail[i].mr_start +
+			    avail[i].mr_size) {
+				if (fdtmap[j].address + fdtmap[j].size < 
+				    avail[i].mr_start + avail[i].mr_size) {
+					avail[asz].mr_start =
+					    roundup2(fdtmap[j].address +
+					    fdtmap[j].size, PAGE_SIZE);
+					avail[asz].mr_size = avail[i].mr_start +
+					     avail[i].mr_size -
+					     avail[asz].mr_start;
+					asz++;
+				}
+
+				avail[i].mr_size =
+				    rounddown2(fdtmap[j].address, PAGE_MASK) -
+				    avail[i].mr_start;
+			}
+
+			/*
+			 * Case 3: Exclusion region ends in available entry.
+			 * Move start point to where the exclusion zone ends.
+			 * The case of a contained exclusion zone has already
+			 * been caught in case 2.
+			 */
+			if (fdtmap[j].address + fdtmap[j].size >=
+			    avail[i].mr_start && fdtmap[j].address +
+			    fdtmap[j].size < avail[i].mr_start +
+			    avail[i].mr_size) {
+				avail[i].mr_start =
+				    roundup2(fdtmap[j].address + fdtmap[j].size,				    PAGE_MASK);
+			}
+		}
+	}
+
+	return (asz);
+}
+
 /*
  * This is called during powerpc_init, before the system is really initialized.
  * It shall provide the total and the available regions of RAM.
- * Both lists must have a zero-size entry as terminator.
- * The available regions need not take the kernel into account, but needs
- * to provide space for two additional entry beyond the terminating one.
+ * The available regions need not take the kernel into account.
  */
 void
 ofw_mem_regions(struct mem_region *memp, int *memsz,
@@ -236,7 +314,8 @@ ofw_mem_regions(struct mem_region *memp, int *memsz,
 	    phandle = OF_peer(phandle)) {
 		if (OF_getprop(phandle, "name", name, sizeof(name)) <= 0)
 			continue;
-		if (strncmp(name, "memory", sizeof(name)) != 0)
+		if (strncmp(name, "memory", sizeof(name)) != 0 &&
+		    strncmp(name, "memory@", strlen("memory@")) != 0)
 			continue;
 
 		res = parse_ofw_memory(phandle, "reg", &memp[msz]);
@@ -249,6 +328,10 @@ ofw_mem_regions(struct mem_region *memp, int *memsz,
 		asz += res/sizeof(struct mem_region);
 	}
 
+	phandle = OF_finddevice("/chosen");
+	if (OF_hasprop(phandle, "fdtmemreserv"))
+		asz = excise_fdt_reserved(availp, asz);
+
 	*memsz = msz;
 	*availsz = asz;
 }

From e2763dca83ee1bcf06aa282b483bcf7751038ab4 Mon Sep 17 00:00:00 2001
From: Ganbold Tsagaankhuu <ganbold@FreeBSD.org>
Date: Tue, 20 Jan 2015 09:07:28 +0000
Subject: [PATCH 097/279] Enable Synopsys DesignWare Mobile Storage Host
 Controller driver on Rockchip boards. It currently supports PIO mode and dma
 mode needs external dma controller to be used.

Submitted by:   jmcneill
Approved by:    stas (mentor)
---
 sys/arm/conf/RK3188                        |   5 +-
 sys/arm/rockchip/files.rk30xx              |   2 +
 sys/boot/fdt/dts/arm/rk3188-radxa-lite.dts |   4 +
 sys/boot/fdt/dts/arm/rk3188-radxa.dts      |   4 +
 sys/boot/fdt/dts/arm/rk3188.dtsi           |  12 +-
 sys/dev/mmc/host/dwmmc.c                   | 169 +++++++++++++++++----
 sys/dev/mmc/host/dwmmc.h                   |   2 +
 7 files changed, 163 insertions(+), 35 deletions(-)

diff --git a/sys/arm/conf/RK3188 b/sys/arm/conf/RK3188
index 13e2ed09a63..2aba68e184d 100644
--- a/sys/arm/conf/RK3188
+++ b/sys/arm/conf/RK3188
@@ -76,8 +76,9 @@ options 	DIAGNOSTIC
 options 	ROOTDEVNAME=\"ufs:/dev/da0s2\"
 
 # MMC/SD/SDIO Card slot support
-#device		mmc			# mmc/sd bus
-#device		mmcsd			# mmc/sd flash cards
+device		mmc			# mmc/sd bus
+device		mmcsd			# mmc/sd flash cards
+device		dwmmc
 
 # Console and misc
 device		uart
diff --git a/sys/arm/rockchip/files.rk30xx b/sys/arm/rockchip/files.rk30xx
index bfbf7617657..0bcc34c3657 100644
--- a/sys/arm/rockchip/files.rk30xx
+++ b/sys/arm/rockchip/files.rk30xx
@@ -19,3 +19,5 @@ arm/rockchip/rk30xx_grf.c		standard
 arm/rockchip/rk30xx_wdog.c		standard
 arm/rockchip/rk30xx_gpio.c		optional	gpio
 arm/rockchip/rk30xx_mp.c		optional	smp
+
+dev/mmc/host/dwmmc.c			optional	dwmmc
diff --git a/sys/boot/fdt/dts/arm/rk3188-radxa-lite.dts b/sys/boot/fdt/dts/arm/rk3188-radxa-lite.dts
index 8f8977683e6..643f5d0ea82 100644
--- a/sys/boot/fdt/dts/arm/rk3188-radxa-lite.dts
+++ b/sys/boot/fdt/dts/arm/rk3188-radxa-lite.dts
@@ -48,6 +48,10 @@
 			status = "okay";
 		};
 
+		mmc@10214000 {
+			status = "okay";
+		};
+
 	};
 
 	chosen {
diff --git a/sys/boot/fdt/dts/arm/rk3188-radxa.dts b/sys/boot/fdt/dts/arm/rk3188-radxa.dts
index ff9ee87a8db..c3740a0a5a4 100644
--- a/sys/boot/fdt/dts/arm/rk3188-radxa.dts
+++ b/sys/boot/fdt/dts/arm/rk3188-radxa.dts
@@ -48,6 +48,10 @@
 			status = "okay";
 		};
 
+		mmc@10214000 {
+			status = "okay";
+		};
+
 	};
 
 	chosen {
diff --git a/sys/boot/fdt/dts/arm/rk3188.dtsi b/sys/boot/fdt/dts/arm/rk3188.dtsi
index c6ac38561a1..f16c1b86818 100644
--- a/sys/boot/fdt/dts/arm/rk3188.dtsi
+++ b/sys/boot/fdt/dts/arm/rk3188.dtsi
@@ -231,22 +231,26 @@
 		};
 
 		mmc@10214000 {
-			compatible = "rockchip,rk30xx-mmc";
+			compatible = "rockchip,rk2928-dw-mshc";
 			reg = <0x10214000 0x1000>;
 			interrupts = <55>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			clock-frequency = <24000000>;	/* TODO: verify freq */
+			bus-frequency = <48000000>;	/* TODO: verify freq */
+			fifo-depth = <0x40>;
+			num-slots = <1>;
 			status = "disabled";
 		};
 
 		mmc@10218000 {
-			compatible = "rockchip,rk30xx-mmc";
+			compatible = "rockchip,rk2928-dw-mshc";
 			reg = <0x10218000 0x1000>;
 			interrupts = <56>;
 			#address-cells = <1>;
 			#size-cells = <0>;
-			clock-frequency = <24000000>;	/* TODO: verify freq */
+			bus-frequency = <48000000>;	/* TODO: verify freq */
+			fifo-depth = <0x40>;
+			num-slots = <1>;
 			status = "disabled";
 		};
 	};
diff --git a/sys/dev/mmc/host/dwmmc.c b/sys/dev/mmc/host/dwmmc.c
index 79fe69b40f6..81e3083a33c 100644
--- a/sys/dev/mmc/host/dwmmc.c
+++ b/sys/dev/mmc/host/dwmmc.c
@@ -129,6 +129,8 @@ struct dwmmc_softc {
 	uint32_t		flags;
 	uint32_t		hwtype;
 	uint32_t		use_auto_stop;
+	uint32_t		use_pio;
+	uint32_t		pwren_inverted;
 
 	bus_dma_tag_t		desc_tag;
 	bus_dmamap_t		desc_map;
@@ -152,6 +154,8 @@ static void dwmmc_next_operation(struct dwmmc_softc *);
 static int dwmmc_setup_bus(struct dwmmc_softc *, int);
 static int dma_done(struct dwmmc_softc *, struct mmc_command *);
 static int dma_stop(struct dwmmc_softc *);
+static void pio_read(struct dwmmc_softc *, struct mmc_command *);
+static void pio_write(struct dwmmc_softc *, struct mmc_command *);
 
 static struct resource_spec dwmmc_spec[] = {
 	{ SYS_RES_MEMORY,	0,	RF_ACTIVE },
@@ -163,6 +167,7 @@ enum {
 	HWTYPE_NONE,
 	HWTYPE_ALTERA,
 	HWTYPE_EXYNOS,
+	HWTYPE_ROCKCHIP,
 };
 
 #define	HWTYPE_MASK		(0x0000ffff)
@@ -171,6 +176,7 @@ enum {
 static struct ofw_compat_data compat_data[] = {
 	{"altr,socfpga-dw-mshc",	HWTYPE_ALTERA},
 	{"samsung,exynos5420-dw-mshc",	HWTYPE_EXYNOS},
+	{"rockchip,rk2928-dw-mshc",	HWTYPE_ROCKCHIP},
 	{NULL,				HWTYPE_NONE},
 };
 
@@ -395,8 +401,10 @@ dwmmc_intr(void *arg)
 			dprintf("data err 0x%08x cmd 0x%08x\n",
 				reg, cmd->opcode);
 			cmd->error = MMC_ERR_FAILED;
-			dma_done(sc, cmd);
-			dma_stop(sc);
+			if (!sc->use_pio) {
+				dma_done(sc, cmd);
+				dma_stop(sc);
+			}
 		}
 
 		if (reg & SDMMC_INTMASK_CMD_DONE) {
@@ -421,15 +429,24 @@ dwmmc_intr(void *arg)
 		}
 	}
 
-	/* Now handle DMA interrupts */
-	reg = READ4(sc, SDMMC_IDSTS);
-	if (reg) {
-		dprintf("dma intr 0x%08x\n", reg);
-		if (reg & (SDMMC_IDINTEN_TI | SDMMC_IDINTEN_RI)) {
-			WRITE4(sc, SDMMC_IDSTS, (SDMMC_IDINTEN_TI |
-						 SDMMC_IDINTEN_RI));
-			WRITE4(sc, SDMMC_IDSTS, SDMMC_IDINTEN_NI);
-			dma_done(sc, cmd);
+	if (sc->use_pio) {
+		if (reg & (SDMMC_INTMASK_RXDR|SDMMC_INTMASK_DTO)) {
+			pio_read(sc, cmd);
+		}
+		if (reg & (SDMMC_INTMASK_TXDR|SDMMC_INTMASK_DTO)) {
+			pio_write(sc, cmd);
+		}
+	} else {
+		/* Now handle DMA interrupts */
+		reg = READ4(sc, SDMMC_IDSTS);
+		if (reg) {
+			dprintf("dma intr 0x%08x\n", reg);
+			if (reg & (SDMMC_IDINTEN_TI | SDMMC_IDINTEN_RI)) {
+				WRITE4(sc, SDMMC_IDSTS, (SDMMC_IDINTEN_TI |
+							 SDMMC_IDINTEN_RI));
+				WRITE4(sc, SDMMC_IDSTS, SDMMC_IDINTEN_NI);
+				dma_done(sc, cmd);
+			}
 		}
 	}
 
@@ -560,17 +577,29 @@ dwmmc_attach(device_t dev)
 	device_printf(dev, "Hardware version ID is %04x\n",
 		READ4(sc, SDMMC_VERID) & 0xffff);
 
-	WRITE4(sc, EMMCP_MPSBEGIN0, 0);
-	WRITE4(sc, EMMCP_SEND0, 0);
-	WRITE4(sc, EMMCP_CTRL0, (MPSCTRL_SECURE_READ_BIT |
-				 MPSCTRL_SECURE_WRITE_BIT |
-				 MPSCTRL_NON_SECURE_READ_BIT |
-				 MPSCTRL_NON_SECURE_WRITE_BIT |
-				 MPSCTRL_VALID));
+	sc->use_pio = 0;
+	sc->pwren_inverted = 0;
+
+	if ((sc->hwtype & HWTYPE_MASK) == HWTYPE_ROCKCHIP) {
+		sc->use_pio = 1;
+		sc->pwren_inverted = 1;
+	} else {
+		WRITE4(sc, EMMCP_MPSBEGIN0, 0);
+		WRITE4(sc, EMMCP_SEND0, 0);
+		WRITE4(sc, EMMCP_CTRL0, (MPSCTRL_SECURE_READ_BIT |
+					 MPSCTRL_SECURE_WRITE_BIT |
+					 MPSCTRL_NON_SECURE_READ_BIT |
+					 MPSCTRL_NON_SECURE_WRITE_BIT |
+					 MPSCTRL_VALID));
+	}
 
 	/* XXX: we support operation for slot index 0 only */
 	slot = 0;
-	WRITE4(sc, SDMMC_PWREN, (1 << slot));
+	if (sc->pwren_inverted) {
+		WRITE4(sc, SDMMC_PWREN, (0 << slot));
+	} else {
+		WRITE4(sc, SDMMC_PWREN, (1 << slot));
+	}
 
 	/* Reset all */
 	if (dwmmc_ctrl_reset(sc, (SDMMC_CTRL_RESET |
@@ -580,17 +609,19 @@ dwmmc_attach(device_t dev)
 
 	dwmmc_setup_bus(sc, sc->host.f_min);
 
-	if (dma_setup(sc))
-		return (ENXIO);
+	if (!sc->use_pio) {
+		if (dma_setup(sc))
+			return (ENXIO);
 
-	/* Install desc base */
-	WRITE4(sc, SDMMC_DBADDR, sc->desc_ring_paddr);
+		/* Install desc base */
+		WRITE4(sc, SDMMC_DBADDR, sc->desc_ring_paddr);
 
-	/* Enable DMA interrupts */
-	WRITE4(sc, SDMMC_IDSTS, SDMMC_IDINTEN_MASK);
-	WRITE4(sc, SDMMC_IDINTEN, (SDMMC_IDINTEN_NI |
-				   SDMMC_IDINTEN_RI |
-				   SDMMC_IDINTEN_TI));
+		/* Enable DMA interrupts */
+		WRITE4(sc, SDMMC_IDSTS, SDMMC_IDINTEN_MASK);
+		WRITE4(sc, SDMMC_IDINTEN, (SDMMC_IDINTEN_NI |
+					   SDMMC_IDINTEN_RI |
+					   SDMMC_IDINTEN_TI));
+	}
 
 	/* Clear and disable interrups for a while */
 	WRITE4(sc, SDMMC_RINTSTS, 0xffffffff);
@@ -797,6 +828,79 @@ dma_prepare(struct dwmmc_softc *sc, struct mmc_command *cmd)
 	return (0);
 }
 
+static int
+pio_prepare(struct dwmmc_softc *sc, struct mmc_command *cmd)
+{
+	struct mmc_data *data;
+	int reg;
+
+	data = cmd->data;
+	data->xfer_len = 0;
+
+	reg = (DEF_MSIZE << SDMMC_FIFOTH_MSIZE_S);
+	reg |= ((sc->fifo_depth / 2) - 1) << SDMMC_FIFOTH_RXWMARK_S;
+	reg |= (sc->fifo_depth / 2) << SDMMC_FIFOTH_TXWMARK_S;
+
+	WRITE4(sc, SDMMC_FIFOTH, reg);
+	wmb();
+
+	return (0);
+}
+
+static void
+pio_read(struct dwmmc_softc *sc, struct mmc_command *cmd)
+{
+	struct mmc_data *data;
+	uint32_t *p, status;
+
+	if (cmd == NULL || cmd->data == NULL)
+		return;
+
+	data = cmd->data;
+	if ((data->flags & MMC_DATA_READ) == 0)
+		return;
+
+	KASSERT((data->xfer_len & 3) == 0, ("xfer_len not aligned"));
+	p = (uint32_t *)data->data + (data->xfer_len >> 2);
+
+	while (data->xfer_len < data->len) {
+		status = READ4(sc, SDMMC_STATUS);
+		if (status & SDMMC_STATUS_FIFO_EMPTY)
+			break;
+		*p++ = READ4(sc, SDMMC_DATA);
+		data->xfer_len += 4;
+	}
+
+	WRITE4(sc, SDMMC_RINTSTS, SDMMC_INTMASK_RXDR);
+}
+
+static void
+pio_write(struct dwmmc_softc *sc, struct mmc_command *cmd)
+{
+	struct mmc_data *data;
+	uint32_t *p, status;
+
+	if (cmd == NULL || cmd->data == NULL)
+		return;
+
+	data = cmd->data;
+	if ((data->flags & MMC_DATA_WRITE) == 0)
+		return;
+
+	KASSERT((data->xfer_len & 3) == 0, ("xfer_len not aligned"));
+	p = (uint32_t *)data->data + (data->xfer_len >> 2);
+
+	while (data->xfer_len < data->len) {
+		status = READ4(sc, SDMMC_STATUS);
+		if (status & SDMMC_STATUS_FIFO_FULL)
+			break;
+		WRITE4(sc, SDMMC_DATA, *p++);
+		data->xfer_len += 4;
+	}
+
+	WRITE4(sc, SDMMC_RINTSTS, SDMMC_INTMASK_TXDR);
+}
+
 static void
 dwmmc_start_cmd(struct dwmmc_softc *sc, struct mmc_command *cmd)
 {
@@ -807,6 +911,9 @@ dwmmc_start_cmd(struct dwmmc_softc *sc, struct mmc_command *cmd)
 	sc->curcmd = cmd;
 	data = cmd->data;
 
+	if ((sc->hwtype & HWTYPE_MASK) == HWTYPE_ROCKCHIP)
+		dwmmc_setup_bus(sc, sc->host.ios.clock);
+
 	/* XXX Upper layers don't always set this */
 	cmd->mrq = sc->req;
 
@@ -861,7 +968,11 @@ dwmmc_start_cmd(struct dwmmc_softc *sc, struct mmc_command *cmd)
 			 data->len : MMC_SECTOR_SIZE;
 		WRITE4(sc, SDMMC_BLKSIZ, blksz);
 
-		dma_prepare(sc, cmd);
+		if (sc->use_pio) {
+			pio_prepare(sc, cmd);
+		} else {
+			dma_prepare(sc, cmd);
+		}
 		wmb();
 	}
 
diff --git a/sys/dev/mmc/host/dwmmc.h b/sys/dev/mmc/host/dwmmc.h
index 8f763cd5041..a075adbbf3c 100644
--- a/sys/dev/mmc/host/dwmmc.h
+++ b/sys/dev/mmc/host/dwmmc.h
@@ -91,6 +91,8 @@
 #define	SDMMC_RINTSTS		0x44	/* Raw Interrupt Status Register */
 #define	SDMMC_STATUS		0x48	/* Status Register */
 #define	 SDMMC_STATUS_DATA_BUSY	(1 << 9) /* card_data[0] */
+#define	 SDMMC_STATUS_FIFO_FULL	(1 << 3) /* FIFO full */
+#define	 SDMMC_STATUS_FIFO_EMPTY (1 << 2) /* FIFO empty */
 #define	SDMMC_FIFOTH		0x4C	/* FIFO Threshold Watermark Register */
 #define	 SDMMC_FIFOTH_MSIZE_S	28	/* Burst size of multiple transaction */
 #define	 SDMMC_FIFOTH_RXWMARK_S	16	/* FIFO threshold watermark level */

From fd3276de58db5bfa7204e424fccf8be6a045fffd Mon Sep 17 00:00:00 2001
From: Ruslan Bukin <br@FreeBSD.org>
Date: Tue, 20 Jan 2015 11:10:25 +0000
Subject: [PATCH 098/279] Add 128-byte cache flushing routines.

Leave CNMIPS untouched as these functions depends on config2
register.
---
 sys/mips/include/cache_mipsNN.h |   2 -
 sys/mips/mips/cache.c           |   4 -
 sys/mips/mips/cache_mipsNN.c    | 219 ++++++++++++++++++++++++++++++++
 3 files changed, 219 insertions(+), 6 deletions(-)

diff --git a/sys/mips/include/cache_mipsNN.h b/sys/mips/include/cache_mipsNN.h
index 435993b9b72..1969ab1a911 100644
--- a/sys/mips/include/cache_mipsNN.h
+++ b/sys/mips/include/cache_mipsNN.h
@@ -57,7 +57,6 @@ void	mipsNN_pdcache_inv_range_16(vm_offset_t, vm_size_t);
 void	mipsNN_pdcache_inv_range_32(vm_offset_t, vm_size_t);
 void	mipsNN_pdcache_wb_range_16(vm_offset_t, vm_size_t);
 void	mipsNN_pdcache_wb_range_32(vm_offset_t, vm_size_t);
-#ifdef CPU_CNMIPS
 void	mipsNN_icache_sync_all_128(void);
 void	mipsNN_icache_sync_range_128(vm_offset_t, vm_size_t);
 void	mipsNN_icache_sync_range_index_128(vm_offset_t, vm_size_t);
@@ -66,7 +65,6 @@ void	mipsNN_pdcache_wbinv_range_128(vm_offset_t, vm_size_t);
 void	mipsNN_pdcache_wbinv_range_index_128(vm_offset_t, vm_size_t);
 void	mipsNN_pdcache_inv_range_128(vm_offset_t, vm_size_t);
 void	mipsNN_pdcache_wb_range_128(vm_offset_t, vm_size_t);
-#endif
 void	mipsNN_sdcache_wbinv_all_32(void);
 void	mipsNN_sdcache_wbinv_range_32(vm_offset_t, vm_size_t);
 void	mipsNN_sdcache_wbinv_range_index_32(vm_offset_t, vm_size_t);
diff --git a/sys/mips/mips/cache.c b/sys/mips/mips/cache.c
index 70a1db38042..59172d705b0 100644
--- a/sys/mips/mips/cache.c
+++ b/sys/mips/mips/cache.c
@@ -104,7 +104,6 @@ mips_config_cache(struct mips_cpuinfo * cpuinfo)
 		mips_cache_ops.mco_icache_sync_range_index =
 		    mipsNN_icache_sync_range_index_32;
 		break;
-#ifdef CPU_CNMIPS
 	case 128:
 		mips_cache_ops.mco_icache_sync_all = mipsNN_icache_sync_all_128;
 		mips_cache_ops.mco_icache_sync_range =
@@ -112,7 +111,6 @@ mips_config_cache(struct mips_cpuinfo * cpuinfo)
 		mips_cache_ops.mco_icache_sync_range_index =
 		    mipsNN_icache_sync_range_index_128;
 		break;
-#endif
 
 #ifdef MIPS_DISABLE_L1_CACHE
 	case 0:
@@ -172,7 +170,6 @@ mips_config_cache(struct mips_cpuinfo * cpuinfo)
 		    mipsNN_pdcache_wb_range_32;
 #endif
 		break;
-#ifdef CPU_CNMIPS
 	case 128:
 		mips_cache_ops.mco_pdcache_wbinv_all =
 		    mips_cache_ops.mco_intern_pdcache_wbinv_all =
@@ -188,7 +185,6 @@ mips_config_cache(struct mips_cpuinfo * cpuinfo)
 		    mips_cache_ops.mco_intern_pdcache_wb_range =
 		    mipsNN_pdcache_wb_range_128;
 		break;
-#endif		
 #ifdef MIPS_DISABLE_L1_CACHE
 	case 0:
 		mips_cache_ops.mco_pdcache_wbinv_all =
diff --git a/sys/mips/mips/cache_mipsNN.c b/sys/mips/mips/cache_mipsNN.c
index ac104c2cfd2..2bb1fa159c7 100644
--- a/sys/mips/mips/cache_mipsNN.c
+++ b/sys/mips/mips/cache_mipsNN.c
@@ -647,6 +647,225 @@ mipsNN_pdcache_wb_range_128(vm_offset_t va, vm_size_t size)
 	SYNC;
 }
 
+#else
+
+void
+mipsNN_icache_sync_all_128(void)
+{
+	vm_offset_t va, eva;
+
+	va = MIPS_PHYS_TO_KSEG0(0);
+	eva = va + picache_size;
+
+	/*
+	 * Since we're hitting the whole thing, we don't have to
+	 * worry about the N different "ways".
+	 */
+
+	mips_intern_dcache_wbinv_all();
+
+	while (va < eva) {
+		cache_r4k_op_32lines_128(va, CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
+		va += (32 * 128);
+	}
+
+	SYNC;
+}
+
+void
+mipsNN_icache_sync_range_128(vm_offset_t va, vm_size_t size)
+{
+	vm_offset_t eva;
+
+	eva = round_line128(va + size);
+	va = trunc_line128(va);
+
+	mips_intern_dcache_wb_range(va, (eva - va));
+
+	while ((eva - va) >= (32 * 128)) {
+		cache_r4k_op_32lines_128(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
+		va += (32 * 128);
+	}
+
+	while (va < eva) {
+		cache_op_r4k_line(va, CACHE_R4K_I|CACHEOP_R4K_HIT_INV);
+		va += 128;
+	}
+
+	SYNC;
+}
+
+void
+mipsNN_icache_sync_range_index_128(vm_offset_t va, vm_size_t size)
+{
+	vm_offset_t eva, tmpva;
+	int i, stride, loopcount;
+
+	/*
+	 * Since we're doing Index ops, we expect to not be able
+	 * to access the address we've been given.  So, get the
+	 * bits that determine the cache index, and make a KSEG0
+	 * address out of them.
+	 */
+	va = MIPS_PHYS_TO_KSEG0(va & picache_way_mask);
+
+	eva = round_line128(va + size);
+	va = trunc_line128(va);
+
+	/*
+	 * GCC generates better code in the loops if we reference local
+	 * copies of these global variables.
+	 */
+	stride = picache_stride;
+	loopcount = picache_loopcount;
+
+	mips_intern_dcache_wbinv_range_index(va, (eva - va));
+
+	while ((eva - va) >= (32 * 128)) {
+		tmpva = va;
+		for (i = 0; i < loopcount; i++, tmpva += stride)
+			cache_r4k_op_32lines_128(tmpva,
+			    CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
+		va += 32 * 128;
+	}
+
+	while (va < eva) {
+		tmpva = va;
+		for (i = 0; i < loopcount; i++, tmpva += stride)
+			cache_op_r4k_line(tmpva,
+			    CACHE_R4K_I|CACHEOP_R4K_INDEX_INV);
+		va += 128;
+	}
+}
+
+void
+mipsNN_pdcache_wbinv_all_128(void)
+{
+	vm_offset_t va, eva;
+
+	va = MIPS_PHYS_TO_KSEG0(0);
+	eva = va + pdcache_size;
+
+	/*
+	 * Since we're hitting the whole thing, we don't have to
+	 * worry about the N different "ways".
+	 */
+
+	while (va < eva) {
+		cache_r4k_op_32lines_128(va,
+		    CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
+		va += (32 * 128);
+	}
+
+	SYNC;
+}
+
+
+void
+mipsNN_pdcache_wbinv_range_128(vm_offset_t va, vm_size_t size)
+{
+	vm_offset_t eva;
+
+	eva = round_line128(va + size);
+	va = trunc_line128(va);
+
+	while ((eva - va) >= (32 * 128)) {
+		cache_r4k_op_32lines_128(va,
+		    CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
+		va += (32 * 128);
+	}
+
+	while (va < eva) {
+		cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB_INV);
+		va += 128;
+	}
+
+	SYNC;
+}
+
+void
+mipsNN_pdcache_wbinv_range_index_128(vm_offset_t va, vm_size_t size)
+{
+	vm_offset_t eva, tmpva;
+	int i, stride, loopcount;
+
+	/*
+	 * Since we're doing Index ops, we expect to not be able
+	 * to access the address we've been given.  So, get the
+	 * bits that determine the cache index, and make a KSEG0
+	 * address out of them.
+	 */
+	va = MIPS_PHYS_TO_KSEG0(va & pdcache_way_mask);
+
+	eva = round_line128(va + size);
+	va = trunc_line128(va);
+
+	/*
+	 * GCC generates better code in the loops if we reference local
+	 * copies of these global variables.
+	 */
+	stride = pdcache_stride;
+	loopcount = pdcache_loopcount;
+
+	while ((eva - va) >= (32 * 128)) {
+		tmpva = va;
+		for (i = 0; i < loopcount; i++, tmpva += stride)
+			cache_r4k_op_32lines_128(tmpva,
+			    CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
+		va += 32 * 128;
+	}
+
+	while (va < eva) {
+		tmpva = va;
+		for (i = 0; i < loopcount; i++, tmpva += stride)
+			cache_op_r4k_line(tmpva,
+			    CACHE_R4K_D|CACHEOP_R4K_INDEX_WB_INV);
+		va += 128;
+	}
+}
+
+void
+mipsNN_pdcache_inv_range_128(vm_offset_t va, vm_size_t size)
+{
+	vm_offset_t eva;
+
+	eva = round_line128(va + size);
+	va = trunc_line128(va);
+
+	while ((eva - va) >= (32 * 128)) {
+		cache_r4k_op_32lines_128(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
+		va += (32 * 128);
+	}
+
+	while (va < eva) {
+		cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_INV);
+		va += 128;
+	}
+
+	SYNC;
+}
+
+void
+mipsNN_pdcache_wb_range_128(vm_offset_t va, vm_size_t size)
+{
+	vm_offset_t eva;
+
+	eva = round_line128(va + size);
+	va = trunc_line128(va);
+
+	while ((eva - va) >= (32 * 128)) {
+		cache_r4k_op_32lines_128(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
+		va += (32 * 128);
+	}
+
+	while (va < eva) {
+		cache_op_r4k_line(va, CACHE_R4K_D|CACHEOP_R4K_HIT_WB);
+		va += 128;
+	}
+
+	SYNC;
+}
+
 #endif
 
 void

From 195237551a9d753801742b9399848a985e2457fa Mon Sep 17 00:00:00 2001
From: Andrew Turner <andrew@FreeBSD.org>
Date: Tue, 20 Jan 2015 11:11:32 +0000
Subject: [PATCH 099/279] Add the User and PL1 read only and reqd write thread
 ID registers.

Sponsored by:	The FreeBSD Foundation
---
 sys/arm/include/cpu-v6.h | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/sys/arm/include/cpu-v6.h b/sys/arm/include/cpu-v6.h
index 8d1a413b9ce..1eafb62da7c 100644
--- a/sys/arm/include/cpu-v6.h
+++ b/sys/arm/include/cpu-v6.h
@@ -179,6 +179,13 @@ _WF1(cp15_pminten_set, CP15_PMINTENSET(%0))
 _WF1(cp15_pminten_clr, CP15_PMINTENCLR(%0))
 #endif
 
+_RF0(cp15_tpidrurw_get, CP15_TPIDRURW(%0))
+_WF1(cp15_tpidrurw_set, CP15_TPIDRURW(%0))
+_RF0(cp15_tpidruro_get, CP15_TPIDRURO(%0))
+_WF1(cp15_tpidruro_set, CP15_TPIDRURO(%0))
+_RF0(cp15_tpidrpwr_get, CP15_TPIDRPRW(%0))
+_WF1(cp15_tpidrpwr_set, CP15_TPIDRPRW(%0))
+
 #undef	_FX
 #undef	_RF0
 #undef	_WF0

From 79b52c387e6705e65d3020b554add5234bcb0a2e Mon Sep 17 00:00:00 2001
From: Andrew Turner <andrew@FreeBSD.org>
Date: Tue, 20 Jan 2015 11:32:48 +0000
Subject: [PATCH 100/279] Remove the SMP code from locore-v4. These will never
 use the SMP code as there is no multi-core hardware prior to ARMv6.

Sponsored by:	The FreeBSD Foundation
---
 sys/arm/arm/locore-v4.S | 92 -----------------------------------------
 1 file changed, 92 deletions(-)

diff --git a/sys/arm/arm/locore-v4.S b/sys/arm/arm/locore-v4.S
index 6c20b683fd2..8ef53e93fa0 100644
--- a/sys/arm/arm/locore-v4.S
+++ b/sys/arm/arm/locore-v4.S
@@ -218,9 +218,6 @@ Lunmapped:
 	bl	build_pagetables
 #endif
 
-#if defined(SMP)
-	orr 	r0, r0, #2		/* Set TTB shared memory flag */
-#endif
 	mcr	p15, 0, r0, c2, c0, 0	/* Set TTB */
 	mcr	p15, 0, r0, c8, c7, 0	/* Flush TLB */
 
@@ -338,9 +335,6 @@ translate_va_to_pa:
 build_pagetables:
 	/* Set the required page attributed */
 	ldr	r4, =(L1_TYPE_S|L1_S_C|L1_S_AP(AP_KRW))
-#if defined(SMP)
-	orr	r4, #(L1_SHARED)
-#endif
 	orr	r1, r4
 
 	/* Move the virtual address to the correct bit location */
@@ -394,92 +388,6 @@ pagetable:
 .Lcpufuncs:
 	.word	_C_LABEL(cpufuncs)
 
-#if defined(SMP)
-
-.Lmpvirt_done:
-	.word	mpvirt_done
-VA_TO_PA_POINTER(Lstartup_pagetable_secondary, temp_pagetable)
-
-ASENTRY_NP(mpentry)
-
-	/* Make sure interrupts are disabled. */
-	mrs	r7, cpsr
-	orr	r7, r7, #(PSR_I | PSR_F)
-	msr	cpsr_c, r7
-
-	/* Disable MMU.  It should be disabled already, but make sure. */
-	mrc	p15, 0, r2, c1, c0, 0
-	bic	r2, r2, #(CPU_CONTROL_MMU_ENABLE | CPU_CONTROL_DC_ENABLE |\
-	    CPU_CONTROL_WBUF_ENABLE)
-	bic	r2, r2, #(CPU_CONTROL_IC_ENABLE)
-	bic	r2, r2, #(CPU_CONTROL_BPRD_ENABLE)
-	mcr	p15, 0, r2, c1, c0, 0
-	nop
-	nop
-	nop
-	CPWAIT(r0)
-
-#if ARM_MMU_V6
-	bl	armv6_idcache_inv_all	/* Modifies r0 only */
-#elif ARM_MMU_V7
-	bl	armv7_idcache_inv_all	/* Modifies r0-r3, ip */
-#endif
-
-	/* Load the page table physical address */
-	adr	r0, Lstartup_pagetable_secondary
-	bl	translate_va_to_pa
-	/* Load the address the secondary page table */
-	ldr	r0, [r0]
-
-	orr 	r0, r0, #2		/* Set TTB shared memory flag */
-	mcr	p15, 0, r0, c2, c0, 0	/* Set TTB */
-	mcr	p15, 0, r0, c8, c7, 0	/* Flush TLB */
-
-	mov	r0, #0
-	mcr	p15, 0, r0, c13, c0, 1	/* Set ASID to 0 */
-
-	/* Set the Domain Access register.  Very important! */
-	mov	r0, #((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) | DOMAIN_CLIENT)
-	mcr	p15, 0, r0, c3, c0, 0
-	/* Enable MMU */
-	mrc	p15, 0, r0, c1, c0, 0
-	orr	r0, r0, #CPU_CONTROL_V6_EXTPAGE
-	orr	r0, r0, #CPU_CONTROL_AF_ENABLE
-	orr	r0, r0, #(CPU_CONTROL_MMU_ENABLE | CPU_CONTROL_DC_ENABLE |\
-	    CPU_CONTROL_WBUF_ENABLE)
-	orr	r0, r0, #(CPU_CONTROL_IC_ENABLE)
-	orr	r0, r0, #(CPU_CONTROL_BPRD_ENABLE)
-	mcr	p15, 0, r0, c1, c0, 0
-	nop
-	nop
-	nop
-	CPWAIT(r0)
-
-	adr	r1, .Lstart
-	ldmia	r1, {r1, r2, sp}	/* Set initial stack and */
-	mrc	p15, 0, r0, c0, c0, 5
-	and	r0, r0, #15
-	mov	r1, #2048
-	mul	r2, r1, r0
-	sub	sp, sp, r2
-	str	r1, [sp]
-	ldr	pc, .Lmpvirt_done
-
-mpvirt_done:
-
-	mov	fp, #0			/* trace back starts here */
-	bl	_C_LABEL(init_secondary)	/* Off we go */
-
-	adr	r0, .Lmpreturned
-	b	_C_LABEL(panic)
-	/* NOTREACHED */
-
-.Lmpreturned:
-	.asciz	"init_secondary() returned"
-	.align	2
-END(mpentry)
-#endif
-
 ENTRY_NP(cpu_halt)
 	mrs     r2, cpsr
 	bic	r2, r2, #(PSR_MODE)

From 84e3f97e6d3caaa67bf00a4d92612e108d61de4a Mon Sep 17 00:00:00 2001
From: Hans Petter Selasky <hselasky@FreeBSD.org>
Date: Tue, 20 Jan 2015 11:43:16 +0000
Subject: [PATCH 101/279] Fix returned data for the USB_GET_DEV_PORT_PATH IOCTL
 in particular the value returned in the "udp_port_level" field.

Reported by:	Uffe Jakobsen <uffe@uffe.org>
MFC after:	1 week
---
 sys/dev/usb/usb_generic.c | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/sys/dev/usb/usb_generic.c b/sys/dev/usb/usb_generic.c
index 96dfe8bceb8..17956cbc799 100644
--- a/sys/dev/usb/usb_generic.c
+++ b/sys/dev/usb/usb_generic.c
@@ -1849,14 +1849,13 @@ ugen_get_port_path(struct usb_fifo *f, struct usb_device_port_path *dpp)
 	if (nlevel > USB_DEVICE_PORT_PATH_MAX)
 		goto error;
 
+	/* store total level of ports */
+	dpp->udp_port_level = nlevel;
+
 	/* store port index array */
 	next = udev;
 	while (next->parent_hub != NULL) {
-		nlevel--;
-
-		dpp->udp_port_no[nlevel] = next->port_no;
-		dpp->udp_port_level = nlevel;
-
+		dpp->udp_port_no[--nlevel] = next->port_no;
 		next = next->parent_hub;
 	}
 	return (0);	/* success */

From b829c841add97704fecae3491768b5e9d766e193 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Roger=20Pau=20Monn=C3=A9?= <royger@FreeBSD.org>
Date: Tue, 20 Jan 2015 12:28:24 +0000
Subject: [PATCH 102/279] loader: fix the size of MODINFOMD_MODULEP

The data in MODINFOMD_MODULEP is packed by the loader as a 4 byte type, but
the amd64 kernel expects a vm_paddr_t, which is of size 8 bytes. Fix this by
saving it as 8 bytes in the loader and retrieving it using the proper type
in the kernel.

Sponsored by: Citrix Systems R&D
---
 sys/boot/i386/libi386/bootinfo64.c | 5 +++--
 sys/x86/xen/pv.c                   | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/sys/boot/i386/libi386/bootinfo64.c b/sys/boot/i386/libi386/bootinfo64.c
index a97ab84ed7e..751e806e1b8 100644
--- a/sys/boot/i386/libi386/bootinfo64.c
+++ b/sys/boot/i386/libi386/bootinfo64.c
@@ -185,6 +185,7 @@ bi_load64(char *args, vm_offset_t addr, vm_offset_t *modulep,
     struct file_metadata	*md;
     u_int64_t			kernend;
     u_int64_t			envp;
+    u_int64_t			module;
     vm_offset_t			size;
     char			*rootdevname;
     int				howto;
@@ -222,7 +223,7 @@ bi_load64(char *args, vm_offset_t addr, vm_offset_t *modulep,
     addr = roundup(addr, PAGE_SIZE);
 
     /* place the metadata before anything */
-    *modulep = addr;
+    module = *modulep = addr;
 
     kfp = file_findfile(NULL, "elf kernel");
     if (kfp == NULL)
@@ -233,7 +234,7 @@ bi_load64(char *args, vm_offset_t addr, vm_offset_t *modulep,
     file_addmetadata(kfp, MODINFOMD_HOWTO, sizeof howto, &howto);
     file_addmetadata(kfp, MODINFOMD_ENVP, sizeof envp, &envp);
     file_addmetadata(kfp, MODINFOMD_KERNEND, sizeof kernend, &kernend);
-    file_addmetadata(kfp, MODINFOMD_MODULEP, sizeof modulep, modulep);
+    file_addmetadata(kfp, MODINFOMD_MODULEP, sizeof module, &module);
     if (add_smap != 0)
         bios_addsmapdata(kfp);
 
diff --git a/sys/x86/xen/pv.c b/sys/x86/xen/pv.c
index e8cdca4f0bc..233bf3e013b 100644
--- a/sys/x86/xen/pv.c
+++ b/sys/x86/xen/pv.c
@@ -402,7 +402,7 @@ xen_pv_parse_preload_data(u_int64_t modulep)
 		 * then calculating the offset with mod_start,
 		 * which contains the relocated modulep address.
 		 */
-		metadata = MD_FETCH(kmdp, MODINFOMD_MODULEP, int);
+		metadata = MD_FETCH(kmdp, MODINFOMD_MODULEP, vm_paddr_t);
 		off = HYPERVISOR_start_info->mod_start - metadata;
 
 		preload_bootstrap_relocate(off);

From d6245e3d441b3b0c00658543bf1c2162dd1a7b57 Mon Sep 17 00:00:00 2001
From: Alexander Motin <mav@FreeBSD.org>
Date: Tue, 20 Jan 2015 13:09:12 +0000
Subject: [PATCH 103/279] Allow skipping dmu_buf_will_dirty() call in
 dsl_dir_transfer_space().

dsl_dir_transfer_space() is mostly called after dsl_dir_diduse_space(),
which already calls dmu_buf_will_dirty() for the same dbuf and tx, so
its duplicate call in those cases will change nothing, only spend time.

Skipping this call by four times reduces time spent in dbuf_write_done()
and descendants, updating dataset statistics with several congested lock
acquisitions.  When rewriting 8K zvol blocks at 1GB/s rate, this reduces
CPU time spent inside dbuf_write_done(), according to profiling, from 45%
of 683K samples to 18% of 422K.

MFC after:	2 weeks
---
 .../contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c    | 6 +++---
 sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c   | 7 ++++---
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
index 34bef4ee569..ce22f4efb52 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dataset.c
@@ -136,7 +136,7 @@ dsl_dataset_block_born(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx)
 	dsl_dir_diduse_space(ds->ds_dir, DD_USED_HEAD, delta,
 	    compressed, uncompressed, tx);
 	dsl_dir_transfer_space(ds->ds_dir, used - delta,
-	    DD_USED_REFRSRV, DD_USED_HEAD, tx);
+	    DD_USED_REFRSRV, DD_USED_HEAD, NULL);
 }
 
 int
@@ -179,7 +179,7 @@ dsl_dataset_block_kill(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx,
 		dsl_dir_diduse_space(ds->ds_dir, DD_USED_HEAD,
 		    delta, -compressed, -uncompressed, tx);
 		dsl_dir_transfer_space(ds->ds_dir, -used - delta,
-		    DD_USED_REFRSRV, DD_USED_HEAD, tx);
+		    DD_USED_REFRSRV, DD_USED_HEAD, NULL);
 	} else {
 		dprintf_bp(bp, "putting on dead list: %s", "");
 		if (async) {
@@ -2837,7 +2837,7 @@ dsl_dataset_clone_swap_sync_impl(dsl_dataset_t *clone,
 		    origin_head->ds_dir->dd_origin_txg, UINT64_MAX,
 		    &odl_used, &odl_comp, &odl_uncomp);
 		dsl_dir_transfer_space(origin_head->ds_dir, cdl_used - odl_used,
-		    DD_USED_HEAD, DD_USED_SNAP, tx);
+		    DD_USED_HEAD, DD_USED_SNAP, NULL);
 	}
 
 	/* swap ds_*_bytes */
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c
index 5a891d1ba3d..1862f8cb8d7 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/dsl_dir.c
@@ -1389,7 +1389,7 @@ dsl_dir_diduse_space(dsl_dir_t *dd, dd_used_t type,
 		    accounted_delta, compressed, uncompressed, tx);
 		dsl_dir_transfer_space(dd->dd_parent,
 		    used - accounted_delta,
-		    DD_USED_CHILD_RSRV, DD_USED_CHILD, tx);
+		    DD_USED_CHILD_RSRV, DD_USED_CHILD, NULL);
 	}
 }
 
@@ -1397,7 +1397,7 @@ void
 dsl_dir_transfer_space(dsl_dir_t *dd, int64_t delta,
     dd_used_t oldtype, dd_used_t newtype, dmu_tx_t *tx)
 {
-	ASSERT(dmu_tx_is_syncing(tx));
+	ASSERT(tx == NULL || dmu_tx_is_syncing(tx));
 	ASSERT(oldtype < DD_USED_NUM);
 	ASSERT(newtype < DD_USED_NUM);
 
@@ -1405,7 +1405,8 @@ dsl_dir_transfer_space(dsl_dir_t *dd, int64_t delta,
 	    !(dsl_dir_phys(dd)->dd_flags & DD_FLAG_USED_BREAKDOWN))
 		return;
 
-	dmu_buf_will_dirty(dd->dd_dbuf, tx);
+	if (tx != NULL)
+		dmu_buf_will_dirty(dd->dd_dbuf, tx);
 	mutex_enter(&dd->dd_lock);
 	ASSERT(delta > 0 ?
 	    dsl_dir_phys(dd)->dd_used_breakdown[oldtype] >= delta :

From 7d502f32856f198a27e038649b3e90965a9f0491 Mon Sep 17 00:00:00 2001
From: Ruslan Bukin <br@FreeBSD.org>
Date: Tue, 20 Jan 2015 15:45:09 +0000
Subject: [PATCH 104/279] o Do notify USB host each time we receive 'set packet
 filter' request.   This makes Mac OS X happy when it returns back from
 suspending. o Switch notify state after data is transferred, but not before.
 o Consider there is also Super Speed mode. o Do not set stall bit on any
 pipes in device mode as Mac OS X seems   don't support it.

In collaboration with:	hselasky@
---
 sys/dev/usb/net/if_cdce.c | 93 +++++++++++++++++++++++++++++----------
 1 file changed, 69 insertions(+), 24 deletions(-)

diff --git a/sys/dev/usb/net/if_cdce.c b/sys/dev/usb/net/if_cdce.c
index 76134baa10a..6824fbcdf16 100644
--- a/sys/dev/usb/net/if_cdce.c
+++ b/sys/dev/usb/net/if_cdce.c
@@ -857,9 +857,12 @@ tr_setup:
 		if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
 
 		if (error != USB_ERR_CANCELLED) {
-			/* try to clear stall first */
-			usbd_xfer_set_stall(xfer);
-			goto tr_setup;
+			if (usbd_get_mode(sc->sc_ue.ue_udev) == USB_MODE_HOST) {
+				/* try to clear stall first */
+				usbd_xfer_set_stall(xfer);
+			} else {
+				goto tr_setup;
+			}
 		}
 		break;
 	}
@@ -1024,10 +1027,12 @@ cdce_bulk_read_callback(struct usb_xfer *xfer, usb_error_t error)
 
 		if (error != USB_ERR_CANCELLED) {
 tr_stall:
-			/* try to clear stall first */
-			usbd_xfer_set_stall(xfer);
-			usbd_xfer_set_frames(xfer, 0);
-			usbd_transfer_submit(xfer);
+			if (usbd_get_mode(sc->sc_ue.ue_udev) == USB_MODE_HOST) {
+				/* try to clear stall first */
+				usbd_xfer_set_stall(xfer);
+				usbd_xfer_set_frames(xfer, 0);
+				usbd_transfer_submit(xfer);
+			}
 			break;
 		}
 
@@ -1040,6 +1045,7 @@ tr_stall:
 static void
 cdce_intr_read_callback(struct usb_xfer *xfer, usb_error_t error)
 {
+	struct cdce_softc *sc = usbd_xfer_softc(xfer);
 	int actlen;
 
 	usbd_xfer_status(xfer, &actlen, NULL, NULL, NULL);
@@ -1061,8 +1067,10 @@ tr_setup:
 	default:			/* Error */
 		if (error != USB_ERR_CANCELLED) {
 			/* start clear stall */
-			usbd_xfer_set_stall(xfer);
-			goto tr_setup;
+			if (usbd_get_mode(sc->sc_ue.ue_udev) == USB_MODE_HOST)
+				usbd_xfer_set_stall(xfer);
+			else
+				goto tr_setup;
 		}
 		break;
 	}
@@ -1084,6 +1092,17 @@ cdce_intr_write_callback(struct usb_xfer *xfer, usb_error_t error)
 
 		DPRINTF("Transferred %d bytes\n", actlen);
 
+		switch (sc->sc_notify_state) {
+		case (CDCE_NOTIFY_NETWORK_CONNECTION):
+			sc->sc_notify_state = CDCE_NOTIFY_SPEED_CHANGE;
+			break;
+		case (CDCE_NOTIFY_SPEED_CHANGE):
+			sc->sc_notify_state = CDCE_NOTIFY_DONE;
+			break;
+		default:
+			break;
+		}
+
 		/* FALLTHROUGH */
 	case USB_ST_SETUP:
 tr_setup:
@@ -1105,7 +1124,6 @@ tr_setup:
 			usbd_xfer_set_frame_len(xfer, 0, sizeof(req));
 			usbd_xfer_set_frames(xfer, 1);
 			usbd_transfer_submit(xfer); 
-			sc->sc_notify_state = CDCE_NOTIFY_SPEED_CHANGE;
 
 		} else if (sc->sc_notify_state == CDCE_NOTIFY_SPEED_CHANGE) {
 			req.bmRequestType = UCDC_NOTIFICATION;
@@ -1116,7 +1134,7 @@ tr_setup:
 			USETW(req.wLength, 8);
 
 			/* Peak theoretical bulk trasfer rate in bits/s */
-			if (usbd_get_speed(sc->sc_ue.ue_udev) == USB_SPEED_HIGH)
+			if (usbd_get_speed(sc->sc_ue.ue_udev) != USB_SPEED_FULL)
 				speed = (13 * 512 * 8 * 1000 * 8);
 			else
 				speed = (19 * 64 * 1 * 1000 * 8);
@@ -1129,15 +1147,17 @@ tr_setup:
 			usbd_xfer_set_frame_len(xfer, 0, sizeof(req));
 			usbd_xfer_set_frames(xfer, 1);
 			usbd_transfer_submit(xfer); 
-			sc->sc_notify_state = CDCE_NOTIFY_DONE;
 		}
 		break;
 
 	default:			/* Error */
 		if (error != USB_ERR_CANCELLED) {
-			/* start clear stall */
-			usbd_xfer_set_stall(xfer);
-			goto tr_setup;
+			if (usbd_get_mode(sc->sc_ue.ue_udev) == USB_MODE_HOST) {
+				/* start clear stall */
+				usbd_xfer_set_stall(xfer);
+			} else {
+				goto tr_setup;
+			}
 		}
 		break;
 	}
@@ -1145,9 +1165,30 @@ tr_setup:
 
 static int
 cdce_handle_request(device_t dev,
-    const void *req, void **pptr, uint16_t *plen,
+    const void *preq, void **pptr, uint16_t *plen,
     uint16_t offset, uint8_t *pstate)
 {
+	struct cdce_softc *sc = device_get_softc(dev);
+	const struct usb_device_request *req = preq;
+	uint8_t is_complete = *pstate;
+
+	/*
+	 * When Mac OS X resumes after suspending it expects
+	 * to be notified again after this request.
+	 */
+	if (req->bmRequestType == UT_WRITE_CLASS_INTERFACE && \
+	    req->bRequest == UCDC_NCM_SET_ETHERNET_PACKET_FILTER) {
+
+		if (is_complete == 1) {
+			mtx_lock(&sc->sc_mtx);
+			sc->sc_notify_state = CDCE_NOTIFY_SPEED_CHANGE;
+			usbd_transfer_start(sc->sc_xfer[CDCE_INTR_TX]);
+			mtx_unlock(&sc->sc_mtx);
+		}
+
+		return (0);
+	}
+
 	return (ENXIO);			/* use builtin handler */
 }
 
@@ -1363,10 +1404,12 @@ cdce_ncm_bulk_write_callback(struct usb_xfer *xfer, usb_error_t error)
 		if_inc_counter(ifp, IFCOUNTER_OERRORS, 1);
 
 		if (error != USB_ERR_CANCELLED) {
-			/* try to clear stall first */
-			usbd_xfer_set_stall(xfer);
-			usbd_xfer_set_frames(xfer, 0);
-			usbd_transfer_submit(xfer);
+			if (usbd_get_mode(sc->sc_ue.ue_udev) == USB_MODE_HOST) {
+				/* try to clear stall first */
+				usbd_xfer_set_stall(xfer);
+				usbd_xfer_set_frames(xfer, 0);
+				usbd_transfer_submit(xfer);
+			}
 		}
 		break;
 	}
@@ -1524,10 +1567,12 @@ tr_setup:
 
 		if (error != USB_ERR_CANCELLED) {
 tr_stall:
-			/* try to clear stall first */
-			usbd_xfer_set_stall(xfer);
-			usbd_xfer_set_frames(xfer, 0);
-			usbd_transfer_submit(xfer);
+			if (usbd_get_mode(sc->sc_ue.ue_udev) == USB_MODE_HOST) {
+				/* try to clear stall first */
+				usbd_xfer_set_stall(xfer);
+				usbd_xfer_set_frames(xfer, 0);
+				usbd_transfer_submit(xfer);
+			}
 		}
 		break;
 	}

From e1e25f45a4231fb39eab4d7c3dc7b4ad932fb82f Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Tue, 20 Jan 2015 16:21:59 +0000
Subject: [PATCH 105/279] There does not seem to be any reason to acquire GIANT
 here. Follow amd64 in removing it.

MFC after:	1 month
---
 sys/powerpc/powerpc/mem.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/sys/powerpc/powerpc/mem.c b/sys/powerpc/powerpc/mem.c
index 358871cbd55..a7fbfa3edf8 100644
--- a/sys/powerpc/powerpc/mem.c
+++ b/sys/powerpc/powerpc/mem.c
@@ -100,8 +100,6 @@ memrw(struct cdev *dev, struct uio *uio, int flags)
 	cnt = 0;
 	error = 0;
 
-	GIANT_REQUIRED;
-
 	while (uio->uio_resid > 0 && !error) {
 		iov = uio->uio_iov;
 		if (iov->iov_len == 0) {

From d69eefebd21ba36899b893956d59e60e42cdf20f Mon Sep 17 00:00:00 2001
From: Ruslan Bukin <br@FreeBSD.org>
Date: Tue, 20 Jan 2015 16:30:02 +0000
Subject: [PATCH 106/279] o Restore 'goto tr_setup;' when operating in host
 mode mistakenly   removed in r277414. o Remove extra parentheses around
 cases.

---
 sys/dev/usb/net/if_cdce.c | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/sys/dev/usb/net/if_cdce.c b/sys/dev/usb/net/if_cdce.c
index 6824fbcdf16..13fca9359f2 100644
--- a/sys/dev/usb/net/if_cdce.c
+++ b/sys/dev/usb/net/if_cdce.c
@@ -860,9 +860,8 @@ tr_setup:
 			if (usbd_get_mode(sc->sc_ue.ue_udev) == USB_MODE_HOST) {
 				/* try to clear stall first */
 				usbd_xfer_set_stall(xfer);
-			} else {
-				goto tr_setup;
 			}
+			goto tr_setup;
 		}
 		break;
 	}
@@ -1069,8 +1068,7 @@ tr_setup:
 			/* start clear stall */
 			if (usbd_get_mode(sc->sc_ue.ue_udev) == USB_MODE_HOST)
 				usbd_xfer_set_stall(xfer);
-			else
-				goto tr_setup;
+			goto tr_setup;
 		}
 		break;
 	}
@@ -1093,10 +1091,10 @@ cdce_intr_write_callback(struct usb_xfer *xfer, usb_error_t error)
 		DPRINTF("Transferred %d bytes\n", actlen);
 
 		switch (sc->sc_notify_state) {
-		case (CDCE_NOTIFY_NETWORK_CONNECTION):
+		case CDCE_NOTIFY_NETWORK_CONNECTION:
 			sc->sc_notify_state = CDCE_NOTIFY_SPEED_CHANGE;
 			break;
-		case (CDCE_NOTIFY_SPEED_CHANGE):
+		case CDCE_NOTIFY_SPEED_CHANGE:
 			sc->sc_notify_state = CDCE_NOTIFY_DONE;
 			break;
 		default:
@@ -1155,9 +1153,8 @@ tr_setup:
 			if (usbd_get_mode(sc->sc_ue.ue_udev) == USB_MODE_HOST) {
 				/* start clear stall */
 				usbd_xfer_set_stall(xfer);
-			} else {
-				goto tr_setup;
 			}
+			goto tr_setup;
 		}
 		break;
 	}

From 20ef3b88cda81ac021cd3978c80182481b56a601 Mon Sep 17 00:00:00 2001
From: Sean Bruno <sbruno@FreeBSD.org>
Date: Tue, 20 Jan 2015 17:00:28 +0000
Subject: [PATCH 107/279] Allow clang to be built for mips/mips64 backend types
 by adding our mips triple ids

This only allows testing and does not change the defaults for mips/mips64.
They still build/use gcc by default.

Differential Revision:	https://reviews.freebsd.org/D1190
Reviewed by:	dim
---
 contrib/llvm/tools/clang/lib/Driver/Tools.cpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/contrib/llvm/tools/clang/lib/Driver/Tools.cpp b/contrib/llvm/tools/clang/lib/Driver/Tools.cpp
index 89db52e6071..f33fb32b06a 100644
--- a/contrib/llvm/tools/clang/lib/Driver/Tools.cpp
+++ b/contrib/llvm/tools/clang/lib/Driver/Tools.cpp
@@ -6587,6 +6587,17 @@ void freebsd::Link::ConstructJob(Compilation &C, const JobAction &JA,
     CmdArgs.push_back("elf32ppc_fbsd");
   }
 
+  if (Arg *A = Args.getLastArg(options::OPT_G)) {
+    if (ToolChain.getArch() == llvm::Triple::mips ||
+      ToolChain.getArch() == llvm::Triple::mipsel ||
+      ToolChain.getArch() == llvm::Triple::mips64 ||
+      ToolChain.getArch() == llvm::Triple::mips64el) {
+      StringRef v = A->getValue();
+      CmdArgs.push_back(Args.MakeArgString("-G" + v));
+      A->claim();
+    }
+  }
+
   if (Output.isFilename()) {
     CmdArgs.push_back("-o");
     CmdArgs.push_back(Output.getFilename());

From bcbf8c21057c8aa83df4c7d3f290a7cc041b1deb Mon Sep 17 00:00:00 2001
From: Michael Tuexen <tuexen@FreeBSD.org>
Date: Tue, 20 Jan 2015 19:08:55 +0000
Subject: [PATCH 108/279] Remove comparisons which are not necessary.

Reported by:	Coverity
CID:		1237826, 1237844, 1237847
MFC after:	1 week
---
 sys/netinet/sctp_sysctl.c | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/sys/netinet/sctp_sysctl.c b/sys/netinet/sctp_sysctl.c
index 1ca5983f684..285d7fb3f9b 100644
--- a/sys/netinet/sctp_sysctl.c
+++ b/sys/netinet/sctp_sysctl.c
@@ -569,8 +569,12 @@ sctp_sysctl_handle_udp_tunneling(SYSCTL_HANDLER_ARGS)
 	error = sysctl_handle_int(oidp, &new, 0, req);
 	if ((error == 0) &&
 	    (req->newptr != NULL)) {
+#if (SCTPCTL_UDP_TUNNELING_PORT_MIN == 0)
+		if (new > SCTPCTL_UDP_TUNNELING_PORT_MAX) {
+#else
 		if ((new < SCTPCTL_UDP_TUNNELING_PORT_MIN) ||
 		    (new > SCTPCTL_UDP_TUNNELING_PORT_MAX)) {
+#endif
 			error = EINVAL;
 		} else {
 			SCTP_INP_INFO_WLOCK();
@@ -598,9 +602,14 @@ sctp_sysctl_handle_auth(SYSCTL_HANDLER_ARGS)
 	error = sysctl_handle_int(oidp, &new, 0, req);
 	if ((error == 0) &&
 	    (req->newptr != NULL)) {
+#if (SCTPCTL_AUTH_ENABLE_MIN == 0)
+		if ((new > SCTPCTL_AUTH_ENABLE_MAX) ||
+		    ((new == 0) && (SCTP_BASE_SYSCTL(sctp_asconf_enable) == 1))) {
+#else
 		if ((new < SCTPCTL_AUTH_ENABLE_MIN) ||
 		    (new > SCTPCTL_AUTH_ENABLE_MAX) ||
 		    ((new == 0) && (SCTP_BASE_SYSCTL(sctp_asconf_enable) == 1))) {
+#endif
 			error = EINVAL;
 		} else {
 			SCTP_BASE_SYSCTL(sctp_auth_enable) = new;
@@ -619,9 +628,14 @@ sctp_sysctl_handle_asconf(SYSCTL_HANDLER_ARGS)
 	error = sysctl_handle_int(oidp, &new, 0, req);
 	if ((error == 0) &&
 	    (req->newptr != NULL)) {
+#if (SCTPCTL_ASCONF_ENABLE_MIN == 0)
+		if ((new > SCTPCTL_ASCONF_ENABLE_MAX) ||
+		    ((new == 1) && (SCTP_BASE_SYSCTL(sctp_auth_enable) == 0))) {
+#else
 		if ((new < SCTPCTL_ASCONF_ENABLE_MIN) ||
 		    (new > SCTPCTL_ASCONF_ENABLE_MAX) ||
 		    ((new == 1) && (SCTP_BASE_SYSCTL(sctp_auth_enable) == 0))) {
+#endif
 			error = EINVAL;
 		} else {
 			SCTP_BASE_SYSCTL(sctp_asconf_enable) = new;

From cd60f2acd9327feba131aad8a5f4535fe859176a Mon Sep 17 00:00:00 2001
From: Xin LI <delphij@FreeBSD.org>
Date: Tue, 20 Jan 2015 20:04:51 +0000
Subject: [PATCH 109/279] 5376 arc_kmem_reap_now() should not result in
 clearing arc_no_grow Reviewed by: Christopher Siden
 <christopher.siden@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Steven Hartland
 <killing@multiplay.co.uk> Reviewed by: Richard Elling
 <richard.elling@richardelling.com> Approved by: Dan McDonald
 <danmcd@omniti.com> Author: Matthew Ahrens <mahrens@delphix.com>

illumos/illumos-gate@2ec99e3e987d8aa273f1e9ba2b983557d058198c
---
 uts/common/fs/zfs/arc.c | 246 ++++++++++++++++++++++++++--------------
 1 file changed, 162 insertions(+), 84 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 435096becc7..656b4212feb 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -149,13 +149,7 @@ static kmutex_t		arc_reclaim_thr_lock;
 static kcondvar_t	arc_reclaim_thr_cv;	/* used to signal reclaim thr */
 static uint8_t		arc_thread_exit;
 
-#define	ARC_REDUCE_DNLC_PERCENT	3
-uint_t arc_reduce_dnlc_percent = ARC_REDUCE_DNLC_PERCENT;
-
-typedef enum arc_reclaim_strategy {
-	ARC_RECLAIM_AGGR,		/* Aggressive reclaim strategy */
-	ARC_RECLAIM_CONS		/* Conservative reclaim strategy */
-} arc_reclaim_strategy_t;
+uint_t arc_reduce_dnlc_percent = 3;
 
 /*
  * The number of iterations through arc_evict_*() before we
@@ -170,7 +164,19 @@ static int		arc_grow_retry = 60;
 static int		arc_p_min_shift = 4;
 
 /* log2(fraction of arc to reclaim) */
-static int		arc_shrink_shift = 5;
+static int		arc_shrink_shift = 7;
+
+/*
+ * log2(fraction of ARC which must be free to allow growing).
+ * I.e. If there is less than arc_c >> arc_no_grow_shift free memory,
+ * when reading a new block into the ARC, we will evict an equal-sized block
+ * from the ARC.
+ *
+ * This must be less than arc_shrink_shift, so that when we shrink the ARC,
+ * we will still not allow it to grow.
+ */
+int			arc_no_grow_shift = 5;
+
 
 /*
  * minimum lifespan of a prefetch block in clock ticks
@@ -2194,16 +2200,10 @@ arc_flush(spa_t *spa)
 }
 
 void
-arc_shrink(void)
+arc_shrink(int64_t to_free)
 {
 	if (arc_c > arc_c_min) {
-		uint64_t to_free;
 
-#ifdef _KERNEL
-		to_free = MAX(arc_c >> arc_shrink_shift, ptob(needfree));
-#else
-		to_free = arc_c >> arc_shrink_shift;
-#endif
 		if (arc_c > arc_c_min + to_free)
 			atomic_add_64(&arc_c, -to_free);
 		else
@@ -2222,25 +2222,49 @@ arc_shrink(void)
 		arc_adjust();
 }
 
+typedef enum free_memory_reason_t {
+	FMR_UNKNOWN,
+	FMR_NEEDFREE,
+	FMR_LOTSFREE,
+	FMR_SWAPFS_MINFREE,
+	FMR_PAGES_PP_MAXIMUM,
+	FMR_HEAP_ARENA,
+	FMR_ZIO_ARENA,
+} free_memory_reason_t;
+
+int64_t last_free_memory;
+free_memory_reason_t last_free_reason;
+
 /*
- * Determine if the system is under memory pressure and is asking
- * to reclaim memory. A return value of 1 indicates that the system
- * is under memory pressure and that the arc should adjust accordingly.
+ * Additional reserve of pages for pp_reserve.
  */
-static int
-arc_reclaim_needed(void)
+int64_t arc_pages_pp_reserve = 64;
+
+/*
+ * Additional reserve of pages for swapfs.
+ */
+int64_t arc_swapfs_reserve = 64;
+
+/*
+ * Return the amount of memory that can be consumed before reclaim will be
+ * needed.  Positive if there is sufficient free memory, negative indicates
+ * the amount of memory that needs to be freed up.
+ */
+static int64_t
+arc_available_memory(void)
 {
-	uint64_t extra;
+	int64_t lowest = INT64_MAX;
+	int64_t n;
+	free_memory_reason_t r = FMR_UNKNOWN;
 
 #ifdef _KERNEL
-
-	if (needfree)
-		return (1);
-
-	/*
-	 * take 'desfree' extra pages, so we reclaim sooner, rather than later
-	 */
-	extra = desfree;
+	if (needfree > 0) {
+		n = PAGESIZE * (-needfree);
+		if (n < lowest) {
+			lowest = n;
+			r = FMR_NEEDFREE;
+		}
+	}
 
 	/*
 	 * check that we're out of range of the pageout scanner.  It starts to
@@ -2249,8 +2273,11 @@ arc_reclaim_needed(void)
 	 * number of needed free pages.  We add extra pages here to make sure
 	 * the scanner doesn't start up while we're freeing memory.
 	 */
-	if (freemem < lotsfree + needfree + extra)
-		return (1);
+	n = PAGESIZE * (freemem - lotsfree - needfree - desfree);
+	if (n < lowest) {
+		lowest = n;
+		r = FMR_LOTSFREE;
+	}
 
 	/*
 	 * check to make sure that swapfs has enough space so that anon
@@ -2259,8 +2286,13 @@ arc_reclaim_needed(void)
 	 * swap pages.  We also add a bit of extra here just to prevent
 	 * circumstances from getting really dire.
 	 */
-	if (availrmem < swapfs_minfree + swapfs_reserve + extra)
-		return (1);
+	n = PAGESIZE * (availrmem - swapfs_minfree - swapfs_reserve -
+	    desfree - arc_swapfs_reserve);
+	if (n < lowest) {
+		lowest = n;
+		r = FMR_SWAPFS_MINFREE;
+	}
+
 
 	/*
 	 * Check that we have enough availrmem that memory locking (e.g., via
@@ -2269,8 +2301,12 @@ arc_reclaim_needed(void)
 	 * drops below pages_pp_maximum, page locking mechanisms such as
 	 * page_pp_lock() will fail.)
 	 */
-	if (availrmem <= pages_pp_maximum)
-		return (1);
+	n = PAGESIZE * (availrmem - pages_pp_maximum -
+	    arc_pages_pp_reserve);
+	if (n < lowest) {
+		lowest = n;
+		r = FMR_PAGES_PP_MAXIMUM;
+	}
 
 #if defined(__i386)
 	/*
@@ -2284,9 +2320,12 @@ arc_reclaim_needed(void)
 	 * heap is allocated.  (Or, in the calculation, if less than 1/4th is
 	 * free)
 	 */
-	if (vmem_size(heap_arena, VMEM_FREE) <
-	    (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2))
-		return (1);
+	n = vmem_size(heap_arena, VMEM_FREE) -
+	    (vmem_size(heap_arena, VMEM_FREE | VMEM_ALLOC) >> 2);
+	if (n < lowest) {
+		lowest = n;
+		r = FMR_HEAP_ARENA;
+	}
 #endif
 
 	/*
@@ -2298,19 +2337,40 @@ arc_reclaim_needed(void)
 	 * to aggressively evict memory from the arc in order to avoid
 	 * memory fragmentation issues.
 	 */
-	if (zio_arena != NULL &&
-	    vmem_size(zio_arena, VMEM_FREE) <
-	    (vmem_size(zio_arena, VMEM_ALLOC) >> 4))
-		return (1);
+	if (zio_arena != NULL) {
+		n = vmem_size(zio_arena, VMEM_FREE) -
+		    (vmem_size(zio_arena, VMEM_ALLOC) >> 4);
+		if (n < lowest) {
+			lowest = n;
+			r = FMR_ZIO_ARENA;
+		}
+	}
 #else
+	/* Every 100 calls, free a small amount */
 	if (spa_get_random(100) == 0)
-		return (1);
+		lowest = -1024;
 #endif
-	return (0);
+
+	last_free_memory = lowest;
+	last_free_reason = r;
+
+	return (lowest);
+}
+
+
+/*
+ * Determine if the system is under memory pressure and is asking
+ * to reclaim memory. A return value of TRUE indicates that the system
+ * is under memory pressure and that the arc should adjust accordingly.
+ */
+static boolean_t
+arc_reclaim_needed(void)
+{
+	return (arc_available_memory() < 0);
 }
 
 static void
-arc_kmem_reap_now(arc_reclaim_strategy_t strat)
+arc_kmem_reap_now(void)
 {
 	size_t			i;
 	kmem_cache_t		*prev_cache = NULL;
@@ -2335,13 +2395,6 @@ arc_kmem_reap_now(arc_reclaim_strategy_t strat)
 #endif
 #endif
 
-	/*
-	 * An aggressive reclamation will shrink the cache size as well as
-	 * reap free buffers from the arc kmem caches.
-	 */
-	if (strat == ARC_RECLAIM_AGGR)
-		arc_shrink();
-
 	for (i = 0; i < SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT; i++) {
 		if (zio_buf_cache[i] != prev_cache) {
 			prev_cache = zio_buf_cache[i];
@@ -2356,47 +2409,57 @@ arc_kmem_reap_now(arc_reclaim_strategy_t strat)
 	kmem_cache_reap_now(hdr_cache);
 	kmem_cache_reap_now(range_seg_cache);
 
-	/*
-	 * Ask the vmem areana to reclaim unused memory from its
-	 * quantum caches.
-	 */
-	if (zio_arena != NULL && strat == ARC_RECLAIM_AGGR)
+	if (zio_arena != NULL) {
+		/*
+		 * Ask the vmem arena to reclaim unused memory from its
+		 * quantum caches.
+		 */
 		vmem_qcache_reap(zio_arena);
+	}
 }
 
 static void
 arc_reclaim_thread(void)
 {
 	clock_t			growtime = 0;
-	arc_reclaim_strategy_t	last_reclaim = ARC_RECLAIM_CONS;
 	callb_cpr_t		cpr;
 
 	CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG);
 
 	mutex_enter(&arc_reclaim_thr_lock);
 	while (arc_thread_exit == 0) {
-		if (arc_reclaim_needed()) {
+		int64_t free_memory = arc_available_memory();
+		if (free_memory < 0) {
 
-			if (arc_no_grow) {
-				if (last_reclaim == ARC_RECLAIM_CONS) {
-					last_reclaim = ARC_RECLAIM_AGGR;
-				} else {
-					last_reclaim = ARC_RECLAIM_CONS;
-				}
-			} else {
-				arc_no_grow = TRUE;
-				last_reclaim = ARC_RECLAIM_AGGR;
-				membar_producer();
-			}
-
-			/* reset the growth delay for every reclaim */
-			growtime = ddi_get_lbolt() + (arc_grow_retry * hz);
-
-			arc_kmem_reap_now(last_reclaim);
+			arc_no_grow = B_TRUE;
 			arc_warm = B_TRUE;
 
-		} else if (arc_no_grow && ddi_get_lbolt() >= growtime) {
-			arc_no_grow = FALSE;
+			/*
+			 * Wait at least zfs_grow_retry (default 60) seconds
+			 * before considering growing.
+			 */
+			growtime = ddi_get_lbolt() + (arc_grow_retry * hz);
+
+			arc_kmem_reap_now();
+
+			/*
+			 * If we are still low on memory, shrink the ARC
+			 * so that we have arc_shrink_min free space.
+			 */
+			free_memory = arc_available_memory();
+
+			int64_t to_free =
+			    (arc_c >> arc_shrink_shift) - free_memory;
+			if (to_free > 0) {
+#ifdef _KERNEL
+				to_free = MAX(to_free, ptob(needfree));
+#endif
+				arc_shrink(to_free);
+			}
+		} else if (free_memory < arc_c >> arc_no_grow_shift) {
+			arc_no_grow = B_TRUE;
+		} else if (ddi_get_lbolt() >= growtime) {
+			arc_no_grow = B_FALSE;
 		}
 
 		arc_adjust();
@@ -3739,6 +3802,15 @@ arc_tempreserve_space(uint64_t reserve, uint64_t txg)
 void
 arc_init(void)
 {
+	/*
+	 * allmem is "all memory that we could possibly use".
+	 */
+#ifdef _KERNEL
+	uint64_t allmem = ptob(physmem - swapfs_minfree);
+#else
+	uint64_t allmem = (physmem * PAGESIZE) / 2;
+#endif
+
 	mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL);
 
@@ -3746,7 +3818,7 @@ arc_init(void)
 	arc_min_prefetch_lifespan = 1 * hz;
 
 	/* Start out with 1/8 of all memory */
-	arc_c = physmem * PAGESIZE / 8;
+	arc_c = allmem / 8;
 
 #ifdef _KERNEL
 	/*
@@ -3758,21 +3830,21 @@ arc_init(void)
 #endif
 
 	/* set min cache to 1/32 of all memory, or 64MB, whichever is more */
-	arc_c_min = MAX(arc_c / 4, 64<<20);
+	arc_c_min = MAX(allmem / 32, 64 << 20);
 	/* set max to 3/4 of all memory, or all but 1GB, whichever is more */
-	if (arc_c * 8 >= 1<<30)
-		arc_c_max = (arc_c * 8) - (1<<30);
+	if (allmem >= 1 << 30)
+		arc_c_max = allmem - (1 << 30);
 	else
 		arc_c_max = arc_c_min;
-	arc_c_max = MAX(arc_c * 6, arc_c_max);
+	arc_c_max = MAX(allmem * 3 / 4, arc_c_max);
 
 	/*
 	 * Allow the tunables to override our calculations if they are
 	 * reasonable (ie. over 64MB)
 	 */
-	if (zfs_arc_max > 64<<20 && zfs_arc_max < physmem * PAGESIZE)
+	if (zfs_arc_max > 64 << 20 && zfs_arc_max < allmem)
 		arc_c_max = zfs_arc_max;
-	if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max)
+	if (zfs_arc_min > 64 << 20 && zfs_arc_min <= arc_c_max)
 		arc_c_min = zfs_arc_min;
 
 	arc_c = arc_c_max;
@@ -3800,6 +3872,12 @@ arc_init(void)
 	if (zfs_arc_shrink_shift > 0)
 		arc_shrink_shift = zfs_arc_shrink_shift;
 
+	/*
+	 * Ensure that arc_no_grow_shift is less than arc_shrink_shift.
+	 */
+	if (arc_no_grow_shift >= arc_shrink_shift)
+		arc_no_grow_shift = arc_shrink_shift - 1;
+
 	if (zfs_arc_p_min_shift > 0)
 		arc_p_min_shift = zfs_arc_p_min_shift;
 

From f8d150572c165e81ce2c392d94fbd17253fad481 Mon Sep 17 00:00:00 2001
From: Xin LI <delphij@FreeBSD.org>
Date: Tue, 20 Jan 2015 20:08:50 +0000
Subject: [PATCH 110/279] 5408 managing ZFS cache devices requires lots of RAM
 Reviewed by: Christopher Siden <christopher.siden@delphix.com> Reviewed by:
 George Wilson <george.wilson@delphix.com> Reviewed by: Matthew Ahrens
 <mahrens@delphix.com> Reviewed by: Don Brady <dev.fs.zfs@gmail.com> Reviewed
 by: Josef 'Jeff' Sipek <josef.sipek@nexenta.com> Approved by: Garrett D'Amore
 <garrett@damore.org> Author: Chris Williamson <Chris.Williamson@delphix.com>

illumos/illumos-gate@89c86e32293a30cdd7af530c38b2073fee01411c
---
 cmd/ztest/ztest.c           |    9 +-
 uts/common/fs/zfs/arc.c     | 1406 +++++++++++++++++++++--------------
 uts/common/fs/zfs/sys/arc.h |   27 +-
 3 files changed, 878 insertions(+), 564 deletions(-)

diff --git a/cmd/ztest/ztest.c b/cmd/ztest/ztest.c
index a5f96008046..9fdbd2118c5 100644
--- a/cmd/ztest/ztest.c
+++ b/cmd/ztest/ztest.c
@@ -3905,7 +3905,7 @@ ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
 		 * assign an arcbuf to a dbuf.
 		 */
 		for (j = 0; j < s; j++) {
-			if (i != 5) {
+			if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
 				bigbuf_arcbufs[j] =
 				    dmu_request_arcbuf(bonus_db, chunksize);
 			} else {
@@ -3929,7 +3929,8 @@ ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
 			umem_free(packbuf, packsize);
 			umem_free(bigbuf, bigsize);
 			for (j = 0; j < s; j++) {
-				if (i != 5) {
+				if (i != 5 ||
+				    chunksize < (SPA_MINBLOCKSIZE * 2)) {
 					dmu_return_arcbuf(bigbuf_arcbufs[j]);
 				} else {
 					dmu_return_arcbuf(
@@ -3973,7 +3974,7 @@ ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
 		}
 		for (off = bigoff, j = 0; j < s; j++, off += chunksize) {
 			dmu_buf_t *dbt;
-			if (i != 5) {
+			if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
 				bcopy((caddr_t)bigbuf + (off - bigoff),
 				    bigbuf_arcbufs[j]->b_data, chunksize);
 			} else {
@@ -3990,7 +3991,7 @@ ztest_dmu_read_write_zcopy(ztest_ds_t *zd, uint64_t id)
 				VERIFY(dmu_buf_hold(os, bigobj, off,
 				    FTAG, &dbt, DMU_READ_NO_PREFETCH) == 0);
 			}
-			if (i != 5) {
+			if (i != 5 || chunksize < (SPA_MINBLOCKSIZE * 2)) {
 				dmu_assign_arcbuf(bonus_db, off,
 				    bigbuf_arcbufs[j], tx);
 			} else {
diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 656b4212feb..012dd83cd7c 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -111,7 +111,7 @@
  * Note that the majority of the performance stats are manipulated
  * with atomic operations.
  *
- * The L2ARC uses the l2arc_buflist_mtx global mutex for the following:
+ * The L2ARC uses the l2ad_mtx on each vdev for the following:
  *
  *	- L2ARC buflist creation
  *	- L2ARC buflist eviction
@@ -314,6 +314,7 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_l2_writes_hdr_miss;
 	kstat_named_t arcstat_l2_evict_lock_retry;
 	kstat_named_t arcstat_l2_evict_reading;
+	kstat_named_t arcstat_l2_evict_l1cached;
 	kstat_named_t arcstat_l2_free_on_write;
 	kstat_named_t arcstat_l2_abort_lowmem;
 	kstat_named_t arcstat_l2_cksum_bad;
@@ -381,6 +382,7 @@ static arc_stats_t arc_stats = {
 	{ "l2_writes_hdr_miss",		KSTAT_DATA_UINT64 },
 	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
 	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
+	{ "l2_evict_l1cached",		KSTAT_DATA_UINT64 },
 	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
 	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
 	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
@@ -472,8 +474,6 @@ static int		arc_no_grow;	/* Don't try to grow cache size */
 static uint64_t		arc_tempreserve;
 static uint64_t		arc_loaned_bytes;
 
-typedef struct l2arc_buf_hdr l2arc_buf_hdr_t;
-
 typedef struct arc_callback arc_callback_t;
 
 struct arc_callback {
@@ -494,29 +494,53 @@ struct arc_write_callback {
 	arc_buf_t	*awcb_buf;
 };
 
-struct arc_buf_hdr {
-	/* protected by hash lock */
-	dva_t			b_dva;
-	uint64_t		b_birth;
-	uint64_t		b_cksum0;
-
+/*
+ * ARC buffers are separated into multiple structs as a memory saving measure:
+ *   - Common fields struct, always defined, and embedded within it:
+ *       - L2-only fields, always allocated but undefined when not in L2ARC
+ *       - L1-only fields, only allocated when in L1ARC
+ *
+ *           Buffer in L1                     Buffer only in L2
+ *    +------------------------+          +------------------------+
+ *    | arc_buf_hdr_t          |          | arc_buf_hdr_t          |
+ *    |                        |          |                        |
+ *    |                        |          |                        |
+ *    |                        |          |                        |
+ *    +------------------------+          +------------------------+
+ *    | l2arc_buf_hdr_t        |          | l2arc_buf_hdr_t        |
+ *    | (undefined if L1-only) |          |                        |
+ *    +------------------------+          +------------------------+
+ *    | l1arc_buf_hdr_t        |
+ *    |                        |
+ *    |                        |
+ *    |                        |
+ *    |                        |
+ *    +------------------------+
+ *
+ * Because it's possible for the L2ARC to become extremely large, we can wind
+ * up eating a lot of memory in L2ARC buffer headers, so the size of a header
+ * is minimized by only allocating the fields necessary for an L1-cached buffer
+ * when a header is actually in the L1 cache. The sub-headers (l1arc_buf_hdr and
+ * l2arc_buf_hdr) are embedded rather than allocated separately to save a couple
+ * words in pointers. arc_hdr_realloc() is used to switch a header between
+ * these two allocation states.
+ */
+typedef struct l1arc_buf_hdr {
 	kmutex_t		b_freeze_lock;
-	zio_cksum_t		*b_freeze_cksum;
+#ifdef ZFS_DEBUG
+	/*
+	 * used for debugging wtih kmem_flags - by allocating and freeing
+	 * b_thawed when the buffer is thawed, we get a record of the stack
+	 * trace that thawed it.
+	 */
 	void			*b_thawed;
+#endif
 
-	arc_buf_hdr_t		*b_hash_next;
 	arc_buf_t		*b_buf;
-	arc_flags_t		b_flags;
 	uint32_t		b_datacnt;
-
-	arc_callback_t		*b_acb;
+	/* for waiting on writes to complete */
 	kcondvar_t		b_cv;
 
-	/* immutable */
-	arc_buf_contents_t	b_type;
-	uint64_t		b_size;
-	uint64_t		b_spa;
-
 	/* protected by arc state mutex */
 	arc_state_t		*b_state;
 	list_node_t		b_arc_node;
@@ -527,8 +551,46 @@ struct arc_buf_hdr {
 	/* self protecting */
 	refcount_t		b_refcnt;
 
-	l2arc_buf_hdr_t		*b_l2hdr;
+	arc_callback_t		*b_acb;
+	/* temporary buffer holder for in-flight compressed data */
+	void			*b_tmp_cdata;
+} l1arc_buf_hdr_t;
+
+typedef struct l2arc_dev l2arc_dev_t;
+
+typedef struct l2arc_buf_hdr {
+	/* protected by arc_buf_hdr mutex */
+	l2arc_dev_t		*b_dev;		/* L2ARC device */
+	uint64_t		b_daddr;	/* disk address, offset byte */
+	/* real alloc'd buffer size depending on b_compress applied */
+	int32_t			b_asize;
+
 	list_node_t		b_l2node;
+} l2arc_buf_hdr_t;
+
+struct arc_buf_hdr {
+	/* protected by hash lock */
+	dva_t			b_dva;
+	uint64_t		b_birth;
+	/*
+	 * Even though this checksum is only set/verified when a buffer is in
+	 * the L1 cache, it needs to be in the set of common fields because it
+	 * must be preserved from the time before a buffer is written out to
+	 * L2ARC until after it is read back in.
+	 */
+	zio_cksum_t		*b_freeze_cksum;
+
+	arc_buf_hdr_t		*b_hash_next;
+	arc_flags_t		b_flags;
+
+	/* immutable */
+	int32_t			b_size;
+	uint64_t		b_spa;
+
+	/* L2ARC fields. Undefined when not in L2ARC. */
+	l2arc_buf_hdr_t		b_l2hdr;
+	/* L1ARC fields. Undefined when in l2arc_only state */
+	l1arc_buf_hdr_t		b_l1hdr;
 };
 
 static arc_buf_t *arc_eviction_list;
@@ -545,22 +607,38 @@ static arc_buf_hdr_t arc_eviction_hdr;
 #define	HDR_PREFETCH(hdr)	((hdr)->b_flags & ARC_FLAG_PREFETCH)
 #define	HDR_FREED_IN_READ(hdr)	((hdr)->b_flags & ARC_FLAG_FREED_IN_READ)
 #define	HDR_BUF_AVAILABLE(hdr)	((hdr)->b_flags & ARC_FLAG_BUF_AVAILABLE)
-#define	HDR_FREE_IN_PROGRESS(hdr)	\
-	((hdr)->b_flags & ARC_FLAG_FREE_IN_PROGRESS)
+
 #define	HDR_L2CACHE(hdr)	((hdr)->b_flags & ARC_FLAG_L2CACHE)
+#define	HDR_L2COMPRESS(hdr)	((hdr)->b_flags & ARC_FLAG_L2COMPRESS)
 #define	HDR_L2_READING(hdr)	\
-	((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS &&	\
-	    (hdr)->b_l2hdr != NULL)
+	    (((hdr)->b_flags & ARC_FLAG_IO_IN_PROGRESS) &&	\
+	    ((hdr)->b_flags & ARC_FLAG_HAS_L2HDR))
 #define	HDR_L2_WRITING(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITING)
 #define	HDR_L2_EVICTED(hdr)	((hdr)->b_flags & ARC_FLAG_L2_EVICTED)
 #define	HDR_L2_WRITE_HEAD(hdr)	((hdr)->b_flags & ARC_FLAG_L2_WRITE_HEAD)
 
+#define	HDR_ISTYPE_METADATA(hdr)	\
+	    ((hdr)->b_flags & ARC_FLAG_BUFC_METADATA)
+#define	HDR_ISTYPE_DATA(hdr)	(!HDR_ISTYPE_METADATA(hdr))
+
+#define	HDR_HAS_L1HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L1HDR)
+#define	HDR_HAS_L2HDR(hdr)	((hdr)->b_flags & ARC_FLAG_HAS_L2HDR)
+
+/* For storing compression mode in b_flags */
+#define	HDR_COMPRESS_OFFSET	24
+#define	HDR_COMPRESS_NBITS	7
+
+#define	HDR_GET_COMPRESS(hdr)	((enum zio_compress)BF32_GET(hdr->b_flags, \
+	    HDR_COMPRESS_OFFSET, HDR_COMPRESS_NBITS))
+#define	HDR_SET_COMPRESS(hdr, cmp) BF32_SET(hdr->b_flags, \
+	    HDR_COMPRESS_OFFSET, HDR_COMPRESS_NBITS, (cmp))
+
 /*
  * Other sizes
  */
 
-#define	HDR_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
-#define	L2HDR_SIZE ((int64_t)sizeof (l2arc_buf_hdr_t))
+#define	HDR_FULL_SIZE ((int64_t)sizeof (arc_buf_hdr_t))
+#define	HDR_L2ONLY_SIZE ((int64_t)offsetof(arc_buf_hdr_t, b_l1hdr))
 
 /*
  * Hash table routines
@@ -624,7 +702,7 @@ boolean_t l2arc_norw = B_TRUE;			/* no reads during writes */
 /*
  * L2ARC Internals
  */
-typedef struct l2arc_dev {
+struct l2arc_dev {
 	vdev_t			*l2ad_vdev;	/* vdev */
 	spa_t			*l2ad_spa;	/* spa */
 	uint64_t		l2ad_hand;	/* next write location */
@@ -633,15 +711,15 @@ typedef struct l2arc_dev {
 	uint64_t		l2ad_evict;	/* last addr eviction reached */
 	boolean_t		l2ad_first;	/* first sweep through */
 	boolean_t		l2ad_writing;	/* currently writing */
-	list_t			*l2ad_buflist;	/* buffer list */
+	kmutex_t		l2ad_mtx;	/* lock for buffer list */
+	list_t			l2ad_buflist;	/* buffer list */
 	list_node_t		l2ad_node;	/* device list node */
-} l2arc_dev_t;
+};
 
 static list_t L2ARC_dev_list;			/* device list */
 static list_t *l2arc_dev_list;			/* device list pointer */
 static kmutex_t l2arc_dev_mtx;			/* device list mutex */
 static l2arc_dev_t *l2arc_dev_last;		/* last device used */
-static kmutex_t l2arc_buflist_mtx;		/* mutex for all buflists */
 static list_t L2ARC_free_on_write;		/* free after write buf list */
 static list_t *l2arc_free_on_write;		/* free after write list ptr */
 static kmutex_t l2arc_free_on_write_mtx;	/* mutex for list */
@@ -661,18 +739,6 @@ typedef struct l2arc_write_callback {
 	arc_buf_hdr_t	*l2wcb_head;		/* head of write buflist */
 } l2arc_write_callback_t;
 
-struct l2arc_buf_hdr {
-	/* protected by arc_buf_hdr  mutex */
-	l2arc_dev_t		*b_dev;		/* L2ARC device */
-	uint64_t		b_daddr;	/* disk address, offset byte */
-	/* compression applied to buffer data */
-	enum zio_compress	b_compress;
-	/* real alloc'd buffer size depending on b_compress applied */
-	int			b_asize;
-	/* temporary buffer holder for in-flight compressed data */
-	void			*b_tmp_cdata;
-};
-
 typedef struct l2arc_data_free {
 	/* protected by l2arc_free_on_write_mtx */
 	void		*l2df_data;
@@ -691,12 +757,13 @@ static int arc_evict_needed(arc_buf_contents_t);
 static void arc_evict_ghost(arc_state_t *, uint64_t, int64_t);
 static void arc_buf_watch(arc_buf_t *);
 
+static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
+static uint32_t arc_bufc_to_flags(arc_buf_contents_t);
+
 static boolean_t l2arc_write_eligible(uint64_t, arc_buf_hdr_t *);
 static void l2arc_read_done(zio_t *);
-static void l2arc_hdr_stat_add(void);
-static void l2arc_hdr_stat_remove(void);
 
-static boolean_t l2arc_compress_buf(l2arc_buf_hdr_t *);
+static boolean_t l2arc_compress_buf(arc_buf_hdr_t *);
 static void l2arc_decompress_zio(zio_t *, arc_buf_hdr_t *, enum zio_compress);
 static void l2arc_release_cdata_buf(arc_buf_hdr_t *);
 
@@ -719,8 +786,7 @@ buf_hash(uint64_t spa, const dva_t *dva, uint64_t birth)
 
 #define	BUF_EMPTY(buf)						\
 	((buf)->b_dva.dva_word[0] == 0 &&			\
-	(buf)->b_dva.dva_word[1] == 0 &&			\
-	(buf)->b_cksum0 == 0)
+	(buf)->b_dva.dva_word[1] == 0)
 
 #define	BUF_EQUAL(spa, dva, birth, buf)				\
 	((buf)->b_dva.dva_word[0] == (dva)->dva_word[0]) &&	\
@@ -733,7 +799,6 @@ buf_discard_identity(arc_buf_hdr_t *hdr)
 	hdr->b_dva.dva_word[0] = 0;
 	hdr->b_dva.dva_word[1] = 0;
 	hdr->b_birth = 0;
-	hdr->b_cksum0 = 0;
 }
 
 static arc_buf_hdr_t *
@@ -763,6 +828,7 @@ buf_hash_find(uint64_t spa, const blkptr_t *bp, kmutex_t **lockp)
  * equal to elem in the hash table, then the already existing element
  * will be returned and the new element will not be inserted.
  * Otherwise returns NULL.
+ * If lockp == NULL, the caller is assumed to already hold the hash lock.
  */
 static arc_buf_hdr_t *
 buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
@@ -775,8 +841,14 @@ buf_hash_insert(arc_buf_hdr_t *hdr, kmutex_t **lockp)
 	ASSERT(!DVA_IS_EMPTY(&hdr->b_dva));
 	ASSERT(hdr->b_birth != 0);
 	ASSERT(!HDR_IN_HASH_TABLE(hdr));
-	*lockp = hash_lock;
-	mutex_enter(hash_lock);
+
+	if (lockp != NULL) {
+		*lockp = hash_lock;
+		mutex_enter(hash_lock);
+	} else {
+		ASSERT(MUTEX_HELD(hash_lock));
+	}
+
 	for (fhdr = buf_hash_table.ht_table[idx], i = 0; fhdr != NULL;
 	    fhdr = fhdr->b_hash_next, i++) {
 		if (BUF_EQUAL(hdr->b_spa, &hdr->b_dva, hdr->b_birth, fhdr))
@@ -831,7 +903,8 @@ buf_hash_remove(arc_buf_hdr_t *hdr)
 /*
  * Global data structures and functions for the buf kmem cache.
  */
-static kmem_cache_t *hdr_cache;
+static kmem_cache_t *hdr_full_cache;
+static kmem_cache_t *hdr_l2only_cache;
 static kmem_cache_t *buf_cache;
 
 static void
@@ -843,7 +916,8 @@ buf_fini(void)
 	    (buf_hash_table.ht_mask + 1) * sizeof (void *));
 	for (i = 0; i < BUF_LOCKS; i++)
 		mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
-	kmem_cache_destroy(hdr_cache);
+	kmem_cache_destroy(hdr_full_cache);
+	kmem_cache_destroy(hdr_l2only_cache);
 	kmem_cache_destroy(buf_cache);
 }
 
@@ -853,15 +927,27 @@ buf_fini(void)
  */
 /* ARGSUSED */
 static int
-hdr_cons(void *vbuf, void *unused, int kmflag)
+hdr_full_cons(void *vbuf, void *unused, int kmflag)
 {
 	arc_buf_hdr_t *hdr = vbuf;
 
-	bzero(hdr, sizeof (arc_buf_hdr_t));
-	refcount_create(&hdr->b_refcnt);
-	cv_init(&hdr->b_cv, NULL, CV_DEFAULT, NULL);
-	mutex_init(&hdr->b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
-	arc_space_consume(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
+	bzero(hdr, HDR_FULL_SIZE);
+	cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
+	refcount_create(&hdr->b_l1hdr.b_refcnt);
+	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
+	arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
+
+	return (0);
+}
+
+/* ARGSUSED */
+static int
+hdr_l2only_cons(void *vbuf, void *unused, int kmflag)
+{
+	arc_buf_hdr_t *hdr = vbuf;
+
+	bzero(hdr, HDR_L2ONLY_SIZE);
+	arc_space_consume(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
 
 	return (0);
 }
@@ -885,15 +971,25 @@ buf_cons(void *vbuf, void *unused, int kmflag)
  */
 /* ARGSUSED */
 static void
-hdr_dest(void *vbuf, void *unused)
+hdr_full_dest(void *vbuf, void *unused)
 {
 	arc_buf_hdr_t *hdr = vbuf;
 
 	ASSERT(BUF_EMPTY(hdr));
-	refcount_destroy(&hdr->b_refcnt);
-	cv_destroy(&hdr->b_cv);
-	mutex_destroy(&hdr->b_freeze_lock);
-	arc_space_return(sizeof (arc_buf_hdr_t), ARC_SPACE_HDRS);
+	cv_destroy(&hdr->b_l1hdr.b_cv);
+	refcount_destroy(&hdr->b_l1hdr.b_refcnt);
+	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
+	arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
+}
+
+/* ARGSUSED */
+static void
+hdr_l2only_dest(void *vbuf, void *unused)
+{
+	arc_buf_hdr_t *hdr = vbuf;
+
+	ASSERT(BUF_EMPTY(hdr));
+	arc_space_return(HDR_L2ONLY_SIZE, ARC_SPACE_L2HDRS);
 }
 
 /* ARGSUSED */
@@ -947,8 +1043,11 @@ retry:
 		goto retry;
 	}
 
-	hdr_cache = kmem_cache_create("arc_buf_hdr_t", sizeof (arc_buf_hdr_t),
-	    0, hdr_cons, hdr_dest, hdr_recl, NULL, NULL, 0);
+	hdr_full_cache = kmem_cache_create("arc_buf_hdr_t_full", HDR_FULL_SIZE,
+	    0, hdr_full_cons, hdr_full_dest, hdr_recl, NULL, NULL, 0);
+	hdr_l2only_cache = kmem_cache_create("arc_buf_hdr_t_l2only",
+	    HDR_L2ONLY_SIZE, 0, hdr_l2only_cons, hdr_l2only_dest, hdr_recl,
+	    NULL, NULL, 0);
 	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
 	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
 
@@ -962,6 +1061,81 @@ retry:
 	}
 }
 
+/*
+ * Transition between the two allocation states for the arc_buf_hdr struct.
+ * The arc_buf_hdr struct can be allocated with (hdr_full_cache) or without
+ * (hdr_l2only_cache) the fields necessary for the L1 cache - the smaller
+ * version is used when a cache buffer is only in the L2ARC in order to reduce
+ * memory usage.
+ */
+static arc_buf_hdr_t *
+arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
+{
+	ASSERT(HDR_HAS_L2HDR(hdr));
+
+	arc_buf_hdr_t *nhdr;
+	l2arc_dev_t *dev = hdr->b_l2hdr.b_dev;
+
+	ASSERT((old == hdr_full_cache && new == hdr_l2only_cache) ||
+	    (old == hdr_l2only_cache && new == hdr_full_cache));
+
+	nhdr = kmem_cache_alloc(new, KM_PUSHPAGE);
+
+	ASSERT(MUTEX_HELD(HDR_LOCK(hdr)));
+	buf_hash_remove(hdr);
+
+	bcopy(hdr, nhdr, HDR_L2ONLY_SIZE);
+	if (new == hdr_full_cache) {
+		nhdr->b_flags |= ARC_FLAG_HAS_L1HDR;
+		/*
+		 * arc_access and arc_change_state need to be aware that a
+		 * header has just come out of L2ARC, so we set its state to
+		 * l2c_only even though it's about to change.
+		 */
+		nhdr->b_l1hdr.b_state = arc_l2c_only;
+	} else {
+		ASSERT(hdr->b_l1hdr.b_buf == NULL);
+		ASSERT0(hdr->b_l1hdr.b_datacnt);
+		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
+		/*
+		 * We might be removing the L1hdr of a buffer which was just
+		 * written out to L2ARC. If such a buffer is compressed then we
+		 * need to free its b_tmp_cdata before destroying the header.
+		 */
+		if (hdr->b_l1hdr.b_tmp_cdata != NULL &&
+		    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF)
+			l2arc_release_cdata_buf(hdr);
+		nhdr->b_flags &= ~ARC_FLAG_HAS_L1HDR;
+	}
+	/*
+	 * The header has been reallocated so we need to re-insert it into any
+	 * lists it was on.
+	 */
+	(void) buf_hash_insert(nhdr, NULL);
+
+	ASSERT(list_link_active(&hdr->b_l2hdr.b_l2node));
+
+	mutex_enter(&dev->l2ad_mtx);
+
+	/*
+	 * We must place the realloc'ed header back into the list at
+	 * the same spot. Otherwise, if it's placed earlier in the list,
+	 * l2arc_write_buffers() could find it during the function's
+	 * write phase, and try to write it out to the l2arc.
+	 */
+	list_insert_after(&dev->l2ad_buflist, hdr, nhdr);
+	list_remove(&dev->l2ad_buflist, hdr);
+
+	mutex_exit(&dev->l2ad_mtx);
+
+	buf_discard_identity(hdr);
+	hdr->b_freeze_cksum = NULL;
+	kmem_cache_free(old, hdr);
+
+	return (nhdr);
+}
+
+
 #define	ARC_MINTIME	(hz>>4) /* 62 ms */
 
 static void
@@ -972,16 +1146,15 @@ arc_cksum_verify(arc_buf_t *buf)
 	if (!(zfs_flags & ZFS_DEBUG_MODIFY))
 		return;
 
-	mutex_enter(&buf->b_hdr->b_freeze_lock);
-	if (buf->b_hdr->b_freeze_cksum == NULL ||
-	    (buf->b_hdr->b_flags & ARC_FLAG_IO_ERROR)) {
-		mutex_exit(&buf->b_hdr->b_freeze_lock);
+	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
+	if (buf->b_hdr->b_freeze_cksum == NULL || HDR_IO_ERROR(buf->b_hdr)) {
+		mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 		return;
 	}
 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
 	if (!ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc))
 		panic("buffer modified while frozen!");
-	mutex_exit(&buf->b_hdr->b_freeze_lock);
+	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 }
 
 static int
@@ -990,10 +1163,10 @@ arc_cksum_equal(arc_buf_t *buf)
 	zio_cksum_t zc;
 	int equal;
 
-	mutex_enter(&buf->b_hdr->b_freeze_lock);
+	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size, &zc);
 	equal = ZIO_CHECKSUM_EQUAL(*buf->b_hdr->b_freeze_cksum, zc);
-	mutex_exit(&buf->b_hdr->b_freeze_lock);
+	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 
 	return (equal);
 }
@@ -1004,15 +1177,15 @@ arc_cksum_compute(arc_buf_t *buf, boolean_t force)
 	if (!force && !(zfs_flags & ZFS_DEBUG_MODIFY))
 		return;
 
-	mutex_enter(&buf->b_hdr->b_freeze_lock);
+	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 	if (buf->b_hdr->b_freeze_cksum != NULL) {
-		mutex_exit(&buf->b_hdr->b_freeze_lock);
+		mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 		return;
 	}
 	buf->b_hdr->b_freeze_cksum = kmem_alloc(sizeof (zio_cksum_t), KM_SLEEP);
 	fletcher_2_native(buf->b_data, buf->b_hdr->b_size,
 	    buf->b_hdr->b_freeze_cksum);
-	mutex_exit(&buf->b_hdr->b_freeze_lock);
+	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 	arc_buf_watch(buf);
 }
 
@@ -1059,30 +1232,58 @@ arc_buf_watch(arc_buf_t *buf)
 #endif
 }
 
+static arc_buf_contents_t
+arc_buf_type(arc_buf_hdr_t *hdr)
+{
+	if (HDR_ISTYPE_METADATA(hdr)) {
+		return (ARC_BUFC_METADATA);
+	} else {
+		return (ARC_BUFC_DATA);
+	}
+}
+
+static uint32_t
+arc_bufc_to_flags(arc_buf_contents_t type)
+{
+	switch (type) {
+	case ARC_BUFC_DATA:
+		/* metadata field is 0 if buffer contains normal data */
+		return (0);
+	case ARC_BUFC_METADATA:
+		return (ARC_FLAG_BUFC_METADATA);
+	default:
+		break;
+	}
+	panic("undefined ARC buffer type!");
+	return ((uint32_t)-1);
+}
+
 void
 arc_buf_thaw(arc_buf_t *buf)
 {
 	if (zfs_flags & ZFS_DEBUG_MODIFY) {
-		if (buf->b_hdr->b_state != arc_anon)
+		if (buf->b_hdr->b_l1hdr.b_state != arc_anon)
 			panic("modifying non-anon buffer!");
-		if (buf->b_hdr->b_flags & ARC_FLAG_IO_IN_PROGRESS)
+		if (HDR_IO_IN_PROGRESS(buf->b_hdr))
 			panic("modifying buffer while i/o in progress!");
 		arc_cksum_verify(buf);
 	}
 
-	mutex_enter(&buf->b_hdr->b_freeze_lock);
+	mutex_enter(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 	if (buf->b_hdr->b_freeze_cksum != NULL) {
 		kmem_free(buf->b_hdr->b_freeze_cksum, sizeof (zio_cksum_t));
 		buf->b_hdr->b_freeze_cksum = NULL;
 	}
 
+#ifdef ZFS_DEBUG
 	if (zfs_flags & ZFS_DEBUG_MODIFY) {
-		if (buf->b_hdr->b_thawed)
-			kmem_free(buf->b_hdr->b_thawed, 1);
-		buf->b_hdr->b_thawed = kmem_alloc(1, KM_SLEEP);
+		if (buf->b_hdr->b_l1hdr.b_thawed != NULL)
+			kmem_free(buf->b_hdr->b_l1hdr.b_thawed, 1);
+		buf->b_hdr->b_l1hdr.b_thawed = kmem_alloc(1, KM_SLEEP);
 	}
+#endif
 
-	mutex_exit(&buf->b_hdr->b_freeze_lock);
+	mutex_exit(&buf->b_hdr->b_l1hdr.b_freeze_lock);
 
 	arc_buf_unwatch(buf);
 }
@@ -1099,7 +1300,7 @@ arc_buf_freeze(arc_buf_t *buf)
 	mutex_enter(hash_lock);
 
 	ASSERT(buf->b_hdr->b_freeze_cksum != NULL ||
-	    buf->b_hdr->b_state == arc_anon);
+	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
 	arc_cksum_compute(buf, B_FALSE);
 	mutex_exit(hash_lock);
 
@@ -1108,30 +1309,34 @@ arc_buf_freeze(arc_buf_t *buf)
 static void
 add_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
 {
+	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT(MUTEX_HELD(hash_lock));
+	arc_state_t *state = hdr->b_l1hdr.b_state;
 
-	if ((refcount_add(&hdr->b_refcnt, tag) == 1) &&
-	    (hdr->b_state != arc_anon)) {
-		uint64_t delta = hdr->b_size * hdr->b_datacnt;
-		list_t *list = &hdr->b_state->arcs_list[hdr->b_type];
-		uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
+	if ((refcount_add(&hdr->b_l1hdr.b_refcnt, tag) == 1) &&
+	    (state != arc_anon)) {
+		/* We don't use the L2-only state list. */
+		if (state != arc_l2c_only) {
+			uint64_t delta = hdr->b_size * hdr->b_l1hdr.b_datacnt;
+			list_t *list = &state->arcs_list[arc_buf_type(hdr)];
+			uint64_t *size = &state->arcs_lsize[arc_buf_type(hdr)];
 
-		ASSERT(!MUTEX_HELD(&hdr->b_state->arcs_mtx));
-		mutex_enter(&hdr->b_state->arcs_mtx);
-		ASSERT(list_link_active(&hdr->b_arc_node));
-		list_remove(list, hdr);
-		if (GHOST_STATE(hdr->b_state)) {
-			ASSERT0(hdr->b_datacnt);
-			ASSERT3P(hdr->b_buf, ==, NULL);
-			delta = hdr->b_size;
+			ASSERT(!MUTEX_HELD(&state->arcs_mtx));
+			mutex_enter(&state->arcs_mtx);
+			ASSERT(list_link_active(&hdr->b_l1hdr.b_arc_node));
+			list_remove(list, hdr);
+			if (GHOST_STATE(state)) {
+				ASSERT0(hdr->b_l1hdr.b_datacnt);
+				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
+				delta = hdr->b_size;
+			}
+			ASSERT(delta > 0);
+			ASSERT3U(*size, >=, delta);
+			atomic_add_64(size, -delta);
+			mutex_exit(&state->arcs_mtx);
 		}
-		ASSERT(delta > 0);
-		ASSERT3U(*size, >=, delta);
-		atomic_add_64(size, -delta);
-		mutex_exit(&hdr->b_state->arcs_mtx);
 		/* remove the prefetch flag if we get a reference */
-		if (hdr->b_flags & ARC_FLAG_PREFETCH)
-			hdr->b_flags &= ~ARC_FLAG_PREFETCH;
+		hdr->b_flags &= ~ARC_FLAG_PREFETCH;
 	}
 }
 
@@ -1139,21 +1344,27 @@ static int
 remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
 {
 	int cnt;
-	arc_state_t *state = hdr->b_state;
+	arc_state_t *state = hdr->b_l1hdr.b_state;
 
+	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT(state == arc_anon || MUTEX_HELD(hash_lock));
 	ASSERT(!GHOST_STATE(state));
 
-	if (((cnt = refcount_remove(&hdr->b_refcnt, tag)) == 0) &&
+	/*
+	 * arc_l2c_only counts as a ghost state so we don't need to explicitly
+	 * check to prevent usage of the arc_l2c_only list.
+	 */
+	if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
 	    (state != arc_anon)) {
-		uint64_t *size = &state->arcs_lsize[hdr->b_type];
+		uint64_t *size = &state->arcs_lsize[arc_buf_type(hdr)];
 
 		ASSERT(!MUTEX_HELD(&state->arcs_mtx));
 		mutex_enter(&state->arcs_mtx);
-		ASSERT(!list_link_active(&hdr->b_arc_node));
-		list_insert_head(&state->arcs_list[hdr->b_type], hdr);
-		ASSERT(hdr->b_datacnt > 0);
-		atomic_add_64(size, hdr->b_size * hdr->b_datacnt);
+		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
+		list_insert_head(&state->arcs_list[arc_buf_type(hdr)], hdr);
+		ASSERT(hdr->b_l1hdr.b_datacnt > 0);
+		atomic_add_64(size, hdr->b_size *
+		    hdr->b_l1hdr.b_datacnt);
 		mutex_exit(&state->arcs_mtx);
 	}
 	return (cnt);
@@ -1167,40 +1378,60 @@ static void
 arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
     kmutex_t *hash_lock)
 {
-	arc_state_t *old_state = hdr->b_state;
-	int64_t refcnt = refcount_count(&hdr->b_refcnt);
+	arc_state_t *old_state;
+	int64_t refcnt;
+	uint32_t datacnt;
 	uint64_t from_delta, to_delta;
+	arc_buf_contents_t buftype = arc_buf_type(hdr);
+
+	/*
+	 * We almost always have an L1 hdr here, since we call arc_hdr_realloc()
+	 * in arc_read() when bringing a buffer out of the L2ARC.  However, the
+	 * L1 hdr doesn't always exist when we change state to arc_anon before
+	 * destroying a header, in which case reallocating to add the L1 hdr is
+	 * pointless.
+	 */
+	if (HDR_HAS_L1HDR(hdr)) {
+		old_state = hdr->b_l1hdr.b_state;
+		refcnt = refcount_count(&hdr->b_l1hdr.b_refcnt);
+		datacnt = hdr->b_l1hdr.b_datacnt;
+	} else {
+		old_state = arc_l2c_only;
+		refcnt = 0;
+		datacnt = 0;
+	}
 
 	ASSERT(MUTEX_HELD(hash_lock));
 	ASSERT3P(new_state, !=, old_state);
-	ASSERT(refcnt == 0 || hdr->b_datacnt > 0);
-	ASSERT(hdr->b_datacnt == 0 || !GHOST_STATE(new_state));
-	ASSERT(hdr->b_datacnt <= 1 || old_state != arc_anon);
+	ASSERT(refcnt == 0 || datacnt > 0);
+	ASSERT(!GHOST_STATE(new_state) || datacnt == 0);
+	ASSERT(old_state != arc_anon || datacnt <= 1);
 
-	from_delta = to_delta = hdr->b_datacnt * hdr->b_size;
+	from_delta = to_delta = datacnt * hdr->b_size;
 
 	/*
 	 * If this buffer is evictable, transfer it from the
 	 * old state list to the new state list.
 	 */
 	if (refcnt == 0) {
-		if (old_state != arc_anon) {
+		if (old_state != arc_anon && old_state != arc_l2c_only) {
 			int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx);
-			uint64_t *size = &old_state->arcs_lsize[hdr->b_type];
+			uint64_t *size = &old_state->arcs_lsize[buftype];
 
 			if (use_mutex)
 				mutex_enter(&old_state->arcs_mtx);
 
-			ASSERT(list_link_active(&hdr->b_arc_node));
-			list_remove(&old_state->arcs_list[hdr->b_type], hdr);
+			ASSERT(HDR_HAS_L1HDR(hdr));
+			ASSERT(list_link_active(&hdr->b_l1hdr.b_arc_node));
+			list_remove(&old_state->arcs_list[buftype], hdr);
 
 			/*
 			 * If prefetching out of the ghost cache,
 			 * we will have a non-zero datacnt.
 			 */
-			if (GHOST_STATE(old_state) && hdr->b_datacnt == 0) {
+			if (GHOST_STATE(old_state) && datacnt == 0) {
 				/* ghost elements have a ghost size */
-				ASSERT(hdr->b_buf == NULL);
+				ASSERT(hdr->b_l1hdr.b_buf == NULL);
 				from_delta = hdr->b_size;
 			}
 			ASSERT3U(*size, >=, from_delta);
@@ -1209,20 +1440,26 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 			if (use_mutex)
 				mutex_exit(&old_state->arcs_mtx);
 		}
-		if (new_state != arc_anon) {
+		if (new_state != arc_anon && new_state != arc_l2c_only) {
 			int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx);
-			uint64_t *size = &new_state->arcs_lsize[hdr->b_type];
+			uint64_t *size = &new_state->arcs_lsize[buftype];
 
+			/*
+			 * An L1 header always exists here, since if we're
+			 * moving to some L1-cached state (i.e. not l2c_only or
+			 * anonymous), we realloc the header to add an L1hdr
+			 * beforehand.
+			 */
+			ASSERT(HDR_HAS_L1HDR(hdr));
 			if (use_mutex)
 				mutex_enter(&new_state->arcs_mtx);
 
-			list_insert_head(&new_state->arcs_list[hdr->b_type],
-			    hdr);
+			list_insert_head(&new_state->arcs_list[buftype], hdr);
 
 			/* ghost elements have a ghost size */
 			if (GHOST_STATE(new_state)) {
-				ASSERT(hdr->b_datacnt == 0);
-				ASSERT(hdr->b_buf == NULL);
+				ASSERT0(datacnt);
+				ASSERT(hdr->b_l1hdr.b_buf == NULL);
 				to_delta = hdr->b_size;
 			}
 			atomic_add_64(size, to_delta);
@@ -1236,20 +1473,22 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 	if (new_state == arc_anon && HDR_IN_HASH_TABLE(hdr))
 		buf_hash_remove(hdr);
 
-	/* adjust state sizes */
-	if (to_delta)
+	/* adjust state sizes (ignore arc_l2c_only) */
+	if (to_delta && new_state != arc_l2c_only)
 		atomic_add_64(&new_state->arcs_size, to_delta);
-	if (from_delta) {
+	if (from_delta && old_state != arc_l2c_only) {
 		ASSERT3U(old_state->arcs_size, >=, from_delta);
 		atomic_add_64(&old_state->arcs_size, -from_delta);
 	}
-	hdr->b_state = new_state;
+	if (HDR_HAS_L1HDR(hdr))
+		hdr->b_l1hdr.b_state = new_state;
 
-	/* adjust l2arc hdr stats */
-	if (new_state == arc_l2c_only)
-		l2arc_hdr_stat_add();
-	else if (old_state == arc_l2c_only)
-		l2arc_hdr_stat_remove();
+	/*
+	 * L2 headers should never be on the L2 state list since they don't
+	 * have L1 headers allocated.
+	 */
+	ASSERT(list_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
+	    list_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
 }
 
 void
@@ -1305,31 +1544,36 @@ arc_space_return(uint64_t space, arc_space_type_t type)
 }
 
 arc_buf_t *
-arc_buf_alloc(spa_t *spa, int size, void *tag, arc_buf_contents_t type)
+arc_buf_alloc(spa_t *spa, int32_t size, void *tag, arc_buf_contents_t type)
 {
 	arc_buf_hdr_t *hdr;
 	arc_buf_t *buf;
 
 	ASSERT3U(size, >, 0);
-	hdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
+	hdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
 	ASSERT(BUF_EMPTY(hdr));
+	ASSERT3P(hdr->b_freeze_cksum, ==, NULL);
 	hdr->b_size = size;
-	hdr->b_type = type;
 	hdr->b_spa = spa_load_guid(spa);
-	hdr->b_state = arc_anon;
-	hdr->b_arc_access = 0;
+
 	buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
 	buf->b_hdr = hdr;
 	buf->b_data = NULL;
 	buf->b_efunc = NULL;
 	buf->b_private = NULL;
 	buf->b_next = NULL;
-	hdr->b_buf = buf;
+
+	hdr->b_flags = arc_bufc_to_flags(type);
+	hdr->b_flags |= ARC_FLAG_HAS_L1HDR;
+
+	hdr->b_l1hdr.b_buf = buf;
+	hdr->b_l1hdr.b_state = arc_anon;
+	hdr->b_l1hdr.b_arc_access = 0;
+	hdr->b_l1hdr.b_datacnt = 1;
+
 	arc_get_data_buf(buf);
-	hdr->b_datacnt = 1;
-	hdr->b_flags = 0;
-	ASSERT(refcount_is_zero(&hdr->b_refcnt));
-	(void) refcount_add(&hdr->b_refcnt, tag);
+	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
+	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
 
 	return (buf);
 }
@@ -1362,8 +1606,9 @@ arc_return_buf(arc_buf_t *buf, void *tag)
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	ASSERT(buf->b_data != NULL);
-	(void) refcount_add(&hdr->b_refcnt, tag);
-	(void) refcount_remove(&hdr->b_refcnt, arc_onloan_tag);
+	ASSERT(HDR_HAS_L1HDR(hdr));
+	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, tag);
+	(void) refcount_remove(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
 
 	atomic_add_64(&arc_loaned_bytes, -hdr->b_size);
 }
@@ -1372,12 +1617,12 @@ arc_return_buf(arc_buf_t *buf, void *tag)
 void
 arc_loan_inuse_buf(arc_buf_t *buf, void *tag)
 {
-	arc_buf_hdr_t *hdr;
+	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	ASSERT(buf->b_data != NULL);
-	hdr = buf->b_hdr;
-	(void) refcount_add(&hdr->b_refcnt, arc_onloan_tag);
-	(void) refcount_remove(&hdr->b_refcnt, tag);
+	ASSERT(HDR_HAS_L1HDR(hdr));
+	(void) refcount_add(&hdr->b_l1hdr.b_refcnt, arc_onloan_tag);
+	(void) refcount_remove(&hdr->b_l1hdr.b_refcnt, tag);
 	buf->b_efunc = NULL;
 	buf->b_private = NULL;
 
@@ -1391,15 +1636,16 @@ arc_buf_clone(arc_buf_t *from)
 	arc_buf_hdr_t *hdr = from->b_hdr;
 	uint64_t size = hdr->b_size;
 
-	ASSERT(hdr->b_state != arc_anon);
+	ASSERT(HDR_HAS_L1HDR(hdr));
+	ASSERT(hdr->b_l1hdr.b_state != arc_anon);
 
 	buf = kmem_cache_alloc(buf_cache, KM_PUSHPAGE);
 	buf->b_hdr = hdr;
 	buf->b_data = NULL;
 	buf->b_efunc = NULL;
 	buf->b_private = NULL;
-	buf->b_next = hdr->b_buf;
-	hdr->b_buf = buf;
+	buf->b_next = hdr->b_l1hdr.b_buf;
+	hdr->b_l1hdr.b_buf = buf;
 	arc_get_data_buf(buf);
 	bcopy(from->b_data, buf->b_data, size);
 
@@ -1409,11 +1655,11 @@ arc_buf_clone(arc_buf_t *from)
 	 * then track the size and number of duplicates.  These stats will be
 	 * updated as duplicate buffers are created and destroyed.
 	 */
-	if (hdr->b_type == ARC_BUFC_DATA) {
+	if (HDR_ISTYPE_DATA(hdr)) {
 		ARCSTAT_BUMP(arcstat_duplicate_buffers);
 		ARCSTAT_INCR(arcstat_duplicate_buffers_size, size);
 	}
-	hdr->b_datacnt += 1;
+	hdr->b_l1hdr.b_datacnt += 1;
 	return (buf);
 }
 
@@ -1436,17 +1682,20 @@ arc_buf_add_ref(arc_buf_t *buf, void* tag)
 	hash_lock = HDR_LOCK(buf->b_hdr);
 	mutex_enter(hash_lock);
 	hdr = buf->b_hdr;
+	ASSERT(HDR_HAS_L1HDR(hdr));
 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
 	mutex_exit(&buf->b_evict_lock);
 
-	ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
+	ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
+	    hdr->b_l1hdr.b_state == arc_mfu);
+
 	add_reference(hdr, hash_lock, tag);
 	DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
 	arc_access(hdr, hash_lock);
 	mutex_exit(hash_lock);
 	ARCSTAT_BUMP(arcstat_hits);
-	ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_FLAG_PREFETCH),
-	    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
+	ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
+	    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
 	    data, metadata, hits);
 }
 
@@ -1484,10 +1733,10 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 	arc_buf_t **bufp;
 
 	/* free up data associated with the buf */
-	if (buf->b_data) {
-		arc_state_t *state = buf->b_hdr->b_state;
+	if (buf->b_data != NULL) {
+		arc_state_t *state = buf->b_hdr->b_l1hdr.b_state;
 		uint64_t size = buf->b_hdr->b_size;
-		arc_buf_contents_t type = buf->b_hdr->b_type;
+		arc_buf_contents_t type = arc_buf_type(buf->b_hdr);
 
 		arc_cksum_verify(buf);
 		arc_buf_unwatch(buf);
@@ -1503,11 +1752,12 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 				atomic_add_64(&arc_size, -size);
 			}
 		}
-		if (list_link_active(&buf->b_hdr->b_arc_node)) {
+		if (list_link_active(&buf->b_hdr->b_l1hdr.b_arc_node)) {
 			uint64_t *cnt = &state->arcs_lsize[type];
 
-			ASSERT(refcount_is_zero(&buf->b_hdr->b_refcnt));
-			ASSERT(state != arc_anon);
+			ASSERT(refcount_is_zero(
+			    &buf->b_hdr->b_l1hdr.b_refcnt));
+			ASSERT(state != arc_anon && state != arc_l2c_only);
 
 			ASSERT3U(*cnt, >=, size);
 			atomic_add_64(cnt, -size);
@@ -1520,13 +1770,13 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 		 * If we're destroying a duplicate buffer make sure
 		 * that the appropriate statistics are updated.
 		 */
-		if (buf->b_hdr->b_datacnt > 1 &&
-		    buf->b_hdr->b_type == ARC_BUFC_DATA) {
+		if (buf->b_hdr->b_l1hdr.b_datacnt > 1 &&
+		    HDR_ISTYPE_DATA(buf->b_hdr)) {
 			ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
 			ARCSTAT_INCR(arcstat_duplicate_buffers_size, -size);
 		}
-		ASSERT(buf->b_hdr->b_datacnt > 0);
-		buf->b_hdr->b_datacnt -= 1;
+		ASSERT(buf->b_hdr->b_l1hdr.b_datacnt > 0);
+		buf->b_hdr->b_l1hdr.b_datacnt -= 1;
 	}
 
 	/* only remove the buf if requested */
@@ -1534,7 +1784,8 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 		return;
 
 	/* remove the buf from the hdr list */
-	for (bufp = &buf->b_hdr->b_buf; *bufp != buf; bufp = &(*bufp)->b_next)
+	for (bufp = &buf->b_hdr->b_l1hdr.b_buf; *bufp != buf;
+	    bufp = &(*bufp)->b_next)
 		continue;
 	*bufp = buf->b_next;
 	buf->b_next = NULL;
@@ -1549,86 +1800,87 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 static void
 arc_hdr_destroy(arc_buf_hdr_t *hdr)
 {
-	ASSERT(refcount_is_zero(&hdr->b_refcnt));
-	ASSERT3P(hdr->b_state, ==, arc_anon);
+	if (HDR_HAS_L1HDR(hdr)) {
+		ASSERT(hdr->b_l1hdr.b_buf == NULL ||
+		    hdr->b_l1hdr.b_datacnt > 0);
+		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
+		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
+	}
 	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
-	l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr;
+	ASSERT(!HDR_IN_HASH_TABLE(hdr));
+
+	if (HDR_HAS_L2HDR(hdr)) {
+		l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
+		boolean_t buflist_held = MUTEX_HELD(&l2hdr->b_dev->l2ad_mtx);
 
-	if (l2hdr != NULL) {
-		boolean_t buflist_held = MUTEX_HELD(&l2arc_buflist_mtx);
-		/*
-		 * To prevent arc_free() and l2arc_evict() from
-		 * attempting to free the same buffer at the same time,
-		 * a FREE_IN_PROGRESS flag is given to arc_free() to
-		 * give it priority.  l2arc_evict() can't destroy this
-		 * header while we are waiting on l2arc_buflist_mtx.
-		 *
-		 * The hdr may be removed from l2ad_buflist before we
-		 * grab l2arc_buflist_mtx, so b_l2hdr is rechecked.
-		 */
 		if (!buflist_held) {
-			mutex_enter(&l2arc_buflist_mtx);
-			l2hdr = hdr->b_l2hdr;
+			mutex_enter(&l2hdr->b_dev->l2ad_mtx);
+			l2hdr = &hdr->b_l2hdr;
 		}
 
-		if (l2hdr != NULL) {
-			list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
-			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
-			ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
-			vdev_space_update(l2hdr->b_dev->l2ad_vdev,
-			    -l2hdr->b_asize, 0, 0);
-			kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
-			if (hdr->b_state == arc_l2c_only)
-				l2arc_hdr_stat_remove();
-			hdr->b_l2hdr = NULL;
-		}
+		list_remove(&l2hdr->b_dev->l2ad_buflist, hdr);
+
+		ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
+		ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
 
 		if (!buflist_held)
-			mutex_exit(&l2arc_buflist_mtx);
+			mutex_exit(&l2hdr->b_dev->l2ad_mtx);
+
+		hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
 	}
 
-	if (!BUF_EMPTY(hdr)) {
-		ASSERT(!HDR_IN_HASH_TABLE(hdr));
+	if (!BUF_EMPTY(hdr))
 		buf_discard_identity(hdr);
-	}
-	while (hdr->b_buf) {
-		arc_buf_t *buf = hdr->b_buf;
 
-		if (buf->b_efunc) {
-			mutex_enter(&arc_eviction_mtx);
-			mutex_enter(&buf->b_evict_lock);
-			ASSERT(buf->b_hdr != NULL);
-			arc_buf_destroy(hdr->b_buf, FALSE, FALSE);
-			hdr->b_buf = buf->b_next;
-			buf->b_hdr = &arc_eviction_hdr;
-			buf->b_next = arc_eviction_list;
-			arc_eviction_list = buf;
-			mutex_exit(&buf->b_evict_lock);
-			mutex_exit(&arc_eviction_mtx);
-		} else {
-			arc_buf_destroy(hdr->b_buf, FALSE, TRUE);
-		}
-	}
 	if (hdr->b_freeze_cksum != NULL) {
 		kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
 		hdr->b_freeze_cksum = NULL;
 	}
-	if (hdr->b_thawed) {
-		kmem_free(hdr->b_thawed, 1);
-		hdr->b_thawed = NULL;
+
+	if (HDR_HAS_L1HDR(hdr)) {
+		while (hdr->b_l1hdr.b_buf) {
+			arc_buf_t *buf = hdr->b_l1hdr.b_buf;
+
+			if (buf->b_efunc != NULL) {
+				mutex_enter(&arc_eviction_mtx);
+				mutex_enter(&buf->b_evict_lock);
+				ASSERT(buf->b_hdr != NULL);
+				arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE,
+				    FALSE);
+				hdr->b_l1hdr.b_buf = buf->b_next;
+				buf->b_hdr = &arc_eviction_hdr;
+				buf->b_next = arc_eviction_list;
+				arc_eviction_list = buf;
+				mutex_exit(&buf->b_evict_lock);
+				mutex_exit(&arc_eviction_mtx);
+			} else {
+				arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE,
+				    TRUE);
+			}
+		}
+#ifdef ZFS_DEBUG
+		if (hdr->b_l1hdr.b_thawed != NULL) {
+			kmem_free(hdr->b_l1hdr.b_thawed, 1);
+			hdr->b_l1hdr.b_thawed = NULL;
+		}
+#endif
 	}
 
-	ASSERT(!list_link_active(&hdr->b_arc_node));
 	ASSERT3P(hdr->b_hash_next, ==, NULL);
-	ASSERT3P(hdr->b_acb, ==, NULL);
-	kmem_cache_free(hdr_cache, hdr);
+	if (HDR_HAS_L1HDR(hdr)) {
+		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
+		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
+		kmem_cache_free(hdr_full_cache, hdr);
+	} else {
+		kmem_cache_free(hdr_l2only_cache, hdr);
+	}
 }
 
 void
 arc_buf_free(arc_buf_t *buf, void *tag)
 {
 	arc_buf_hdr_t *hdr = buf->b_hdr;
-	int hashed = hdr->b_state != arc_anon;
+	int hashed = hdr->b_l1hdr.b_state != arc_anon;
 
 	ASSERT(buf->b_efunc == NULL);
 	ASSERT(buf->b_data != NULL);
@@ -1641,10 +1893,10 @@ arc_buf_free(arc_buf_t *buf, void *tag)
 		ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
 
 		(void) remove_reference(hdr, hash_lock, tag);
-		if (hdr->b_datacnt > 1) {
+		if (hdr->b_l1hdr.b_datacnt > 1) {
 			arc_buf_destroy(buf, FALSE, TRUE);
 		} else {
-			ASSERT(buf == hdr->b_buf);
+			ASSERT(buf == hdr->b_l1hdr.b_buf);
 			ASSERT(buf->b_efunc == NULL);
 			hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
 		}
@@ -1658,7 +1910,7 @@ arc_buf_free(arc_buf_t *buf, void *tag)
 		 */
 		mutex_enter(&arc_eviction_mtx);
 		(void) remove_reference(hdr, NULL, tag);
-		ASSERT(refcount_is_zero(&hdr->b_refcnt));
+		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 		destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
 		mutex_exit(&arc_eviction_mtx);
 		if (destroy_hdr)
@@ -1678,34 +1930,35 @@ arc_buf_remove_ref(arc_buf_t *buf, void* tag)
 	kmutex_t *hash_lock = HDR_LOCK(hdr);
 	boolean_t no_callback = (buf->b_efunc == NULL);
 
-	if (hdr->b_state == arc_anon) {
-		ASSERT(hdr->b_datacnt == 1);
+	if (hdr->b_l1hdr.b_state == arc_anon) {
+		ASSERT(hdr->b_l1hdr.b_datacnt == 1);
 		arc_buf_free(buf, tag);
 		return (no_callback);
 	}
 
 	mutex_enter(hash_lock);
 	hdr = buf->b_hdr;
+	ASSERT(hdr->b_l1hdr.b_datacnt > 0);
 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
-	ASSERT(hdr->b_state != arc_anon);
+	ASSERT(hdr->b_l1hdr.b_state != arc_anon);
 	ASSERT(buf->b_data != NULL);
 
 	(void) remove_reference(hdr, hash_lock, tag);
-	if (hdr->b_datacnt > 1) {
+	if (hdr->b_l1hdr.b_datacnt > 1) {
 		if (no_callback)
 			arc_buf_destroy(buf, FALSE, TRUE);
 	} else if (no_callback) {
-		ASSERT(hdr->b_buf == buf && buf->b_next == NULL);
+		ASSERT(hdr->b_l1hdr.b_buf == buf && buf->b_next == NULL);
 		ASSERT(buf->b_efunc == NULL);
 		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
 	}
-	ASSERT(no_callback || hdr->b_datacnt > 1 ||
-	    refcount_is_zero(&hdr->b_refcnt));
+	ASSERT(no_callback || hdr->b_l1hdr.b_datacnt > 1 ||
+	    refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 	mutex_exit(hash_lock);
 	return (no_callback);
 }
 
-int
+int32_t
 arc_buf_size(arc_buf_t *buf)
 {
 	return (buf->b_hdr->b_size);
@@ -1746,7 +1999,7 @@ arc_buf_eviction_needed(arc_buf_t *buf)
 		return (B_TRUE);
 	}
 
-	if (hdr->b_datacnt > 1 && hdr->b_type == ARC_BUFC_DATA)
+	if (hdr->b_l1hdr.b_datacnt > 1 && HDR_ISTYPE_DATA(hdr))
 		evict_needed = B_TRUE;
 
 	mutex_exit(&buf->b_evict_lock);
@@ -1783,8 +2036,20 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 
 	evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
 
-	mutex_enter(&state->arcs_mtx);
+	/*
+	 * The ghost list lock must be acquired first in order to prevent
+	 * a 3 party deadlock:
+	 *
+	 *  - arc_evict_ghost acquires arc_*_ghost->arcs_mtx, followed by
+	 *    l2ad_mtx in arc_hdr_realloc
+	 *  - l2arc_write_buffers acquires l2ad_mtx, followed by arc_*->arcs_mtx
+	 *  - arc_evict acquires arc_*_ghost->arcs_mtx, followed by
+	 *    arc_*_ghost->arcs_mtx and forms a deadlock cycle.
+	 *
+	 * This situation is avoided by acquiring the ghost list lock first.
+	 */
 	mutex_enter(&evicted_state->arcs_mtx);
+	mutex_enter(&state->arcs_mtx);
 
 	/*
 	 * Decide which "type" (data vs metadata) to recycle from.
@@ -1800,6 +2065,7 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 		arc_buf_hdr_t *metadata_hdr =
 		    list_tail(&state->arcs_list[ARC_BUFC_METADATA]);
 		arc_buf_contents_t realtype;
+
 		if (data_hdr == NULL) {
 			realtype = ARC_BUFC_METADATA;
 		} else if (metadata_hdr == NULL) {
@@ -1808,13 +2074,13 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 			realtype = ARC_BUFC_METADATA;
 		} else if (arc_meta_used <= arc_meta_min) {
 			realtype = ARC_BUFC_DATA;
+		} else if (HDR_HAS_L1HDR(data_hdr) &&
+		    HDR_HAS_L1HDR(metadata_hdr) &&
+		    data_hdr->b_l1hdr.b_arc_access <
+		    metadata_hdr->b_l1hdr.b_arc_access) {
+			realtype = ARC_BUFC_DATA;
 		} else {
-			if (data_hdr->b_arc_access <
-			    metadata_hdr->b_arc_access) {
-				realtype = ARC_BUFC_DATA;
-			} else {
-				realtype = ARC_BUFC_METADATA;
-			}
+			realtype = ARC_BUFC_METADATA;
 		}
 		if (realtype != type) {
 			/*
@@ -1835,8 +2101,8 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 		/* prefetch buffers have a minimum lifespan */
 		if (HDR_IO_IN_PROGRESS(hdr) ||
 		    (spa && hdr->b_spa != spa) ||
-		    (hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT) &&
-		    ddi_get_lbolt() - hdr->b_arc_access <
+		    ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
+		    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
 		    arc_min_prefetch_lifespan)) {
 			skipped++;
 			continue;
@@ -1861,11 +2127,11 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 		 */
 		if (!recycle && count++ > arc_evict_iterations) {
 			list_insert_after(list, hdr, &marker);
-			mutex_exit(&evicted_state->arcs_mtx);
 			mutex_exit(&state->arcs_mtx);
+			mutex_exit(&evicted_state->arcs_mtx);
 			kpreempt(KPREEMPT_SYNC);
-			mutex_enter(&state->arcs_mtx);
 			mutex_enter(&evicted_state->arcs_mtx);
+			mutex_enter(&state->arcs_mtx);
 			hdr_prev = list_prev(list, &marker);
 			list_remove(list, &marker);
 			count = 0;
@@ -1875,28 +2141,29 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 		hash_lock = HDR_LOCK(hdr);
 		have_lock = MUTEX_HELD(hash_lock);
 		if (have_lock || mutex_tryenter(hash_lock)) {
-			ASSERT0(refcount_count(&hdr->b_refcnt));
-			ASSERT(hdr->b_datacnt > 0);
-			while (hdr->b_buf) {
-				arc_buf_t *buf = hdr->b_buf;
+			ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
+			ASSERT3U(hdr->b_l1hdr.b_datacnt, >, 0);
+			while (hdr->b_l1hdr.b_buf) {
+				arc_buf_t *buf = hdr->b_l1hdr.b_buf;
 				if (!mutex_tryenter(&buf->b_evict_lock)) {
 					missed += 1;
 					break;
 				}
-				if (buf->b_data) {
+				if (buf->b_data != NULL) {
 					bytes_evicted += hdr->b_size;
-					if (recycle && hdr->b_type == type &&
+					if (recycle &&
+					    arc_buf_type(hdr) == type &&
 					    hdr->b_size == bytes &&
 					    !HDR_L2_WRITING(hdr)) {
 						stolen = buf->b_data;
 						recycle = FALSE;
 					}
 				}
-				if (buf->b_efunc) {
+				if (buf->b_efunc != NULL) {
 					mutex_enter(&arc_eviction_mtx);
 					arc_buf_destroy(buf,
 					    buf->b_data == stolen, FALSE);
-					hdr->b_buf = buf->b_next;
+					hdr->b_l1hdr.b_buf = buf->b_next;
 					buf->b_hdr = &arc_eviction_hdr;
 					buf->b_next = arc_eviction_list;
 					arc_eviction_list = buf;
@@ -1909,7 +2176,7 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 				}
 			}
 
-			if (hdr->b_l2hdr) {
+			if (HDR_HAS_L2HDR(hdr)) {
 				ARCSTAT_INCR(arcstat_evict_l2_cached,
 				    hdr->b_size);
 			} else {
@@ -1923,7 +2190,7 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 				}
 			}
 
-			if (hdr->b_datacnt == 0) {
+			if (hdr->b_l1hdr.b_datacnt == 0) {
 				arc_change_state(evicted_state, hdr, hash_lock);
 				ASSERT(HDR_IN_HASH_TABLE(hdr));
 				hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
@@ -1939,8 +2206,8 @@ arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
 		}
 	}
 
-	mutex_exit(&evicted_state->arcs_mtx);
 	mutex_exit(&state->arcs_mtx);
+	mutex_exit(&evicted_state->arcs_mtx);
 
 	if (bytes_evicted < bytes)
 		dprintf("only evicted %lld bytes from %x",
@@ -1982,7 +2249,7 @@ top:
 	mutex_enter(&state->arcs_mtx);
 	for (hdr = list_tail(list); hdr; hdr = hdr_prev) {
 		hdr_prev = list_prev(list, hdr);
-		if (hdr->b_type > ARC_BUFC_NUMTYPES)
+		if (arc_buf_type(hdr) >= ARC_BUFC_NUMTYPES)
 			panic("invalid hdr=%p", (void *)hdr);
 		if (spa && hdr->b_spa != spa)
 			continue;
@@ -2014,16 +2281,23 @@ top:
 		}
 		if (mutex_tryenter(hash_lock)) {
 			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
-			ASSERT(hdr->b_buf == NULL);
+			ASSERT(!HDR_HAS_L1HDR(hdr) ||
+			    hdr->b_l1hdr.b_buf == NULL);
 			ARCSTAT_BUMP(arcstat_deleted);
 			bytes_deleted += hdr->b_size;
 
-			if (hdr->b_l2hdr != NULL) {
+			if (HDR_HAS_L2HDR(hdr)) {
 				/*
 				 * This buffer is cached on the 2nd Level ARC;
 				 * don't destroy the header.
 				 */
 				arc_change_state(arc_l2c_only, hdr, hash_lock);
+				/*
+				 * dropping from L1+L2 cached to L2-only,
+				 * realloc to remove the L1 header.
+				 */
+				hdr = arc_hdr_realloc(hdr, hdr_full_cache,
+				    hdr_l2only_cache);
 				mutex_exit(hash_lock);
 			} else {
 				arc_change_state(arc_anon, hdr, hash_lock);
@@ -2166,27 +2440,27 @@ arc_flush(spa_t *spa)
 {
 	uint64_t guid = 0;
 
-	if (spa)
+	if (spa != NULL)
 		guid = spa_load_guid(spa);
 
 	while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) {
 		(void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA);
-		if (spa)
+		if (spa != NULL)
 			break;
 	}
 	while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) {
 		(void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA);
-		if (spa)
+		if (spa != NULL)
 			break;
 	}
 	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) {
 		(void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA);
-		if (spa)
+		if (spa != NULL)
 			break;
 	}
 	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) {
 		(void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA);
-		if (spa)
+		if (spa != NULL)
 			break;
 	}
 
@@ -2406,7 +2680,8 @@ arc_kmem_reap_now(void)
 		}
 	}
 	kmem_cache_reap_now(buf_cache);
-	kmem_cache_reap_now(hdr_cache);
+	kmem_cache_reap_now(hdr_full_cache);
+	kmem_cache_reap_now(hdr_l2only_cache);
 	kmem_cache_reap_now(range_seg_cache);
 
 	if (zio_arena != NULL) {
@@ -2590,9 +2865,9 @@ arc_evict_needed(arc_buf_contents_t type)
 static void
 arc_get_data_buf(arc_buf_t *buf)
 {
-	arc_state_t		*state = buf->b_hdr->b_state;
+	arc_state_t		*state = buf->b_hdr->b_l1hdr.b_state;
 	uint64_t		size = buf->b_hdr->b_size;
-	arc_buf_contents_t	type = buf->b_hdr->b_type;
+	arc_buf_contents_t	type = arc_buf_type(buf->b_hdr);
 
 	arc_adapt(size, state);
 
@@ -2618,8 +2893,7 @@ arc_get_data_buf(arc_buf_t *buf)
 	 * will end up on the mru list; so steal space from there.
 	 */
 	if (state == arc_mfu_ghost)
-		state = buf->b_hdr->b_flags & ARC_FLAG_PREFETCH ?
-		    arc_mru : arc_mfu;
+		state = HDR_PREFETCH(buf->b_hdr) ? arc_mru : arc_mfu;
 	else if (state == arc_mru_ghost)
 		state = arc_mru;
 
@@ -2651,19 +2925,20 @@ out:
 	 * Update the state size.  Note that ghost states have a
 	 * "ghost size" and so don't need to be updated.
 	 */
-	if (!GHOST_STATE(buf->b_hdr->b_state)) {
+	if (!GHOST_STATE(buf->b_hdr->b_l1hdr.b_state)) {
 		arc_buf_hdr_t *hdr = buf->b_hdr;
 
-		atomic_add_64(&hdr->b_state->arcs_size, size);
-		if (list_link_active(&hdr->b_arc_node)) {
-			ASSERT(refcount_is_zero(&hdr->b_refcnt));
-			atomic_add_64(&hdr->b_state->arcs_lsize[type], size);
+		atomic_add_64(&hdr->b_l1hdr.b_state->arcs_size, size);
+		if (list_link_active(&hdr->b_l1hdr.b_arc_node)) {
+			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
+			atomic_add_64(&hdr->b_l1hdr.b_state->arcs_lsize[type],
+			    size);
 		}
 		/*
 		 * If we are growing the cache, and we are adding anonymous
 		 * data, and we have outgrown arc_p, update arc_p
 		 */
-		if (arc_size < arc_c && hdr->b_state == arc_anon &&
+		if (arc_size < arc_c && hdr->b_l1hdr.b_state == arc_anon &&
 		    arc_anon->arcs_size + arc_mru->arcs_size > arc_p)
 			arc_p = MIN(arc_c, arc_p + size);
 	}
@@ -2679,20 +2954,21 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 	clock_t now;
 
 	ASSERT(MUTEX_HELD(hash_lock));
+	ASSERT(HDR_HAS_L1HDR(hdr));
 
-	if (hdr->b_state == arc_anon) {
+	if (hdr->b_l1hdr.b_state == arc_anon) {
 		/*
 		 * This buffer is not in the cache, and does not
 		 * appear in our "ghost" list.  Add the new buffer
 		 * to the MRU state.
 		 */
 
-		ASSERT(hdr->b_arc_access == 0);
-		hdr->b_arc_access = ddi_get_lbolt();
+		ASSERT0(hdr->b_l1hdr.b_arc_access);
+		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
 		DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
 		arc_change_state(arc_mru, hdr, hash_lock);
 
-	} else if (hdr->b_state == arc_mru) {
+	} else if (hdr->b_l1hdr.b_state == arc_mru) {
 		now = ddi_get_lbolt();
 
 		/*
@@ -2703,14 +2979,15 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 		 * - move the buffer to the head of the list if this is
 		 *   another prefetch (to make it less likely to be evicted).
 		 */
-		if ((hdr->b_flags & ARC_FLAG_PREFETCH) != 0) {
-			if (refcount_count(&hdr->b_refcnt) == 0) {
-				ASSERT(list_link_active(&hdr->b_arc_node));
+		if (HDR_PREFETCH(hdr)) {
+			if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
+				ASSERT(list_link_active(
+				    &hdr->b_l1hdr.b_arc_node));
 			} else {
 				hdr->b_flags &= ~ARC_FLAG_PREFETCH;
 				ARCSTAT_BUMP(arcstat_mru_hits);
 			}
-			hdr->b_arc_access = now;
+			hdr->b_l1hdr.b_arc_access = now;
 			return;
 		}
 
@@ -2719,18 +2996,18 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 		 * but it is still in the cache. Move it to the MFU
 		 * state.
 		 */
-		if (now > hdr->b_arc_access + ARC_MINTIME) {
+		if (now > hdr->b_l1hdr.b_arc_access + ARC_MINTIME) {
 			/*
 			 * More than 125ms have passed since we
 			 * instantiated this buffer.  Move it to the
 			 * most frequently used state.
 			 */
-			hdr->b_arc_access = now;
+			hdr->b_l1hdr.b_arc_access = now;
 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
 			arc_change_state(arc_mfu, hdr, hash_lock);
 		}
 		ARCSTAT_BUMP(arcstat_mru_hits);
-	} else if (hdr->b_state == arc_mru_ghost) {
+	} else if (hdr->b_l1hdr.b_state == arc_mru_ghost) {
 		arc_state_t	*new_state;
 		/*
 		 * This buffer has been "accessed" recently, but
@@ -2738,9 +3015,9 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 		 * MFU state.
 		 */
 
-		if (hdr->b_flags & ARC_FLAG_PREFETCH) {
+		if (HDR_PREFETCH(hdr)) {
 			new_state = arc_mru;
-			if (refcount_count(&hdr->b_refcnt) > 0)
+			if (refcount_count(&hdr->b_l1hdr.b_refcnt) > 0)
 				hdr->b_flags &= ~ARC_FLAG_PREFETCH;
 			DTRACE_PROBE1(new_state__mru, arc_buf_hdr_t *, hdr);
 		} else {
@@ -2748,11 +3025,11 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 			DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
 		}
 
-		hdr->b_arc_access = ddi_get_lbolt();
+		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
 		arc_change_state(new_state, hdr, hash_lock);
 
 		ARCSTAT_BUMP(arcstat_mru_ghost_hits);
-	} else if (hdr->b_state == arc_mfu) {
+	} else if (hdr->b_l1hdr.b_state == arc_mfu) {
 		/*
 		 * This buffer has been accessed more than once and is
 		 * still in the cache.  Keep it in the MFU state.
@@ -2762,13 +3039,13 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 		 * If it was a prefetch, we will explicitly move it to
 		 * the head of the list now.
 		 */
-		if ((hdr->b_flags & ARC_FLAG_PREFETCH) != 0) {
-			ASSERT(refcount_count(&hdr->b_refcnt) == 0);
-			ASSERT(list_link_active(&hdr->b_arc_node));
+		if ((HDR_PREFETCH(hdr)) != 0) {
+			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
+			ASSERT(list_link_active(&hdr->b_l1hdr.b_arc_node));
 		}
 		ARCSTAT_BUMP(arcstat_mfu_hits);
-		hdr->b_arc_access = ddi_get_lbolt();
-	} else if (hdr->b_state == arc_mfu_ghost) {
+		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
+	} else if (hdr->b_l1hdr.b_state == arc_mfu_ghost) {
 		arc_state_t	*new_state = arc_mfu;
 		/*
 		 * This buffer has been accessed more than once but has
@@ -2776,26 +3053,26 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 		 * MFU state.
 		 */
 
-		if (hdr->b_flags & ARC_FLAG_PREFETCH) {
+		if (HDR_PREFETCH(hdr)) {
 			/*
 			 * This is a prefetch access...
 			 * move this block back to the MRU state.
 			 */
-			ASSERT0(refcount_count(&hdr->b_refcnt));
+			ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
 			new_state = arc_mru;
 		}
 
-		hdr->b_arc_access = ddi_get_lbolt();
+		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
 		arc_change_state(new_state, hdr, hash_lock);
 
 		ARCSTAT_BUMP(arcstat_mfu_ghost_hits);
-	} else if (hdr->b_state == arc_l2c_only) {
+	} else if (hdr->b_l1hdr.b_state == arc_l2c_only) {
 		/*
 		 * This buffer is on the 2nd Level ARC.
 		 */
 
-		hdr->b_arc_access = ddi_get_lbolt();
+		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
 		DTRACE_PROBE1(new_state__mfu, arc_buf_hdr_t *, hdr);
 		arc_change_state(arc_mfu, hdr, hash_lock);
 	} else {
@@ -2866,11 +3143,11 @@ arc_read_done(zio_t *zio)
 	}
 
 	hdr->b_flags &= ~ARC_FLAG_L2_EVICTED;
-	if (l2arc_noprefetch && (hdr->b_flags & ARC_FLAG_PREFETCH))
+	if (l2arc_noprefetch && HDR_PREFETCH(hdr))
 		hdr->b_flags &= ~ARC_FLAG_L2CACHE;
 
 	/* byteswap if necessary */
-	callback_list = hdr->b_acb;
+	callback_list = hdr->b_l1hdr.b_acb;
 	ASSERT(callback_list != NULL);
 	if (BP_SHOULD_BYTESWAP(zio->io_bp) && zio->io_error == 0) {
 		dmu_object_byteswap_t bswap =
@@ -2884,7 +3161,8 @@ arc_read_done(zio_t *zio)
 	arc_cksum_compute(buf, B_FALSE);
 	arc_buf_watch(buf);
 
-	if (hash_lock && zio->io_error == 0 && hdr->b_state == arc_anon) {
+	if (hash_lock && zio->io_error == 0 &&
+	    hdr->b_l1hdr.b_state == arc_anon) {
 		/*
 		 * Only call arc_access on anonymous buffers.  This is because
 		 * if we've issued an I/O for an evicted buffer, we've already
@@ -2906,24 +3184,25 @@ arc_read_done(zio_t *zio)
 			abuf = NULL;
 		}
 	}
-	hdr->b_acb = NULL;
+	hdr->b_l1hdr.b_acb = NULL;
 	hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
 	ASSERT(!HDR_BUF_AVAILABLE(hdr));
 	if (abuf == buf) {
 		ASSERT(buf->b_efunc == NULL);
-		ASSERT(hdr->b_datacnt == 1);
+		ASSERT(hdr->b_l1hdr.b_datacnt == 1);
 		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
 	}
 
-	ASSERT(refcount_is_zero(&hdr->b_refcnt) || callback_list != NULL);
+	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt) ||
+	    callback_list != NULL);
 
 	if (zio->io_error != 0) {
 		hdr->b_flags |= ARC_FLAG_IO_ERROR;
-		if (hdr->b_state != arc_anon)
+		if (hdr->b_l1hdr.b_state != arc_anon)
 			arc_change_state(arc_anon, hdr, hash_lock);
 		if (HDR_IN_HASH_TABLE(hdr))
 			buf_hash_remove(hdr);
-		freeable = refcount_is_zero(&hdr->b_refcnt);
+		freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
 	}
 
 	/*
@@ -2931,9 +3210,9 @@ arc_read_done(zio_t *zio)
 	 * that the hdr (and hence the cv) might be freed before we get to
 	 * the cv_broadcast().
 	 */
-	cv_broadcast(&hdr->b_cv);
+	cv_broadcast(&hdr->b_l1hdr.b_cv);
 
-	if (hash_lock) {
+	if (hash_lock != NULL) {
 		mutex_exit(hash_lock);
 	} else {
 		/*
@@ -2942,8 +3221,8 @@ arc_read_done(zio_t *zio)
 		 * moved to the anonymous state (so that it won't show up
 		 * in the cache).
 		 */
-		ASSERT3P(hdr->b_state, ==, arc_anon);
-		freeable = refcount_is_zero(&hdr->b_refcnt);
+		ASSERT3P(hdr->b_l1hdr.b_state, ==, arc_anon);
+		freeable = refcount_is_zero(&hdr->b_l1hdr.b_refcnt);
 	}
 
 	/* execute each callback and free its structure */
@@ -3005,14 +3284,14 @@ top:
 		hdr = buf_hash_find(guid, bp, &hash_lock);
 	}
 
-	if (hdr != NULL && hdr->b_datacnt > 0) {
+	if (hdr != NULL && HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_datacnt > 0) {
 
 		*arc_flags |= ARC_FLAG_CACHED;
 
 		if (HDR_IO_IN_PROGRESS(hdr)) {
 
 			if (*arc_flags & ARC_FLAG_WAIT) {
-				cv_wait(&hdr->b_cv, hash_lock);
+				cv_wait(&hdr->b_l1hdr.b_cv, hash_lock);
 				mutex_exit(hash_lock);
 				goto top;
 			}
@@ -3030,8 +3309,8 @@ top:
 					    spa, NULL, NULL, NULL, zio_flags);
 
 				ASSERT(acb->acb_done != NULL);
-				acb->acb_next = hdr->b_acb;
-				hdr->b_acb = acb;
+				acb->acb_next = hdr->b_l1hdr.b_acb;
+				hdr->b_l1hdr.b_acb = acb;
 				add_reference(hdr, hash_lock, private);
 				mutex_exit(hash_lock);
 				return (0);
@@ -3040,7 +3319,8 @@ top:
 			return (0);
 		}
 
-		ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
+		ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
+		    hdr->b_l1hdr.b_state == arc_mfu);
 
 		if (done) {
 			add_reference(hdr, hash_lock, private);
@@ -3049,7 +3329,7 @@ top:
 			 * copy of the data so that we will be guaranteed
 			 * that arc_release() will always succeed.
 			 */
-			buf = hdr->b_buf;
+			buf = hdr->b_l1hdr.b_buf;
 			ASSERT(buf);
 			ASSERT(buf->b_data);
 			if (HDR_BUF_AVAILABLE(hdr)) {
@@ -3060,7 +3340,7 @@ top:
 			}
 
 		} else if (*arc_flags & ARC_FLAG_PREFETCH &&
-		    refcount_count(&hdr->b_refcnt) == 0) {
+		    refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
 			hdr->b_flags |= ARC_FLAG_PREFETCH;
 		}
 		DTRACE_PROBE1(arc__hit, arc_buf_hdr_t *, hdr);
@@ -3071,8 +3351,8 @@ top:
 			hdr->b_flags |= ARC_FLAG_L2COMPRESS;
 		mutex_exit(hash_lock);
 		ARCSTAT_BUMP(arcstat_hits);
-		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_FLAG_PREFETCH),
-		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
+		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
+		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
 		    data, metadata, hits);
 
 		if (done)
@@ -3084,7 +3364,7 @@ top:
 		uint64_t addr = 0;
 		boolean_t devw = B_FALSE;
 		enum zio_compress b_compress = ZIO_COMPRESS_OFF;
-		uint64_t b_asize = 0;
+		int32_t b_asize = 0;
 
 		if (hdr == NULL) {
 			/* this block is not in the cache */
@@ -3095,7 +3375,6 @@ top:
 			if (!BP_IS_EMBEDDED(bp)) {
 				hdr->b_dva = *BP_IDENTITY(bp);
 				hdr->b_birth = BP_PHYSICAL_BIRTH(bp);
-				hdr->b_cksum0 = bp->blk_cksum.zc_word[0];
 				exists = buf_hash_insert(hdr, &hash_lock);
 			}
 			if (exists != NULL) {
@@ -3119,11 +3398,20 @@ top:
 			if (BP_GET_LEVEL(bp) > 0)
 				hdr->b_flags |= ARC_FLAG_INDIRECT;
 		} else {
-			/* this block is in the ghost cache */
-			ASSERT(GHOST_STATE(hdr->b_state));
+			/*
+			 * This block is in the ghost cache. If it was L2-only
+			 * (and thus didn't have an L1 hdr), we realloc the
+			 * header to add an L1 hdr.
+			 */
+			if (!HDR_HAS_L1HDR(hdr)) {
+				hdr = arc_hdr_realloc(hdr, hdr_l2only_cache,
+				    hdr_full_cache);
+			}
+
+			ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
 			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
-			ASSERT0(refcount_count(&hdr->b_refcnt));
-			ASSERT(hdr->b_buf == NULL);
+			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
+			ASSERT(hdr->b_l1hdr.b_buf == NULL);
 
 			/* if this is a prefetch, we don't have a reference */
 			if (*arc_flags & ARC_FLAG_PREFETCH)
@@ -3140,29 +3428,29 @@ top:
 			buf->b_efunc = NULL;
 			buf->b_private = NULL;
 			buf->b_next = NULL;
-			hdr->b_buf = buf;
-			ASSERT(hdr->b_datacnt == 0);
-			hdr->b_datacnt = 1;
+			hdr->b_l1hdr.b_buf = buf;
+			ASSERT0(hdr->b_l1hdr.b_datacnt);
+			hdr->b_l1hdr.b_datacnt = 1;
 			arc_get_data_buf(buf);
 			arc_access(hdr, hash_lock);
 		}
 
-		ASSERT(!GHOST_STATE(hdr->b_state));
+		ASSERT(!GHOST_STATE(hdr->b_l1hdr.b_state));
 
 		acb = kmem_zalloc(sizeof (arc_callback_t), KM_SLEEP);
 		acb->acb_done = done;
 		acb->acb_private = private;
 
-		ASSERT(hdr->b_acb == NULL);
-		hdr->b_acb = acb;
+		ASSERT(hdr->b_l1hdr.b_acb == NULL);
+		hdr->b_l1hdr.b_acb = acb;
 		hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS;
 
-		if (hdr->b_l2hdr != NULL &&
-		    (vd = hdr->b_l2hdr->b_dev->l2ad_vdev) != NULL) {
-			devw = hdr->b_l2hdr->b_dev->l2ad_writing;
-			addr = hdr->b_l2hdr->b_daddr;
-			b_compress = hdr->b_l2hdr->b_compress;
-			b_asize = hdr->b_l2hdr->b_asize;
+		if (HDR_HAS_L2HDR(hdr) &&
+		    (vd = hdr->b_l2hdr.b_dev->l2ad_vdev) != NULL) {
+			devw = hdr->b_l2hdr.b_dev->l2ad_writing;
+			addr = hdr->b_l2hdr.b_daddr;
+			b_compress = HDR_GET_COMPRESS(hdr);
+			b_asize = hdr->b_l2hdr.b_asize;
 			/*
 			 * Lock out device removal.
 			 */
@@ -3182,8 +3470,8 @@ top:
 		DTRACE_PROBE4(arc__miss, arc_buf_hdr_t *, hdr, blkptr_t *, bp,
 		    uint64_t, size, zbookmark_phys_t *, zb);
 		ARCSTAT_BUMP(arcstat_misses);
-		ARCSTAT_CONDSTAT(!(hdr->b_flags & ARC_FLAG_PREFETCH),
-		    demand, prefetch, hdr->b_type != ARC_BUFC_METADATA,
+		ARCSTAT_CONDSTAT(!HDR_PREFETCH(hdr),
+		    demand, prefetch, !HDR_ISTYPE_METADATA(hdr),
 		    data, metadata, misses);
 
 		if (vd != NULL && l2arc_ndev != 0 && !(l2arc_norw && devw)) {
@@ -3196,7 +3484,7 @@ top:
 			 *    also have invalidated the vdev.
 			 * 5. This isn't prefetch and l2arc_noprefetch is set.
 			 */
-			if (hdr->b_l2hdr != NULL &&
+			if (HDR_HAS_L2HDR(hdr) &&
 			    !HDR_L2_WRITING(hdr) && !HDR_L2_EVICTED(hdr) &&
 			    !(l2arc_noprefetch && HDR_PREFETCH(hdr))) {
 				l2arc_read_callback_t *cb;
@@ -3288,8 +3576,9 @@ void
 arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private)
 {
 	ASSERT(buf->b_hdr != NULL);
-	ASSERT(buf->b_hdr->b_state != arc_anon);
-	ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt) || func == NULL);
+	ASSERT(buf->b_hdr->b_l1hdr.b_state != arc_anon);
+	ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt) ||
+	    func == NULL);
 	ASSERT(buf->b_efunc == NULL);
 	ASSERT(!HDR_BUF_AVAILABLE(buf->b_hdr));
 
@@ -3313,7 +3602,7 @@ arc_freed(spa_t *spa, const blkptr_t *bp)
 	if (hdr == NULL)
 		return;
 	if (HDR_BUF_AVAILABLE(hdr)) {
-		arc_buf_t *buf = hdr->b_buf;
+		arc_buf_t *buf = hdr->b_l1hdr.b_buf;
 		add_reference(hdr, hash_lock, FTAG);
 		hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
 		mutex_exit(hash_lock);
@@ -3371,17 +3660,19 @@ arc_clear_callback(arc_buf_t *buf)
 	hdr = buf->b_hdr;
 	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
 
-	ASSERT3U(refcount_count(&hdr->b_refcnt), <, hdr->b_datacnt);
-	ASSERT(hdr->b_state == arc_mru || hdr->b_state == arc_mfu);
+	ASSERT3U(refcount_count(&hdr->b_l1hdr.b_refcnt), <,
+	    hdr->b_l1hdr.b_datacnt);
+	ASSERT(hdr->b_l1hdr.b_state == arc_mru ||
+	    hdr->b_l1hdr.b_state == arc_mfu);
 
 	buf->b_efunc = NULL;
 	buf->b_private = NULL;
 
-	if (hdr->b_datacnt > 1) {
+	if (hdr->b_l1hdr.b_datacnt > 1) {
 		mutex_exit(&buf->b_evict_lock);
 		arc_buf_destroy(buf, FALSE, TRUE);
 	} else {
-		ASSERT(buf == hdr->b_buf);
+		ASSERT(buf == hdr->b_l1hdr.b_buf);
 		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
 		mutex_exit(&buf->b_evict_lock);
 	}
@@ -3400,10 +3691,7 @@ arc_clear_callback(arc_buf_t *buf)
 void
 arc_release(arc_buf_t *buf, void *tag)
 {
-	arc_buf_hdr_t *hdr;
-	kmutex_t *hash_lock = NULL;
-	l2arc_buf_hdr_t *l2hdr;
-	uint64_t buf_size;
+	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	/*
 	 * It would be nice to assert that if it's DMU metadata (level >
@@ -3412,56 +3700,87 @@ arc_release(arc_buf_t *buf, void *tag)
 	 */
 
 	mutex_enter(&buf->b_evict_lock);
-	hdr = buf->b_hdr;
+
+	/*
+	 * We don't grab the hash lock prior to this check, because if
+	 * the buffer's header is in the arc_anon state, it won't be
+	 * linked into the hash table.
+	 */
+	if (hdr->b_l1hdr.b_state == arc_anon) {
+		mutex_exit(&buf->b_evict_lock);
+		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
+		ASSERT(!HDR_IN_HASH_TABLE(hdr));
+		ASSERT(!HDR_HAS_L2HDR(hdr));
+		ASSERT(BUF_EMPTY(hdr));
+
+		ASSERT3U(hdr->b_l1hdr.b_datacnt, ==, 1);
+		ASSERT3S(refcount_count(&hdr->b_l1hdr.b_refcnt), ==, 1);
+		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
+
+		ASSERT3P(buf->b_efunc, ==, NULL);
+		ASSERT3P(buf->b_private, ==, NULL);
+
+		hdr->b_l1hdr.b_arc_access = 0;
+		arc_buf_thaw(buf);
+
+		return;
+	}
+
+	kmutex_t *hash_lock = HDR_LOCK(hdr);
+	mutex_enter(hash_lock);
+
+	/*
+	 * This assignment is only valid as long as the hash_lock is
+	 * held, we must be careful not to reference state or the
+	 * b_state field after dropping the lock.
+	 */
+	arc_state_t *state = hdr->b_l1hdr.b_state;
+	ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
+	ASSERT3P(state, !=, arc_anon);
 
 	/* this buffer is not on any list */
-	ASSERT(refcount_count(&hdr->b_refcnt) > 0);
+	ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) > 0);
 
-	if (hdr->b_state == arc_anon) {
-		/* this buffer is already released */
-		ASSERT(buf->b_efunc == NULL);
-	} else {
-		hash_lock = HDR_LOCK(hdr);
-		mutex_enter(hash_lock);
-		hdr = buf->b_hdr;
-		ASSERT3P(hash_lock, ==, HDR_LOCK(hdr));
-	}
+	if (HDR_HAS_L2HDR(hdr)) {
+		ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
+		ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
 
-	l2hdr = hdr->b_l2hdr;
-	if (l2hdr) {
-		mutex_enter(&l2arc_buflist_mtx);
-		hdr->b_l2hdr = NULL;
-		list_remove(l2hdr->b_dev->l2ad_buflist, hdr);
+		mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
+		list_remove(&hdr->b_l2hdr.b_dev->l2ad_buflist, hdr);
+		mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
+
+		hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
 	}
-	buf_size = hdr->b_size;
 
 	/*
 	 * Do we have more than one buf?
 	 */
-	if (hdr->b_datacnt > 1) {
+	if (hdr->b_l1hdr.b_datacnt > 1) {
 		arc_buf_hdr_t *nhdr;
 		arc_buf_t **bufp;
 		uint64_t blksz = hdr->b_size;
 		uint64_t spa = hdr->b_spa;
-		arc_buf_contents_t type = hdr->b_type;
+		arc_buf_contents_t type = arc_buf_type(hdr);
 		uint32_t flags = hdr->b_flags;
 
-		ASSERT(hdr->b_buf != buf || buf->b_next != NULL);
+		ASSERT(hdr->b_l1hdr.b_buf != buf || buf->b_next != NULL);
 		/*
 		 * Pull the data off of this hdr and attach it to
 		 * a new anonymous hdr.
 		 */
 		(void) remove_reference(hdr, hash_lock, tag);
-		bufp = &hdr->b_buf;
+		bufp = &hdr->b_l1hdr.b_buf;
 		while (*bufp != buf)
 			bufp = &(*bufp)->b_next;
 		*bufp = buf->b_next;
 		buf->b_next = NULL;
 
-		ASSERT3U(hdr->b_state->arcs_size, >=, hdr->b_size);
-		atomic_add_64(&hdr->b_state->arcs_size, -hdr->b_size);
-		if (refcount_is_zero(&hdr->b_refcnt)) {
-			uint64_t *size = &hdr->b_state->arcs_lsize[hdr->b_type];
+		ASSERT3P(state, !=, arc_l2c_only);
+		ASSERT3U(state->arcs_size, >=, hdr->b_size);
+		atomic_add_64(&state->arcs_size, -hdr->b_size);
+		if (refcount_is_zero(&hdr->b_l1hdr.b_refcnt)) {
+			ASSERT3P(state, !=, arc_l2c_only);
+			uint64_t *size = &state->arcs_lsize[type];
 			ASSERT3U(*size, >=, hdr->b_size);
 			atomic_add_64(size, -hdr->b_size);
 		}
@@ -3470,57 +3789,50 @@ arc_release(arc_buf_t *buf, void *tag)
 		 * We're releasing a duplicate user data buffer, update
 		 * our statistics accordingly.
 		 */
-		if (hdr->b_type == ARC_BUFC_DATA) {
+		if (HDR_ISTYPE_DATA(hdr)) {
 			ARCSTAT_BUMPDOWN(arcstat_duplicate_buffers);
 			ARCSTAT_INCR(arcstat_duplicate_buffers_size,
 			    -hdr->b_size);
 		}
-		hdr->b_datacnt -= 1;
+		hdr->b_l1hdr.b_datacnt -= 1;
 		arc_cksum_verify(buf);
 		arc_buf_unwatch(buf);
 
 		mutex_exit(hash_lock);
 
-		nhdr = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
+		nhdr = kmem_cache_alloc(hdr_full_cache, KM_PUSHPAGE);
 		nhdr->b_size = blksz;
 		nhdr->b_spa = spa;
-		nhdr->b_type = type;
-		nhdr->b_buf = buf;
-		nhdr->b_state = arc_anon;
-		nhdr->b_arc_access = 0;
+
 		nhdr->b_flags = flags & ARC_FLAG_L2_WRITING;
-		nhdr->b_l2hdr = NULL;
-		nhdr->b_datacnt = 1;
+		nhdr->b_flags |= arc_bufc_to_flags(type);
+		nhdr->b_flags |= ARC_FLAG_HAS_L1HDR;
+
+		nhdr->b_l1hdr.b_buf = buf;
+		nhdr->b_l1hdr.b_datacnt = 1;
+		nhdr->b_l1hdr.b_state = arc_anon;
+		nhdr->b_l1hdr.b_arc_access = 0;
 		nhdr->b_freeze_cksum = NULL;
-		(void) refcount_add(&nhdr->b_refcnt, tag);
+
+		(void) refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
 		buf->b_hdr = nhdr;
 		mutex_exit(&buf->b_evict_lock);
 		atomic_add_64(&arc_anon->arcs_size, blksz);
 	} else {
 		mutex_exit(&buf->b_evict_lock);
-		ASSERT(refcount_count(&hdr->b_refcnt) == 1);
-		ASSERT(!list_link_active(&hdr->b_arc_node));
+		ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
+		/* protected by hash lock */
+		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
-		if (hdr->b_state != arc_anon)
-			arc_change_state(arc_anon, hdr, hash_lock);
-		hdr->b_arc_access = 0;
-		if (hash_lock)
-			mutex_exit(hash_lock);
+		arc_change_state(arc_anon, hdr, hash_lock);
+		hdr->b_l1hdr.b_arc_access = 0;
+		mutex_exit(hash_lock);
 
 		buf_discard_identity(hdr);
 		arc_buf_thaw(buf);
 	}
 	buf->b_efunc = NULL;
 	buf->b_private = NULL;
-
-	if (l2hdr) {
-		ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
-		vdev_space_update(l2hdr->b_dev->l2ad_vdev,
-		    -l2hdr->b_asize, 0, 0);
-		kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
-		ARCSTAT_INCR(arcstat_l2_size, -buf_size);
-		mutex_exit(&l2arc_buflist_mtx);
-	}
 }
 
 int
@@ -3529,7 +3841,8 @@ arc_released(arc_buf_t *buf)
 	int released;
 
 	mutex_enter(&buf->b_evict_lock);
-	released = (buf->b_data != NULL && buf->b_hdr->b_state == arc_anon);
+	released = (buf->b_data != NULL &&
+	    buf->b_hdr->b_l1hdr.b_state == arc_anon);
 	mutex_exit(&buf->b_evict_lock);
 	return (released);
 }
@@ -3541,7 +3854,7 @@ arc_referenced(arc_buf_t *buf)
 	int referenced;
 
 	mutex_enter(&buf->b_evict_lock);
-	referenced = (refcount_count(&buf->b_hdr->b_refcnt));
+	referenced = (refcount_count(&buf->b_hdr->b_l1hdr.b_refcnt));
 	mutex_exit(&buf->b_evict_lock);
 	return (referenced);
 }
@@ -3554,7 +3867,9 @@ arc_write_ready(zio_t *zio)
 	arc_buf_t *buf = callback->awcb_buf;
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
-	ASSERT(!refcount_is_zero(&buf->b_hdr->b_refcnt));
+	ASSERT(HDR_HAS_L1HDR(hdr));
+	ASSERT(!refcount_is_zero(&buf->b_hdr->b_l1hdr.b_refcnt));
+	ASSERT(hdr->b_l1hdr.b_datacnt > 0);
 	callback->awcb_ready(zio, buf, callback->awcb_private);
 
 	/*
@@ -3564,12 +3879,12 @@ arc_write_ready(zio_t *zio)
 	 * accounting for any re-write attempt.
 	 */
 	if (HDR_IO_IN_PROGRESS(hdr)) {
-		mutex_enter(&hdr->b_freeze_lock);
+		mutex_enter(&hdr->b_l1hdr.b_freeze_lock);
 		if (hdr->b_freeze_cksum != NULL) {
 			kmem_free(hdr->b_freeze_cksum, sizeof (zio_cksum_t));
 			hdr->b_freeze_cksum = NULL;
 		}
-		mutex_exit(&hdr->b_freeze_lock);
+		mutex_exit(&hdr->b_l1hdr.b_freeze_lock);
 	}
 	arc_cksum_compute(buf, B_FALSE);
 	hdr->b_flags |= ARC_FLAG_IO_IN_PROGRESS;
@@ -3594,7 +3909,7 @@ arc_write_done(zio_t *zio)
 	arc_buf_t *buf = callback->awcb_buf;
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
-	ASSERT(hdr->b_acb == NULL);
+	ASSERT(hdr->b_l1hdr.b_acb == NULL);
 
 	if (zio->io_error == 0) {
 		if (BP_IS_HOLE(zio->io_bp) || BP_IS_EMBEDDED(zio->io_bp)) {
@@ -3602,7 +3917,6 @@ arc_write_done(zio_t *zio)
 		} else {
 			hdr->b_dva = *BP_IDENTITY(zio->io_bp);
 			hdr->b_birth = BP_PHYSICAL_BIRTH(zio->io_bp);
-			hdr->b_cksum0 = zio->io_bp->blk_cksum.zc_word[0];
 		}
 	} else {
 		ASSERT(BUF_EMPTY(hdr));
@@ -3623,7 +3937,7 @@ arc_write_done(zio_t *zio)
 		arc_cksum_verify(buf);
 
 		exists = buf_hash_insert(hdr, &hash_lock);
-		if (exists) {
+		if (exists != NULL) {
 			/*
 			 * This can only happen if we overwrite for
 			 * sync-to-convergence, because we remove
@@ -3633,7 +3947,8 @@ arc_write_done(zio_t *zio)
 				if (!BP_EQUAL(&zio->io_bp_orig, zio->io_bp))
 					panic("bad overwrite, hdr=%p exists=%p",
 					    (void *)hdr, (void *)exists);
-				ASSERT(refcount_is_zero(&exists->b_refcnt));
+				ASSERT(refcount_is_zero(
+				    &exists->b_l1hdr.b_refcnt));
 				arc_change_state(arc_anon, exists, hash_lock);
 				mutex_exit(hash_lock);
 				arc_hdr_destroy(exists);
@@ -3647,22 +3962,22 @@ arc_write_done(zio_t *zio)
 					    (void *)hdr, (void *)exists);
 			} else {
 				/* Dedup */
-				ASSERT(hdr->b_datacnt == 1);
-				ASSERT(hdr->b_state == arc_anon);
+				ASSERT(hdr->b_l1hdr.b_datacnt == 1);
+				ASSERT(hdr->b_l1hdr.b_state == arc_anon);
 				ASSERT(BP_GET_DEDUP(zio->io_bp));
 				ASSERT(BP_GET_LEVEL(zio->io_bp) == 0);
 			}
 		}
 		hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
 		/* if it's not anon, we are doing a scrub */
-		if (!exists && hdr->b_state == arc_anon)
+		if (exists == NULL && hdr->b_l1hdr.b_state == arc_anon)
 			arc_access(hdr, hash_lock);
 		mutex_exit(hash_lock);
 	} else {
 		hdr->b_flags &= ~ARC_FLAG_IO_IN_PROGRESS;
 	}
 
-	ASSERT(!refcount_is_zero(&hdr->b_refcnt));
+	ASSERT(!refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 	callback->awcb_done(zio, buf, callback->awcb_private);
 
 	kmem_free(callback, sizeof (arc_write_callback_t));
@@ -3682,8 +3997,9 @@ arc_write(zio_t *pio, spa_t *spa, uint64_t txg,
 	ASSERT(ready != NULL);
 	ASSERT(done != NULL);
 	ASSERT(!HDR_IO_ERROR(hdr));
-	ASSERT((hdr->b_flags & ARC_FLAG_IO_IN_PROGRESS) == 0);
-	ASSERT(hdr->b_acb == NULL);
+	ASSERT(!HDR_IO_IN_PROGRESS(hdr));
+	ASSERT(hdr->b_l1hdr.b_acb == NULL);
+	ASSERT(hdr->b_l1hdr.b_datacnt > 0);
 	if (l2arc)
 		hdr->b_flags |= ARC_FLAG_L2CACHE;
 	if (l2arc_compress)
@@ -3903,25 +4219,35 @@ arc_init(void)
 	mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
 
 	list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 	list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
-	    sizeof (arc_buf_hdr_t), offsetof(arc_buf_hdr_t, b_arc_node));
+	    sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
 
 	buf_init();
 
@@ -4000,7 +4326,7 @@ arc_fini(void)
 
 	buf_fini();
 
-	ASSERT(arc_loaned_bytes == 0);
+	ASSERT0(arc_loaned_bytes);
 }
 
 /*
@@ -4158,7 +4484,7 @@ l2arc_write_eligible(uint64_t spa_guid, arc_buf_hdr_t *hdr)
 	 * 3. has an I/O in progress (it may be an incomplete read).
 	 * 4. is flagged not eligible (zfs property).
 	 */
-	if (hdr->b_spa != spa_guid || hdr->b_l2hdr != NULL ||
+	if (hdr->b_spa != spa_guid || HDR_HAS_L2HDR(hdr) ||
 	    HDR_IO_IN_PROGRESS(hdr) || !HDR_L2CACHE(hdr))
 		return (B_FALSE);
 
@@ -4211,20 +4537,6 @@ l2arc_write_interval(clock_t began, uint64_t wanted, uint64_t wrote)
 	return (next);
 }
 
-static void
-l2arc_hdr_stat_add(void)
-{
-	ARCSTAT_INCR(arcstat_l2_hdr_size, HDR_SIZE + L2HDR_SIZE);
-	ARCSTAT_INCR(arcstat_hdr_size, -HDR_SIZE);
-}
-
-static void
-l2arc_hdr_stat_remove(void)
-{
-	ARCSTAT_INCR(arcstat_l2_hdr_size, -(HDR_SIZE + L2HDR_SIZE));
-	ARCSTAT_INCR(arcstat_hdr_size, HDR_SIZE);
-}
-
 /*
  * Cycle through L2ARC devices.  This is how L2ARC load balances.
  * If a device is returned, this also returns holding the spa config lock.
@@ -4321,7 +4633,6 @@ l2arc_write_done(zio_t *zio)
 	l2arc_dev_t *dev;
 	list_t *buflist;
 	arc_buf_hdr_t *head, *hdr, *hdr_prev;
-	l2arc_buf_hdr_t *abl2;
 	kmutex_t *hash_lock;
 	int64_t bytes_dropped = 0;
 
@@ -4331,7 +4642,7 @@ l2arc_write_done(zio_t *zio)
 	ASSERT(dev != NULL);
 	head = cb->l2wcb_head;
 	ASSERT(head != NULL);
-	buflist = dev->l2ad_buflist;
+	buflist = &dev->l2ad_buflist;
 	ASSERT(buflist != NULL);
 	DTRACE_PROBE2(l2arc__iodone, zio_t *, zio,
 	    l2arc_write_callback_t *, cb);
@@ -4339,41 +4650,43 @@ l2arc_write_done(zio_t *zio)
 	if (zio->io_error != 0)
 		ARCSTAT_BUMP(arcstat_l2_writes_error);
 
-	mutex_enter(&l2arc_buflist_mtx);
+	mutex_enter(&dev->l2ad_mtx);
 
 	/*
 	 * All writes completed, or an error was hit.
 	 */
 	for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
 		hdr_prev = list_prev(buflist, hdr);
-		abl2 = hdr->b_l2hdr;
-
-		/*
-		 * Release the temporary compressed buffer as soon as possible.
-		 */
-		if (abl2->b_compress != ZIO_COMPRESS_OFF)
-			l2arc_release_cdata_buf(hdr);
 
 		hash_lock = HDR_LOCK(hdr);
 		if (!mutex_tryenter(hash_lock)) {
 			/*
 			 * This buffer misses out.  It may be in a stage
-			 * of eviction.  Its ARC_L2_WRITING flag will be
+			 * of eviction.  Its ARC_FLAG_L2_WRITING flag will be
 			 * left set, denying reads to this buffer.
 			 */
 			ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss);
 			continue;
 		}
 
+		/*
+		 * It's possible that this buffer got evicted from the L1 cache
+		 * before we grabbed the vdev + hash locks, in which case
+		 * arc_hdr_realloc freed b_tmp_cdata for us if it was allocated.
+		 * Only free the buffer if we still have an L1 hdr.
+		 */
+		if (HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_tmp_cdata != NULL &&
+		    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF)
+			l2arc_release_cdata_buf(hdr);
+
 		if (zio->io_error != 0) {
 			/*
 			 * Error - drop L2ARC entry.
 			 */
 			list_remove(buflist, hdr);
-			ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
-			bytes_dropped += abl2->b_asize;
-			hdr->b_l2hdr = NULL;
-			kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
+			hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
+
+			ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
 			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
 		}
 
@@ -4387,8 +4700,9 @@ l2arc_write_done(zio_t *zio)
 
 	atomic_inc_64(&l2arc_writes_done);
 	list_remove(buflist, head);
-	kmem_cache_free(hdr_cache, head);
-	mutex_exit(&l2arc_buflist_mtx);
+	ASSERT(!HDR_HAS_L1HDR(head));
+	kmem_cache_free(hdr_l2only_cache, head);
+	mutex_exit(&dev->l2ad_mtx);
 
 	vdev_space_update(dev->l2ad_vdev, -bytes_dropped, 0, 0);
 
@@ -4526,16 +4840,12 @@ static void
 l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
 {
 	list_t *buflist;
-	l2arc_buf_hdr_t *abl2;
 	arc_buf_hdr_t *hdr, *hdr_prev;
 	kmutex_t *hash_lock;
 	uint64_t taddr;
 	int64_t bytes_evicted = 0;
 
-	buflist = dev->l2ad_buflist;
-
-	if (buflist == NULL)
-		return;
+	buflist = &dev->l2ad_buflist;
 
 	if (!all && dev->l2ad_first) {
 		/*
@@ -4558,7 +4868,7 @@ l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
 	    uint64_t, taddr, boolean_t, all);
 
 top:
-	mutex_enter(&l2arc_buflist_mtx);
+	mutex_enter(&dev->l2ad_mtx);
 	for (hdr = list_tail(buflist); hdr; hdr = hdr_prev) {
 		hdr_prev = list_prev(buflist, hdr);
 
@@ -4568,7 +4878,7 @@ top:
 			 * Missed the hash lock.  Retry.
 			 */
 			ARCSTAT_BUMP(arcstat_l2_evict_lock_retry);
-			mutex_exit(&l2arc_buflist_mtx);
+			mutex_exit(&dev->l2ad_mtx);
 			mutex_enter(hash_lock);
 			mutex_exit(hash_lock);
 			goto top;
@@ -4584,9 +4894,9 @@ top:
 			continue;
 		}
 
-		if (!all && hdr->b_l2hdr != NULL &&
-		    (hdr->b_l2hdr->b_daddr > taddr ||
-		    hdr->b_l2hdr->b_daddr < dev->l2ad_hand)) {
+		if (!all && HDR_HAS_L2HDR(hdr) &&
+		    (hdr->b_l2hdr.b_daddr > taddr ||
+		    hdr->b_l2hdr.b_daddr < dev->l2ad_hand)) {
 			/*
 			 * We've evicted to the target address,
 			 * or the end of the device.
@@ -4595,15 +4905,8 @@ top:
 			break;
 		}
 
-		if (HDR_FREE_IN_PROGRESS(hdr)) {
-			/*
-			 * Already on the path to destruction.
-			 */
-			mutex_exit(hash_lock);
-			continue;
-		}
-
-		if (hdr->b_state == arc_l2c_only) {
+		ASSERT(HDR_HAS_L2HDR(hdr));
+		if (!HDR_HAS_L1HDR(hdr)) {
 			ASSERT(!HDR_L2_READING(hdr));
 			/*
 			 * This doesn't exist in the ARC.  Destroy.
@@ -4613,6 +4916,8 @@ top:
 			arc_change_state(arc_anon, hdr, hash_lock);
 			arc_hdr_destroy(hdr);
 		} else {
+			ASSERT(hdr->b_l1hdr.b_state != arc_l2c_only);
+			ARCSTAT_BUMP(arcstat_l2_evict_l1cached);
 			/*
 			 * Invalidate issued or about to be issued
 			 * reads, since we may be about to write
@@ -4623,28 +4928,18 @@ top:
 				hdr->b_flags |= ARC_FLAG_L2_EVICTED;
 			}
 
-			/*
-			 * Tell ARC this no longer exists in L2ARC.
-			 */
-			if (hdr->b_l2hdr != NULL) {
-				abl2 = hdr->b_l2hdr;
-				ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
-				bytes_evicted += abl2->b_asize;
-				hdr->b_l2hdr = NULL;
-				kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
-				ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
-			}
+			/* Tell ARC this no longer exists in L2ARC. */
+			ARCSTAT_INCR(arcstat_l2_asize, -hdr->b_l2hdr.b_asize);
+			ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
+			hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
 			list_remove(buflist, hdr);
 
-			/*
-			 * This may have been leftover after a
-			 * failed write.
-			 */
+			/* This may have been leftover after a failed write. */
 			hdr->b_flags &= ~ARC_FLAG_L2_WRITING;
 		}
 		mutex_exit(hash_lock);
 	}
-	mutex_exit(&l2arc_buflist_mtx);
+	mutex_exit(&dev->l2ad_mtx);
 
 	vdev_space_update(dev->l2ad_vdev, -bytes_evicted, 0, 0);
 	dev->l2ad_evict = taddr;
@@ -4685,8 +4980,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	pio = NULL;
 	write_sz = write_asize = write_psize = 0;
 	full = B_FALSE;
-	head = kmem_cache_alloc(hdr_cache, KM_PUSHPAGE);
+	head = kmem_cache_alloc(hdr_l2only_cache, KM_PUSHPAGE);
 	head->b_flags |= ARC_FLAG_L2_WRITE_HEAD;
+	head->b_flags |= ARC_FLAG_HAS_L2HDR;
 
 	/*
 	 * We will want to try to compress buffers that are at least 2x the
@@ -4697,7 +4993,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	/*
 	 * Copy buffers for L2ARC writing.
 	 */
-	mutex_enter(&l2arc_buflist_mtx);
+	mutex_enter(&dev->l2ad_mtx);
 	for (int try = 0; try <= 3; try++) {
 		uint64_t passed_sz = 0;
 
@@ -4719,7 +5015,6 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			headroom = (headroom * l2arc_headroom_boost) / 100;
 
 		for (; hdr; hdr = hdr_prev) {
-			l2arc_buf_hdr_t *l2hdr;
 			kmutex_t *hash_lock;
 			uint64_t buf_sz;
 
@@ -4762,7 +5057,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 				 * l2arc_write_done() can find where the
 				 * write buffers begin without searching.
 				 */
-				list_insert_head(dev->l2ad_buflist, head);
+				list_insert_head(&dev->l2ad_buflist, head);
 
 				cb = kmem_alloc(
 				    sizeof (l2arc_write_callback_t), KM_SLEEP);
@@ -4775,33 +5070,31 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			/*
 			 * Create and add a new L2ARC header.
 			 */
-			l2hdr = kmem_zalloc(sizeof (l2arc_buf_hdr_t), KM_SLEEP);
-			l2hdr->b_dev = dev;
+			hdr->b_l2hdr.b_dev = dev;
 			hdr->b_flags |= ARC_FLAG_L2_WRITING;
-
 			/*
 			 * Temporarily stash the data buffer in b_tmp_cdata.
 			 * The subsequent write step will pick it up from
-			 * there. This is because can't access hdr->b_buf
+			 * there. This is because can't access b_l1hdr.b_buf
 			 * without holding the hash_lock, which we in turn
 			 * can't access without holding the ARC list locks
 			 * (which we want to avoid during compression/writing).
 			 */
-			l2hdr->b_compress = ZIO_COMPRESS_OFF;
-			l2hdr->b_asize = hdr->b_size;
-			l2hdr->b_tmp_cdata = hdr->b_buf->b_data;
+			HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_OFF);
+			hdr->b_l2hdr.b_asize = hdr->b_size;
+			hdr->b_l1hdr.b_tmp_cdata = hdr->b_l1hdr.b_buf->b_data;
 
 			buf_sz = hdr->b_size;
-			hdr->b_l2hdr = l2hdr;
+			hdr->b_flags |= ARC_FLAG_HAS_L2HDR;
 
-			list_insert_head(dev->l2ad_buflist, hdr);
+			list_insert_head(&dev->l2ad_buflist, hdr);
 
 			/*
 			 * Compute and store the buffer cksum before
 			 * writing.  On debug the cksum is verified first.
 			 */
-			arc_cksum_verify(hdr->b_buf);
-			arc_cksum_compute(hdr->b_buf, B_TRUE);
+			arc_cksum_verify(hdr->b_l1hdr.b_buf);
+			arc_cksum_compute(hdr->b_l1hdr.b_buf, B_TRUE);
 
 			mutex_exit(hash_lock);
 
@@ -4817,8 +5110,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	/* No buffers selected for writing? */
 	if (pio == NULL) {
 		ASSERT0(write_sz);
-		mutex_exit(&l2arc_buflist_mtx);
-		kmem_cache_free(hdr_cache, head);
+		mutex_exit(&dev->l2ad_mtx);
+		ASSERT(!HDR_HAS_L1HDR(head));
+		kmem_cache_free(hdr_l2only_cache, head);
 		return (0);
 	}
 
@@ -4827,24 +5121,22 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	 * and work backwards, retracing the course of the buffer selector
 	 * loop above.
 	 */
-	for (hdr = list_prev(dev->l2ad_buflist, head); hdr;
-	    hdr = list_prev(dev->l2ad_buflist, hdr)) {
-		l2arc_buf_hdr_t *l2hdr;
+	for (hdr = list_prev(&dev->l2ad_buflist, head); hdr;
+	    hdr = list_prev(&dev->l2ad_buflist, hdr)) {
 		uint64_t buf_sz;
 
 		/*
 		 * We shouldn't need to lock the buffer here, since we flagged
 		 * it as ARC_FLAG_L2_WRITING in the previous step, but we must
 		 * take care to only access its L2 cache parameters. In
-		 * particular, hdr->b_buf may be invalid by now due to
+		 * particular, hdr->l1hdr.b_buf may be invalid by now due to
 		 * ARC eviction.
 		 */
-		l2hdr = hdr->b_l2hdr;
-		l2hdr->b_daddr = dev->l2ad_hand;
+		hdr->b_l2hdr.b_daddr = dev->l2ad_hand;
 
-		if ((hdr->b_flags & ARC_FLAG_L2COMPRESS) &&
-		    l2hdr->b_asize >= buf_compress_minsz) {
-			if (l2arc_compress_buf(l2hdr)) {
+		if ((HDR_L2COMPRESS(hdr)) &&
+		    hdr->b_l2hdr.b_asize >= buf_compress_minsz) {
+			if (l2arc_compress_buf(hdr)) {
 				/*
 				 * If compression succeeded, enable headroom
 				 * boost on the next scan cycle.
@@ -4857,8 +5149,8 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 		 * Pick up the buffer data we had previously stashed away
 		 * (and now potentially also compressed).
 		 */
-		buf_data = l2hdr->b_tmp_cdata;
-		buf_sz = l2hdr->b_asize;
+		buf_data = hdr->b_l1hdr.b_tmp_cdata;
+		buf_sz = hdr->b_l2hdr.b_asize;
 
 		/* Compression may have squashed the buffer to zero length. */
 		if (buf_sz != 0) {
@@ -4883,7 +5175,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 		}
 	}
 
-	mutex_exit(&l2arc_buflist_mtx);
+	mutex_exit(&dev->l2ad_mtx);
 
 	ASSERT3U(write_asize, <=, target_sz);
 	ARCSTAT_BUMP(arcstat_l2_writes_sent);
@@ -4911,7 +5203,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 
 /*
  * Compresses an L2ARC buffer.
- * The data to be compressed must be prefilled in l2hdr->b_tmp_cdata and its
+ * The data to be compressed must be prefilled in l1hdr.b_tmp_cdata and its
  * size in l2hdr->b_asize. This routine tries to compress the data and
  * depending on the compression result there are three possible outcomes:
  * *) The buffer was incompressible. The original l2hdr contents were left
@@ -4929,17 +5221,21 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
  * buffer was incompressible).
  */
 static boolean_t
-l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr)
+l2arc_compress_buf(arc_buf_hdr_t *hdr)
 {
 	void *cdata;
 	size_t csize, len, rounded;
+	ASSERT(HDR_HAS_L2HDR(hdr));
+	l2arc_buf_hdr_t *l2hdr = &hdr->b_l2hdr;
 
-	ASSERT(l2hdr->b_compress == ZIO_COMPRESS_OFF);
-	ASSERT(l2hdr->b_tmp_cdata != NULL);
+	ASSERT(HDR_HAS_L1HDR(hdr));
+	ASSERT(HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF);
+	ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
 
 	len = l2hdr->b_asize;
 	cdata = zio_data_buf_alloc(len);
-	csize = zio_compress_data(ZIO_COMPRESS_LZ4, l2hdr->b_tmp_cdata,
+	ASSERT3P(cdata, !=, NULL);
+	csize = zio_compress_data(ZIO_COMPRESS_LZ4, hdr->b_l1hdr.b_tmp_cdata,
 	    cdata, l2hdr->b_asize);
 
 	rounded = P2ROUNDUP(csize, (size_t)SPA_MINBLOCKSIZE);
@@ -4951,9 +5247,9 @@ l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr)
 	if (csize == 0) {
 		/* zero block, indicate that there's nothing to write */
 		zio_data_buf_free(cdata, len);
-		l2hdr->b_compress = ZIO_COMPRESS_EMPTY;
+		HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_EMPTY);
 		l2hdr->b_asize = 0;
-		l2hdr->b_tmp_cdata = NULL;
+		hdr->b_l1hdr.b_tmp_cdata = NULL;
 		ARCSTAT_BUMP(arcstat_l2_compress_zeros);
 		return (B_TRUE);
 	} else if (csize > 0 && csize < len) {
@@ -4961,9 +5257,9 @@ l2arc_compress_buf(l2arc_buf_hdr_t *l2hdr)
 		 * Compression succeeded, we'll keep the cdata around for
 		 * writing and release it afterwards.
 		 */
-		l2hdr->b_compress = ZIO_COMPRESS_LZ4;
+		HDR_SET_COMPRESS(hdr, ZIO_COMPRESS_LZ4);
 		l2hdr->b_asize = csize;
-		l2hdr->b_tmp_cdata = cdata;
+		hdr->b_l1hdr.b_tmp_cdata = cdata;
 		ARCSTAT_BUMP(arcstat_l2_compress_successes);
 		return (B_TRUE);
 	} else {
@@ -5008,9 +5304,9 @@ l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
 		 * need to fill its io_data after we're done restoring the
 		 * buffer's contents.
 		 */
-		ASSERT(hdr->b_buf != NULL);
-		bzero(hdr->b_buf->b_data, hdr->b_size);
-		zio->io_data = zio->io_orig_data = hdr->b_buf->b_data;
+		ASSERT(hdr->b_l1hdr.b_buf != NULL);
+		bzero(hdr->b_l1hdr.b_buf->b_data, hdr->b_size);
+		zio->io_data = zio->io_orig_data = hdr->b_l1hdr.b_buf->b_data;
 	} else {
 		ASSERT(zio->io_data != NULL);
 		/*
@@ -5048,17 +5344,17 @@ l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
 static void
 l2arc_release_cdata_buf(arc_buf_hdr_t *hdr)
 {
-	l2arc_buf_hdr_t *l2hdr = hdr->b_l2hdr;
-
-	if (l2hdr->b_compress == ZIO_COMPRESS_LZ4) {
+	ASSERT(HDR_HAS_L1HDR(hdr));
+	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_EMPTY) {
 		/*
 		 * If the data was compressed, then we've allocated a
 		 * temporary buffer for it, so now we need to release it.
 		 */
-		ASSERT(l2hdr->b_tmp_cdata != NULL);
-		zio_data_buf_free(l2hdr->b_tmp_cdata, hdr->b_size);
+		ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
+		zio_data_buf_free(hdr->b_l1hdr.b_tmp_cdata,
+		    hdr->b_size);
 	}
-	l2hdr->b_tmp_cdata = NULL;
+	hdr->b_l1hdr.b_tmp_cdata = NULL;
 }
 
 /*
@@ -5199,13 +5495,13 @@ l2arc_add_vdev(spa_t *spa, vdev_t *vd)
 	adddev->l2ad_first = B_TRUE;
 	adddev->l2ad_writing = B_FALSE;
 
+	mutex_init(&adddev->l2ad_mtx, NULL, MUTEX_DEFAULT, NULL);
 	/*
 	 * This is a list of all ARC buffers that are still valid on the
 	 * device.
 	 */
-	adddev->l2ad_buflist = kmem_zalloc(sizeof (list_t), KM_SLEEP);
-	list_create(adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l2node));
+	list_create(&adddev->l2ad_buflist, sizeof (arc_buf_hdr_t),
+	    offsetof(arc_buf_hdr_t, b_l2hdr.b_l2node));
 
 	vdev_space_update(vd, 0, 0, adddev->l2ad_end - adddev->l2ad_hand);
 
@@ -5251,8 +5547,8 @@ l2arc_remove_vdev(vdev_t *vd)
 	 * Clear all buflists and ARC references.  L2ARC device flush.
 	 */
 	l2arc_evict(remdev, 0, B_TRUE);
-	list_destroy(remdev->l2ad_buflist);
-	kmem_free(remdev->l2ad_buflist, sizeof (list_t));
+	list_destroy(&remdev->l2ad_buflist);
+	mutex_destroy(&remdev->l2ad_mtx);
 	kmem_free(remdev, sizeof (l2arc_dev_t));
 }
 
@@ -5267,7 +5563,6 @@ l2arc_init(void)
 	mutex_init(&l2arc_feed_thr_lock, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&l2arc_feed_thr_cv, NULL, CV_DEFAULT, NULL);
 	mutex_init(&l2arc_dev_mtx, NULL, MUTEX_DEFAULT, NULL);
-	mutex_init(&l2arc_buflist_mtx, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&l2arc_free_on_write_mtx, NULL, MUTEX_DEFAULT, NULL);
 
 	l2arc_dev_list = &L2ARC_dev_list;
@@ -5292,7 +5587,6 @@ l2arc_fini(void)
 	mutex_destroy(&l2arc_feed_thr_lock);
 	cv_destroy(&l2arc_feed_thr_cv);
 	mutex_destroy(&l2arc_dev_mtx);
-	mutex_destroy(&l2arc_buflist_mtx);
 	mutex_destroy(&l2arc_free_on_write_mtx);
 
 	list_destroy(l2arc_dev_list);
diff --git a/uts/common/fs/zfs/sys/arc.h b/uts/common/fs/zfs/sys/arc.h
index 83ea6f7fbc7..4e9e8e202a7 100644
--- a/uts/common/fs/zfs/sys/arc.h
+++ b/uts/common/fs/zfs/sys/arc.h
@@ -70,10 +70,29 @@ typedef enum arc_flags
 	ARC_FLAG_FREED_IN_READ		= 1 << 10,	/* freed during read */
 	ARC_FLAG_BUF_AVAILABLE		= 1 << 11,	/* block not in use */
 	ARC_FLAG_INDIRECT		= 1 << 12,	/* indirect block */
-	ARC_FLAG_FREE_IN_PROGRESS	= 1 << 13,	/*  about to be freed */
-	ARC_FLAG_L2_WRITING		= 1 << 14,	/* write in progress */
-	ARC_FLAG_L2_EVICTED		= 1 << 15,	/* evicted during I/O */
-	ARC_FLAG_L2_WRITE_HEAD		= 1 << 16,	/* head of write list */
+	ARC_FLAG_L2_WRITING		= 1 << 13,	/* write in progress */
+	ARC_FLAG_L2_EVICTED		= 1 << 14,	/* evicted during I/O */
+	ARC_FLAG_L2_WRITE_HEAD		= 1 << 15,	/* head of write list */
+	/* indicates that the buffer contains metadata (otherwise, data) */
+	ARC_FLAG_BUFC_METADATA		= 1 << 16,
+
+	/* Flags specifying whether optional hdr struct fields are defined */
+	ARC_FLAG_HAS_L1HDR		= 1 << 17,
+	ARC_FLAG_HAS_L2HDR		= 1 << 18,
+
+	/*
+	 * The arc buffer's compression mode is stored in the top 7 bits of the
+	 * flags field, so these dummy flags are included so that MDB can
+	 * interpret the enum properly.
+	 */
+	ARC_FLAG_COMPRESS_0		= 1 << 24,
+	ARC_FLAG_COMPRESS_1		= 1 << 25,
+	ARC_FLAG_COMPRESS_2		= 1 << 26,
+	ARC_FLAG_COMPRESS_3		= 1 << 27,
+	ARC_FLAG_COMPRESS_4		= 1 << 28,
+	ARC_FLAG_COMPRESS_5		= 1 << 29,
+	ARC_FLAG_COMPRESS_6		= 1 << 30
+
 } arc_flags_t;
 
 struct arc_buf {

From 8f4cea4c428b6f0e59c03488ff66ee9788de95ec Mon Sep 17 00:00:00 2001
From: Xin LI <delphij@FreeBSD.org>
Date: Tue, 20 Jan 2015 20:10:03 +0000
Subject: [PATCH 111/279] 5445 Add more visibility via arcstats; specifically
 arc_state_t stats and differentiate between "data" and "metadata" Reviewed
 by: Basil Crow <basil.crow@delphix.com> Reviewed by: George Wilson
 <george.wilson@delphix.com> Reviewed by: Matthew Ahrens <mahrens@delphix.com>
 Reviewed by: Bayard Bell <bayard.bell@nexenta.com> Approved by: Robert
 Mustacchi <rm@joyent.com> Author: Prakash Surya <prakash.surya@delphix.com>

illumos/illumos-gate@4076b1bf41cfd9f968a33ed54a7ae76d9e996fe8
---
 uts/common/fs/zfs/arc.c     | 237 +++++++++++++++++++++++++++++++++---
 uts/common/fs/zfs/sys/arc.h |   1 +
 2 files changed, 224 insertions(+), 14 deletions(-)

diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 012dd83cd7c..24198579a89 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -299,9 +299,137 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_c_min;
 	kstat_named_t arcstat_c_max;
 	kstat_named_t arcstat_size;
+	/*
+	 * Number of bytes consumed by internal ARC structures necessary
+	 * for tracking purposes; these structures are not actually
+	 * backed by ARC buffers. This includes arc_buf_hdr_t structures
+	 * (allocated via arc_buf_hdr_t_full and arc_buf_hdr_t_l2only
+	 * caches), and arc_buf_t structures (allocated via arc_buf_t
+	 * cache).
+	 */
 	kstat_named_t arcstat_hdr_size;
+	/*
+	 * Number of bytes consumed by ARC buffers of type equal to
+	 * ARC_BUFC_DATA. This is generally consumed by buffers backing
+	 * on disk user data (e.g. plain file contents).
+	 */
 	kstat_named_t arcstat_data_size;
+	/*
+	 * Number of bytes consumed by ARC buffers of type equal to
+	 * ARC_BUFC_METADATA. This is generally consumed by buffers
+	 * backing on disk data that is used for internal ZFS
+	 * structures (e.g. ZAP, dnode, indirect blocks, etc).
+	 */
+	kstat_named_t arcstat_metadata_size;
+	/*
+	 * Number of bytes consumed by various buffers and structures
+	 * not actually backed with ARC buffers. This includes bonus
+	 * buffers (allocated directly via zio_buf_* functions),
+	 * dmu_buf_impl_t structures (allocated via dmu_buf_impl_t
+	 * cache), and dnode_t structures (allocated via dnode_t cache).
+	 */
 	kstat_named_t arcstat_other_size;
+	/*
+	 * Total number of bytes consumed by ARC buffers residing in the
+	 * arc_anon state. This includes *all* buffers in the arc_anon
+	 * state; e.g. data, metadata, evictable, and unevictable buffers
+	 * are all included in this value.
+	 */
+	kstat_named_t arcstat_anon_size;
+	/*
+	 * Number of bytes consumed by ARC buffers that meet the
+	 * following criteria: backing buffers of type ARC_BUFC_DATA,
+	 * residing in the arc_anon state, and are eligible for eviction
+	 * (e.g. have no outstanding holds on the buffer).
+	 */
+	kstat_named_t arcstat_anon_evictable_data;
+	/*
+	 * Number of bytes consumed by ARC buffers that meet the
+	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
+	 * residing in the arc_anon state, and are eligible for eviction
+	 * (e.g. have no outstanding holds on the buffer).
+	 */
+	kstat_named_t arcstat_anon_evictable_metadata;
+	/*
+	 * Total number of bytes consumed by ARC buffers residing in the
+	 * arc_mru state. This includes *all* buffers in the arc_mru
+	 * state; e.g. data, metadata, evictable, and unevictable buffers
+	 * are all included in this value.
+	 */
+	kstat_named_t arcstat_mru_size;
+	/*
+	 * Number of bytes consumed by ARC buffers that meet the
+	 * following criteria: backing buffers of type ARC_BUFC_DATA,
+	 * residing in the arc_mru state, and are eligible for eviction
+	 * (e.g. have no outstanding holds on the buffer).
+	 */
+	kstat_named_t arcstat_mru_evictable_data;
+	/*
+	 * Number of bytes consumed by ARC buffers that meet the
+	 * following criteria: backing buffers of type ARC_BUFC_METADATA,
+	 * residing in the arc_mru state, and are eligible for eviction
+	 * (e.g. have no outstanding holds on the buffer).
+	 */
+	kstat_named_t arcstat_mru_evictable_metadata;
+	/*
+	 * Total number of bytes that *would have been* consumed by ARC
+	 * buffers in the arc_mru_ghost state. The key thing to note
+	 * here, is the fact that this size doesn't actually indicate
+	 * RAM consumption. The ghost lists only consist of headers and
+	 * don't actually have ARC buffers linked off of these headers.
+	 * Thus, *if* the headers had associated ARC buffers, these
+	 * buffers *would have* consumed this number of bytes.
+	 */
+	kstat_named_t arcstat_mru_ghost_size;
+	/*
+	 * Number of bytes that *would have been* consumed by ARC
+	 * buffers that are eligible for eviction, of type
+	 * ARC_BUFC_DATA, and linked off the arc_mru_ghost state.
+	 */
+	kstat_named_t arcstat_mru_ghost_evictable_data;
+	/*
+	 * Number of bytes that *would have been* consumed by ARC
+	 * buffers that are eligible for eviction, of type
+	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
+	 */
+	kstat_named_t arcstat_mru_ghost_evictable_metadata;
+	/*
+	 * Total number of bytes consumed by ARC buffers residing in the
+	 * arc_mfu state. This includes *all* buffers in the arc_mfu
+	 * state; e.g. data, metadata, evictable, and unevictable buffers
+	 * are all included in this value.
+	 */
+	kstat_named_t arcstat_mfu_size;
+	/*
+	 * Number of bytes consumed by ARC buffers that are eligible for
+	 * eviction, of type ARC_BUFC_DATA, and reside in the arc_mfu
+	 * state.
+	 */
+	kstat_named_t arcstat_mfu_evictable_data;
+	/*
+	 * Number of bytes consumed by ARC buffers that are eligible for
+	 * eviction, of type ARC_BUFC_METADATA, and reside in the
+	 * arc_mfu state.
+	 */
+	kstat_named_t arcstat_mfu_evictable_metadata;
+	/*
+	 * Total number of bytes that *would have been* consumed by ARC
+	 * buffers in the arc_mfu_ghost state. See the comment above
+	 * arcstat_mru_ghost_size for more details.
+	 */
+	kstat_named_t arcstat_mfu_ghost_size;
+	/*
+	 * Number of bytes that *would have been* consumed by ARC
+	 * buffers that are eligible for eviction, of type
+	 * ARC_BUFC_DATA, and linked off the arc_mfu_ghost state.
+	 */
+	kstat_named_t arcstat_mfu_ghost_evictable_data;
+	/*
+	 * Number of bytes that *would have been* consumed by ARC
+	 * buffers that are eligible for eviction, of type
+	 * ARC_BUFC_METADATA, and linked off the arc_mru_ghost state.
+	 */
+	kstat_named_t arcstat_mfu_ghost_evictable_metadata;
 	kstat_named_t arcstat_l2_hits;
 	kstat_named_t arcstat_l2_misses;
 	kstat_named_t arcstat_l2_feeds;
@@ -369,7 +497,23 @@ static arc_stats_t arc_stats = {
 	{ "size",			KSTAT_DATA_UINT64 },
 	{ "hdr_size",			KSTAT_DATA_UINT64 },
 	{ "data_size",			KSTAT_DATA_UINT64 },
+	{ "metadata_size",		KSTAT_DATA_UINT64 },
 	{ "other_size",			KSTAT_DATA_UINT64 },
+	{ "anon_size",			KSTAT_DATA_UINT64 },
+	{ "anon_evictable_data",	KSTAT_DATA_UINT64 },
+	{ "anon_evictable_metadata",	KSTAT_DATA_UINT64 },
+	{ "mru_size",			KSTAT_DATA_UINT64 },
+	{ "mru_evictable_data",		KSTAT_DATA_UINT64 },
+	{ "mru_evictable_metadata",	KSTAT_DATA_UINT64 },
+	{ "mru_ghost_size",		KSTAT_DATA_UINT64 },
+	{ "mru_ghost_evictable_data",	KSTAT_DATA_UINT64 },
+	{ "mru_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
+	{ "mfu_size",			KSTAT_DATA_UINT64 },
+	{ "mfu_evictable_data",		KSTAT_DATA_UINT64 },
+	{ "mfu_evictable_metadata",	KSTAT_DATA_UINT64 },
+	{ "mfu_ghost_size",		KSTAT_DATA_UINT64 },
+	{ "mfu_ghost_evictable_data",	KSTAT_DATA_UINT64 },
+	{ "mfu_ghost_evictable_metadata", KSTAT_DATA_UINT64 },
 	{ "l2_hits",			KSTAT_DATA_UINT64 },
 	{ "l2_misses",			KSTAT_DATA_UINT64 },
 	{ "l2_feeds",			KSTAT_DATA_UINT64 },
@@ -1500,6 +1644,9 @@ arc_space_consume(uint64_t space, arc_space_type_t type)
 	case ARC_SPACE_DATA:
 		ARCSTAT_INCR(arcstat_data_size, space);
 		break;
+	case ARC_SPACE_META:
+		ARCSTAT_INCR(arcstat_metadata_size, space);
+		break;
 	case ARC_SPACE_OTHER:
 		ARCSTAT_INCR(arcstat_other_size, space);
 		break;
@@ -1511,7 +1658,9 @@ arc_space_consume(uint64_t space, arc_space_type_t type)
 		break;
 	}
 
-	ARCSTAT_INCR(arcstat_meta_used, space);
+	if (type != ARC_SPACE_DATA)
+		ARCSTAT_INCR(arcstat_meta_used, space);
+
 	atomic_add_64(&arc_size, space);
 }
 
@@ -1524,6 +1673,9 @@ arc_space_return(uint64_t space, arc_space_type_t type)
 	case ARC_SPACE_DATA:
 		ARCSTAT_INCR(arcstat_data_size, -space);
 		break;
+	case ARC_SPACE_META:
+		ARCSTAT_INCR(arcstat_metadata_size, -space);
+		break;
 	case ARC_SPACE_OTHER:
 		ARCSTAT_INCR(arcstat_other_size, -space);
 		break;
@@ -1535,10 +1687,13 @@ arc_space_return(uint64_t space, arc_space_type_t type)
 		break;
 	}
 
-	ASSERT(arc_meta_used >= space);
-	if (arc_meta_max < arc_meta_used)
-		arc_meta_max = arc_meta_used;
-	ARCSTAT_INCR(arcstat_meta_used, -space);
+	if (type != ARC_SPACE_DATA) {
+		ASSERT(arc_meta_used >= space);
+		if (arc_meta_max < arc_meta_used)
+			arc_meta_max = arc_meta_used;
+		ARCSTAT_INCR(arcstat_meta_used, -space);
+	}
+
 	ASSERT(arc_size >= space);
 	atomic_add_64(&arc_size, -space);
 }
@@ -1744,12 +1899,11 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 		if (!recycle) {
 			if (type == ARC_BUFC_METADATA) {
 				arc_buf_data_free(buf, zio_buf_free);
-				arc_space_return(size, ARC_SPACE_DATA);
+				arc_space_return(size, ARC_SPACE_META);
 			} else {
 				ASSERT(type == ARC_BUFC_DATA);
 				arc_buf_data_free(buf, zio_data_buf_free);
-				ARCSTAT_INCR(arcstat_data_size, -size);
-				atomic_add_64(&arc_size, -size);
+				arc_space_return(size, ARC_SPACE_DATA);
 			}
 		}
 		if (list_link_active(&buf->b_hdr->b_l1hdr.b_arc_node)) {
@@ -2742,6 +2896,20 @@ arc_reclaim_thread(void)
 		if (arc_eviction_list != NULL)
 			arc_do_user_evicts();
 
+		/*
+		 * This is necessary in order for the mdb ::arc dcmd to
+		 * show up to date information. Since the ::arc command
+		 * does not call the kstat's update function, without
+		 * this call, the command may show stale stats for the
+		 * anon, mru, mru_ghost, mfu, and mfu_ghost lists. Even
+		 * with this change, the data might be up to 1 second
+		 * out of date; but that should suffice. The arc_state_t
+		 * structures can be queried directly if more accurate
+		 * information is needed.
+		 */
+		if (arc_ksp != NULL)
+			arc_ksp->ks_update(arc_ksp, KSTAT_READ);
+
 		/* block until needed, or one second, whichever is shorter */
 		CALLB_CPR_SAFE_BEGIN(&cpr);
 		(void) cv_timedwait(&arc_reclaim_thr_cv,
@@ -2878,12 +3046,11 @@ arc_get_data_buf(arc_buf_t *buf)
 	if (!arc_evict_needed(type)) {
 		if (type == ARC_BUFC_METADATA) {
 			buf->b_data = zio_buf_alloc(size);
-			arc_space_consume(size, ARC_SPACE_DATA);
+			arc_space_consume(size, ARC_SPACE_META);
 		} else {
 			ASSERT(type == ARC_BUFC_DATA);
 			buf->b_data = zio_data_buf_alloc(size);
-			ARCSTAT_INCR(arcstat_data_size, size);
-			atomic_add_64(&arc_size, size);
+			arc_space_consume(size, ARC_SPACE_DATA);
 		}
 		goto out;
 	}
@@ -2910,12 +3077,11 @@ arc_get_data_buf(arc_buf_t *buf)
 	if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) {
 		if (type == ARC_BUFC_METADATA) {
 			buf->b_data = zio_buf_alloc(size);
-			arc_space_consume(size, ARC_SPACE_DATA);
+			arc_space_consume(size, ARC_SPACE_META);
 		} else {
 			ASSERT(type == ARC_BUFC_DATA);
 			buf->b_data = zio_data_buf_alloc(size);
-			ARCSTAT_INCR(arcstat_data_size, size);
-			atomic_add_64(&arc_size, size);
+			arc_space_consume(size, ARC_SPACE_DATA);
 		}
 		ARCSTAT_BUMP(arcstat_recycle_miss);
 	}
@@ -4115,6 +4281,48 @@ arc_tempreserve_space(uint64_t reserve, uint64_t txg)
 	return (0);
 }
 
+static void
+arc_kstat_update_state(arc_state_t *state, kstat_named_t *size,
+    kstat_named_t *evict_data, kstat_named_t *evict_metadata)
+{
+	size->value.ui64 = state->arcs_size;
+	evict_data->value.ui64 = state->arcs_lsize[ARC_BUFC_DATA];
+	evict_metadata->value.ui64 = state->arcs_lsize[ARC_BUFC_METADATA];
+}
+
+static int
+arc_kstat_update(kstat_t *ksp, int rw)
+{
+	arc_stats_t *as = ksp->ks_data;
+
+	if (rw == KSTAT_WRITE) {
+		return (EACCES);
+	} else {
+		arc_kstat_update_state(arc_anon,
+		    &as->arcstat_anon_size,
+		    &as->arcstat_anon_evictable_data,
+		    &as->arcstat_anon_evictable_metadata);
+		arc_kstat_update_state(arc_mru,
+		    &as->arcstat_mru_size,
+		    &as->arcstat_mru_evictable_data,
+		    &as->arcstat_mru_evictable_metadata);
+		arc_kstat_update_state(arc_mru_ghost,
+		    &as->arcstat_mru_ghost_size,
+		    &as->arcstat_mru_ghost_evictable_data,
+		    &as->arcstat_mru_ghost_evictable_metadata);
+		arc_kstat_update_state(arc_mfu,
+		    &as->arcstat_mfu_size,
+		    &as->arcstat_mfu_evictable_data,
+		    &as->arcstat_mfu_evictable_metadata);
+		arc_kstat_update_state(arc_mfu_ghost,
+		    &as->arcstat_mfu_ghost_size,
+		    &as->arcstat_mfu_ghost_evictable_data,
+		    &as->arcstat_mfu_ghost_evictable_metadata);
+	}
+
+	return (0);
+}
+
 void
 arc_init(void)
 {
@@ -4261,6 +4469,7 @@ arc_init(void)
 
 	if (arc_ksp != NULL) {
 		arc_ksp->ks_data = &arc_stats;
+		arc_ksp->ks_update = arc_kstat_update;
 		kstat_install(arc_ksp);
 	}
 
diff --git a/uts/common/fs/zfs/sys/arc.h b/uts/common/fs/zfs/sys/arc.h
index 4e9e8e202a7..60112baaba5 100644
--- a/uts/common/fs/zfs/sys/arc.h
+++ b/uts/common/fs/zfs/sys/arc.h
@@ -115,6 +115,7 @@ typedef enum arc_buf_contents {
  */
 typedef enum arc_space_type {
 	ARC_SPACE_DATA,
+	ARC_SPACE_META,
 	ARC_SPACE_HDRS,
 	ARC_SPACE_L2HDRS,
 	ARC_SPACE_OTHER,

From 871e4ea882f72e65f100b5700773632fbe603b80 Mon Sep 17 00:00:00 2001
From: Xin LI <delphij@FreeBSD.org>
Date: Tue, 20 Jan 2015 20:11:30 +0000
Subject: [PATCH 112/279] 5056 ZFS deadlock on db_mtx and dn_holds Reviewed by:
 Will Andrews <willa@spectralogic.com> Reviewed by: Matt Ahrens
 <mahrens@delphix.com> Reviewed by: George Wilson <george.wilson@delphix.com>
 Approved by: Dan McDonald <danmcd@omniti.com> Author: Justin Gibbs
 <justing@spectralogic.com>

illumos/illumos-gate@bc9014e6a81272073b9854d9f65dd59e18d18c35
---
 uts/common/fs/zfs/dbuf.c            | 182 ++++++++++++++++++++--------
 uts/common/fs/zfs/dmu_objset.c      | 111 ++++++++++-------
 uts/common/fs/zfs/dmu_send.c        |   6 +-
 uts/common/fs/zfs/dmu_traverse.c    |   2 +-
 uts/common/fs/zfs/dnode.c           |  85 ++++++++-----
 uts/common/fs/zfs/dnode_sync.c      |  62 ++++------
 uts/common/fs/zfs/dsl_bookmark.c    |   2 +-
 uts/common/fs/zfs/dsl_dataset.c     |  53 ++++----
 uts/common/fs/zfs/dsl_deadlist.c    |   3 +
 uts/common/fs/zfs/dsl_deleg.c       |   2 +-
 uts/common/fs/zfs/dsl_destroy.c     |   8 +-
 uts/common/fs/zfs/dsl_dir.c         |  38 ++++--
 uts/common/fs/zfs/dsl_pool.c        |   3 +
 uts/common/fs/zfs/dsl_prop.c        |  14 +--
 uts/common/fs/zfs/dsl_scan.c        |   8 +-
 uts/common/fs/zfs/dsl_userhold.c    |   2 +-
 uts/common/fs/zfs/sa.c              |  42 ++++---
 uts/common/fs/zfs/spa.c             |  17 +++
 uts/common/fs/zfs/spa_misc.c        |  48 ++++++++
 uts/common/fs/zfs/sys/dbuf.h        |   6 +-
 uts/common/fs/zfs/sys/dmu.h         | 147 +++++++++++++++++-----
 uts/common/fs/zfs/sys/dmu_objset.h  |  14 ++-
 uts/common/fs/zfs/sys/dnode.h       |   4 +-
 uts/common/fs/zfs/sys/dsl_dataset.h |  10 +-
 uts/common/fs/zfs/sys/dsl_dir.h     |   4 +
 uts/common/fs/zfs/sys/sa.h          |   1 -
 uts/common/fs/zfs/sys/sa_impl.h     |   4 +-
 uts/common/fs/zfs/sys/spa.h         |   5 +
 uts/common/fs/zfs/sys/spa_impl.h    |   4 +
 uts/common/fs/zfs/sys/zap_impl.h    |   4 +-
 uts/common/fs/zfs/sys/zap_leaf.h    |   2 +
 uts/common/fs/zfs/zap.c             |  35 +++---
 uts/common/fs/zfs/zap_micro.c       |   9 +-
 uts/common/fs/zfs/zfs_sa.c          |   3 +-
 uts/common/fs/zfs/zil.c             |   2 +-
 35 files changed, 633 insertions(+), 309 deletions(-)

diff --git a/uts/common/fs/zfs/dbuf.c b/uts/common/fs/zfs/dbuf.c
index fd539c838d0..49c659a51b4 100644
--- a/uts/common/fs/zfs/dbuf.c
+++ b/uts/common/fs/zfs/dbuf.c
@@ -24,6 +24,7 @@
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -54,10 +55,16 @@ static void dbuf_destroy(dmu_buf_impl_t *db);
 static boolean_t dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx);
 static void dbuf_write(dbuf_dirty_record_t *dr, arc_buf_t *data, dmu_tx_t *tx);
 
+#ifndef __lint
+extern inline void dmu_buf_init_user(dmu_buf_user_t *dbu,
+    dmu_buf_evict_func_t *evict_func, dmu_buf_t **clear_on_evict_dbufp);
+#endif /* ! __lint */
+
 /*
  * Global data structures and functions for the dbuf cache.
  */
 static kmem_cache_t *dbuf_cache;
+static taskq_t *dbu_evict_taskq;
 
 /* ARGSUSED */
 static int
@@ -215,17 +222,72 @@ dbuf_hash_remove(dmu_buf_impl_t *db)
 
 static arc_evict_func_t dbuf_do_evict;
 
+typedef enum {
+	DBVU_EVICTING,
+	DBVU_NOT_EVICTING
+} dbvu_verify_type_t;
+
+static void
+dbuf_verify_user(dmu_buf_impl_t *db, dbvu_verify_type_t verify_type)
+{
+#ifdef ZFS_DEBUG
+	int64_t holds;
+
+	if (db->db_user == NULL)
+		return;
+
+	/* Only data blocks support the attachment of user data. */
+	ASSERT(db->db_level == 0);
+
+	/* Clients must resolve a dbuf before attaching user data. */
+	ASSERT(db->db.db_data != NULL);
+	ASSERT3U(db->db_state, ==, DB_CACHED);
+
+	holds = refcount_count(&db->db_holds);
+	if (verify_type == DBVU_EVICTING) {
+		/*
+		 * Immediate eviction occurs when holds == dirtycnt.
+		 * For normal eviction buffers, holds is zero on
+		 * eviction, except when dbuf_fix_old_data() calls
+		 * dbuf_clear_data().  However, the hold count can grow
+		 * during eviction even though db_mtx is held (see
+		 * dmu_bonus_hold() for an example), so we can only
+		 * test the generic invariant that holds >= dirtycnt.
+		 */
+		ASSERT3U(holds, >=, db->db_dirtycnt);
+	} else {
+		if (db->db_immediate_evict == TRUE)
+			ASSERT3U(holds, >=, db->db_dirtycnt);
+		else
+			ASSERT3U(holds, >, 0);
+	}
+#endif
+}
+
 static void
 dbuf_evict_user(dmu_buf_impl_t *db)
 {
+	dmu_buf_user_t *dbu = db->db_user;
+
 	ASSERT(MUTEX_HELD(&db->db_mtx));
 
-	if (db->db_level != 0 || db->db_evict_func == NULL)
+	if (dbu == NULL)
 		return;
 
-	db->db_evict_func(&db->db, db->db_user_ptr);
-	db->db_user_ptr = NULL;
-	db->db_evict_func = NULL;
+	dbuf_verify_user(db, DBVU_EVICTING);
+	db->db_user = NULL;
+
+#ifdef ZFS_DEBUG
+	if (dbu->dbu_clear_on_evict_dbufp != NULL)
+		*dbu->dbu_clear_on_evict_dbufp = NULL;
+#endif
+
+	/*
+	 * Invoke the callback from a taskq to avoid lock order reversals
+	 * and limit stack depth.
+	 */
+	taskq_dispatch_ent(dbu_evict_taskq, dbu->dbu_evict_func, dbu, 0,
+	    &dbu->dbu_tqent);
 }
 
 boolean_t
@@ -286,6 +348,12 @@ retry:
 
 	for (i = 0; i < DBUF_MUTEXES; i++)
 		mutex_init(&h->hash_mutexes[i], NULL, MUTEX_DEFAULT, NULL);
+
+	/*
+	 * All entries are queued via taskq_dispatch_ent(), so min/maxalloc
+	 * configuration is not required.
+	 */
+	dbu_evict_taskq = taskq_create("dbu_evict", 1, minclsyspri, 0, 0, 0);
 }
 
 void
@@ -298,6 +366,7 @@ dbuf_fini(void)
 		mutex_destroy(&h->hash_mutexes[i]);
 	kmem_free(h->hash_table, (h->hash_table_mask + 1) * sizeof (void *));
 	kmem_cache_destroy(dbuf_cache);
+	taskq_destroy(dbu_evict_taskq);
 }
 
 /*
@@ -414,22 +483,28 @@ dbuf_verify(dmu_buf_impl_t *db)
 }
 #endif
 
+static void
+dbuf_clear_data(dmu_buf_impl_t *db)
+{
+	ASSERT(MUTEX_HELD(&db->db_mtx));
+	dbuf_evict_user(db);
+	db->db_buf = NULL;
+	db->db.db_data = NULL;
+	if (db->db_state != DB_NOFILL)
+		db->db_state = DB_UNCACHED;
+}
+
 static void
 dbuf_set_data(dmu_buf_impl_t *db, arc_buf_t *buf)
 {
 	ASSERT(MUTEX_HELD(&db->db_mtx));
+	ASSERT(buf != NULL);
+
 	db->db_buf = buf;
-	if (buf != NULL) {
-		ASSERT(buf->b_data != NULL);
-		db->db.db_data = buf->b_data;
-		if (!arc_released(buf))
-			arc_set_callback(buf, dbuf_do_evict, db);
-	} else {
-		dbuf_evict_user(db);
-		db->db.db_data = NULL;
-		if (db->db_state != DB_NOFILL)
-			db->db_state = DB_UNCACHED;
-	}
+	ASSERT(buf->b_data != NULL);
+	db->db.db_data = buf->b_data;
+	if (!arc_released(buf))
+		arc_set_callback(buf, dbuf_do_evict, db);
 }
 
 /*
@@ -451,7 +526,7 @@ dbuf_loan_arcbuf(dmu_buf_impl_t *db)
 	} else {
 		abuf = db->db_buf;
 		arc_loan_inuse_buf(abuf, db);
-		dbuf_set_data(db, NULL);
+		dbuf_clear_data(db);
 		mutex_exit(&db->db_mtx);
 	}
 	return (abuf);
@@ -687,7 +762,7 @@ dbuf_noread(dmu_buf_impl_t *db)
 		dbuf_set_data(db, arc_buf_alloc(spa, db->db.db_size, db, type));
 		db->db_state = DB_FILL;
 	} else if (db->db_state == DB_NOFILL) {
-		dbuf_set_data(db, NULL);
+		dbuf_clear_data(db);
 	} else {
 		ASSERT3U(db->db_state, ==, DB_CACHED);
 	}
@@ -743,7 +818,7 @@ dbuf_fix_old_data(dmu_buf_impl_t *db, uint64_t txg)
 		dr->dt.dl.dr_data = arc_buf_alloc(spa, size, db, type);
 		bcopy(db->db.db_data, dr->dt.dl.dr_data->b_data, size);
 	} else {
-		dbuf_set_data(db, NULL);
+		dbuf_clear_data(db);
 	}
 }
 
@@ -794,7 +869,8 @@ void
 dbuf_free_range(dnode_t *dn, uint64_t start_blkid, uint64_t end_blkid,
     dmu_tx_t *tx)
 {
-	dmu_buf_impl_t *db, *db_next, db_search;
+	dmu_buf_impl_t db_search;
+	dmu_buf_impl_t *db, *db_next;
 	uint64_t txg = tx->tx_txg;
 	avl_index_t where;
 
@@ -1370,7 +1446,7 @@ dbuf_undirty(dmu_buf_impl_t *db, dmu_tx_t *tx)
 		arc_buf_t *buf = db->db_buf;
 
 		ASSERT(db->db_state == DB_NOFILL || arc_released(buf));
-		dbuf_set_data(db, NULL);
+		dbuf_clear_data(db);
 		VERIFY(arc_buf_remove_ref(buf, db));
 		dbuf_evict(db);
 		return (B_TRUE);
@@ -1710,8 +1786,7 @@ dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid,
 	db->db_parent = parent;
 	db->db_blkptr = blkptr;
 
-	db->db_user_ptr = NULL;
-	db->db_evict_func = NULL;
+	db->db_user = NULL;
 	db->db_immediate_evict = 0;
 	db->db_freed_in_flight = 0;
 
@@ -2114,7 +2189,7 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 			/*
 			 * This dbuf has anonymous data associated with it.
 			 */
-			dbuf_set_data(db, NULL);
+			dbuf_clear_data(db);
 			VERIFY(arc_buf_remove_ref(buf, db));
 			dbuf_evict(db);
 		} else {
@@ -2147,7 +2222,8 @@ dbuf_rele_and_unlock(dmu_buf_impl_t *db, void *tag)
 				} else {
 					dbuf_clear(db);
 				}
-			} else if (arc_buf_eviction_needed(db->db_buf)) {
+			} else if (db->db_objset->os_evicting ||
+			    arc_buf_eviction_needed(db->db_buf)) {
 				dbuf_clear(db);
 			} else {
 				mutex_exit(&db->db_mtx);
@@ -2166,51 +2242,57 @@ dbuf_refcount(dmu_buf_impl_t *db)
 }
 
 void *
-dmu_buf_set_user(dmu_buf_t *db_fake, void *user_ptr,
-    dmu_buf_evict_func_t *evict_func)
+dmu_buf_replace_user(dmu_buf_t *db_fake, dmu_buf_user_t *old_user,
+    dmu_buf_user_t *new_user)
 {
-	return (dmu_buf_update_user(db_fake, NULL, user_ptr, evict_func));
+	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
+
+	mutex_enter(&db->db_mtx);
+	dbuf_verify_user(db, DBVU_NOT_EVICTING);
+	if (db->db_user == old_user)
+		db->db_user = new_user;
+	else
+		old_user = db->db_user;
+	dbuf_verify_user(db, DBVU_NOT_EVICTING);
+	mutex_exit(&db->db_mtx);
+
+	return (old_user);
 }
 
 void *
-dmu_buf_set_user_ie(dmu_buf_t *db_fake, void *user_ptr,
-    dmu_buf_evict_func_t *evict_func)
+dmu_buf_set_user(dmu_buf_t *db_fake, dmu_buf_user_t *user)
+{
+	return (dmu_buf_replace_user(db_fake, NULL, user));
+}
+
+void *
+dmu_buf_set_user_ie(dmu_buf_t *db_fake, dmu_buf_user_t *user)
 {
 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
 
 	db->db_immediate_evict = TRUE;
-	return (dmu_buf_update_user(db_fake, NULL, user_ptr, evict_func));
+	return (dmu_buf_set_user(db_fake, user));
 }
 
 void *
-dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr, void *user_ptr,
-    dmu_buf_evict_func_t *evict_func)
+dmu_buf_remove_user(dmu_buf_t *db_fake, dmu_buf_user_t *user)
 {
-	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
-	ASSERT(db->db_level == 0);
-
-	ASSERT((user_ptr == NULL) == (evict_func == NULL));
-
-	mutex_enter(&db->db_mtx);
-
-	if (db->db_user_ptr == old_user_ptr) {
-		db->db_user_ptr = user_ptr;
-		db->db_evict_func = evict_func;
-	} else {
-		old_user_ptr = db->db_user_ptr;
-	}
-
-	mutex_exit(&db->db_mtx);
-	return (old_user_ptr);
+	return (dmu_buf_replace_user(db_fake, user, NULL));
 }
 
 void *
 dmu_buf_get_user(dmu_buf_t *db_fake)
 {
 	dmu_buf_impl_t *db = (dmu_buf_impl_t *)db_fake;
-	ASSERT(!refcount_is_zero(&db->db_holds));
 
-	return (db->db_user_ptr);
+	dbuf_verify_user(db, DBVU_NOT_EVICTING);
+	return (db->db_user);
+}
+
+void
+dmu_buf_user_evict_wait()
+{
+	taskq_wait(dbu_evict_taskq);
 }
 
 boolean_t
diff --git a/uts/common/fs/zfs/dmu_objset.c b/uts/common/fs/zfs/dmu_objset.c
index 36ac27a5e0d..8593e2c94f6 100644
--- a/uts/common/fs/zfs/dmu_objset.c
+++ b/uts/common/fs/zfs/dmu_objset.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -355,7 +356,7 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 			    zfs_prop_to_name(ZFS_PROP_SECONDARYCACHE),
 			    secondary_cache_changed_cb, os);
 		}
-		if (!dsl_dataset_is_snapshot(ds)) {
+		if (!ds->ds_is_snapshot) {
 			if (err == 0) {
 				err = dsl_prop_register(ds,
 				    zfs_prop_to_name(ZFS_PROP_CHECKSUM),
@@ -417,7 +418,7 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 		os->os_secondary_cache = ZFS_CACHE_ALL;
 	}
 
-	if (ds == NULL || !dsl_dataset_is_snapshot(ds))
+	if (ds == NULL || !ds->ds_is_snapshot)
 		os->os_zil_header = os->os_phys->os_zil_header;
 	os->os_zil = zil_alloc(os, &os->os_zil_header);
 
@@ -436,16 +437,13 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp,
 	mutex_init(&os->os_obj_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&os->os_user_ptr_lock, NULL, MUTEX_DEFAULT, NULL);
 
-	DMU_META_DNODE(os) = dnode_special_open(os,
-	    &os->os_phys->os_meta_dnode, DMU_META_DNODE_OBJECT,
-	    &os->os_meta_dnode);
+	dnode_special_open(os, &os->os_phys->os_meta_dnode,
+	    DMU_META_DNODE_OBJECT, &os->os_meta_dnode);
 	if (arc_buf_size(os->os_phys_buf) >= sizeof (objset_phys_t)) {
-		DMU_USERUSED_DNODE(os) = dnode_special_open(os,
-		    &os->os_phys->os_userused_dnode, DMU_USERUSED_OBJECT,
-		    &os->os_userused_dnode);
-		DMU_GROUPUSED_DNODE(os) = dnode_special_open(os,
-		    &os->os_phys->os_groupused_dnode, DMU_GROUPUSED_OBJECT,
-		    &os->os_groupused_dnode);
+		dnode_special_open(os, &os->os_phys->os_userused_dnode,
+		    DMU_USERUSED_OBJECT, &os->os_userused_dnode);
+		dnode_special_open(os, &os->os_phys->os_groupused_dnode,
+		    DMU_GROUPUSED_OBJECT, &os->os_groupused_dnode);
 	}
 
 	*osp = os;
@@ -533,7 +531,7 @@ dmu_objset_own(const char *name, dmu_objset_type_t type,
 	} else if (type != DMU_OST_ANY && type != (*osp)->os_phys->os_type) {
 		dsl_dataset_disown(ds, tag);
 		return (SET_ERROR(EINVAL));
-	} else if (!readonly && dsl_dataset_is_snapshot(ds)) {
+	} else if (!readonly && ds->ds_is_snapshot) {
 		dsl_dataset_disown(ds, tag);
 		return (SET_ERROR(EROFS));
 	}
@@ -589,41 +587,53 @@ dmu_objset_disown(objset_t *os, void *tag)
 void
 dmu_objset_evict_dbufs(objset_t *os)
 {
+	dnode_t dn_marker;
 	dnode_t *dn;
 
 	mutex_enter(&os->os_lock);
+	dn = list_head(&os->os_dnodes);
+	while (dn != NULL) {
+		/*
+		 * Skip dnodes without holds.  We have to do this dance
+		 * because dnode_add_ref() only works if there is already a
+		 * hold.  If the dnode has no holds, then it has no dbufs.
+		 */
+		if (dnode_add_ref(dn, FTAG)) {
+			list_insert_after(&os->os_dnodes, dn, &dn_marker);
+			mutex_exit(&os->os_lock);
 
-	/* process the mdn last, since the other dnodes have holds on it */
-	list_remove(&os->os_dnodes, DMU_META_DNODE(os));
-	list_insert_tail(&os->os_dnodes, DMU_META_DNODE(os));
+			dnode_evict_dbufs(dn);
+			dnode_rele(dn, FTAG);
 
-	/*
-	 * Find the first dnode with holds.  We have to do this dance
-	 * because dnode_add_ref() only works if you already have a
-	 * hold.  If there are no holds then it has no dbufs so OK to
-	 * skip.
-	 */
-	for (dn = list_head(&os->os_dnodes);
-	    dn && !dnode_add_ref(dn, FTAG);
-	    dn = list_next(&os->os_dnodes, dn))
-		continue;
-
-	while (dn) {
-		dnode_t *next_dn = dn;
-
-		do {
-			next_dn = list_next(&os->os_dnodes, next_dn);
-		} while (next_dn && !dnode_add_ref(next_dn, FTAG));
-
-		mutex_exit(&os->os_lock);
-		dnode_evict_dbufs(dn);
-		dnode_rele(dn, FTAG);
-		mutex_enter(&os->os_lock);
-		dn = next_dn;
+			mutex_enter(&os->os_lock);
+			dn = list_next(&os->os_dnodes, &dn_marker);
+			list_remove(&os->os_dnodes, &dn_marker);
+		} else {
+			dn = list_next(&os->os_dnodes, dn);
+		}
 	}
 	mutex_exit(&os->os_lock);
+
+	if (DMU_USERUSED_DNODE(os) != NULL) {
+		dnode_evict_dbufs(DMU_GROUPUSED_DNODE(os));
+		dnode_evict_dbufs(DMU_USERUSED_DNODE(os));
+	}
+	dnode_evict_dbufs(DMU_META_DNODE(os));
 }
 
+/*
+ * Objset eviction processing is split into into two pieces.
+ * The first marks the objset as evicting, evicts any dbufs that
+ * have a refcount of zero, and then queues up the objset for the
+ * second phase of eviction.  Once os->os_dnodes has been cleared by
+ * dnode_buf_pageout()->dnode_destroy(), the second phase is executed.
+ * The second phase closes the special dnodes, dequeues the objset from
+ * the list of those undergoing eviction, and finally frees the objset.
+ *
+ * NOTE: Due to asynchronous eviction processing (invocation of
+ *       dnode_buf_pageout()), it is possible for the meta dnode for the
+ *       objset to have no holds even though os->os_dnodes is not empty.
+ */
 void
 dmu_objset_evict(objset_t *os)
 {
@@ -633,7 +643,7 @@ dmu_objset_evict(objset_t *os)
 		ASSERT(!dmu_objset_is_dirty(os, t));
 
 	if (ds) {
-		if (!dsl_dataset_is_snapshot(ds)) {
+		if (!ds->ds_is_snapshot) {
 			VERIFY0(dsl_prop_unregister(ds,
 			    zfs_prop_to_name(ZFS_PROP_CHECKSUM),
 			    checksum_changed_cb, os));
@@ -670,8 +680,24 @@ dmu_objset_evict(objset_t *os)
 	if (os->os_sa)
 		sa_tear_down(os);
 
+	os->os_evicting = B_TRUE;
 	dmu_objset_evict_dbufs(os);
 
+	mutex_enter(&os->os_lock);
+	spa_evicting_os_register(os->os_spa, os);
+	if (list_is_empty(&os->os_dnodes)) {
+		mutex_exit(&os->os_lock);
+		dmu_objset_evict_done(os);
+	} else {
+		mutex_exit(&os->os_lock);
+	}
+}
+
+void
+dmu_objset_evict_done(objset_t *os)
+{
+	ASSERT3P(list_head(&os->os_dnodes), ==, NULL);
+
 	dnode_special_close(&os->os_meta_dnode);
 	if (DMU_USERUSED_DNODE(os)) {
 		dnode_special_close(&os->os_userused_dnode);
@@ -679,8 +705,6 @@ dmu_objset_evict(objset_t *os)
 	}
 	zil_free(os->os_zil);
 
-	ASSERT3P(list_head(&os->os_dnodes), ==, NULL);
-
 	VERIFY(arc_buf_remove_ref(os->os_phys_buf, &os->os_phys_buf));
 
 	/*
@@ -695,6 +719,7 @@ dmu_objset_evict(objset_t *os)
 	mutex_destroy(&os->os_lock);
 	mutex_destroy(&os->os_obj_lock);
 	mutex_destroy(&os->os_user_ptr_lock);
+	spa_evicting_os_deregister(os->os_spa, os);
 	kmem_free(os, sizeof (objset_t));
 }
 
@@ -903,7 +928,7 @@ dmu_objset_clone_check(void *arg, dmu_tx_t *tx)
 	}
 
 	/* You can only clone snapshots, not the head datasets. */
-	if (!dsl_dataset_is_snapshot(origin)) {
+	if (!origin->ds_is_snapshot) {
 		dsl_dataset_rele(origin, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
@@ -1467,7 +1492,7 @@ int
 dmu_objset_is_snapshot(objset_t *os)
 {
 	if (os->os_dsl_dataset != NULL)
-		return (dsl_dataset_is_snapshot(os->os_dsl_dataset));
+		return (os->os_dsl_dataset->ds_is_snapshot);
 	else
 		return (B_FALSE);
 }
diff --git a/uts/common/fs/zfs/dmu_send.c b/uts/common/fs/zfs/dmu_send.c
index 4bb1290a07c..3c156054877 100644
--- a/uts/common/fs/zfs/dmu_send.c
+++ b/uts/common/fs/zfs/dmu_send.c
@@ -611,7 +611,7 @@ dmu_send_impl(void *tag, dsl_pool_t *dp, dsl_dataset_t *ds,
 		fromtxg = fromzb->zbm_creation_txg;
 	}
 	dsl_dataset_name(ds, drr->drr_u.drr_begin.drr_toname);
-	if (!dsl_dataset_is_snapshot(ds)) {
+	if (!ds->ds_is_snapshot) {
 		(void) strlcat(drr->drr_u.drr_begin.drr_toname, "@--head--",
 		    sizeof (drr->drr_u.drr_begin.drr_toname));
 	}
@@ -818,7 +818,7 @@ dmu_send_estimate(dsl_dataset_t *ds, dsl_dataset_t *fromds, uint64_t *sizep)
 	ASSERT(dsl_pool_config_held(dp));
 
 	/* tosnap must be a snapshot */
-	if (!dsl_dataset_is_snapshot(ds))
+	if (!ds->ds_is_snapshot)
 		return (SET_ERROR(EINVAL));
 
 	/*
@@ -1065,7 +1065,7 @@ dmu_recv_begin_check(void *arg, dmu_tx_t *tx)
 				dsl_dataset_rele(ds, FTAG);
 				return (error);
 			}
-			if (!dsl_dataset_is_snapshot(origin)) {
+			if (!origin->ds_is_snapshot) {
 				dsl_dataset_rele(origin, FTAG);
 				dsl_dataset_rele(ds, FTAG);
 				return (SET_ERROR(EINVAL));
diff --git a/uts/common/fs/zfs/dmu_traverse.c b/uts/common/fs/zfs/dmu_traverse.c
index 7b1d6befa6a..14eef2e7516 100644
--- a/uts/common/fs/zfs/dmu_traverse.c
+++ b/uts/common/fs/zfs/dmu_traverse.c
@@ -532,7 +532,7 @@ traverse_impl(spa_t *spa, dsl_dataset_t *ds, uint64_t objset, blkptr_t *rootbp,
 	cv_init(&pd.pd_cv, NULL, CV_DEFAULT, NULL);
 
 	/* See comment on ZIL traversal in dsl_scan_visitds. */
-	if (ds != NULL && !dsl_dataset_is_snapshot(ds) && !BP_IS_HOLE(rootbp)) {
+	if (ds != NULL && !ds->ds_is_snapshot && !BP_IS_HOLE(rootbp)) {
 		arc_flags_t flags = ARC_FLAG_WAIT;
 		objset_phys_t *osp;
 		arc_buf_t *buf;
diff --git a/uts/common/fs/zfs/dnode.c b/uts/common/fs/zfs/dnode.c
index 3303ee709ed..4e376fc4d92 100644
--- a/uts/common/fs/zfs/dnode.c
+++ b/uts/common/fs/zfs/dnode.c
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -402,8 +403,9 @@ static dnode_t *
 dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db,
     uint64_t object, dnode_handle_t *dnh)
 {
-	dnode_t *dn = kmem_cache_alloc(dnode_cache, KM_SLEEP);
+	dnode_t *dn;
 
+	dn = kmem_cache_alloc(dnode_cache, KM_SLEEP);
 	ASSERT(!POINTER_IS_VALID(dn->dn_objset));
 	dn->dn_moved = 0;
 
@@ -440,13 +442,31 @@ dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db,
 	ASSERT(DMU_OT_IS_VALID(dn->dn_phys->dn_type));
 
 	mutex_enter(&os->os_lock);
-	list_insert_head(&os->os_dnodes, dn);
-	membar_producer();
+	if (dnh->dnh_dnode != NULL) {
+		/* Lost the allocation race. */
+		mutex_exit(&os->os_lock);
+		kmem_cache_free(dnode_cache, dn);
+		return (dnh->dnh_dnode);
+	}
+
 	/*
-	 * Everything else must be valid before assigning dn_objset makes the
-	 * dnode eligible for dnode_move().
+	 * Exclude special dnodes from os_dnodes so an empty os_dnodes
+	 * signifies that the special dnodes have no references from
+	 * their children (the entries in os_dnodes).  This allows
+	 * dnode_destroy() to easily determine if the last child has
+	 * been removed and then complete eviction of the objset.
+	 */
+	if (!DMU_OBJECT_IS_SPECIAL(object))
+		list_insert_head(&os->os_dnodes, dn);
+	membar_producer();
+
+	/*
+	 * Everything else must be valid before assigning dn_objset
+	 * makes the dnode eligible for dnode_move().
 	 */
 	dn->dn_objset = os;
+
+	dnh->dnh_dnode = dn;
 	mutex_exit(&os->os_lock);
 
 	arc_space_consume(sizeof (dnode_t), ARC_SPACE_OTHER);
@@ -460,12 +480,18 @@ static void
 dnode_destroy(dnode_t *dn)
 {
 	objset_t *os = dn->dn_objset;
+	boolean_t complete_os_eviction = B_FALSE;
 
 	ASSERT((dn->dn_id_flags & DN_ID_NEW_EXIST) == 0);
 
 	mutex_enter(&os->os_lock);
 	POINTER_INVALIDATE(&dn->dn_objset);
-	list_remove(&os->os_dnodes, dn);
+	if (!DMU_OBJECT_IS_SPECIAL(dn->dn_object)) {
+		list_remove(&os->os_dnodes, dn);
+		complete_os_eviction =
+		    list_is_empty(&os->os_dnodes) &&
+		    list_link_active(&os->os_evicting_node);
+	}
 	mutex_exit(&os->os_lock);
 
 	/* the dnode can no longer move, so we can release the handle */
@@ -500,6 +526,9 @@ dnode_destroy(dnode_t *dn)
 	dmu_zfetch_rele(&dn->dn_zfetch);
 	kmem_cache_free(dnode_cache, dn);
 	arc_space_return(sizeof (dnode_t), ARC_SPACE_OTHER);
+
+	if (complete_os_eviction)
+		dmu_objset_evict_done(os);
 }
 
 void
@@ -966,33 +995,32 @@ dnode_special_close(dnode_handle_t *dnh)
 	 */
 	while (refcount_count(&dn->dn_holds) > 0)
 		delay(1);
+	ASSERT(dn->dn_dbuf == NULL ||
+	    dmu_buf_get_user(&dn->dn_dbuf->db) == NULL);
 	zrl_add(&dnh->dnh_zrlock);
 	dnode_destroy(dn); /* implicit zrl_remove() */
 	zrl_destroy(&dnh->dnh_zrlock);
 	dnh->dnh_dnode = NULL;
 }
 
-dnode_t *
+void
 dnode_special_open(objset_t *os, dnode_phys_t *dnp, uint64_t object,
     dnode_handle_t *dnh)
 {
-	dnode_t *dn = dnode_create(os, dnp, NULL, object, dnh);
-	dnh->dnh_dnode = dn;
+	dnode_t *dn;
+
+	dn = dnode_create(os, dnp, NULL, object, dnh);
 	zrl_init(&dnh->dnh_zrlock);
 	DNODE_VERIFY(dn);
-	return (dn);
 }
 
 static void
-dnode_buf_pageout(dmu_buf_t *db, void *arg)
+dnode_buf_pageout(void *dbu)
 {
-	dnode_children_t *children_dnodes = arg;
+	dnode_children_t *children_dnodes = dbu;
 	int i;
-	int epb = db->db_size >> DNODE_SHIFT;
 
-	ASSERT(epb == children_dnodes->dnc_count);
-
-	for (i = 0; i < epb; i++) {
+	for (i = 0; i < children_dnodes->dnc_count; i++) {
 		dnode_handle_t *dnh = &children_dnodes->dnc_children[i];
 		dnode_t *dn;
 
@@ -1022,7 +1050,7 @@ dnode_buf_pageout(dmu_buf_t *db, void *arg)
 		dnh->dnh_dnode = NULL;
 	}
 	kmem_free(children_dnodes, sizeof (dnode_children_t) +
-	    epb * sizeof (dnode_handle_t));
+	    children_dnodes->dnc_count * sizeof (dnode_handle_t));
 }
 
 /*
@@ -1106,16 +1134,17 @@ dnode_hold_impl(objset_t *os, uint64_t object, int flag,
 	if (children_dnodes == NULL) {
 		int i;
 		dnode_children_t *winner;
-		children_dnodes = kmem_alloc(sizeof (dnode_children_t) +
+		children_dnodes = kmem_zalloc(sizeof (dnode_children_t) +
 		    epb * sizeof (dnode_handle_t), KM_SLEEP);
 		children_dnodes->dnc_count = epb;
 		dnh = &children_dnodes->dnc_children[0];
 		for (i = 0; i < epb; i++) {
 			zrl_init(&dnh[i].dnh_zrlock);
-			dnh[i].dnh_dnode = NULL;
 		}
-		if (winner = dmu_buf_set_user(&db->db, children_dnodes,
-		    dnode_buf_pageout)) {
+		dmu_buf_init_user(&children_dnodes->dnc_dbu,
+		    dnode_buf_pageout, NULL);
+		winner = dmu_buf_set_user(&db->db, &children_dnodes->dnc_dbu);
+		if (winner != NULL) {
 
 			for (i = 0; i < epb; i++) {
 				zrl_destroy(&dnh[i].dnh_zrlock);
@@ -1130,17 +1159,11 @@ dnode_hold_impl(objset_t *os, uint64_t object, int flag,
 
 	dnh = &children_dnodes->dnc_children[idx];
 	zrl_add(&dnh->dnh_zrlock);
-	if ((dn = dnh->dnh_dnode) == NULL) {
+	dn = dnh->dnh_dnode;
+	if (dn == NULL) {
 		dnode_phys_t *phys = (dnode_phys_t *)db->db.db_data+idx;
-		dnode_t *winner;
 
 		dn = dnode_create(os, phys, db, object, dnh);
-		winner = atomic_cas_ptr(&dnh->dnh_dnode, NULL, dn);
-		if (winner != NULL) {
-			zrl_add(&dnh->dnh_zrlock);
-			dnode_destroy(dn); /* implicit zrl_remove() */
-			dn = winner;
-		}
 	}
 
 	mutex_enter(&dn->dn_mtx);
@@ -1154,10 +1177,10 @@ dnode_hold_impl(objset_t *os, uint64_t object, int flag,
 		dbuf_rele(db, FTAG);
 		return (type == DMU_OT_NONE ? ENOENT : EEXIST);
 	}
-	mutex_exit(&dn->dn_mtx);
-
 	if (refcount_add(&dn->dn_holds, tag) == 1)
 		dbuf_add_ref(db, dnh);
+	mutex_exit(&dn->dn_mtx);
+
 	/* Now we can rely on the hold to prevent the dnode from moving. */
 	zrl_remove(&dnh->dnh_zrlock);
 
diff --git a/uts/common/fs/zfs/dnode_sync.c b/uts/common/fs/zfs/dnode_sync.c
index 63bfc94f9ae..99ecb9e3ff7 100644
--- a/uts/common/fs/zfs/dnode_sync.c
+++ b/uts/common/fs/zfs/dnode_sync.c
@@ -22,6 +22,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -396,49 +397,37 @@ dnode_sync_free_range(void *arg, uint64_t blkid, uint64_t nblks)
 void
 dnode_evict_dbufs(dnode_t *dn)
 {
-	int progress;
-	int pass = 0;
+	dmu_buf_impl_t db_marker;
+	dmu_buf_impl_t *db, *db_next;
 
-	do {
-		dmu_buf_impl_t *db, *db_next;
-		int evicting = FALSE;
+	mutex_enter(&dn->dn_dbufs_mtx);
+	for (db = avl_first(&dn->dn_dbufs); db != NULL; db = db_next) {
 
-		progress = FALSE;
-		mutex_enter(&dn->dn_dbufs_mtx);
-		for (db = avl_first(&dn->dn_dbufs); db != NULL; db = db_next) {
-			db_next = AVL_NEXT(&dn->dn_dbufs, db);
 #ifdef	DEBUG
-			DB_DNODE_ENTER(db);
-			ASSERT3P(DB_DNODE(db), ==, dn);
-			DB_DNODE_EXIT(db);
+		DB_DNODE_ENTER(db);
+		ASSERT3P(DB_DNODE(db), ==, dn);
+		DB_DNODE_EXIT(db);
 #endif	/* DEBUG */
 
-			mutex_enter(&db->db_mtx);
-			if (db->db_state == DB_EVICTING) {
-				progress = TRUE;
-				evicting = TRUE;
-				mutex_exit(&db->db_mtx);
-			} else if (refcount_is_zero(&db->db_holds)) {
-				progress = TRUE;
-				dbuf_clear(db); /* exits db_mtx for us */
-			} else {
-				mutex_exit(&db->db_mtx);
-			}
+		mutex_enter(&db->db_mtx);
+		if (db->db_state != DB_EVICTING &&
+		    refcount_is_zero(&db->db_holds)) {
+			db_marker.db_level = db->db_level;
+			db_marker.db_blkid = db->db_blkid;
+			db_marker.db_state = DB_SEARCH;
+			avl_insert_here(&dn->dn_dbufs, &db_marker, db,
+			    AVL_BEFORE);
 
+			dbuf_clear(db);
+
+			db_next = AVL_NEXT(&dn->dn_dbufs, &db_marker);
+			avl_remove(&dn->dn_dbufs, &db_marker);
+		} else {
+			mutex_exit(&db->db_mtx);
+			db_next = AVL_NEXT(&dn->dn_dbufs, db);
 		}
-		/*
-		 * NB: we need to drop dn_dbufs_mtx between passes so
-		 * that any DB_EVICTING dbufs can make progress.
-		 * Ideally, we would have some cv we could wait on, but
-		 * since we don't, just wait a bit to give the other
-		 * thread a chance to run.
-		 */
-		mutex_exit(&dn->dn_dbufs_mtx);
-		if (evicting)
-			delay(1);
-		pass++;
-		ASSERT(pass < 100); /* sanity check */
-	} while (progress);
+	}
+	mutex_exit(&dn->dn_dbufs_mtx);
 
 	rw_enter(&dn->dn_struct_rwlock, RW_WRITER);
 	if (dn->dn_bonus && refcount_is_zero(&dn->dn_bonus->db_holds)) {
@@ -497,7 +486,6 @@ dnode_sync_free(dnode_t *dn, dmu_tx_t *tx)
 	dnode_undirty_dbufs(&dn->dn_dirty_records[txgoff]);
 	dnode_evict_dbufs(dn);
 	ASSERT(avl_is_empty(&dn->dn_dbufs));
-	ASSERT3P(dn->dn_bonus, ==, NULL);
 
 	/*
 	 * XXX - It would be nice to assert this, but we may still
diff --git a/uts/common/fs/zfs/dsl_bookmark.c b/uts/common/fs/zfs/dsl_bookmark.c
index 5fb7f96600d..bc08d1c10e8 100644
--- a/uts/common/fs/zfs/dsl_bookmark.c
+++ b/uts/common/fs/zfs/dsl_bookmark.c
@@ -120,7 +120,7 @@ dsl_bookmark_create_check_impl(dsl_dataset_t *snapds, const char *bookmark_name,
 	int error;
 	zfs_bookmark_phys_t bmark_phys;
 
-	if (!dsl_dataset_is_snapshot(snapds))
+	if (!snapds->ds_is_snapshot)
 		return (SET_ERROR(EINVAL));
 
 	error = dsl_bookmark_hold_ds(dp, bookmark_name,
diff --git a/uts/common/fs/zfs/dsl_dataset.c b/uts/common/fs/zfs/dsl_dataset.c
index 5baf5c3c0a1..d3ca43af47d 100644
--- a/uts/common/fs/zfs/dsl_dataset.c
+++ b/uts/common/fs/zfs/dsl_dataset.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
  * Copyright (c) 2014 RackTop Systems.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/dmu_objset.h>
@@ -71,7 +72,6 @@ int zfs_max_recordsize = 1 * 1024 * 1024;
 #define	DS_REF_MAX	(1ULL << 62)
 
 extern inline dsl_dataset_phys_t *dsl_dataset_phys(dsl_dataset_t *ds);
-extern inline boolean_t dsl_dataset_is_snapshot(dsl_dataset_t *ds);
 
 /*
  * Figure out how much of this delta should be propogated to the dsl_dir
@@ -155,7 +155,7 @@ dsl_dataset_block_kill(dsl_dataset_t *ds, const blkptr_t *bp, dmu_tx_t *tx,
 	}
 	ASSERT3P(tx->tx_pool, ==, ds->ds_dir->dd_pool);
 
-	ASSERT(!dsl_dataset_is_snapshot(ds));
+	ASSERT(!ds->ds_is_snapshot);
 	dmu_buf_will_dirty(ds->ds_dbuf, tx);
 
 	if (bp->blk_birth > dsl_dataset_phys(ds)->ds_prev_snap_txg) {
@@ -253,14 +253,15 @@ dsl_dataset_block_freeable(dsl_dataset_t *ds, const blkptr_t *bp,
 	return (B_TRUE);
 }
 
-/* ARGSUSED */
 static void
-dsl_dataset_evict(dmu_buf_t *db, void *dsv)
+dsl_dataset_evict(void *dbu)
 {
-	dsl_dataset_t *ds = dsv;
+	dsl_dataset_t *ds = dbu;
 
 	ASSERT(ds->ds_owner == NULL);
 
+	ds->ds_dbuf = NULL;
+
 	unique_remove(ds->ds_fsid_guid);
 
 	if (ds->ds_objset != NULL)
@@ -272,10 +273,10 @@ dsl_dataset_evict(dmu_buf_t *db, void *dsv)
 	}
 
 	bplist_destroy(&ds->ds_pending_deadlist);
-	if (dsl_dataset_phys(ds)->ds_deadlist_obj != 0)
+	if (ds->ds_deadlist.dl_os != NULL)
 		dsl_deadlist_close(&ds->ds_deadlist);
 	if (ds->ds_dir)
-		dsl_dir_rele(ds->ds_dir, ds);
+		dsl_dir_async_rele(ds->ds_dir, ds);
 
 	ASSERT(!list_link_active(&ds->ds_synced_link));
 
@@ -389,6 +390,7 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 		ds = kmem_zalloc(sizeof (dsl_dataset_t), KM_SLEEP);
 		ds->ds_dbuf = dbuf;
 		ds->ds_object = dsobj;
+		ds->ds_is_snapshot = dsl_dataset_phys(ds)->ds_num_children != 0;
 
 		mutex_init(&ds->ds_lock, NULL, MUTEX_DEFAULT, NULL);
 		mutex_init(&ds->ds_opening_lock, NULL, MUTEX_DEFAULT, NULL);
@@ -427,7 +429,7 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 			return (err);
 		}
 
-		if (!dsl_dataset_is_snapshot(ds)) {
+		if (!ds->ds_is_snapshot) {
 			ds->ds_snapname[0] = '\0';
 			if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0) {
 				err = dsl_dataset_hold_obj(dp,
@@ -454,7 +456,7 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 			}
 		}
 
-		if (err == 0 && !dsl_dataset_is_snapshot(ds)) {
+		if (err == 0 && !ds->ds_is_snapshot) {
 			err = dsl_prop_get_int_ds(ds,
 			    zfs_prop_to_name(ZFS_PROP_REFRESERVATION),
 			    &ds->ds_reserved);
@@ -467,8 +469,11 @@ dsl_dataset_hold_obj(dsl_pool_t *dp, uint64_t dsobj, void *tag,
 			ds->ds_reserved = ds->ds_quota = 0;
 		}
 
-		if (err != 0 || (winner = dmu_buf_set_user_ie(dbuf, ds,
-		    dsl_dataset_evict)) != NULL) {
+		dmu_buf_init_user(&ds->ds_dbu, dsl_dataset_evict, &ds->ds_dbuf);
+		if (err == 0)
+			winner = dmu_buf_set_user_ie(dbuf, &ds->ds_dbu);
+
+		if (err != 0 || winner != NULL) {
 			bplist_destroy(&ds->ds_pending_deadlist);
 			dsl_deadlist_close(&ds->ds_deadlist);
 			if (ds->ds_prev)
@@ -848,7 +853,7 @@ dsl_dataset_recalc_head_uniq(dsl_dataset_t *ds)
 	uint64_t mrs_used;
 	uint64_t dlused, dlcomp, dluncomp;
 
-	ASSERT(!dsl_dataset_is_snapshot(ds));
+	ASSERT(!ds->ds_is_snapshot);
 
 	if (dsl_dataset_phys(ds)->ds_prev_snap_obj != 0)
 		mrs_used = dsl_dataset_phys(ds->ds_prev)->ds_referenced_bytes;
@@ -1591,7 +1596,7 @@ dsl_dataset_stats(dsl_dataset_t *ds, nvlist_t *nv)
 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_LOGICALREFERENCED,
 	    dsl_dataset_phys(ds)->ds_uncompressed_bytes);
 
-	if (dsl_dataset_is_snapshot(ds)) {
+	if (ds->ds_is_snapshot) {
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_COMPRESSRATIO, ratio);
 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_USED,
 		    dsl_dataset_phys(ds)->ds_unique_bytes);
@@ -1659,7 +1664,7 @@ dsl_dataset_fast_stat(dsl_dataset_t *ds, dmu_objset_stats_t *stat)
 	    dsl_dataset_phys(ds)->ds_flags & DS_FLAG_INCONSISTENT;
 	stat->dds_guid = dsl_dataset_phys(ds)->ds_guid;
 	stat->dds_origin[0] = '\0';
-	if (dsl_dataset_is_snapshot(ds)) {
+	if (ds->ds_is_snapshot) {
 		stat->dds_is_snapshot = B_TRUE;
 		stat->dds_num_clones =
 		    dsl_dataset_phys(ds)->ds_num_children - 1;
@@ -1919,7 +1924,7 @@ dsl_dataset_rollback_check(void *arg, dmu_tx_t *tx)
 		return (error);
 
 	/* must not be a snapshot */
-	if (dsl_dataset_is_snapshot(ds)) {
+	if (ds->ds_is_snapshot) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
@@ -2491,7 +2496,7 @@ promote_hold(dsl_dataset_promote_arg_t *ddpa, dsl_pool_t *dp, void *tag)
 		return (error);
 	dd = ddpa->ddpa_clone->ds_dir;
 
-	if (dsl_dataset_is_snapshot(ddpa->ddpa_clone) ||
+	if (ddpa->ddpa_clone->ds_is_snapshot ||
 	    !dsl_dir_is_clone(dd)) {
 		dsl_dataset_rele(ddpa->ddpa_clone, tag);
 		return (SET_ERROR(EINVAL));
@@ -2583,8 +2588,8 @@ dsl_dataset_clone_swap_check_impl(dsl_dataset_t *clone,
 	int64_t unused_refres_delta;
 
 	/* they should both be heads */
-	if (dsl_dataset_is_snapshot(clone) ||
-	    dsl_dataset_is_snapshot(origin_head))
+	if (clone->ds_is_snapshot ||
+	    origin_head->ds_is_snapshot)
 		return (SET_ERROR(EINVAL));
 
 	/* if we are not forcing, the branch point should be just before them */
@@ -2863,7 +2868,7 @@ dsl_dataset_set_refquota_check(void *arg, dmu_tx_t *tx)
 	if (error != 0)
 		return (error);
 
-	if (dsl_dataset_is_snapshot(ds)) {
+	if (ds->ds_is_snapshot) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
@@ -2946,7 +2951,7 @@ dsl_dataset_set_refreservation_check(void *arg, dmu_tx_t *tx)
 	if (error != 0)
 		return (error);
 
-	if (dsl_dataset_is_snapshot(ds)) {
+	if (ds->ds_is_snapshot) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(EINVAL));
 	}
@@ -3160,8 +3165,8 @@ dsl_dataset_space_wouldfree(dsl_dataset_t *firstsnap,
 	uint64_t snapobj;
 	dsl_pool_t *dp = firstsnap->ds_dir->dd_pool;
 
-	ASSERT(dsl_dataset_is_snapshot(firstsnap));
-	ASSERT(dsl_dataset_is_snapshot(lastsnap));
+	ASSERT(firstsnap->ds_is_snapshot);
+	ASSERT(lastsnap->ds_is_snapshot);
 
 	/*
 	 * Check that the snapshots are in the same dsl_dir, and firstsnap
@@ -3286,12 +3291,12 @@ dsl_dataset_is_before(dsl_dataset_t *later, dsl_dataset_t *earlier,
 	boolean_t ret;
 
 	ASSERT(dsl_pool_config_held(dp));
-	ASSERT(dsl_dataset_is_snapshot(earlier) || earlier_txg != 0);
+	ASSERT(earlier->ds_is_snapshot || earlier_txg != 0);
 
 	if (earlier_txg == 0)
 		earlier_txg = dsl_dataset_phys(earlier)->ds_creation_txg;
 
-	if (dsl_dataset_is_snapshot(later) &&
+	if (later->ds_is_snapshot &&
 	    earlier_txg >= dsl_dataset_phys(later)->ds_creation_txg)
 		return (B_FALSE);
 
diff --git a/uts/common/fs/zfs/dsl_deadlist.c b/uts/common/fs/zfs/dsl_deadlist.c
index 4ac562bfdba..d26c6cd3579 100644
--- a/uts/common/fs/zfs/dsl_deadlist.c
+++ b/uts/common/fs/zfs/dsl_deadlist.c
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/dsl_dataset.h>
@@ -119,6 +120,8 @@ dsl_deadlist_close(dsl_deadlist_t *dl)
 	void *cookie = NULL;
 	dsl_deadlist_entry_t *dle;
 
+	dl->dl_os = NULL;
+
 	if (dl->dl_oldfmt) {
 		dl->dl_oldfmt = B_FALSE;
 		bpobj_close(&dl->dl_bpobj);
diff --git a/uts/common/fs/zfs/dsl_deleg.c b/uts/common/fs/zfs/dsl_deleg.c
index dc405ebe58b..32fe03803f2 100644
--- a/uts/common/fs/zfs/dsl_deleg.c
+++ b/uts/common/fs/zfs/dsl_deleg.c
@@ -561,7 +561,7 @@ dsl_deleg_access_impl(dsl_dataset_t *ds, const char *perm, cred_t *cr)
 	    SPA_VERSION_DELEGATED_PERMS)
 		return (SET_ERROR(EPERM));
 
-	if (dsl_dataset_is_snapshot(ds)) {
+	if (ds->ds_is_snapshot) {
 		/*
 		 * Snapshots are treated as descendents only,
 		 * local permissions do not apply.
diff --git a/uts/common/fs/zfs/dsl_destroy.c b/uts/common/fs/zfs/dsl_destroy.c
index a776e144f1d..7f9046957f9 100644
--- a/uts/common/fs/zfs/dsl_destroy.c
+++ b/uts/common/fs/zfs/dsl_destroy.c
@@ -51,7 +51,7 @@ typedef struct dmu_snapshots_destroy_arg {
 int
 dsl_destroy_snapshot_check_impl(dsl_dataset_t *ds, boolean_t defer)
 {
-	if (!dsl_dataset_is_snapshot(ds))
+	if (!ds->ds_is_snapshot)
 		return (SET_ERROR(EINVAL));
 
 	if (dsl_dataset_long_held(ds))
@@ -354,7 +354,7 @@ dsl_destroy_snapshot_sync_impl(dsl_dataset_t *ds, boolean_t defer, dmu_tx_t *tx)
 	dsl_dataset_remove_clones_key(ds,
 	    dsl_dataset_phys(ds)->ds_creation_txg, tx);
 
-	if (dsl_dataset_is_snapshot(ds_next)) {
+	if (ds_next->ds_is_snapshot) {
 		dsl_dataset_t *ds_nextnext;
 
 		/*
@@ -604,8 +604,8 @@ dsl_destroy_head_check_impl(dsl_dataset_t *ds, int expected_holds)
 	uint64_t count;
 	objset_t *mos;
 
-	ASSERT(!dsl_dataset_is_snapshot(ds));
-	if (dsl_dataset_is_snapshot(ds))
+	ASSERT(!ds->ds_is_snapshot);
+	if (ds->ds_is_snapshot)
 		return (SET_ERROR(EINVAL));
 
 	if (refcount_count(&ds->ds_longholds) != expected_holds)
diff --git a/uts/common/fs/zfs/dsl_dir.c b/uts/common/fs/zfs/dsl_dir.c
index f4958416834..ee69cc8dbff 100644
--- a/uts/common/fs/zfs/dsl_dir.c
+++ b/uts/common/fs/zfs/dsl_dir.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 Martin Matuska. All rights reserved.
  * Copyright (c) 2014 Joyent, Inc. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/dmu.h>
@@ -125,14 +126,15 @@ extern inline dsl_dir_phys_t *dsl_dir_phys(dsl_dir_t *dd);
 
 static uint64_t dsl_dir_space_towrite(dsl_dir_t *dd);
 
-/* ARGSUSED */
 static void
-dsl_dir_evict(dmu_buf_t *db, void *arg)
+dsl_dir_evict(void *dbu)
 {
-	dsl_dir_t *dd = arg;
+	dsl_dir_t *dd = dbu;
 	dsl_pool_t *dp = dd->dd_pool;
 	int t;
 
+	dd->dd_dbuf = NULL;
+
 	for (t = 0; t < TXG_SIZE; t++) {
 		ASSERT(!txg_list_member(&dp->dp_dirty_dirs, dd, t));
 		ASSERT(dd->dd_tempreserved[t] == 0);
@@ -140,9 +142,9 @@ dsl_dir_evict(dmu_buf_t *db, void *arg)
 	}
 
 	if (dd->dd_parent)
-		dsl_dir_rele(dd->dd_parent, dd);
+		dsl_dir_async_rele(dd->dd_parent, dd);
 
-	spa_close(dd->dd_pool->dp_spa, dd);
+	spa_async_close(dd->dd_pool->dp_spa, dd);
 
 	/*
 	 * The props callback list should have been cleaned up by
@@ -238,8 +240,9 @@ dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
 			dmu_buf_rele(origin_bonus, FTAG);
 		}
 
-		winner = dmu_buf_set_user_ie(dbuf, dd, dsl_dir_evict);
-		if (winner) {
+		dmu_buf_init_user(&dd->dd_dbu, dsl_dir_evict, &dd->dd_dbuf);
+		winner = dmu_buf_set_user_ie(dbuf, &dd->dd_dbu);
+		if (winner != NULL) {
 			if (dd->dd_parent)
 				dsl_dir_rele(dd->dd_parent, dd);
 			mutex_destroy(&dd->dd_lock);
@@ -283,6 +286,21 @@ dsl_dir_rele(dsl_dir_t *dd, void *tag)
 	dmu_buf_rele(dd->dd_dbuf, tag);
 }
 
+/*
+ * Remove a reference to the given dsl dir that is being asynchronously
+ * released.  Async releases occur from a taskq performing eviction of
+ * dsl datasets and dirs.  This process is identical to a normal release
+ * with the exception of using the async API for releasing the reference on
+ * the spa.
+ */
+void
+dsl_dir_async_rele(dsl_dir_t *dd, void *tag)
+{
+	dprintf_dd(dd, "%s\n", "");
+	spa_async_close(dd->dd_pool->dp_spa, tag);
+	dmu_buf_rele(dd->dd_dbuf, tag);
+}
+
 /* buf must be long enough (MAXNAMELEN + strlen(MOS_DIR_NAME) + 1 should do) */
 void
 dsl_dir_name(dsl_dir_t *dd, char *buf)
@@ -413,7 +431,7 @@ dsl_dir_hold(dsl_pool_t *dp, const char *name, void *tag,
 	}
 
 	while (next != NULL) {
-		dsl_dir_t *child_ds;
+		dsl_dir_t *child_dd;
 		err = getcomponent(next, buf, &nextnext);
 		if (err != 0)
 			break;
@@ -432,11 +450,11 @@ dsl_dir_hold(dsl_pool_t *dp, const char *name, void *tag,
 			break;
 		}
 
-		err = dsl_dir_hold_obj(dp, ddobj, buf, tag, &child_ds);
+		err = dsl_dir_hold_obj(dp, ddobj, buf, tag, &child_dd);
 		if (err != 0)
 			break;
 		dsl_dir_rele(dd, tag);
-		dd = child_ds;
+		dd = child_dd;
 		next = nextnext;
 	}
 
diff --git a/uts/common/fs/zfs/dsl_pool.c b/uts/common/fs/zfs/dsl_pool.c
index 39a797c2795..30b7b4c8f02 100644
--- a/uts/common/fs/zfs/dsl_pool.c
+++ b/uts/common/fs/zfs/dsl_pool.c
@@ -22,6 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/dsl_pool.h>
@@ -317,6 +318,8 @@ dsl_pool_close(dsl_pool_t *dp)
 	arc_flush(dp->dp_spa);
 	txg_fini(dp);
 	dsl_scan_fini(dp);
+	dmu_buf_user_evict_wait();
+
 	rrw_destroy(&dp->dp_config_rwlock);
 	mutex_destroy(&dp->dp_lock);
 	taskq_destroy(dp->dp_vnrele_taskq);
diff --git a/uts/common/fs/zfs/dsl_prop.c b/uts/common/fs/zfs/dsl_prop.c
index 4e93e43f969..e795d735c6f 100644
--- a/uts/common/fs/zfs/dsl_prop.c
+++ b/uts/common/fs/zfs/dsl_prop.c
@@ -163,19 +163,17 @@ dsl_prop_get_ds(dsl_dataset_t *ds, const char *propname,
 {
 	zfs_prop_t prop = zfs_name_to_prop(propname);
 	boolean_t inheritable;
-	boolean_t snapshot;
 	uint64_t zapobj;
 
 	ASSERT(dsl_pool_config_held(ds->ds_dir->dd_pool));
 	inheritable = (prop == ZPROP_INVAL || zfs_prop_inheritable(prop));
-	snapshot = dsl_dataset_is_snapshot(ds);
 	zapobj = dsl_dataset_phys(ds)->ds_props_obj;
 
 	if (zapobj != 0) {
 		objset_t *mos = ds->ds_dir->dd_pool->dp_meta_objset;
 		int err;
 
-		ASSERT(snapshot);
+		ASSERT(ds->ds_is_snapshot);
 
 		/* Check for a local value. */
 		err = zap_lookup(mos, zapobj, propname, intsz, numints, buf);
@@ -215,7 +213,7 @@ dsl_prop_get_ds(dsl_dataset_t *ds, const char *propname,
 	}
 
 	return (dsl_prop_get_dd(ds->ds_dir, propname,
-	    intsz, numints, buf, setpoint, snapshot));
+	    intsz, numints, buf, setpoint, ds->ds_is_snapshot));
 }
 
 /*
@@ -545,7 +543,7 @@ dsl_prop_set_sync_impl(dsl_dataset_t *ds, const char *propname,
 
 	isint = (dodefault(propname, 8, 1, &intval) == 0);
 
-	if (dsl_dataset_is_snapshot(ds)) {
+	if (ds->ds_is_snapshot) {
 		ASSERT(version >= SPA_VERSION_SNAP_PROPS);
 		if (dsl_dataset_phys(ds)->ds_props_obj == 0) {
 			dmu_buf_will_dirty(ds->ds_dbuf, tx);
@@ -642,7 +640,7 @@ dsl_prop_set_sync_impl(dsl_dataset_t *ds, const char *propname,
 	if (isint) {
 		VERIFY0(dsl_prop_get_int_ds(ds, propname, &intval));
 
-		if (dsl_dataset_is_snapshot(ds)) {
+		if (ds->ds_is_snapshot) {
 			dsl_prop_cb_record_t *cbr;
 			/*
 			 * It's a snapshot; nothing can inherit this
@@ -760,7 +758,7 @@ dsl_props_set_check(void *arg, dmu_tx_t *tx)
 		}
 	}
 
-	if (dsl_dataset_is_snapshot(ds) && version < SPA_VERSION_SNAP_PROPS) {
+	if (ds->ds_is_snapshot && version < SPA_VERSION_SNAP_PROPS) {
 		dsl_dataset_rele(ds, FTAG);
 		return (SET_ERROR(ENOTSUP));
 	}
@@ -983,7 +981,7 @@ dsl_prop_get_all_ds(dsl_dataset_t *ds, nvlist_t **nvp,
 
 	VERIFY(nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP) == 0);
 
-	if (dsl_dataset_is_snapshot(ds))
+	if (ds->ds_is_snapshot)
 		flags |= DSL_PROP_GET_SNAPSHOT;
 
 	ASSERT(dsl_pool_config_held(dp));
diff --git a/uts/common/fs/zfs/dsl_scan.c b/uts/common/fs/zfs/dsl_scan.c
index a372b2cdc0c..0894ebd4545 100644
--- a/uts/common/fs/zfs/dsl_scan.c
+++ b/uts/common/fs/zfs/dsl_scan.c
@@ -375,7 +375,7 @@ static uint64_t
 dsl_scan_ds_maxtxg(dsl_dataset_t *ds)
 {
 	uint64_t smt = ds->ds_dir->dd_pool->dp_scan->scn_phys.scn_max_txg;
-	if (dsl_dataset_is_snapshot(ds))
+	if (ds->ds_is_snapshot)
 		return (MIN(smt, dsl_dataset_phys(ds)->ds_creation_txg));
 	return (smt);
 }
@@ -811,7 +811,7 @@ dsl_scan_ds_destroyed(dsl_dataset_t *ds, dmu_tx_t *tx)
 		return;
 
 	if (scn->scn_phys.scn_bookmark.zb_objset == ds->ds_object) {
-		if (dsl_dataset_is_snapshot(ds)) {
+		if (ds->ds_is_snapshot) {
 			/* Note, scn_cur_{min,max}_txg stays the same. */
 			scn->scn_phys.scn_bookmark.zb_objset =
 			    dsl_dataset_phys(ds)->ds_next_snap_obj;
@@ -833,7 +833,7 @@ dsl_scan_ds_destroyed(dsl_dataset_t *ds, dmu_tx_t *tx)
 		ASSERT3U(dsl_dataset_phys(ds)->ds_num_children, <=, 1);
 		VERIFY3U(0, ==, zap_remove_int(dp->dp_meta_objset,
 		    scn->scn_phys.scn_queue_obj, ds->ds_object, tx));
-		if (dsl_dataset_is_snapshot(ds)) {
+		if (ds->ds_is_snapshot) {
 			/*
 			 * We keep the same mintxg; it could be >
 			 * ds_creation_txg if the previous snapshot was
@@ -1017,7 +1017,7 @@ dsl_scan_visitds(dsl_scan_t *scn, uint64_t dsobj, dmu_tx_t *tx)
 	 * ZIL here, rather than in scan_recurse(), because the regular
 	 * snapshot block-sharing rules don't apply to it.
 	 */
-	if (DSL_SCAN_IS_SCRUB_RESILVER(scn) && !dsl_dataset_is_snapshot(ds))
+	if (DSL_SCAN_IS_SCRUB_RESILVER(scn) && !ds->ds_is_snapshot)
 		dsl_scan_zil(dp, &os->os_zil_header);
 
 	/*
diff --git a/uts/common/fs/zfs/dsl_userhold.c b/uts/common/fs/zfs/dsl_userhold.c
index 0985ec91fd5..a67db5f19e3 100644
--- a/uts/common/fs/zfs/dsl_userhold.c
+++ b/uts/common/fs/zfs/dsl_userhold.c
@@ -352,7 +352,7 @@ dsl_dataset_user_release_check_one(dsl_dataset_user_release_arg_t *ddura,
 	objset_t *mos;
 	int numholds;
 
-	if (!dsl_dataset_is_snapshot(ds))
+	if (!ds->ds_is_snapshot)
 		return (SET_ERROR(EINVAL));
 
 	if (nvlist_empty(holds))
diff --git a/uts/common/fs/zfs/sa.c b/uts/common/fs/zfs/sa.c
index 27d8513541a..2e3156c00a4 100644
--- a/uts/common/fs/zfs/sa.c
+++ b/uts/common/fs/zfs/sa.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  * Portions Copyright 2011 iXsystems, Inc
  * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -210,6 +211,7 @@ sa_cache_constructor(void *buf, void *unused, int kmflag)
 {
 	sa_handle_t *hdl = buf;
 
+	hdl->sa_dbu.dbu_evict_func = NULL;
 	hdl->sa_bonus_tab = NULL;
 	hdl->sa_spill_tab = NULL;
 	hdl->sa_os = NULL;
@@ -225,6 +227,7 @@ static void
 sa_cache_destructor(void *buf, void *unused)
 {
 	sa_handle_t *hdl = buf;
+	hdl->sa_dbu.dbu_evict_func = NULL;
 	mutex_destroy(&hdl->sa_lock);
 }
 
@@ -1301,10 +1304,10 @@ sa_build_index(sa_handle_t *hdl, sa_buf_type_t buftype)
 }
 
 /*ARGSUSED*/
-void
-sa_evict(dmu_buf_t *db, void *sap)
+static void
+sa_evict(void *dbu)
 {
-	panic("evicting sa dbuf %p\n", (void *)db);
+	panic("evicting sa dbuf\n");
 }
 
 static void
@@ -1343,9 +1346,10 @@ sa_idx_tab_hold(objset_t *os, sa_idx_tab_t *idx_tab)
 void
 sa_handle_destroy(sa_handle_t *hdl)
 {
+	dmu_buf_t *db = hdl->sa_bonus;
+
 	mutex_enter(&hdl->sa_lock);
-	(void) dmu_buf_update_user((dmu_buf_t *)hdl->sa_bonus, hdl,
-	    NULL, NULL);
+	(void) dmu_buf_remove_user(db, &hdl->sa_dbu);
 
 	if (hdl->sa_bonus_tab) {
 		sa_idx_tab_rele(hdl->sa_os, hdl->sa_bonus_tab);
@@ -1371,7 +1375,7 @@ sa_handle_get_from_db(objset_t *os, dmu_buf_t *db, void *userp,
 {
 	int error = 0;
 	dmu_object_info_t doi;
-	sa_handle_t *handle;
+	sa_handle_t *handle = NULL;
 
 #ifdef ZFS_DEBUG
 	dmu_object_info_from_db(db, &doi);
@@ -1381,9 +1385,12 @@ sa_handle_get_from_db(objset_t *os, dmu_buf_t *db, void *userp,
 	/* find handle, if it exists */
 	/* if one doesn't exist then create a new one, and initialize it */
 
-	handle = (hdl_type == SA_HDL_SHARED) ? dmu_buf_get_user(db) : NULL;
+	if (hdl_type == SA_HDL_SHARED)
+		handle = dmu_buf_get_user(db);
+
 	if (handle == NULL) {
-		sa_handle_t *newhandle;
+		sa_handle_t *winner = NULL;
+
 		handle = kmem_cache_alloc(sa_cache, KM_SLEEP);
 		handle->sa_userp = userp;
 		handle->sa_bonus = db;
@@ -1391,12 +1398,15 @@ sa_handle_get_from_db(objset_t *os, dmu_buf_t *db, void *userp,
 		handle->sa_spill = NULL;
 
 		error = sa_build_index(handle, SA_BONUS);
-		newhandle = (hdl_type == SA_HDL_SHARED) ?
-		    dmu_buf_set_user_ie(db, handle, sa_evict) : NULL;
 
-		if (newhandle != NULL) {
+		if (hdl_type == SA_HDL_SHARED) {
+			dmu_buf_init_user(&handle->sa_dbu, sa_evict, NULL);
+			winner = dmu_buf_set_user_ie(db, &handle->sa_dbu);
+		}
+
+		if (winner != NULL) {
 			kmem_cache_free(sa_cache, handle);
-			handle = newhandle;
+			handle = winner;
 		}
 	}
 	*handlepp = handle;
@@ -1906,14 +1916,6 @@ sa_object_size(sa_handle_t *hdl, uint32_t *blksize, u_longlong_t *nblocks)
 	    blksize, nblocks);
 }
 
-void
-sa_update_user(sa_handle_t *newhdl, sa_handle_t *oldhdl)
-{
-	(void) dmu_buf_update_user((dmu_buf_t *)newhdl->sa_bonus,
-	    oldhdl, newhdl, sa_evict);
-	oldhdl->sa_bonus = NULL;
-}
-
 void
 sa_set_userp(sa_handle_t *hdl, void *ptr)
 {
diff --git a/uts/common/fs/zfs/spa.c b/uts/common/fs/zfs/spa.c
index d0408a58023..7947d25f48d 100644
--- a/uts/common/fs/zfs/spa.c
+++ b/uts/common/fs/zfs/spa.c
@@ -23,6 +23,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013, 2014, Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 /*
@@ -1088,6 +1089,8 @@ spa_activate(spa_t *spa, int mode)
 
 	list_create(&spa->spa_config_dirty_list, sizeof (vdev_t),
 	    offsetof(vdev_t, vdev_config_dirty_node));
+	list_create(&spa->spa_evicting_os_list, sizeof (objset_t),
+	    offsetof(objset_t, os_evicting_node));
 	list_create(&spa->spa_state_dirty_list, sizeof (vdev_t),
 	    offsetof(vdev_t, vdev_state_dirty_node));
 
@@ -1114,9 +1117,12 @@ spa_deactivate(spa_t *spa)
 	ASSERT(spa->spa_async_zio_root == NULL);
 	ASSERT(spa->spa_state != POOL_STATE_UNINITIALIZED);
 
+	spa_evicting_os_wait(spa);
+
 	txg_list_destroy(&spa->spa_vdev_txg_list);
 
 	list_destroy(&spa->spa_config_dirty_list);
+	list_destroy(&spa->spa_evicting_os_list);
 	list_destroy(&spa->spa_state_dirty_list);
 
 	for (int t = 0; t < ZIO_TYPES; t++) {
@@ -2107,6 +2113,11 @@ spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type,
 		    mosconfig, &ereport);
 	}
 
+	/*
+	 * Don't count references from objsets that are already closed
+	 * and are making their way through the eviction process.
+	 */
+	spa_evicting_os_wait(spa);
 	spa->spa_minref = refcount_count(&spa->spa_refcount);
 	if (error) {
 		if (error != EEXIST) {
@@ -3675,6 +3686,11 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props,
 
 	spa_history_log_version(spa, "create");
 
+	/*
+	 * Don't count references from objsets that are already closed
+	 * and are making their way through the eviction process.
+	 */
+	spa_evicting_os_wait(spa);
 	spa->spa_minref = refcount_count(&spa->spa_refcount);
 
 	mutex_exit(&spa_namespace_lock);
@@ -4207,6 +4223,7 @@ spa_export_common(char *pool, int new_state, nvlist_t **oldconfig,
 		 * have to force it to sync before checking spa_refcnt.
 		 */
 		txg_wait_synced(spa->spa_dsl_pool, 0);
+		spa_evicting_os_wait(spa);
 
 		/*
 		 * A pool cannot be exported or destroyed if there are active
diff --git a/uts/common/fs/zfs/spa_misc.c b/uts/common/fs/zfs/spa_misc.c
index 1729ba0503b..56ad27cf9ae 100644
--- a/uts/common/fs/zfs/spa_misc.c
+++ b/uts/common/fs/zfs/spa_misc.c
@@ -22,6 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -551,6 +552,7 @@ spa_add(const char *name, nvlist_t *config, const char *altroot)
 	mutex_init(&spa->spa_async_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_errlist_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_errlog_lock, NULL, MUTEX_DEFAULT, NULL);
+	mutex_init(&spa->spa_evicting_os_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_history_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_proc_lock, NULL, MUTEX_DEFAULT, NULL);
 	mutex_init(&spa->spa_props_lock, NULL, MUTEX_DEFAULT, NULL);
@@ -560,6 +562,7 @@ spa_add(const char *name, nvlist_t *config, const char *altroot)
 	mutex_init(&spa->spa_iokstat_lock, NULL, MUTEX_DEFAULT, NULL);
 
 	cv_init(&spa->spa_async_cv, NULL, CV_DEFAULT, NULL);
+	cv_init(&spa->spa_evicting_os_cv, NULL, CV_DEFAULT, NULL);
 	cv_init(&spa->spa_proc_cv, NULL, CV_DEFAULT, NULL);
 	cv_init(&spa->spa_scrub_io_cv, NULL, CV_DEFAULT, NULL);
 	cv_init(&spa->spa_suspend_cv, NULL, CV_DEFAULT, NULL);
@@ -669,6 +672,7 @@ spa_remove(spa_t *spa)
 
 	ASSERT(MUTEX_HELD(&spa_namespace_lock));
 	ASSERT(spa->spa_state == POOL_STATE_UNINITIALIZED);
+	ASSERT3U(refcount_count(&spa->spa_refcount), ==, 0);
 
 	nvlist_free(spa->spa_config_splitting);
 
@@ -710,6 +714,7 @@ spa_remove(spa_t *spa)
 		bplist_destroy(&spa->spa_free_bplist[t]);
 
 	cv_destroy(&spa->spa_async_cv);
+	cv_destroy(&spa->spa_evicting_os_cv);
 	cv_destroy(&spa->spa_proc_cv);
 	cv_destroy(&spa->spa_scrub_io_cv);
 	cv_destroy(&spa->spa_suspend_cv);
@@ -717,6 +722,7 @@ spa_remove(spa_t *spa)
 	mutex_destroy(&spa->spa_async_lock);
 	mutex_destroy(&spa->spa_errlist_lock);
 	mutex_destroy(&spa->spa_errlog_lock);
+	mutex_destroy(&spa->spa_evicting_os_lock);
 	mutex_destroy(&spa->spa_history_lock);
 	mutex_destroy(&spa->spa_proc_lock);
 	mutex_destroy(&spa->spa_props_lock);
@@ -773,6 +779,20 @@ spa_close(spa_t *spa, void *tag)
 	(void) refcount_remove(&spa->spa_refcount, tag);
 }
 
+/*
+ * Remove a reference to the given spa_t held by a dsl dir that is
+ * being asynchronously released.  Async releases occur from a taskq
+ * performing eviction of dsl datasets and dirs.  The namespace lock
+ * isn't held and the hold by the object being evicted may contribute to
+ * spa_minref (e.g. dataset or directory released during pool export),
+ * so the asserts in spa_close() do not apply.
+ */
+void
+spa_async_close(spa_t *spa, void *tag)
+{
+	(void) refcount_remove(&spa->spa_refcount, tag);
+}
+
 /*
  * Check to see if the spa refcount is zero.  Must be called with
  * spa_namespace_lock held.  We really compare against spa_minref, which is the
@@ -1669,6 +1689,34 @@ spa_log_class(spa_t *spa)
 	return (spa->spa_log_class);
 }
 
+void
+spa_evicting_os_register(spa_t *spa, objset_t *os)
+{
+	mutex_enter(&spa->spa_evicting_os_lock);
+	list_insert_head(&spa->spa_evicting_os_list, os);
+	mutex_exit(&spa->spa_evicting_os_lock);
+}
+
+void
+spa_evicting_os_deregister(spa_t *spa, objset_t *os)
+{
+	mutex_enter(&spa->spa_evicting_os_lock);
+	list_remove(&spa->spa_evicting_os_list, os);
+	cv_broadcast(&spa->spa_evicting_os_cv);
+	mutex_exit(&spa->spa_evicting_os_lock);
+}
+
+void
+spa_evicting_os_wait(spa_t *spa)
+{
+	mutex_enter(&spa->spa_evicting_os_lock);
+	while (!list_is_empty(&spa->spa_evicting_os_list))
+		cv_wait(&spa->spa_evicting_os_cv, &spa->spa_evicting_os_lock);
+	mutex_exit(&spa->spa_evicting_os_lock);
+
+	dmu_buf_user_evict_wait();
+}
+
 int
 spa_max_replication(spa_t *spa)
 {
diff --git a/uts/common/fs/zfs/sys/dbuf.h b/uts/common/fs/zfs/sys/dbuf.h
index 8be8ed6bc85..6da5dccc05f 100644
--- a/uts/common/fs/zfs/sys/dbuf.h
+++ b/uts/common/fs/zfs/sys/dbuf.h
@@ -22,6 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_DBUF_H
@@ -226,9 +227,8 @@ typedef struct dmu_buf_impl {
 
 	/* Data which is unique to data (leaf) blocks: */
 
-	/* stuff we store for the user (see dmu_buf_set_user) */
-	void *db_user_ptr;
-	dmu_buf_evict_func_t *db_evict_func;
+	/* User callback information. */
+	dmu_buf_user_t *db_user;
 
 	uint8_t db_immediate_evict;
 	uint8_t db_freed_in_flight;
diff --git a/uts/common/fs/zfs/sys/dmu.h b/uts/common/fs/zfs/sys/dmu.h
index 24473daa6d2..9b8b9ddc201 100644
--- a/uts/common/fs/zfs/sys/dmu.h
+++ b/uts/common/fs/zfs/sys/dmu.h
@@ -26,6 +26,7 @@
  * Copyright (c) 2012, Joyent, Inc. All rights reserved.
  * Copyright 2013 DEY Storage Systems, Inc.
  * Copyright 2014 HybridCluster. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -41,11 +42,9 @@
  * dmu_spa.h.
  */
 
+#include <sys/zfs_context.h>
 #include <sys/inttypes.h>
-#include <sys/types.h>
-#include <sys/param.h>
 #include <sys/cred.h>
-#include <sys/time.h>
 #include <sys/fs/zfs.h>
 
 #ifdef	__cplusplus
@@ -291,8 +290,6 @@ typedef struct dmu_buf {
 	void *db_data;			/* data in buffer */
 } dmu_buf_t;
 
-typedef void dmu_buf_evict_func_t(struct dmu_buf *db, void *user_ptr);
-
 /*
  * The names of zap entries in the DIRECTORY_OBJECT of the MOS.
  */
@@ -478,36 +475,126 @@ int dmu_buf_hold_array_by_bonus(dmu_buf_t *db, uint64_t offset,
     uint64_t length, int read, void *tag, int *numbufsp, dmu_buf_t ***dbpp);
 void dmu_buf_rele_array(dmu_buf_t **, int numbufs, void *tag);
 
-/*
- * Returns NULL on success, or the existing user ptr if it's already
- * been set.
- *
- * user_ptr is for use by the user and can be obtained via dmu_buf_get_user().
- *
- * If non-NULL, pageout func will be called when this buffer is being
- * excised from the cache, so that you can clean up the data structure
- * pointed to by user_ptr.
- *
- * dmu_evict_user() will call the pageout func for all buffers in a
- * objset with a given pageout func.
- */
-void *dmu_buf_set_user(dmu_buf_t *db, void *user_ptr,
-    dmu_buf_evict_func_t *pageout_func);
-/*
- * set_user_ie is the same as set_user, but request immediate eviction
- * when hold count goes to zero.
- */
-void *dmu_buf_set_user_ie(dmu_buf_t *db, void *user_ptr,
-    dmu_buf_evict_func_t *pageout_func);
-void *dmu_buf_update_user(dmu_buf_t *db_fake, void *old_user_ptr,
-    void *user_ptr, dmu_buf_evict_func_t *pageout_func);
-void dmu_evict_user(objset_t *os, dmu_buf_evict_func_t *func);
+typedef void dmu_buf_evict_func_t(void *user_ptr);
 
 /*
- * Returns the user_ptr set with dmu_buf_set_user(), or NULL if not set.
+ * A DMU buffer user object may be associated with a dbuf for the
+ * duration of its lifetime.  This allows the user of a dbuf (client)
+ * to attach private data to a dbuf (e.g. in-core only data such as a
+ * dnode_children_t, zap_t, or zap_leaf_t) and be optionally notified
+ * when that dbuf has been evicted.  Clients typically respond to the
+ * eviction notification by freeing their private data, thus ensuring
+ * the same lifetime for both dbuf and private data.
+ *
+ * The mapping from a dmu_buf_user_t to any client private data is the
+ * client's responsibility.  All current consumers of the API with private
+ * data embed a dmu_buf_user_t as the first member of the structure for
+ * their private data.  This allows conversions between the two types
+ * with a simple cast.  Since the DMU buf user API never needs access
+ * to the private data, other strategies can be employed if necessary
+ * or convenient for the client (e.g. using container_of() to do the
+ * conversion for private data that cannot have the dmu_buf_user_t as
+ * its first member).
+ *
+ * Eviction callbacks are executed without the dbuf mutex held or any
+ * other type of mechanism to guarantee that the dbuf is still available.
+ * For this reason, users must assume the dbuf has already been freed
+ * and not reference the dbuf from the callback context.
+ *
+ * Users requesting "immediate eviction" are notified as soon as the dbuf
+ * is only referenced by dirty records (dirties == holds).  Otherwise the
+ * notification occurs after eviction processing for the dbuf begins.
+ */
+typedef struct dmu_buf_user {
+	/*
+	 * Asynchronous user eviction callback state.
+	 */
+	taskq_ent_t	dbu_tqent;
+
+	/* This instance's eviction function pointer. */
+	dmu_buf_evict_func_t *dbu_evict_func;
+#ifdef ZFS_DEBUG
+	/*
+	 * Pointer to user's dbuf pointer.  NULL for clients that do
+	 * not associate a dbuf with their user data.
+	 *
+	 * The dbuf pointer is cleared upon eviction so as to catch
+	 * use-after-evict bugs in clients.
+	 */
+	dmu_buf_t **dbu_clear_on_evict_dbufp;
+#endif
+} dmu_buf_user_t;
+
+/*
+ * Initialize the given dmu_buf_user_t instance with the eviction function
+ * evict_func, to be called when the user is evicted.
+ *
+ * NOTE: This function should only be called once on a given dmu_buf_user_t.
+ *       To allow enforcement of this, dbu must already be zeroed on entry.
+ */
+#ifdef __lint
+/* Very ugly, but it beats issuing suppression directives in many Makefiles. */
+extern void
+dmu_buf_init_user(dmu_buf_user_t *dbu, dmu_buf_evict_func_t *evict_func,
+    dmu_buf_t **clear_on_evict_dbufp);
+#else /* __lint */
+inline void
+dmu_buf_init_user(dmu_buf_user_t *dbu, dmu_buf_evict_func_t *evict_func,
+    dmu_buf_t **clear_on_evict_dbufp)
+{
+	ASSERT(dbu->dbu_evict_func == NULL);
+	ASSERT(evict_func != NULL);
+	dbu->dbu_evict_func = evict_func;
+#ifdef ZFS_DEBUG
+	dbu->dbu_clear_on_evict_dbufp = clear_on_evict_dbufp;
+#endif
+}
+#endif /* __lint */
+
+/*
+ * Attach user data to a dbuf and mark it for normal (when the dbuf's
+ * data is cleared or its reference count goes to zero) eviction processing.
+ *
+ * Returns NULL on success, or the existing user if another user currently
+ * owns the buffer.
+ */
+void *dmu_buf_set_user(dmu_buf_t *db, dmu_buf_user_t *user);
+
+/*
+ * Attach user data to a dbuf and mark it for immediate (its dirty and
+ * reference counts are equal) eviction processing.
+ *
+ * Returns NULL on success, or the existing user if another user currently
+ * owns the buffer.
+ */
+void *dmu_buf_set_user_ie(dmu_buf_t *db, dmu_buf_user_t *user);
+
+/*
+ * Replace the current user of a dbuf.
+ *
+ * If given the current user of a dbuf, replaces the dbuf's user with
+ * "new_user" and returns the user data pointer that was replaced.
+ * Otherwise returns the current, and unmodified, dbuf user pointer.
+ */
+void *dmu_buf_replace_user(dmu_buf_t *db,
+    dmu_buf_user_t *old_user, dmu_buf_user_t *new_user);
+
+/*
+ * Remove the specified user data for a DMU buffer.
+ *
+ * Returns the user that was removed on success, or the current user if
+ * another user currently owns the buffer.
+ */
+void *dmu_buf_remove_user(dmu_buf_t *db, dmu_buf_user_t *user);
+
+/*
+ * Returns the user data (dmu_buf_user_t *) associated with this dbuf.
  */
 void *dmu_buf_get_user(dmu_buf_t *db);
 
+/* Block until any in-progress dmu buf user evictions complete. */
+void dmu_buf_user_evict_wait(void);
+
 /*
  * Returns the blkptr associated with this dbuf, or NULL if not set.
  */
diff --git a/uts/common/fs/zfs/sys/dmu_objset.h b/uts/common/fs/zfs/sys/dmu_objset.h
index 804f0c182b6..e00847e6cbb 100644
--- a/uts/common/fs/zfs/sys/dmu_objset.h
+++ b/uts/common/fs/zfs/sys/dmu_objset.h
@@ -22,6 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 /* Portions Copyright 2010 Robert Milkowski */
@@ -74,22 +75,25 @@ struct objset {
 	arc_buf_t *os_phys_buf;
 	objset_phys_t *os_phys;
 	/*
-	 * The following "special" dnodes have no parent and are exempt from
-	 * dnode_move(), but they root their descendents in this objset using
-	 * handles anyway, so that all access to dnodes from dbufs consistently
-	 * uses handles.
+	 * The following "special" dnodes have no parent, are exempt
+	 * from dnode_move(), and are not recorded in os_dnodes, but they
+	 * root their descendents in this objset using handles anyway, so
+	 * that all access to dnodes from dbufs consistently uses handles.
 	 */
 	dnode_handle_t os_meta_dnode;
 	dnode_handle_t os_userused_dnode;
 	dnode_handle_t os_groupused_dnode;
 	zilog_t *os_zil;
 
+	list_node_t os_evicting_node;
+
 	/* can change, under dsl_dir's locks: */
 	enum zio_checksum os_checksum;
 	enum zio_compress os_compress;
 	uint8_t os_copies;
 	enum zio_checksum os_dedup_checksum;
 	boolean_t os_dedup_verify;
+	boolean_t os_evicting;
 	zfs_logbias_op_t os_logbias;
 	zfs_cache_type_t os_primary_cache;
 	zfs_cache_type_t os_secondary_cache;
@@ -170,6 +174,8 @@ int dmu_objset_userspace_upgrade(objset_t *os);
 boolean_t dmu_objset_userspace_present(objset_t *os);
 int dmu_fsname(const char *snapname, char *buf);
 
+void dmu_objset_evict_done(objset_t *os);
+
 void dmu_objset_init(void);
 void dmu_objset_fini(void);
 
diff --git a/uts/common/fs/zfs/sys/dnode.h b/uts/common/fs/zfs/sys/dnode.h
index 5668af1fef0..406954a6532 100644
--- a/uts/common/fs/zfs/sys/dnode.h
+++ b/uts/common/fs/zfs/sys/dnode.h
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_DNODE_H
@@ -255,6 +256,7 @@ typedef struct dnode_handle {
 } dnode_handle_t;
 
 typedef struct dnode_children {
+	dmu_buf_user_t dnc_dbu;		/* User evict data */
 	size_t dnc_count;		/* number of children */
 	dnode_handle_t dnc_children[];	/* sized dynamically */
 } dnode_children_t;
@@ -265,7 +267,7 @@ typedef struct free_range {
 	uint64_t fr_nblks;
 } free_range_t;
 
-dnode_t *dnode_special_open(struct objset *dd, dnode_phys_t *dnp,
+void dnode_special_open(struct objset *dd, dnode_phys_t *dnp,
     uint64_t object, dnode_handle_t *dnh);
 void dnode_special_close(dnode_handle_t *dnh);
 
diff --git a/uts/common/fs/zfs/sys/dsl_dataset.h b/uts/common/fs/zfs/sys/dsl_dataset.h
index 3160a05a8c5..f0c78c971be 100644
--- a/uts/common/fs/zfs/sys/dsl_dataset.h
+++ b/uts/common/fs/zfs/sys/dsl_dataset.h
@@ -23,6 +23,7 @@
  * Copyright (c) 2013 by Delphix. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  * Copyright (c) 2013 Steven Hartland. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_DSL_DATASET_H
@@ -132,11 +133,14 @@ typedef struct dsl_dataset_phys {
 } dsl_dataset_phys_t;
 
 typedef struct dsl_dataset {
+	dmu_buf_user_t ds_dbu;
+
 	/* Immutable: */
 	struct dsl_dir *ds_dir;
 	dmu_buf_t *ds_dbuf;
 	uint64_t ds_object;
 	uint64_t ds_fsid_guid;
+	boolean_t ds_is_snapshot;
 
 	/* only used in syncing context, only valid for non-snapshots: */
 	struct dsl_dataset *ds_prev;
@@ -197,12 +201,6 @@ dsl_dataset_phys(dsl_dataset_t *ds)
  */
 #define	MAX_TAG_PREFIX_LEN	17
 
-inline boolean_t
-dsl_dataset_is_snapshot(dsl_dataset_t *ds)
-{
-	return (dsl_dataset_phys(ds)->ds_num_children != 0);
-}
-
 #define	DS_UNIQUE_IS_ACCURATE(ds)	\
 	((dsl_dataset_phys(ds)->ds_flags & DS_FLAG_UNIQUE_ACCURATE) != 0)
 
diff --git a/uts/common/fs/zfs/sys/dsl_dir.h b/uts/common/fs/zfs/sys/dsl_dir.h
index 772bfbe6db1..f50014d95a8 100644
--- a/uts/common/fs/zfs/sys/dsl_dir.h
+++ b/uts/common/fs/zfs/sys/dsl_dir.h
@@ -22,6 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
  * Copyright (c) 2014, Joyent, Inc. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_DSL_DIR_H
@@ -84,6 +85,8 @@ typedef struct dsl_dir_phys {
 } dsl_dir_phys_t;
 
 struct dsl_dir {
+	dmu_buf_user_t dd_dbu;
+
 	/* These are immutable; no lock needed: */
 	uint64_t dd_object;
 	dsl_pool_t *dd_pool;
@@ -119,6 +122,7 @@ dsl_dir_phys(dsl_dir_t *dd)
 }
 
 void dsl_dir_rele(dsl_dir_t *dd, void *tag);
+void dsl_dir_async_rele(dsl_dir_t *dd, void *tag);
 int dsl_dir_hold(dsl_pool_t *dp, const char *name, void *tag,
     dsl_dir_t **, const char **tail);
 int dsl_dir_hold_obj(dsl_pool_t *dp, uint64_t ddobj,
diff --git a/uts/common/fs/zfs/sys/sa.h b/uts/common/fs/zfs/sys/sa.h
index bc89fa07d22..a60eb3772cd 100644
--- a/uts/common/fs/zfs/sys/sa.h
+++ b/uts/common/fs/zfs/sys/sa.h
@@ -133,7 +133,6 @@ int sa_update_from_cb(sa_handle_t *, sa_attr_type_t,
     uint32_t buflen, sa_data_locator_t *, void *userdata, dmu_tx_t *);
 void sa_object_info(sa_handle_t *, dmu_object_info_t *);
 void sa_object_size(sa_handle_t *, uint32_t *, u_longlong_t *);
-void sa_update_user(sa_handle_t *, sa_handle_t *);
 void *sa_get_userdata(sa_handle_t *);
 void sa_set_userp(sa_handle_t *, void *);
 dmu_buf_t *sa_get_db(sa_handle_t *);
diff --git a/uts/common/fs/zfs/sys/sa_impl.h b/uts/common/fs/zfs/sys/sa_impl.h
index 6b9af2ef4f8..50874c6bf08 100644
--- a/uts/common/fs/zfs/sys/sa_impl.h
+++ b/uts/common/fs/zfs/sys/sa_impl.h
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_SA_IMPL_H
@@ -210,11 +211,12 @@ typedef enum sa_data_op {
  */
 
 struct sa_handle {
+	dmu_buf_user_t	sa_dbu;
 	kmutex_t	sa_lock;
 	dmu_buf_t	*sa_bonus;
 	dmu_buf_t	*sa_spill;
 	objset_t	*sa_os;
-	void 		*sa_userp;
+	void		*sa_userp;
 	sa_idx_tab_t	*sa_bonus_tab;	 /* idx of bonus */
 	sa_idx_tab_t	*sa_spill_tab; /* only present if spill activated */
 };
diff --git a/uts/common/fs/zfs/sys/spa.h b/uts/common/fs/zfs/sys/spa.h
index cf307a2ec97..4dd6ce69af9 100644
--- a/uts/common/fs/zfs/sys/spa.h
+++ b/uts/common/fs/zfs/sys/spa.h
@@ -22,6 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef _SYS_SPA_H
@@ -682,6 +683,7 @@ extern spa_t *spa_next(spa_t *prev);
 /* Refcount functions */
 extern void spa_open_ref(spa_t *spa, void *tag);
 extern void spa_close(spa_t *spa, void *tag);
+extern void spa_async_close(spa_t *spa, void *tag);
 extern boolean_t spa_refcount_zero(spa_t *spa);
 
 #define	SCL_NONE	0x00
@@ -754,6 +756,9 @@ extern uint64_t spa_version(spa_t *spa);
 extern boolean_t spa_deflate(spa_t *spa);
 extern metaslab_class_t *spa_normal_class(spa_t *spa);
 extern metaslab_class_t *spa_log_class(spa_t *spa);
+extern void spa_evicting_os_register(spa_t *, objset_t *os);
+extern void spa_evicting_os_deregister(spa_t *, objset_t *os);
+extern void spa_evicting_os_wait(spa_t *spa);
 extern int spa_max_replication(spa_t *spa);
 extern int spa_prev_software_version(spa_t *spa);
 extern int spa_busy(void);
diff --git a/uts/common/fs/zfs/sys/spa_impl.h b/uts/common/fs/zfs/sys/spa_impl.h
index 48b28eb5a13..9bbb240e0a1 100644
--- a/uts/common/fs/zfs/sys/spa_impl.h
+++ b/uts/common/fs/zfs/sys/spa_impl.h
@@ -22,6 +22,7 @@
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef _SYS_SPA_IMPL_H
@@ -144,6 +145,9 @@ struct spa {
 	uint64_t	spa_claim_max_txg;	/* highest claimed birth txg */
 	timespec_t	spa_loaded_ts;		/* 1st successful open time */
 	objset_t	*spa_meta_objset;	/* copy of dp->dp_meta_objset */
+	kmutex_t	spa_evicting_os_lock;	/* Evicting objset list lock */
+	list_t		spa_evicting_os_list;	/* Objsets being evicted. */
+	kcondvar_t	spa_evicting_os_cv;	/* Objset Eviction Completion */
 	txg_list_t	spa_vdev_txg_list;	/* per-txg dirty vdev list */
 	vdev_t		*spa_root_vdev;		/* top-level vdev container */
 	uint64_t	spa_config_guid;	/* config pool guid */
diff --git a/uts/common/fs/zfs/sys/zap_impl.h b/uts/common/fs/zfs/sys/zap_impl.h
index 0d4fc69815f..acc795cbc44 100644
--- a/uts/common/fs/zfs/sys/zap_impl.h
+++ b/uts/common/fs/zfs/sys/zap_impl.h
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_ZAP_IMPL_H
@@ -140,6 +141,7 @@ typedef struct zap_phys {
 typedef struct zap_table_phys zap_table_phys_t;
 
 typedef struct zap {
+	dmu_buf_user_t zap_dbu;
 	objset_t *zap_objset;
 	uint64_t zap_object;
 	struct dmu_buf *zap_dbuf;
@@ -196,7 +198,7 @@ boolean_t zap_match(zap_name_t *zn, const char *matchname);
 int zap_lockdir(objset_t *os, uint64_t obj, dmu_tx_t *tx,
     krw_t lti, boolean_t fatreader, boolean_t adding, zap_t **zapp);
 void zap_unlockdir(zap_t *zap);
-void zap_evict(dmu_buf_t *db, void *vmzap);
+void zap_evict(void *dbu);
 zap_name_t *zap_name_alloc(zap_t *zap, const char *key, matchtype_t mt);
 void zap_name_free(zap_name_t *zn);
 int zap_hashbits(zap_t *zap);
diff --git a/uts/common/fs/zfs/sys/zap_leaf.h b/uts/common/fs/zfs/sys/zap_leaf.h
index cd8b74a77a8..76b3ecc7255 100644
--- a/uts/common/fs/zfs/sys/zap_leaf.h
+++ b/uts/common/fs/zfs/sys/zap_leaf.h
@@ -20,6 +20,7 @@
  */
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #ifndef	_SYS_ZAP_LEAF_H
@@ -152,6 +153,7 @@ typedef union zap_leaf_chunk {
 } zap_leaf_chunk_t;
 
 typedef struct zap_leaf {
+	dmu_buf_user_t l_dbu;
 	krwlock_t l_rwlock;
 	uint64_t l_blkid;		/* 1<<ZAP_BLOCK_SHIFT byte block off */
 	int l_bs;			/* block size shift */
diff --git a/uts/common/fs/zfs/zap.c b/uts/common/fs/zfs/zap.c
index de2d4deb00e..20f34965100 100644
--- a/uts/common/fs/zfs/zap.c
+++ b/uts/common/fs/zfs/zap.c
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2012, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 /*
@@ -52,7 +53,6 @@ int fzap_default_block_shift = 14; /* 16k blocksize */
 
 extern inline zap_phys_t *zap_f_phys(zap_t *zap);
 
-static void zap_leaf_pageout(dmu_buf_t *db, void *vl);
 static uint64_t zap_allocate_blocks(zap_t *zap, int nblocks);
 
 void
@@ -81,7 +81,7 @@ fzap_upgrade(zap_t *zap, dmu_tx_t *tx, zap_flags_t flags)
 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
 	zap->zap_ismicro = FALSE;
 
-	(void) dmu_buf_update_user(zap->zap_dbuf, zap, zap, zap_evict);
+	zap->zap_dbu.dbu_evict_func = zap_evict;
 
 	mutex_init(&zap->zap_f.zap_num_entries_mtx, 0, 0, 0);
 	zap->zap_f.zap_block_shift = highbit64(zap->zap_dbuf->db_size) - 1;
@@ -387,11 +387,20 @@ zap_allocate_blocks(zap_t *zap, int nblocks)
 	return (newblk);
 }
 
+static void
+zap_leaf_pageout(void *dbu)
+{
+	zap_leaf_t *l = dbu;
+
+	rw_destroy(&l->l_rwlock);
+	kmem_free(l, sizeof (zap_leaf_t));
+}
+
 static zap_leaf_t *
 zap_create_leaf(zap_t *zap, dmu_tx_t *tx)
 {
 	void *winner;
-	zap_leaf_t *l = kmem_alloc(sizeof (zap_leaf_t), KM_SLEEP);
+	zap_leaf_t *l = kmem_zalloc(sizeof (zap_leaf_t), KM_SLEEP);
 
 	ASSERT(RW_WRITE_HELD(&zap->zap_rwlock));
 
@@ -403,7 +412,8 @@ zap_create_leaf(zap_t *zap, dmu_tx_t *tx)
 	VERIFY(0 == dmu_buf_hold(zap->zap_objset, zap->zap_object,
 	    l->l_blkid << FZAP_BLOCK_SHIFT(zap), NULL, &l->l_dbuf,
 	    DMU_READ_NO_PREFETCH));
-	winner = dmu_buf_set_user(l->l_dbuf, l, zap_leaf_pageout);
+	dmu_buf_init_user(&l->l_dbu, zap_leaf_pageout, &l->l_dbuf);
+	winner = dmu_buf_set_user(l->l_dbuf, &l->l_dbu);
 	ASSERT(winner == NULL);
 	dmu_buf_will_dirty(l->l_dbuf, tx);
 
@@ -435,16 +445,6 @@ zap_put_leaf(zap_leaf_t *l)
 	dmu_buf_rele(l->l_dbuf, NULL);
 }
 
-_NOTE(ARGSUSED(0))
-static void
-zap_leaf_pageout(dmu_buf_t *db, void *vl)
-{
-	zap_leaf_t *l = vl;
-
-	rw_destroy(&l->l_rwlock);
-	kmem_free(l, sizeof (zap_leaf_t));
-}
-
 static zap_leaf_t *
 zap_open_leaf(uint64_t blkid, dmu_buf_t *db)
 {
@@ -452,19 +452,20 @@ zap_open_leaf(uint64_t blkid, dmu_buf_t *db)
 
 	ASSERT(blkid != 0);
 
-	l = kmem_alloc(sizeof (zap_leaf_t), KM_SLEEP);
+	l = kmem_zalloc(sizeof (zap_leaf_t), KM_SLEEP);
 	rw_init(&l->l_rwlock, 0, 0, 0);
 	rw_enter(&l->l_rwlock, RW_WRITER);
 	l->l_blkid = blkid;
 	l->l_bs = highbit64(db->db_size) - 1;
 	l->l_dbuf = db;
 
-	winner = dmu_buf_set_user(db, l, zap_leaf_pageout);
+	dmu_buf_init_user(&l->l_dbu, zap_leaf_pageout, &l->l_dbuf);
+	winner = dmu_buf_set_user(db, &l->l_dbu);
 
 	rw_exit(&l->l_rwlock);
 	if (winner != NULL) {
 		/* someone else set it first */
-		zap_leaf_pageout(NULL, l);
+		zap_leaf_pageout(&l->l_dbu);
 		l = winner;
 	}
 
diff --git a/uts/common/fs/zfs/zap_micro.c b/uts/common/fs/zfs/zap_micro.c
index 81ea4f730f9..3742471f78e 100644
--- a/uts/common/fs/zfs/zap_micro.c
+++ b/uts/common/fs/zfs/zap_micro.c
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
+ * Copyright (c) 2014 Spectra Logic Corporation, All rights reserved.
  */
 
 #include <sys/zio.h>
@@ -387,7 +388,8 @@ mzap_open(objset_t *os, uint64_t obj, dmu_buf_t *db)
 	 * it, because zap_lockdir() checks zap_ismicro without the lock
 	 * held.
 	 */
-	winner = dmu_buf_set_user(db, zap, zap_evict);
+	dmu_buf_init_user(&zap->zap_dbu, zap_evict, &zap->zap_dbuf);
+	winner = dmu_buf_set_user(db, &zap->zap_dbu);
 
 	if (winner != NULL) {
 		rw_exit(&zap->zap_rwlock);
@@ -679,11 +681,10 @@ zap_destroy(objset_t *os, uint64_t zapobj, dmu_tx_t *tx)
 	return (dmu_object_free(os, zapobj, tx));
 }
 
-_NOTE(ARGSUSED(0))
 void
-zap_evict(dmu_buf_t *db, void *vzap)
+zap_evict(void *dbu)
 {
-	zap_t *zap = vzap;
+	zap_t *zap = dbu;
 
 	rw_destroy(&zap->zap_rwlock);
 
diff --git a/uts/common/fs/zfs/zfs_sa.c b/uts/common/fs/zfs/zfs_sa.c
index ed5f2764751..3a472aa11a4 100644
--- a/uts/common/fs/zfs/zfs_sa.c
+++ b/uts/common/fs/zfs/zfs_sa.c
@@ -22,8 +22,7 @@
  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
  */
 
-#include <sys/types.h>
-#include <sys/param.h>
+#include <sys/zfs_context.h>
 #include <sys/vnode.h>
 #include <sys/sa.h>
 #include <sys/zfs_acl.h>
diff --git a/uts/common/fs/zfs/zil.c b/uts/common/fs/zfs/zil.c
index 53944165efb..f48828dc906 100644
--- a/uts/common/fs/zfs/zil.c
+++ b/uts/common/fs/zfs/zil.c
@@ -474,7 +474,7 @@ zilog_dirty(zilog_t *zilog, uint64_t txg)
 	dsl_pool_t *dp = zilog->zl_dmu_pool;
 	dsl_dataset_t *ds = dmu_objset_ds(zilog->zl_os);
 
-	if (dsl_dataset_is_snapshot(ds))
+	if (ds->ds_is_snapshot)
 		panic("dirtying snapshot!");
 
 	if (txg_list_add(&dp->dp_dirty_zilogs, zilog, txg)) {

From e3bdc85205ba550f8b64e90b896a6457ae9ff5cc Mon Sep 17 00:00:00 2001
From: Xin LI <delphij@FreeBSD.org>
Date: Tue, 20 Jan 2015 20:13:46 +0000
Subject: [PATCH 113/279] 5410 Document -S option to zfs inherit 5412 Mention
 -S option when zfs inherit fails on quota Reviewed by: Matthew Ahrens
 <mahrens@delphix.com> Approved by: Richard Lowe <richlowe@richlowe.net>
 Author: Paul B. Henson <henson@acm.org>

illumos/illumos-gate@5ff8cfa92ec8ea0f8593ad21aa2a04829b0ef5ea
---
 cmd/zfs/zfs_main.c |  6 +++++-
 man/man1m/zfs.1m   | 40 +++++++++++++++++++++++++++++++---------
 2 files changed, 36 insertions(+), 10 deletions(-)

diff --git a/cmd/zfs/zfs_main.c b/cmd/zfs/zfs_main.c
index 109a2cf109f..5037325e9bc 100644
--- a/cmd/zfs/zfs_main.c
+++ b/cmd/zfs/zfs_main.c
@@ -1890,9 +1890,13 @@ zfs_do_inherit(int argc, char **argv)
 			if (prop == ZFS_PROP_QUOTA ||
 			    prop == ZFS_PROP_RESERVATION ||
 			    prop == ZFS_PROP_REFQUOTA ||
-			    prop == ZFS_PROP_REFRESERVATION)
+			    prop == ZFS_PROP_REFRESERVATION) {
 				(void) fprintf(stderr, gettext("use 'zfs set "
 				    "%s=none' to clear\n"), propname);
+				(void) fprintf(stderr, gettext("use 'zfs "
+				    "inherit -S %s' to revert to received "
+				    "value\n"), propname);
+			}
 			return (1);
 		}
 		if (received && (prop == ZFS_PROP_VOLSIZE ||
diff --git a/man/man1m/zfs.1m b/man/man1m/zfs.1m
index 13b360d9ea0..b679b77ba4e 100644
--- a/man/man1m/zfs.1m
+++ b/man/man1m/zfs.1m
@@ -119,7 +119,7 @@ zfs \- configures ZFS file systems
 
 .LP
 .nf
-\fBzfs\fR \fBinherit\fR [\fB-r\fR] \fIproperty\fR \fIfilesystem\fR|\fIvolume|snapshot\fR...
+\fBzfs\fR \fBinherit\fR [\fB-rS\fR] \fIproperty\fR \fIfilesystem\fR|\fIvolume|snapshot\fR...
 .fi
 
 .LP
@@ -2419,7 +2419,7 @@ property, the following columns are displayed:
      property  Property name
      value     Property value
      source    Property source. Can either be local, default,
-               temporary, inherited, or none (-).
+               temporary, inherited, received, or none (-).
 .fi
 .in -2
 .sp
@@ -2484,8 +2484,8 @@ is the default value.
 .RS 4n
 A comma-separated list of sources to display. Those properties coming from a
 source other than those in this list are ignored. Each source must be one of
-the following: \fBlocal,default,inherited,temporary,none\fR. The default value
-is all sources.
+the following: \fBlocal,default,inherited,received,temporary,none\fR. The
+default value is all sources.
 .RE
 
 .sp
@@ -2503,15 +2503,16 @@ Display numbers in parsable (exact) values.
 .sp
 .ne 2
 .na
-\fB\fBzfs inherit\fR [\fB-r\fR] \fIproperty\fR
+\fB\fBzfs inherit\fR [\fB-rS\fR] \fIproperty\fR
 \fIfilesystem\fR|\fIvolume\fR|\fIsnapshot\fR...\fR
 .ad
 .sp .6
 .RS 4n
-Clears the specified property, causing it to be inherited from an ancestor. If
-no ancestor has the property set, then the default value is used. See the
-"Properties" section for a listing of default values, and details on which
-properties can be inherited.
+Clears the specified property, causing it to be inherited from an ancestor,
+restored to default if no ancestor has the property set, or with the \fB-S\fR
+option reverted to the received value if one exists.  See the "Properties"
+section for a listing of default values, and details on which properties can be
+inherited.
 .sp
 .ne 2
 .na
@@ -2521,6 +2522,16 @@ properties can be inherited.
 .RS 4n
 Recursively inherit the given property for all children.
 .RE
+.sp
+.ne 2
+.na
+\fB\fB-S\fR\fR
+.ad
+.sp .6
+.RS 4n
+Revert the property to the received value if one exists; otherwise operate as
+if the \fB-S\fR option was not specified.
+.RE
 
 .RE
 
@@ -3896,6 +3907,17 @@ inherit the \fBchecksum\fR property from their parent.
 .fi
 .in -2
 .sp
+.LP
+The following command causes \fBpool/home/bob\fR to revert to the received
+value for the \fBquota\fR property if it exists.
+
+.sp
+.in +2
+.nf
+# \fBzfs inherit -S quota pool/home/bob
+.fi
+.in -2
+.sp
 
 .LP
 \fBExample 12 \fRRemotely Replicating ZFS Data

From fd97c9198ff6d02b006688355a454e2943f81a8e Mon Sep 17 00:00:00 2001
From: Xin LI <delphij@FreeBSD.org>
Date: Tue, 20 Jan 2015 20:14:50 +0000
Subject: [PATCH 114/279] 5313 Allow I/Os to be aggregated across ZIO priority
 classes Reviewed by: Andriy Gapon <avg@FreeBSD.org> Reviewed by: Will Andrews
 <willa@SpectraLogic.com> Reviewed by: Matt Ahrens <mahrens@delphix.com>
 Reviewed by: George Wilson <george@delphix.com> Approved by: Robert Mustacchi
 <rm@joyent.com> Author: Justin T. Gibbs <justing@spectralogic.com>

illumos/illumos-gate@fe319232d24f4ae183730a5a24a09423d8ab4429
---
 uts/common/fs/zfs/sys/vdev_impl.h |  2 +
 uts/common/fs/zfs/sys/zio.h       |  1 +
 uts/common/fs/zfs/vdev_queue.c    | 78 +++++++++++++++++++------------
 3 files changed, 51 insertions(+), 30 deletions(-)

diff --git a/uts/common/fs/zfs/sys/vdev_impl.h b/uts/common/fs/zfs/sys/vdev_impl.h
index 6d9bcb17d00..3af0e2a552a 100644
--- a/uts/common/fs/zfs/sys/vdev_impl.h
+++ b/uts/common/fs/zfs/sys/vdev_impl.h
@@ -113,6 +113,8 @@ struct vdev_queue {
 	vdev_t		*vq_vdev;
 	vdev_queue_class_t vq_class[ZIO_PRIORITY_NUM_QUEUEABLE];
 	avl_tree_t	vq_active_tree;
+	avl_tree_t	vq_read_offset_tree;
+	avl_tree_t	vq_write_offset_tree;
 	uint64_t	vq_last_offset;
 	hrtime_t	vq_io_complete_ts; /* time last i/o completed */
 	kmutex_t	vq_lock;
diff --git a/uts/common/fs/zfs/sys/zio.h b/uts/common/fs/zfs/sys/zio.h
index f6cf259bf71..71bb6b96a01 100644
--- a/uts/common/fs/zfs/sys/zio.h
+++ b/uts/common/fs/zfs/sys/zio.h
@@ -419,6 +419,7 @@ struct zio {
 	uint64_t	io_offset;
 	hrtime_t	io_timestamp;
 	avl_node_t	io_queue_node;
+	avl_node_t	io_offset_node;
 
 	/* Internal pipeline state */
 	enum zio_flag	io_flags;
diff --git a/uts/common/fs/zfs/vdev_queue.c b/uts/common/fs/zfs/vdev_queue.c
index f47c4cd1e26..79d6e13b3be 100644
--- a/uts/common/fs/zfs/vdev_queue.c
+++ b/uts/common/fs/zfs/vdev_queue.c
@@ -185,6 +185,22 @@ vdev_queue_offset_compare(const void *x1, const void *x2)
 	return (0);
 }
 
+static inline avl_tree_t *
+vdev_queue_class_tree(vdev_queue_t *vq, zio_priority_t p)
+{
+	return (&vq->vq_class[p].vqc_queued_tree);
+}
+
+static inline avl_tree_t *
+vdev_queue_type_tree(vdev_queue_t *vq, zio_type_t t)
+{
+	ASSERT(t == ZIO_TYPE_READ || t == ZIO_TYPE_WRITE);
+	if (t == ZIO_TYPE_READ)
+		return (&vq->vq_read_offset_tree);
+	else
+		return (&vq->vq_write_offset_tree);
+}
+
 int
 vdev_queue_timestamp_compare(const void *x1, const void *x2)
 {
@@ -214,19 +230,27 @@ vdev_queue_init(vdev_t *vd)
 
 	avl_create(&vq->vq_active_tree, vdev_queue_offset_compare,
 	    sizeof (zio_t), offsetof(struct zio, io_queue_node));
+	avl_create(vdev_queue_type_tree(vq, ZIO_TYPE_READ),
+	    vdev_queue_offset_compare, sizeof (zio_t),
+	    offsetof(struct zio, io_offset_node));
+	avl_create(vdev_queue_type_tree(vq, ZIO_TYPE_WRITE),
+	    vdev_queue_offset_compare, sizeof (zio_t),
+	    offsetof(struct zio, io_offset_node));
 
 	for (zio_priority_t p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) {
+		int (*compfn) (const void *, const void *);
+
 		/*
-		 * The synchronous i/o queues are FIFO rather than LBA ordered.
-		 * This provides more consistent latency for these i/os, and
-		 * they tend to not be tightly clustered anyway so there is
-		 * little to no throughput loss.
+		 * The synchronous i/o queues are dispatched in FIFO rather
+		 * than LBA order.  This provides more consistent latency for
+		 * these i/os.
 		 */
-		boolean_t fifo = (p == ZIO_PRIORITY_SYNC_READ ||
-		    p == ZIO_PRIORITY_SYNC_WRITE);
-		avl_create(&vq->vq_class[p].vqc_queued_tree,
-		    fifo ? vdev_queue_timestamp_compare :
-		    vdev_queue_offset_compare,
+		if (p == ZIO_PRIORITY_SYNC_READ || p == ZIO_PRIORITY_SYNC_WRITE)
+			compfn = vdev_queue_timestamp_compare;
+		else
+			compfn = vdev_queue_offset_compare;
+
+		avl_create(vdev_queue_class_tree(vq, p), compfn,
 		    sizeof (zio_t), offsetof(struct zio, io_queue_node));
 	}
 }
@@ -237,8 +261,10 @@ vdev_queue_fini(vdev_t *vd)
 	vdev_queue_t *vq = &vd->vdev_queue;
 
 	for (zio_priority_t p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++)
-		avl_destroy(&vq->vq_class[p].vqc_queued_tree);
+		avl_destroy(vdev_queue_class_tree(vq, p));
 	avl_destroy(&vq->vq_active_tree);
+	avl_destroy(vdev_queue_type_tree(vq, ZIO_TYPE_READ));
+	avl_destroy(vdev_queue_type_tree(vq, ZIO_TYPE_WRITE));
 
 	mutex_destroy(&vq->vq_lock);
 }
@@ -248,7 +274,8 @@ vdev_queue_io_add(vdev_queue_t *vq, zio_t *zio)
 {
 	spa_t *spa = zio->io_spa;
 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
-	avl_add(&vq->vq_class[zio->io_priority].vqc_queued_tree, zio);
+	avl_add(vdev_queue_class_tree(vq, zio->io_priority), zio);
+	avl_add(vdev_queue_type_tree(vq, zio->io_type), zio);
 
 	mutex_enter(&spa->spa_iokstat_lock);
 	spa->spa_queue_stats[zio->io_priority].spa_queued++;
@@ -262,7 +289,8 @@ vdev_queue_io_remove(vdev_queue_t *vq, zio_t *zio)
 {
 	spa_t *spa = zio->io_spa;
 	ASSERT3U(zio->io_priority, <, ZIO_PRIORITY_NUM_QUEUEABLE);
-	avl_remove(&vq->vq_class[zio->io_priority].vqc_queued_tree, zio);
+	avl_remove(vdev_queue_class_tree(vq, zio->io_priority), zio);
+	avl_remove(vdev_queue_type_tree(vq, zio->io_type), zio);
 
 	mutex_enter(&spa->spa_iokstat_lock);
 	ASSERT3U(spa->spa_queue_stats[zio->io_priority].spa_queued, >, 0);
@@ -423,7 +451,7 @@ vdev_queue_class_to_issue(vdev_queue_t *vq)
 
 	/* find a queue that has not reached its minimum # outstanding i/os */
 	for (p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) {
-		if (avl_numnodes(&vq->vq_class[p].vqc_queued_tree) > 0 &&
+		if (avl_numnodes(vdev_queue_class_tree(vq, p)) > 0 &&
 		    vq->vq_class[p].vqc_active <
 		    vdev_queue_class_min_active(p))
 			return (p);
@@ -434,7 +462,7 @@ vdev_queue_class_to_issue(vdev_queue_t *vq)
 	 * maximum # outstanding i/os.
 	 */
 	for (p = 0; p < ZIO_PRIORITY_NUM_QUEUEABLE; p++) {
-		if (avl_numnodes(&vq->vq_class[p].vqc_queued_tree) > 0 &&
+		if (avl_numnodes(vdev_queue_class_tree(vq, p)) > 0 &&
 		    vq->vq_class[p].vqc_active <
 		    vdev_queue_class_max_active(spa, p))
 			return (p);
@@ -460,22 +488,12 @@ vdev_queue_aggregate(vdev_queue_t *vq, zio_t *zio)
 	uint64_t maxgap = 0;
 	uint64_t size;
 	boolean_t stretch = B_FALSE;
-	vdev_queue_class_t *vqc = &vq->vq_class[zio->io_priority];
-	avl_tree_t *t = &vqc->vqc_queued_tree;
+	avl_tree_t *t = vdev_queue_type_tree(vq, zio->io_type);
 	enum zio_flag flags = zio->io_flags & ZIO_FLAG_AGG_INHERIT;
 
 	if (zio->io_flags & ZIO_FLAG_DONT_AGGREGATE)
 		return (NULL);
 
-	/*
-	 * The synchronous i/o queues are not sorted by LBA, so we can't
-	 * find adjacent i/os.  These i/os tend to not be tightly clustered,
-	 * or too large to aggregate, so this has little impact on performance.
-	 */
-	if (zio->io_priority == ZIO_PRIORITY_SYNC_READ ||
-	    zio->io_priority == ZIO_PRIORITY_SYNC_WRITE)
-		return (NULL);
-
 	first = last = zio;
 
 	if (zio->io_type == ZIO_TYPE_READ)
@@ -607,7 +625,7 @@ vdev_queue_io_to_issue(vdev_queue_t *vq)
 	zio_t *zio, *aio;
 	zio_priority_t p;
 	avl_index_t idx;
-	vdev_queue_class_t *vqc;
+	avl_tree_t *tree;
 	zio_t search;
 
 again:
@@ -626,13 +644,13 @@ again:
 	 *
 	 * For FIFO queues (sync), issue the i/o with the lowest timestamp.
 	 */
-	vqc = &vq->vq_class[p];
+	tree = vdev_queue_class_tree(vq, p);
 	search.io_timestamp = 0;
 	search.io_offset = vq->vq_last_offset + 1;
-	VERIFY3P(avl_find(&vqc->vqc_queued_tree, &search, &idx), ==, NULL);
-	zio = avl_nearest(&vqc->vqc_queued_tree, idx, AVL_AFTER);
+	VERIFY3P(avl_find(tree, &search, &idx), ==, NULL);
+	zio = avl_nearest(tree, idx, AVL_AFTER);
 	if (zio == NULL)
-		zio = avl_first(&vqc->vqc_queued_tree);
+		zio = avl_first(tree);
 	ASSERT3U(zio->io_priority, ==, p);
 
 	aio = vdev_queue_aggregate(vq, zio);

From 1a3dd08e88e5b7385b661285a6abc06586236b96 Mon Sep 17 00:00:00 2001
From: Xin LI <delphij@FreeBSD.org>
Date: Tue, 20 Jan 2015 20:17:29 +0000
Subject: [PATCH 115/279] 5497 lock contention on arcs_mtx Reviewed by: George
 Wilson <george.wilson@delphix.com> Reviewed by: Matthew Ahrens
 <mahrens@delphix.com> Reviewed by: Richard Elling
 <richard.elling@richardelling.com> Approved by: Dan McDonald
 <danmcd@omniti.com> Author: Prakash Surya <prakash.surya@delphix.com>

illumos/illumos-gate@244781f10dcd82684fd8163c016540667842f203
---
 lib/libzpool/common/sys/zfs_context.h |    1 +
 uts/common/fs/zfs/arc.c               | 1826 ++++++++++++++++---------
 uts/common/fs/zfs/dsl_pool.c          |    9 +-
 uts/common/fs/zfs/multilist.c         |  366 +++++
 uts/common/fs/zfs/sys/arc.h           |    8 +-
 uts/common/fs/zfs/sys/multilist.h     |  106 ++
 uts/common/fs/zfs/zio_inject.c        |    6 +-
 7 files changed, 1680 insertions(+), 642 deletions(-)
 create mode 100644 uts/common/fs/zfs/multilist.c
 create mode 100644 uts/common/fs/zfs/sys/multilist.h

diff --git a/lib/libzpool/common/sys/zfs_context.h b/lib/libzpool/common/sys/zfs_context.h
index 5a19542b8f7..b1154fb71bf 100644
--- a/lib/libzpool/common/sys/zfs_context.h
+++ b/lib/libzpool/common/sys/zfs_context.h
@@ -505,6 +505,7 @@ extern void delay(clock_t ticks);
 	} while (0);
 
 #define	max_ncpus	64
+#define	boot_ncpus	(sysconf(_SC_NPROCESSORS_ONLN))
 
 #define	minclsyspri	60
 #define	maxclsyspri	99
diff --git a/uts/common/fs/zfs/arc.c b/uts/common/fs/zfs/arc.c
index 24198579a89..a227c424708 100644
--- a/uts/common/fs/zfs/arc.c
+++ b/uts/common/fs/zfs/arc.c
@@ -129,6 +129,7 @@
 #include <sys/vdev.h>
 #include <sys/vdev_impl.h>
 #include <sys/dsl_pool.h>
+#include <sys/multilist.h>
 #ifdef _KERNEL
 #include <sys/vmsystm.h>
 #include <vm/anon.h>
@@ -145,21 +146,39 @@ boolean_t arc_watch = B_FALSE;
 int arc_procfd;
 #endif
 
-static kmutex_t		arc_reclaim_thr_lock;
-static kcondvar_t	arc_reclaim_thr_cv;	/* used to signal reclaim thr */
-static uint8_t		arc_thread_exit;
+static kmutex_t		arc_reclaim_lock;
+static kcondvar_t	arc_reclaim_thread_cv;
+static boolean_t	arc_reclaim_thread_exit;
+static kcondvar_t	arc_reclaim_waiters_cv;
+
+static kmutex_t		arc_user_evicts_lock;
+static kcondvar_t	arc_user_evicts_cv;
+static boolean_t	arc_user_evicts_thread_exit;
 
 uint_t arc_reduce_dnlc_percent = 3;
 
 /*
- * The number of iterations through arc_evict_*() before we
- * drop & reacquire the lock.
+ * The number of headers to evict in arc_evict_state_impl() before
+ * dropping the sublist lock and evicting from another sublist. A lower
+ * value means we're more likely to evict the "correct" header (i.e. the
+ * oldest header in the arc state), but comes with higher overhead
+ * (i.e. more invocations of arc_evict_state_impl()).
  */
-int arc_evict_iterations = 100;
+int zfs_arc_evict_batch_limit = 10;
+
+/*
+ * The number of sublists used for each of the arc state lists. If this
+ * is not set to a suitable value by the user, it will be configured to
+ * the number of CPUs on the system in arc_init().
+ */
+int zfs_arc_num_sublists_per_state = 0;
 
 /* number of seconds before growing cache again */
 static int		arc_grow_retry = 60;
 
+/* shift of arc_c for calculating overflow limit in arc_get_data_buf */
+int		zfs_arc_overflow_shift = 8;
+
 /* shift of arc_c for calculating both min and max arc_p */
 static int		arc_p_min_shift = 4;
 
@@ -242,10 +261,19 @@ int zfs_arc_average_blocksize = 8 * 1024; /* 8KB */
  */
 
 typedef struct arc_state {
-	list_t	arcs_list[ARC_BUFC_NUMTYPES];	/* list of evictable buffers */
-	uint64_t arcs_lsize[ARC_BUFC_NUMTYPES];	/* amount of evictable data */
-	uint64_t arcs_size;	/* total amount of data in this state */
-	kmutex_t arcs_mtx;
+	/*
+	 * list of evictable buffers
+	 */
+	multilist_t arcs_list[ARC_BUFC_NUMTYPES];
+	/*
+	 * total amount of evictable data in this state
+	 */
+	uint64_t arcs_lsize[ARC_BUFC_NUMTYPES];
+	/*
+	 * total amount of data in this state; this includes: evictable,
+	 * non-evictable, ARC_BUFC_DATA, and ARC_BUFC_METADATA.
+	 */
+	uint64_t arcs_size;
 } arc_state_t;
 
 /* The 6 states: */
@@ -272,7 +300,6 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_mfu_hits;
 	kstat_named_t arcstat_mfu_ghost_hits;
 	kstat_named_t arcstat_deleted;
-	kstat_named_t arcstat_recycle_miss;
 	/*
 	 * Number of buffers that could not be evicted because the hash lock
 	 * was held by another thread.  The lock may not necessarily be held
@@ -286,9 +313,15 @@ typedef struct arc_stats {
 	 * not from the spa we're trying to evict from.
 	 */
 	kstat_named_t arcstat_evict_skip;
+	/*
+	 * Number of times arc_evict_state() was unable to evict enough
+	 * buffers to reach it's target amount.
+	 */
+	kstat_named_t arcstat_evict_not_enough;
 	kstat_named_t arcstat_evict_l2_cached;
 	kstat_named_t arcstat_evict_l2_eligible;
 	kstat_named_t arcstat_evict_l2_ineligible;
+	kstat_named_t arcstat_evict_l2_skip;
 	kstat_named_t arcstat_hash_elements;
 	kstat_named_t arcstat_hash_elements_max;
 	kstat_named_t arcstat_hash_collisions;
@@ -439,11 +472,12 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_l2_writes_sent;
 	kstat_named_t arcstat_l2_writes_done;
 	kstat_named_t arcstat_l2_writes_error;
-	kstat_named_t arcstat_l2_writes_hdr_miss;
+	kstat_named_t arcstat_l2_writes_lock_retry;
 	kstat_named_t arcstat_l2_evict_lock_retry;
 	kstat_named_t arcstat_l2_evict_reading;
 	kstat_named_t arcstat_l2_evict_l1cached;
 	kstat_named_t arcstat_l2_free_on_write;
+	kstat_named_t arcstat_l2_cdata_free_on_write;
 	kstat_named_t arcstat_l2_abort_lowmem;
 	kstat_named_t arcstat_l2_cksum_bad;
 	kstat_named_t arcstat_l2_io_error;
@@ -479,12 +513,13 @@ static arc_stats_t arc_stats = {
 	{ "mfu_hits",			KSTAT_DATA_UINT64 },
 	{ "mfu_ghost_hits",		KSTAT_DATA_UINT64 },
 	{ "deleted",			KSTAT_DATA_UINT64 },
-	{ "recycle_miss",		KSTAT_DATA_UINT64 },
 	{ "mutex_miss",			KSTAT_DATA_UINT64 },
 	{ "evict_skip",			KSTAT_DATA_UINT64 },
+	{ "evict_not_enough",		KSTAT_DATA_UINT64 },
 	{ "evict_l2_cached",		KSTAT_DATA_UINT64 },
 	{ "evict_l2_eligible",		KSTAT_DATA_UINT64 },
 	{ "evict_l2_ineligible",	KSTAT_DATA_UINT64 },
+	{ "evict_l2_skip",		KSTAT_DATA_UINT64 },
 	{ "hash_elements",		KSTAT_DATA_UINT64 },
 	{ "hash_elements_max",		KSTAT_DATA_UINT64 },
 	{ "hash_collisions",		KSTAT_DATA_UINT64 },
@@ -523,11 +558,12 @@ static arc_stats_t arc_stats = {
 	{ "l2_writes_sent",		KSTAT_DATA_UINT64 },
 	{ "l2_writes_done",		KSTAT_DATA_UINT64 },
 	{ "l2_writes_error",		KSTAT_DATA_UINT64 },
-	{ "l2_writes_hdr_miss",		KSTAT_DATA_UINT64 },
+	{ "l2_writes_lock_retry",	KSTAT_DATA_UINT64 },
 	{ "l2_evict_lock_retry",	KSTAT_DATA_UINT64 },
 	{ "l2_evict_reading",		KSTAT_DATA_UINT64 },
 	{ "l2_evict_l1cached",		KSTAT_DATA_UINT64 },
 	{ "l2_free_on_write",		KSTAT_DATA_UINT64 },
+	{ "l2_cdata_free_on_write",	KSTAT_DATA_UINT64 },
 	{ "l2_abort_lowmem",		KSTAT_DATA_UINT64 },
 	{ "l2_cksum_bad",		KSTAT_DATA_UINT64 },
 	{ "l2_io_error",		KSTAT_DATA_UINT64 },
@@ -687,7 +723,7 @@ typedef struct l1arc_buf_hdr {
 
 	/* protected by arc state mutex */
 	arc_state_t		*b_state;
-	list_node_t		b_arc_node;
+	multilist_node_t	b_arc_node;
 
 	/* updated atomically */
 	clock_t			b_arc_access;
@@ -738,7 +774,6 @@ struct arc_buf_hdr {
 };
 
 static arc_buf_t *arc_eviction_list;
-static kmutex_t arc_eviction_mtx;
 static arc_buf_hdr_t arc_eviction_hdr;
 
 #define	GHOST_STATE(state)	\
@@ -897,8 +932,7 @@ static uint8_t l2arc_thread_exit;
 
 static void arc_get_data_buf(arc_buf_t *);
 static void arc_access(arc_buf_hdr_t *, kmutex_t *);
-static int arc_evict_needed(arc_buf_contents_t);
-static void arc_evict_ghost(arc_state_t *, uint64_t, int64_t);
+static boolean_t arc_is_overflowing();
 static void arc_buf_watch(arc_buf_t *);
 
 static arc_buf_contents_t arc_buf_type(arc_buf_hdr_t *);
@@ -1079,6 +1113,7 @@ hdr_full_cons(void *vbuf, void *unused, int kmflag)
 	cv_init(&hdr->b_l1hdr.b_cv, NULL, CV_DEFAULT, NULL);
 	refcount_create(&hdr->b_l1hdr.b_refcnt);
 	mutex_init(&hdr->b_l1hdr.b_freeze_lock, NULL, MUTEX_DEFAULT, NULL);
+	multilist_link_init(&hdr->b_l1hdr.b_arc_node);
 	arc_space_consume(HDR_FULL_SIZE, ARC_SPACE_HDRS);
 
 	return (0);
@@ -1123,6 +1158,7 @@ hdr_full_dest(void *vbuf, void *unused)
 	cv_destroy(&hdr->b_l1hdr.b_cv);
 	refcount_destroy(&hdr->b_l1hdr.b_refcnt);
 	mutex_destroy(&hdr->b_l1hdr.b_freeze_lock);
+	ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
 	arc_space_return(HDR_FULL_SIZE, ARC_SPACE_HDRS);
 }
 
@@ -1159,7 +1195,7 @@ hdr_recl(void *unused)
 	 * which is after we do arc_fini().
 	 */
 	if (!arc_dead)
-		cv_signal(&arc_reclaim_thr_cv);
+		cv_signal(&arc_reclaim_thread_cv);
 }
 
 static void
@@ -1237,18 +1273,31 @@ arc_hdr_realloc(arc_buf_hdr_t *hdr, kmem_cache_t *old, kmem_cache_t *new)
 		 * l2c_only even though it's about to change.
 		 */
 		nhdr->b_l1hdr.b_state = arc_l2c_only;
+
+		/* Verify previous threads set to NULL before freeing */
+		ASSERT3P(nhdr->b_l1hdr.b_tmp_cdata, ==, NULL);
 	} else {
 		ASSERT(hdr->b_l1hdr.b_buf == NULL);
 		ASSERT0(hdr->b_l1hdr.b_datacnt);
-		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
+
 		/*
-		 * We might be removing the L1hdr of a buffer which was just
-		 * written out to L2ARC. If such a buffer is compressed then we
-		 * need to free its b_tmp_cdata before destroying the header.
+		 * If we've reached here, We must have been called from
+		 * arc_evict_hdr(), as such we should have already been
+		 * removed from any ghost list we were previously on
+		 * (which protects us from racing with arc_evict_state),
+		 * thus no locking is needed during this check.
 		 */
-		if (hdr->b_l1hdr.b_tmp_cdata != NULL &&
-		    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF)
-			l2arc_release_cdata_buf(hdr);
+		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
+
+		/*
+		 * A buffer must not be moved into the arc_l2c_only
+		 * state if it's not finished being written out to the
+		 * l2arc device. Otherwise, the b_l1hdr.b_tmp_cdata field
+		 * might try to be accessed, even though it was removed.
+		 */
+		VERIFY(!HDR_L2_WRITING(hdr));
+		VERIFY3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
+
 		nhdr->b_flags &= ~ARC_FLAG_HAS_L1HDR;
 	}
 	/*
@@ -1461,14 +1510,13 @@ add_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
 	    (state != arc_anon)) {
 		/* We don't use the L2-only state list. */
 		if (state != arc_l2c_only) {
+			arc_buf_contents_t type = arc_buf_type(hdr);
 			uint64_t delta = hdr->b_size * hdr->b_l1hdr.b_datacnt;
-			list_t *list = &state->arcs_list[arc_buf_type(hdr)];
-			uint64_t *size = &state->arcs_lsize[arc_buf_type(hdr)];
+			multilist_t *list = &state->arcs_list[type];
+			uint64_t *size = &state->arcs_lsize[type];
+
+			multilist_remove(list, hdr);
 
-			ASSERT(!MUTEX_HELD(&state->arcs_mtx));
-			mutex_enter(&state->arcs_mtx);
-			ASSERT(list_link_active(&hdr->b_l1hdr.b_arc_node));
-			list_remove(list, hdr);
 			if (GHOST_STATE(state)) {
 				ASSERT0(hdr->b_l1hdr.b_datacnt);
 				ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
@@ -1477,7 +1525,6 @@ add_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
 			ASSERT(delta > 0);
 			ASSERT3U(*size, >=, delta);
 			atomic_add_64(size, -delta);
-			mutex_exit(&state->arcs_mtx);
 		}
 		/* remove the prefetch flag if we get a reference */
 		hdr->b_flags &= ~ARC_FLAG_PREFETCH;
@@ -1500,22 +1547,21 @@ remove_reference(arc_buf_hdr_t *hdr, kmutex_t *hash_lock, void *tag)
 	 */
 	if (((cnt = refcount_remove(&hdr->b_l1hdr.b_refcnt, tag)) == 0) &&
 	    (state != arc_anon)) {
-		uint64_t *size = &state->arcs_lsize[arc_buf_type(hdr)];
+		arc_buf_contents_t type = arc_buf_type(hdr);
+		multilist_t *list = &state->arcs_list[type];
+		uint64_t *size = &state->arcs_lsize[type];
+
+		multilist_insert(list, hdr);
 
-		ASSERT(!MUTEX_HELD(&state->arcs_mtx));
-		mutex_enter(&state->arcs_mtx);
-		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
-		list_insert_head(&state->arcs_list[arc_buf_type(hdr)], hdr);
 		ASSERT(hdr->b_l1hdr.b_datacnt > 0);
 		atomic_add_64(size, hdr->b_size *
 		    hdr->b_l1hdr.b_datacnt);
-		mutex_exit(&state->arcs_mtx);
 	}
 	return (cnt);
 }
 
 /*
- * Move the supplied buffer to the indicated state.  The mutex
+ * Move the supplied buffer to the indicated state. The hash lock
  * for the buffer must be held by the caller.
  */
 static void
@@ -1559,15 +1605,10 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 	 */
 	if (refcnt == 0) {
 		if (old_state != arc_anon && old_state != arc_l2c_only) {
-			int use_mutex = !MUTEX_HELD(&old_state->arcs_mtx);
 			uint64_t *size = &old_state->arcs_lsize[buftype];
 
-			if (use_mutex)
-				mutex_enter(&old_state->arcs_mtx);
-
 			ASSERT(HDR_HAS_L1HDR(hdr));
-			ASSERT(list_link_active(&hdr->b_l1hdr.b_arc_node));
-			list_remove(&old_state->arcs_list[buftype], hdr);
+			multilist_remove(&old_state->arcs_list[buftype], hdr);
 
 			/*
 			 * If prefetching out of the ghost cache,
@@ -1580,12 +1621,8 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 			}
 			ASSERT3U(*size, >=, from_delta);
 			atomic_add_64(size, -from_delta);
-
-			if (use_mutex)
-				mutex_exit(&old_state->arcs_mtx);
 		}
 		if (new_state != arc_anon && new_state != arc_l2c_only) {
-			int use_mutex = !MUTEX_HELD(&new_state->arcs_mtx);
 			uint64_t *size = &new_state->arcs_lsize[buftype];
 
 			/*
@@ -1595,10 +1632,7 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 			 * beforehand.
 			 */
 			ASSERT(HDR_HAS_L1HDR(hdr));
-			if (use_mutex)
-				mutex_enter(&new_state->arcs_mtx);
-
-			list_insert_head(&new_state->arcs_list[buftype], hdr);
+			multilist_insert(&new_state->arcs_list[buftype], hdr);
 
 			/* ghost elements have a ghost size */
 			if (GHOST_STATE(new_state)) {
@@ -1607,9 +1641,6 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 				to_delta = hdr->b_size;
 			}
 			atomic_add_64(size, to_delta);
-
-			if (use_mutex)
-				mutex_exit(&new_state->arcs_mtx);
 		}
 	}
 
@@ -1631,8 +1662,8 @@ arc_change_state(arc_state_t *new_state, arc_buf_hdr_t *hdr,
 	 * L2 headers should never be on the L2 state list since they don't
 	 * have L1 headers allocated.
 	 */
-	ASSERT(list_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
-	    list_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
+	ASSERT(multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_DATA]) &&
+	    multilist_is_empty(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA]));
 }
 
 void
@@ -1725,6 +1756,7 @@ arc_buf_alloc(spa_t *spa, int32_t size, void *tag, arc_buf_contents_t type)
 	hdr->b_l1hdr.b_state = arc_anon;
 	hdr->b_l1hdr.b_arc_access = 0;
 	hdr->b_l1hdr.b_datacnt = 1;
+	hdr->b_l1hdr.b_tmp_cdata = NULL;
 
 	arc_get_data_buf(buf);
 	ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
@@ -1854,6 +1886,21 @@ arc_buf_add_ref(arc_buf_t *buf, void* tag)
 	    data, metadata, hits);
 }
 
+static void
+arc_buf_free_on_write(void *data, size_t size,
+    void (*free_func)(void *, size_t))
+{
+	l2arc_data_free_t *df;
+
+	df = kmem_alloc(sizeof (*df), KM_SLEEP);
+	df->l2df_data = data;
+	df->l2df_size = size;
+	df->l2df_func = free_func;
+	mutex_enter(&l2arc_free_on_write_mtx);
+	list_insert_head(l2arc_free_on_write, df);
+	mutex_exit(&l2arc_free_on_write_mtx);
+}
+
 /*
  * Free the arc data buffer.  If it is an l2arc write in progress,
  * the buffer is placed on l2arc_free_on_write to be freed later.
@@ -1864,26 +1911,74 @@ arc_buf_data_free(arc_buf_t *buf, void (*free_func)(void *, size_t))
 	arc_buf_hdr_t *hdr = buf->b_hdr;
 
 	if (HDR_L2_WRITING(hdr)) {
-		l2arc_data_free_t *df;
-		df = kmem_alloc(sizeof (l2arc_data_free_t), KM_SLEEP);
-		df->l2df_data = buf->b_data;
-		df->l2df_size = hdr->b_size;
-		df->l2df_func = free_func;
-		mutex_enter(&l2arc_free_on_write_mtx);
-		list_insert_head(l2arc_free_on_write, df);
-		mutex_exit(&l2arc_free_on_write_mtx);
+		arc_buf_free_on_write(buf->b_data, hdr->b_size, free_func);
 		ARCSTAT_BUMP(arcstat_l2_free_on_write);
 	} else {
 		free_func(buf->b_data, hdr->b_size);
 	}
 }
 
+static void
+arc_buf_l2_cdata_free(arc_buf_hdr_t *hdr)
+{
+	ASSERT(HDR_HAS_L2HDR(hdr));
+	ASSERT(MUTEX_HELD(&hdr->b_l2hdr.b_dev->l2ad_mtx));
+
+	/*
+	 * The b_tmp_cdata field is linked off of the b_l1hdr, so if
+	 * that doesn't exist, the header is in the arc_l2c_only state,
+	 * and there isn't anything to free (it's already been freed).
+	 */
+	if (!HDR_HAS_L1HDR(hdr))
+		return;
+
+	/*
+	 * The header isn't being written to the l2arc device, thus it
+	 * shouldn't have a b_tmp_cdata to free.
+	 */
+	if (!HDR_L2_WRITING(hdr)) {
+		ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
+		return;
+	}
+
+	/*
+	 * The header does not have compression enabled. This can be due
+	 * to the buffer not being compressible, or because we're
+	 * freeing the buffer before the second phase of
+	 * l2arc_write_buffer() has started (which does the compression
+	 * step). In either case, b_tmp_cdata does not point to a
+	 * separately compressed buffer, so there's nothing to free (it
+	 * points to the same buffer as the arc_buf_t's b_data field).
+	 */
+	if (HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_OFF) {
+		hdr->b_l1hdr.b_tmp_cdata = NULL;
+		return;
+	}
+
+	/*
+	 * There's nothing to free since the buffer was all zero's and
+	 * compressed to a zero length buffer.
+	 */
+	if (HDR_GET_COMPRESS(hdr) == ZIO_COMPRESS_EMPTY) {
+		ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
+		return;
+	}
+
+	ASSERT(L2ARC_IS_VALID_COMPRESS(HDR_GET_COMPRESS(hdr)));
+
+	arc_buf_free_on_write(hdr->b_l1hdr.b_tmp_cdata,
+	    hdr->b_size, zio_data_buf_free);
+
+	ARCSTAT_BUMP(arcstat_l2_cdata_free_on_write);
+	hdr->b_l1hdr.b_tmp_cdata = NULL;
+}
+
 /*
  * Free up buf->b_data and if 'remove' is set, then pull the
  * arc_buf_t off of the the arc_buf_hdr_t's list and free it.
  */
 static void
-arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
+arc_buf_destroy(arc_buf_t *buf, boolean_t remove)
 {
 	arc_buf_t **bufp;
 
@@ -1896,17 +1991,17 @@ arc_buf_destroy(arc_buf_t *buf, boolean_t recycle, boolean_t remove)
 		arc_cksum_verify(buf);
 		arc_buf_unwatch(buf);
 
-		if (!recycle) {
-			if (type == ARC_BUFC_METADATA) {
-				arc_buf_data_free(buf, zio_buf_free);
-				arc_space_return(size, ARC_SPACE_META);
-			} else {
-				ASSERT(type == ARC_BUFC_DATA);
-				arc_buf_data_free(buf, zio_data_buf_free);
-				arc_space_return(size, ARC_SPACE_DATA);
-			}
+		if (type == ARC_BUFC_METADATA) {
+			arc_buf_data_free(buf, zio_buf_free);
+			arc_space_return(size, ARC_SPACE_META);
+		} else {
+			ASSERT(type == ARC_BUFC_DATA);
+			arc_buf_data_free(buf, zio_data_buf_free);
+			arc_space_return(size, ARC_SPACE_DATA);
 		}
-		if (list_link_active(&buf->b_hdr->b_l1hdr.b_arc_node)) {
+
+		/* protected by hash lock, if in the hash table */
+		if (multilist_link_active(&buf->b_hdr->b_l1hdr.b_arc_node)) {
 			uint64_t *cnt = &state->arcs_lsize[type];
 
 			ASSERT(refcount_is_zero(
@@ -1974,6 +2069,12 @@ arc_hdr_destroy(arc_buf_hdr_t *hdr)
 
 		list_remove(&l2hdr->b_dev->l2ad_buflist, hdr);
 
+		/*
+		 * We don't want to leak the b_tmp_cdata buffer that was
+		 * allocated in l2arc_write_buffers()
+		 */
+		arc_buf_l2_cdata_free(hdr);
+
 		ARCSTAT_INCR(arcstat_l2_size, -hdr->b_size);
 		ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
 
@@ -1996,20 +2097,19 @@ arc_hdr_destroy(arc_buf_hdr_t *hdr)
 			arc_buf_t *buf = hdr->b_l1hdr.b_buf;
 
 			if (buf->b_efunc != NULL) {
-				mutex_enter(&arc_eviction_mtx);
+				mutex_enter(&arc_user_evicts_lock);
 				mutex_enter(&buf->b_evict_lock);
 				ASSERT(buf->b_hdr != NULL);
-				arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE,
-				    FALSE);
+				arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE);
 				hdr->b_l1hdr.b_buf = buf->b_next;
 				buf->b_hdr = &arc_eviction_hdr;
 				buf->b_next = arc_eviction_list;
 				arc_eviction_list = buf;
 				mutex_exit(&buf->b_evict_lock);
-				mutex_exit(&arc_eviction_mtx);
+				cv_signal(&arc_user_evicts_cv);
+				mutex_exit(&arc_user_evicts_lock);
 			} else {
-				arc_buf_destroy(hdr->b_l1hdr.b_buf, FALSE,
-				    TRUE);
+				arc_buf_destroy(hdr->b_l1hdr.b_buf, TRUE);
 			}
 		}
 #ifdef ZFS_DEBUG
@@ -2022,7 +2122,7 @@ arc_hdr_destroy(arc_buf_hdr_t *hdr)
 
 	ASSERT3P(hdr->b_hash_next, ==, NULL);
 	if (HDR_HAS_L1HDR(hdr)) {
-		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
+		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
 		ASSERT3P(hdr->b_l1hdr.b_acb, ==, NULL);
 		kmem_cache_free(hdr_full_cache, hdr);
 	} else {
@@ -2048,7 +2148,7 @@ arc_buf_free(arc_buf_t *buf, void *tag)
 
 		(void) remove_reference(hdr, hash_lock, tag);
 		if (hdr->b_l1hdr.b_datacnt > 1) {
-			arc_buf_destroy(buf, FALSE, TRUE);
+			arc_buf_destroy(buf, TRUE);
 		} else {
 			ASSERT(buf == hdr->b_l1hdr.b_buf);
 			ASSERT(buf->b_efunc == NULL);
@@ -2062,16 +2162,16 @@ arc_buf_free(arc_buf_t *buf, void *tag)
 		 * this buffer unless the write completes before we finish
 		 * decrementing the reference count.
 		 */
-		mutex_enter(&arc_eviction_mtx);
+		mutex_enter(&arc_user_evicts_lock);
 		(void) remove_reference(hdr, NULL, tag);
 		ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 		destroy_hdr = !HDR_IO_IN_PROGRESS(hdr);
-		mutex_exit(&arc_eviction_mtx);
+		mutex_exit(&arc_user_evicts_lock);
 		if (destroy_hdr)
 			arc_hdr_destroy(hdr);
 	} else {
 		if (remove_reference(hdr, NULL, tag) > 0)
-			arc_buf_destroy(buf, FALSE, TRUE);
+			arc_buf_destroy(buf, TRUE);
 		else
 			arc_hdr_destroy(hdr);
 	}
@@ -2100,7 +2200,7 @@ arc_buf_remove_ref(arc_buf_t *buf, void* tag)
 	(void) remove_reference(hdr, hash_lock, tag);
 	if (hdr->b_l1hdr.b_datacnt > 1) {
 		if (no_callback)
-			arc_buf_destroy(buf, FALSE, TRUE);
+			arc_buf_destroy(buf, TRUE);
 	} else if (no_callback) {
 		ASSERT(hdr->b_l1hdr.b_buf == buf && buf->b_next == NULL);
 		ASSERT(buf->b_efunc == NULL);
@@ -2161,418 +2261,675 @@ arc_buf_eviction_needed(arc_buf_t *buf)
 }
 
 /*
- * Evict buffers from list until we've removed the specified number of
- * bytes.  Move the removed buffers to the appropriate evict state.
- * If the recycle flag is set, then attempt to "recycle" a buffer:
- * - look for a buffer to evict that is `bytes' long.
- * - return the data block from this buffer rather than freeing it.
- * This flag is used by callers that are trying to make space for a
- * new buffer in a full arc cache.
+ * Evict the arc_buf_hdr that is provided as a parameter. The resultant
+ * state of the header is dependent on it's state prior to entering this
+ * function. The following transitions are possible:
  *
- * This function makes a "best effort".  It skips over any buffers
- * it can't get a hash_lock on, and so may not catch all candidates.
- * It may also return without evicting as much space as requested.
+ *    - arc_mru -> arc_mru_ghost
+ *    - arc_mfu -> arc_mfu_ghost
+ *    - arc_mru_ghost -> arc_l2c_only
+ *    - arc_mru_ghost -> deleted
+ *    - arc_mfu_ghost -> arc_l2c_only
+ *    - arc_mfu_ghost -> deleted
  */
-static void *
-arc_evict(arc_state_t *state, uint64_t spa, int64_t bytes, boolean_t recycle,
-    arc_buf_contents_t type)
+static int64_t
+arc_evict_hdr(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 {
-	arc_state_t *evicted_state;
-	uint64_t bytes_evicted = 0, skipped = 0, missed = 0;
-	arc_buf_hdr_t *hdr, *hdr_prev = NULL;
-	kmutex_t *hash_lock;
-	boolean_t have_lock;
-	void *stolen = NULL;
-	arc_buf_hdr_t marker = { 0 };
-	int count = 0;
+	arc_state_t *evicted_state, *state;
+	int64_t bytes_evicted = 0;
+
+	ASSERT(MUTEX_HELD(hash_lock));
+	ASSERT(HDR_HAS_L1HDR(hdr));
+
+	state = hdr->b_l1hdr.b_state;
+	if (GHOST_STATE(state)) {
+		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
+		ASSERT(hdr->b_l1hdr.b_buf == NULL);
+
+		/*
+		 * l2arc_write_buffers() relies on a header's L1 portion
+		 * (i.e. it's b_tmp_cdata field) during it's write phase.
+		 * Thus, we cannot push a header onto the arc_l2c_only
+		 * state (removing it's L1 piece) until the header is
+		 * done being written to the l2arc.
+		 */
+		if (HDR_HAS_L2HDR(hdr) && HDR_L2_WRITING(hdr)) {
+			ARCSTAT_BUMP(arcstat_evict_l2_skip);
+			return (bytes_evicted);
+		}
+
+		ARCSTAT_BUMP(arcstat_deleted);
+		bytes_evicted += hdr->b_size;
+
+		DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
+
+		if (HDR_HAS_L2HDR(hdr)) {
+			/*
+			 * This buffer is cached on the 2nd Level ARC;
+			 * don't destroy the header.
+			 */
+			arc_change_state(arc_l2c_only, hdr, hash_lock);
+			/*
+			 * dropping from L1+L2 cached to L2-only,
+			 * realloc to remove the L1 header.
+			 */
+			hdr = arc_hdr_realloc(hdr, hdr_full_cache,
+			    hdr_l2only_cache);
+		} else {
+			arc_change_state(arc_anon, hdr, hash_lock);
+			arc_hdr_destroy(hdr);
+		}
+		return (bytes_evicted);
+	}
 
 	ASSERT(state == arc_mru || state == arc_mfu);
-
 	evicted_state = (state == arc_mru) ? arc_mru_ghost : arc_mfu_ghost;
 
-	/*
-	 * The ghost list lock must be acquired first in order to prevent
-	 * a 3 party deadlock:
-	 *
-	 *  - arc_evict_ghost acquires arc_*_ghost->arcs_mtx, followed by
-	 *    l2ad_mtx in arc_hdr_realloc
-	 *  - l2arc_write_buffers acquires l2ad_mtx, followed by arc_*->arcs_mtx
-	 *  - arc_evict acquires arc_*_ghost->arcs_mtx, followed by
-	 *    arc_*_ghost->arcs_mtx and forms a deadlock cycle.
-	 *
-	 * This situation is avoided by acquiring the ghost list lock first.
-	 */
-	mutex_enter(&evicted_state->arcs_mtx);
-	mutex_enter(&state->arcs_mtx);
+	/* prefetch buffers have a minimum lifespan */
+	if (HDR_IO_IN_PROGRESS(hdr) ||
+	    ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
+	    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
+	    arc_min_prefetch_lifespan)) {
+		ARCSTAT_BUMP(arcstat_evict_skip);
+		return (bytes_evicted);
+	}
 
-	/*
-	 * Decide which "type" (data vs metadata) to recycle from.
-	 *
-	 * If we are over the metadata limit, recycle from metadata.
-	 * If we are under the metadata minimum, recycle from data.
-	 * Otherwise, recycle from whichever type has the oldest (least
-	 * recently accessed) header.
-	 */
-	if (recycle) {
-		arc_buf_hdr_t *data_hdr =
-		    list_tail(&state->arcs_list[ARC_BUFC_DATA]);
-		arc_buf_hdr_t *metadata_hdr =
-		    list_tail(&state->arcs_list[ARC_BUFC_METADATA]);
-		arc_buf_contents_t realtype;
-
-		if (data_hdr == NULL) {
-			realtype = ARC_BUFC_METADATA;
-		} else if (metadata_hdr == NULL) {
-			realtype = ARC_BUFC_DATA;
-		} else if (arc_meta_used >= arc_meta_limit) {
-			realtype = ARC_BUFC_METADATA;
-		} else if (arc_meta_used <= arc_meta_min) {
-			realtype = ARC_BUFC_DATA;
-		} else if (HDR_HAS_L1HDR(data_hdr) &&
-		    HDR_HAS_L1HDR(metadata_hdr) &&
-		    data_hdr->b_l1hdr.b_arc_access <
-		    metadata_hdr->b_l1hdr.b_arc_access) {
-			realtype = ARC_BUFC_DATA;
-		} else {
-			realtype = ARC_BUFC_METADATA;
+	ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
+	ASSERT3U(hdr->b_l1hdr.b_datacnt, >, 0);
+	while (hdr->b_l1hdr.b_buf) {
+		arc_buf_t *buf = hdr->b_l1hdr.b_buf;
+		if (!mutex_tryenter(&buf->b_evict_lock)) {
+			ARCSTAT_BUMP(arcstat_mutex_miss);
+			break;
 		}
-		if (realtype != type) {
-			/*
-			 * If we want to evict from a different list,
-			 * we can not recycle, because DATA vs METADATA
-			 * buffers are segregated into different kmem
-			 * caches (and vmem arenas).
-			 */
-			type = realtype;
-			recycle = B_FALSE;
+		if (buf->b_data != NULL)
+			bytes_evicted += hdr->b_size;
+		if (buf->b_efunc != NULL) {
+			mutex_enter(&arc_user_evicts_lock);
+			arc_buf_destroy(buf, FALSE);
+			hdr->b_l1hdr.b_buf = buf->b_next;
+			buf->b_hdr = &arc_eviction_hdr;
+			buf->b_next = arc_eviction_list;
+			arc_eviction_list = buf;
+			cv_signal(&arc_user_evicts_cv);
+			mutex_exit(&arc_user_evicts_lock);
+			mutex_exit(&buf->b_evict_lock);
+		} else {
+			mutex_exit(&buf->b_evict_lock);
+			arc_buf_destroy(buf, TRUE);
 		}
 	}
 
-	list_t *list = &state->arcs_list[type];
+	if (HDR_HAS_L2HDR(hdr)) {
+		ARCSTAT_INCR(arcstat_evict_l2_cached, hdr->b_size);
+	} else {
+		if (l2arc_write_eligible(hdr->b_spa, hdr))
+			ARCSTAT_INCR(arcstat_evict_l2_eligible, hdr->b_size);
+		else
+			ARCSTAT_INCR(arcstat_evict_l2_ineligible, hdr->b_size);
+	}
 
-	for (hdr = list_tail(list); hdr; hdr = hdr_prev) {
-		hdr_prev = list_prev(list, hdr);
-		/* prefetch buffers have a minimum lifespan */
-		if (HDR_IO_IN_PROGRESS(hdr) ||
-		    (spa && hdr->b_spa != spa) ||
-		    ((hdr->b_flags & (ARC_FLAG_PREFETCH | ARC_FLAG_INDIRECT)) &&
-		    ddi_get_lbolt() - hdr->b_l1hdr.b_arc_access <
-		    arc_min_prefetch_lifespan)) {
-			skipped++;
-			continue;
-		}
-		/* "lookahead" for better eviction candidate */
-		if (recycle && hdr->b_size != bytes &&
-		    hdr_prev && hdr_prev->b_size == bytes)
-			continue;
+	if (hdr->b_l1hdr.b_datacnt == 0) {
+		arc_change_state(evicted_state, hdr, hash_lock);
+		ASSERT(HDR_IN_HASH_TABLE(hdr));
+		hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
+		hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
+		DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
+	}
 
-		/* ignore markers */
+	return (bytes_evicted);
+}
+
+static uint64_t
+arc_evict_state_impl(multilist_t *ml, int idx, arc_buf_hdr_t *marker,
+    uint64_t spa, int64_t bytes)
+{
+	multilist_sublist_t *mls;
+	uint64_t bytes_evicted = 0;
+	arc_buf_hdr_t *hdr;
+	kmutex_t *hash_lock;
+	int evict_count = 0;
+
+	ASSERT3P(marker, !=, NULL);
+	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
+
+	mls = multilist_sublist_lock(ml, idx);
+
+	for (hdr = multilist_sublist_prev(mls, marker); hdr != NULL;
+	    hdr = multilist_sublist_prev(mls, marker)) {
+		if ((bytes != ARC_EVICT_ALL && bytes_evicted >= bytes) ||
+		    (evict_count >= zfs_arc_evict_batch_limit))
+			break;
+
+		/*
+		 * To keep our iteration location, move the marker
+		 * forward. Since we're not holding hdr's hash lock, we
+		 * must be very careful and not remove 'hdr' from the
+		 * sublist. Otherwise, other consumers might mistake the
+		 * 'hdr' as not being on a sublist when they call the
+		 * multilist_link_active() function (they all rely on
+		 * the hash lock protecting concurrent insertions and
+		 * removals). multilist_sublist_move_forward() was
+		 * specifically implemented to ensure this is the case
+		 * (only 'marker' will be removed and re-inserted).
+		 */
+		multilist_sublist_move_forward(mls, marker);
+
+		/*
+		 * The only case where the b_spa field should ever be
+		 * zero, is the marker headers inserted by
+		 * arc_evict_state(). It's possible for multiple threads
+		 * to be calling arc_evict_state() concurrently (e.g.
+		 * dsl_pool_close() and zio_inject_fault()), so we must
+		 * skip any markers we see from these other threads.
+		 */
 		if (hdr->b_spa == 0)
 			continue;
 
-		/*
-		 * It may take a long time to evict all the bufs requested.
-		 * To avoid blocking all arc activity, periodically drop
-		 * the arcs_mtx and give other threads a chance to run
-		 * before reacquiring the lock.
-		 *
-		 * If we are looking for a buffer to recycle, we are in
-		 * the hot code path, so don't sleep.
-		 */
-		if (!recycle && count++ > arc_evict_iterations) {
-			list_insert_after(list, hdr, &marker);
-			mutex_exit(&state->arcs_mtx);
-			mutex_exit(&evicted_state->arcs_mtx);
-			kpreempt(KPREEMPT_SYNC);
-			mutex_enter(&evicted_state->arcs_mtx);
-			mutex_enter(&state->arcs_mtx);
-			hdr_prev = list_prev(list, &marker);
-			list_remove(list, &marker);
-			count = 0;
+		/* we're only interested in evicting buffers of a certain spa */
+		if (spa != 0 && hdr->b_spa != spa) {
+			ARCSTAT_BUMP(arcstat_evict_skip);
 			continue;
 		}
 
 		hash_lock = HDR_LOCK(hdr);
-		have_lock = MUTEX_HELD(hash_lock);
-		if (have_lock || mutex_tryenter(hash_lock)) {
-			ASSERT0(refcount_count(&hdr->b_l1hdr.b_refcnt));
-			ASSERT3U(hdr->b_l1hdr.b_datacnt, >, 0);
-			while (hdr->b_l1hdr.b_buf) {
-				arc_buf_t *buf = hdr->b_l1hdr.b_buf;
-				if (!mutex_tryenter(&buf->b_evict_lock)) {
-					missed += 1;
-					break;
-				}
-				if (buf->b_data != NULL) {
-					bytes_evicted += hdr->b_size;
-					if (recycle &&
-					    arc_buf_type(hdr) == type &&
-					    hdr->b_size == bytes &&
-					    !HDR_L2_WRITING(hdr)) {
-						stolen = buf->b_data;
-						recycle = FALSE;
-					}
-				}
-				if (buf->b_efunc != NULL) {
-					mutex_enter(&arc_eviction_mtx);
-					arc_buf_destroy(buf,
-					    buf->b_data == stolen, FALSE);
-					hdr->b_l1hdr.b_buf = buf->b_next;
-					buf->b_hdr = &arc_eviction_hdr;
-					buf->b_next = arc_eviction_list;
-					arc_eviction_list = buf;
-					mutex_exit(&arc_eviction_mtx);
-					mutex_exit(&buf->b_evict_lock);
-				} else {
-					mutex_exit(&buf->b_evict_lock);
-					arc_buf_destroy(buf,
-					    buf->b_data == stolen, TRUE);
-				}
-			}
 
-			if (HDR_HAS_L2HDR(hdr)) {
-				ARCSTAT_INCR(arcstat_evict_l2_cached,
-				    hdr->b_size);
-			} else {
-				if (l2arc_write_eligible(hdr->b_spa, hdr)) {
-					ARCSTAT_INCR(arcstat_evict_l2_eligible,
-					    hdr->b_size);
-				} else {
-					ARCSTAT_INCR(
-					    arcstat_evict_l2_ineligible,
-					    hdr->b_size);
-				}
-			}
+		/*
+		 * We aren't calling this function from any code path
+		 * that would already be holding a hash lock, so we're
+		 * asserting on this assumption to be defensive in case
+		 * this ever changes. Without this check, it would be
+		 * possible to incorrectly increment arcstat_mutex_miss
+		 * below (e.g. if the code changed such that we called
+		 * this function with a hash lock held).
+		 */
+		ASSERT(!MUTEX_HELD(hash_lock));
 
-			if (hdr->b_l1hdr.b_datacnt == 0) {
-				arc_change_state(evicted_state, hdr, hash_lock);
-				ASSERT(HDR_IN_HASH_TABLE(hdr));
-				hdr->b_flags |= ARC_FLAG_IN_HASH_TABLE;
-				hdr->b_flags &= ~ARC_FLAG_BUF_AVAILABLE;
-				DTRACE_PROBE1(arc__evict, arc_buf_hdr_t *, hdr);
-			}
-			if (!have_lock)
-				mutex_exit(hash_lock);
-			if (bytes >= 0 && bytes_evicted >= bytes)
-				break;
+		if (mutex_tryenter(hash_lock)) {
+			uint64_t evicted = arc_evict_hdr(hdr, hash_lock);
+			mutex_exit(hash_lock);
+
+			bytes_evicted += evicted;
+
+			/*
+			 * If evicted is zero, arc_evict_hdr() must have
+			 * decided to skip this header, don't increment
+			 * evict_count in this case.
+			 */
+			if (evicted != 0)
+				evict_count++;
+
+			/*
+			 * If arc_size isn't overflowing, signal any
+			 * threads that might happen to be waiting.
+			 *
+			 * For each header evicted, we wake up a single
+			 * thread. If we used cv_broadcast, we could
+			 * wake up "too many" threads causing arc_size
+			 * to significantly overflow arc_c; since
+			 * arc_get_data_buf() doesn't check for overflow
+			 * when it's woken up (it doesn't because it's
+			 * possible for the ARC to be overflowing while
+			 * full of un-evictable buffers, and the
+			 * function should proceed in this case).
+			 *
+			 * If threads are left sleeping, due to not
+			 * using cv_broadcast, they will be woken up
+			 * just before arc_reclaim_thread() sleeps.
+			 */
+			mutex_enter(&arc_reclaim_lock);
+			if (!arc_is_overflowing())
+				cv_signal(&arc_reclaim_waiters_cv);
+			mutex_exit(&arc_reclaim_lock);
 		} else {
-			missed += 1;
+			ARCSTAT_BUMP(arcstat_mutex_miss);
 		}
 	}
 
-	mutex_exit(&state->arcs_mtx);
-	mutex_exit(&evicted_state->arcs_mtx);
+	multilist_sublist_unlock(mls);
 
-	if (bytes_evicted < bytes)
-		dprintf("only evicted %lld bytes from %x",
-		    (longlong_t)bytes_evicted, state);
-
-	if (skipped)
-		ARCSTAT_INCR(arcstat_evict_skip, skipped);
-
-	if (missed)
-		ARCSTAT_INCR(arcstat_mutex_miss, missed);
-
-	/*
-	 * Note: we have just evicted some data into the ghost state,
-	 * potentially putting the ghost size over the desired size.  Rather
-	 * that evicting from the ghost list in this hot code path, leave
-	 * this chore to the arc_reclaim_thread().
-	 */
-
-	return (stolen);
+	return (bytes_evicted);
 }
 
 /*
- * Remove buffers from list until we've removed the specified number of
- * bytes.  Destroy the buffers that are removed.
+ * Evict buffers from the given arc state, until we've removed the
+ * specified number of bytes. Move the removed buffers to the
+ * appropriate evict state.
+ *
+ * This function makes a "best effort". It skips over any buffers
+ * it can't get a hash_lock on, and so, may not catch all candidates.
+ * It may also return without evicting as much space as requested.
+ *
+ * If bytes is specified using the special value ARC_EVICT_ALL, this
+ * will evict all available (i.e. unlocked and evictable) buffers from
+ * the given arc state; which is used by arc_flush().
  */
-static void
-arc_evict_ghost(arc_state_t *state, uint64_t spa, int64_t bytes)
+static uint64_t
+arc_evict_state(arc_state_t *state, uint64_t spa, int64_t bytes,
+    arc_buf_contents_t type)
 {
-	arc_buf_hdr_t *hdr, *hdr_prev;
-	arc_buf_hdr_t marker = { 0 };
-	list_t *list = &state->arcs_list[ARC_BUFC_DATA];
-	kmutex_t *hash_lock;
-	uint64_t bytes_deleted = 0;
-	uint64_t bufs_skipped = 0;
-	int count = 0;
+	uint64_t total_evicted = 0;
+	multilist_t *ml = &state->arcs_list[type];
+	int num_sublists;
+	arc_buf_hdr_t **markers;
 
-	ASSERT(GHOST_STATE(state));
-top:
-	mutex_enter(&state->arcs_mtx);
-	for (hdr = list_tail(list); hdr; hdr = hdr_prev) {
-		hdr_prev = list_prev(list, hdr);
-		if (arc_buf_type(hdr) >= ARC_BUFC_NUMTYPES)
-			panic("invalid hdr=%p", (void *)hdr);
-		if (spa && hdr->b_spa != spa)
-			continue;
+	IMPLY(bytes < 0, bytes == ARC_EVICT_ALL);
 
-		/* ignore markers */
-		if (hdr->b_spa == 0)
-			continue;
+	num_sublists = multilist_get_num_sublists(ml);
 
-		hash_lock = HDR_LOCK(hdr);
-		/* caller may be trying to modify this buffer, skip it */
-		if (MUTEX_HELD(hash_lock))
-			continue;
+	/*
+	 * If we've tried to evict from each sublist, made some
+	 * progress, but still have not hit the target number of bytes
+	 * to evict, we want to keep trying. The markers allow us to
+	 * pick up where we left off for each individual sublist, rather
+	 * than starting from the tail each time.
+	 */
+	markers = kmem_zalloc(sizeof (*markers) * num_sublists, KM_SLEEP);
+	for (int i = 0; i < num_sublists; i++) {
+		markers[i] = kmem_cache_alloc(hdr_full_cache, KM_SLEEP);
 
 		/*
-		 * It may take a long time to evict all the bufs requested.
-		 * To avoid blocking all arc activity, periodically drop
-		 * the arcs_mtx and give other threads a chance to run
-		 * before reacquiring the lock.
+		 * A b_spa of 0 is used to indicate that this header is
+		 * a marker. This fact is used in arc_adjust_type() and
+		 * arc_evict_state_impl().
 		 */
-		if (count++ > arc_evict_iterations) {
-			list_insert_after(list, hdr, &marker);
-			mutex_exit(&state->arcs_mtx);
-			kpreempt(KPREEMPT_SYNC);
-			mutex_enter(&state->arcs_mtx);
-			hdr_prev = list_prev(list, &marker);
-			list_remove(list, &marker);
-			count = 0;
-			continue;
-		}
-		if (mutex_tryenter(hash_lock)) {
-			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
-			ASSERT(!HDR_HAS_L1HDR(hdr) ||
-			    hdr->b_l1hdr.b_buf == NULL);
-			ARCSTAT_BUMP(arcstat_deleted);
-			bytes_deleted += hdr->b_size;
+		markers[i]->b_spa = 0;
 
-			if (HDR_HAS_L2HDR(hdr)) {
-				/*
-				 * This buffer is cached on the 2nd Level ARC;
-				 * don't destroy the header.
-				 */
-				arc_change_state(arc_l2c_only, hdr, hash_lock);
-				/*
-				 * dropping from L1+L2 cached to L2-only,
-				 * realloc to remove the L1 header.
-				 */
-				hdr = arc_hdr_realloc(hdr, hdr_full_cache,
-				    hdr_l2only_cache);
-				mutex_exit(hash_lock);
-			} else {
-				arc_change_state(arc_anon, hdr, hash_lock);
-				mutex_exit(hash_lock);
-				arc_hdr_destroy(hdr);
+		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
+		multilist_sublist_insert_tail(mls, markers[i]);
+		multilist_sublist_unlock(mls);
+	}
+
+	/*
+	 * While we haven't hit our target number of bytes to evict, or
+	 * we're evicting all available buffers.
+	 */
+	while (total_evicted < bytes || bytes == ARC_EVICT_ALL) {
+		/*
+		 * Start eviction using a randomly selected sublist,
+		 * this is to try and evenly balance eviction across all
+		 * sublists. Always starting at the same sublist
+		 * (e.g. index 0) would cause evictions to favor certain
+		 * sublists over others.
+		 */
+		int sublist_idx = multilist_get_random_index(ml);
+		uint64_t scan_evicted = 0;
+
+		for (int i = 0; i < num_sublists; i++) {
+			uint64_t bytes_remaining;
+			uint64_t bytes_evicted;
+
+			if (bytes == ARC_EVICT_ALL)
+				bytes_remaining = ARC_EVICT_ALL;
+			else if (total_evicted < bytes)
+				bytes_remaining = bytes - total_evicted;
+			else
+				break;
+
+			bytes_evicted = arc_evict_state_impl(ml, sublist_idx,
+			    markers[sublist_idx], spa, bytes_remaining);
+
+			scan_evicted += bytes_evicted;
+			total_evicted += bytes_evicted;
+
+			/* we've reached the end, wrap to the beginning */
+			if (++sublist_idx >= num_sublists)
+				sublist_idx = 0;
+		}
+
+		/*
+		 * If we didn't evict anything during this scan, we have
+		 * no reason to believe we'll evict more during another
+		 * scan, so break the loop.
+		 */
+		if (scan_evicted == 0) {
+			/* This isn't possible, let's make that obvious */
+			ASSERT3S(bytes, !=, 0);
+
+			/*
+			 * When bytes is ARC_EVICT_ALL, the only way to
+			 * break the loop is when scan_evicted is zero.
+			 * In that case, we actually have evicted enough,
+			 * so we don't want to increment the kstat.
+			 */
+			if (bytes != ARC_EVICT_ALL) {
+				ASSERT3S(total_evicted, <, bytes);
+				ARCSTAT_BUMP(arcstat_evict_not_enough);
 			}
 
-			DTRACE_PROBE1(arc__delete, arc_buf_hdr_t *, hdr);
-			if (bytes >= 0 && bytes_deleted >= bytes)
-				break;
-		} else if (bytes < 0) {
-			/*
-			 * Insert a list marker and then wait for the
-			 * hash lock to become available. Once its
-			 * available, restart from where we left off.
-			 */
-			list_insert_after(list, hdr, &marker);
-			mutex_exit(&state->arcs_mtx);
-			mutex_enter(hash_lock);
-			mutex_exit(hash_lock);
-			mutex_enter(&state->arcs_mtx);
-			hdr_prev = list_prev(list, &marker);
-			list_remove(list, &marker);
-		} else {
-			bufs_skipped += 1;
+			break;
 		}
-
-	}
-	mutex_exit(&state->arcs_mtx);
-
-	if (list == &state->arcs_list[ARC_BUFC_DATA] &&
-	    (bytes < 0 || bytes_deleted < bytes)) {
-		list = &state->arcs_list[ARC_BUFC_METADATA];
-		goto top;
 	}
 
-	if (bufs_skipped) {
-		ARCSTAT_INCR(arcstat_mutex_miss, bufs_skipped);
-		ASSERT(bytes >= 0);
-	}
+	for (int i = 0; i < num_sublists; i++) {
+		multilist_sublist_t *mls = multilist_sublist_lock(ml, i);
+		multilist_sublist_remove(mls, markers[i]);
+		multilist_sublist_unlock(mls);
 
-	if (bytes_deleted < bytes)
-		dprintf("only deleted %lld bytes from %p",
-		    (longlong_t)bytes_deleted, state);
+		kmem_cache_free(hdr_full_cache, markers[i]);
+	}
+	kmem_free(markers, sizeof (*markers) * num_sublists);
+
+	return (total_evicted);
 }
 
-static void
+/*
+ * Flush all "evictable" data of the given type from the arc state
+ * specified. This will not evict any "active" buffers (i.e. referenced).
+ *
+ * When 'retry' is set to FALSE, the function will make a single pass
+ * over the state and evict any buffers that it can. Since it doesn't
+ * continually retry the eviction, it might end up leaving some buffers
+ * in the ARC due to lock misses.
+ *
+ * When 'retry' is set to TRUE, the function will continually retry the
+ * eviction until *all* evictable buffers have been removed from the
+ * state. As a result, if concurrent insertions into the state are
+ * allowed (e.g. if the ARC isn't shutting down), this function might
+ * wind up in an infinite loop, continually trying to evict buffers.
+ */
+static uint64_t
+arc_flush_state(arc_state_t *state, uint64_t spa, arc_buf_contents_t type,
+    boolean_t retry)
+{
+	uint64_t evicted = 0;
+
+	while (state->arcs_lsize[type] != 0) {
+		evicted += arc_evict_state(state, spa, ARC_EVICT_ALL, type);
+
+		if (!retry)
+			break;
+	}
+
+	return (evicted);
+}
+
+/*
+ * Evict the specified number of bytes from the state specified,
+ * restricting eviction to the spa and type given. This function
+ * prevents us from trying to evict more from a state's list than
+ * is "evictable", and to skip evicting altogether when passed a
+ * negative value for "bytes". In contrast, arc_evict_state() will
+ * evict everything it can, when passed a negative value for "bytes".
+ */
+static uint64_t
+arc_adjust_impl(arc_state_t *state, uint64_t spa, int64_t bytes,
+    arc_buf_contents_t type)
+{
+	int64_t delta;
+
+	if (bytes > 0 && state->arcs_lsize[type] > 0) {
+		delta = MIN(state->arcs_lsize[type], bytes);
+		return (arc_evict_state(state, spa, delta, type));
+	}
+
+	return (0);
+}
+
+/*
+ * Evict metadata buffers from the cache, such that arc_meta_used is
+ * capped by the arc_meta_limit tunable.
+ */
+static uint64_t
+arc_adjust_meta(void)
+{
+	uint64_t total_evicted = 0;
+	int64_t target;
+
+	/*
+	 * If we're over the meta limit, we want to evict enough
+	 * metadata to get back under the meta limit. We don't want to
+	 * evict so much that we drop the MRU below arc_p, though. If
+	 * we're over the meta limit more than we're over arc_p, we
+	 * evict some from the MRU here, and some from the MFU below.
+	 */
+	target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
+	    (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size - arc_p));
+
+	total_evicted += arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
+
+	/*
+	 * Similar to the above, we want to evict enough bytes to get us
+	 * below the meta limit, but not so much as to drop us below the
+	 * space alloted to the MFU (which is defined as arc_c - arc_p).
+	 */
+	target = MIN((int64_t)(arc_meta_used - arc_meta_limit),
+	    (int64_t)(arc_mfu->arcs_size - (arc_c - arc_p)));
+
+	total_evicted += arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
+
+	return (total_evicted);
+}
+
+/*
+ * Return the type of the oldest buffer in the given arc state
+ *
+ * This function will select a random sublist of type ARC_BUFC_DATA and
+ * a random sublist of type ARC_BUFC_METADATA. The tail of each sublist
+ * is compared, and the type which contains the "older" buffer will be
+ * returned.
+ */
+static arc_buf_contents_t
+arc_adjust_type(arc_state_t *state)
+{
+	multilist_t *data_ml = &state->arcs_list[ARC_BUFC_DATA];
+	multilist_t *meta_ml = &state->arcs_list[ARC_BUFC_METADATA];
+	int data_idx = multilist_get_random_index(data_ml);
+	int meta_idx = multilist_get_random_index(meta_ml);
+	multilist_sublist_t *data_mls;
+	multilist_sublist_t *meta_mls;
+	arc_buf_contents_t type;
+	arc_buf_hdr_t *data_hdr;
+	arc_buf_hdr_t *meta_hdr;
+
+	/*
+	 * We keep the sublist lock until we're finished, to prevent
+	 * the headers from being destroyed via arc_evict_state().
+	 */
+	data_mls = multilist_sublist_lock(data_ml, data_idx);
+	meta_mls = multilist_sublist_lock(meta_ml, meta_idx);
+
+	/*
+	 * These two loops are to ensure we skip any markers that
+	 * might be at the tail of the lists due to arc_evict_state().
+	 */
+
+	for (data_hdr = multilist_sublist_tail(data_mls); data_hdr != NULL;
+	    data_hdr = multilist_sublist_prev(data_mls, data_hdr)) {
+		if (data_hdr->b_spa != 0)
+			break;
+	}
+
+	for (meta_hdr = multilist_sublist_tail(meta_mls); meta_hdr != NULL;
+	    meta_hdr = multilist_sublist_prev(meta_mls, meta_hdr)) {
+		if (meta_hdr->b_spa != 0)
+			break;
+	}
+
+	if (data_hdr == NULL && meta_hdr == NULL) {
+		type = ARC_BUFC_DATA;
+	} else if (data_hdr == NULL) {
+		ASSERT3P(meta_hdr, !=, NULL);
+		type = ARC_BUFC_METADATA;
+	} else if (meta_hdr == NULL) {
+		ASSERT3P(data_hdr, !=, NULL);
+		type = ARC_BUFC_DATA;
+	} else {
+		ASSERT3P(data_hdr, !=, NULL);
+		ASSERT3P(meta_hdr, !=, NULL);
+
+		/* The headers can't be on the sublist without an L1 header */
+		ASSERT(HDR_HAS_L1HDR(data_hdr));
+		ASSERT(HDR_HAS_L1HDR(meta_hdr));
+
+		if (data_hdr->b_l1hdr.b_arc_access <
+		    meta_hdr->b_l1hdr.b_arc_access) {
+			type = ARC_BUFC_DATA;
+		} else {
+			type = ARC_BUFC_METADATA;
+		}
+	}
+
+	multilist_sublist_unlock(meta_mls);
+	multilist_sublist_unlock(data_mls);
+
+	return (type);
+}
+
+/*
+ * Evict buffers from the cache, such that arc_size is capped by arc_c.
+ */
+static uint64_t
 arc_adjust(void)
 {
-	int64_t adjustment, delta;
+	uint64_t total_evicted = 0;
+	uint64_t bytes;
+	int64_t target;
+
+	/*
+	 * If we're over arc_meta_limit, we want to correct that before
+	 * potentially evicting data buffers below.
+	 */
+	total_evicted += arc_adjust_meta();
 
 	/*
 	 * Adjust MRU size
+	 *
+	 * If we're over the target cache size, we want to evict enough
+	 * from the list to get back to our target size. We don't want
+	 * to evict too much from the MRU, such that it drops below
+	 * arc_p. So, if we're over our target cache size more than
+	 * the MRU is over arc_p, we'll evict enough to get back to
+	 * arc_p here, and then evict more from the MFU below.
 	 */
-
-	adjustment = MIN((int64_t)(arc_size - arc_c),
+	target = MIN((int64_t)(arc_size - arc_c),
 	    (int64_t)(arc_anon->arcs_size + arc_mru->arcs_size + arc_meta_used -
 	    arc_p));
 
-	if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_DATA] > 0) {
-		delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_DATA], adjustment);
-		(void) arc_evict(arc_mru, NULL, delta, FALSE, ARC_BUFC_DATA);
-		adjustment -= delta;
-	}
+	/*
+	 * If we're below arc_meta_min, always prefer to evict data.
+	 * Otherwise, try to satisfy the requested number of bytes to
+	 * evict from the type which contains older buffers; in an
+	 * effort to keep newer buffers in the cache regardless of their
+	 * type. If we cannot satisfy the number of bytes from this
+	 * type, spill over into the next type.
+	 */
+	if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
+	    arc_meta_used > arc_meta_min) {
+		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
+		total_evicted += bytes;
 
-	if (adjustment > 0 && arc_mru->arcs_lsize[ARC_BUFC_METADATA] > 0) {
-		delta = MIN(arc_mru->arcs_lsize[ARC_BUFC_METADATA], adjustment);
-		(void) arc_evict(arc_mru, NULL, delta, FALSE,
-		    ARC_BUFC_METADATA);
+		/*
+		 * If we couldn't evict our target number of bytes from
+		 * metadata, we try to get the rest from data.
+		 */
+		target -= bytes;
+
+		total_evicted +=
+		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
+	} else {
+		bytes = arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_DATA);
+		total_evicted += bytes;
+
+		/*
+		 * If we couldn't evict our target number of bytes from
+		 * data, we try to get the rest from metadata.
+		 */
+		target -= bytes;
+
+		total_evicted +=
+		    arc_adjust_impl(arc_mru, 0, target, ARC_BUFC_METADATA);
 	}
 
 	/*
 	 * Adjust MFU size
+	 *
+	 * Now that we've tried to evict enough from the MRU to get its
+	 * size back to arc_p, if we're still above the target cache
+	 * size, we evict the rest from the MFU.
 	 */
+	target = arc_size - arc_c;
 
-	adjustment = arc_size - arc_c;
+	if (arc_adjust_type(arc_mru) == ARC_BUFC_METADATA &&
+	    arc_meta_used > arc_meta_min) {
+		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
+		total_evicted += bytes;
 
-	if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_DATA] > 0) {
-		delta = MIN(adjustment, arc_mfu->arcs_lsize[ARC_BUFC_DATA]);
-		(void) arc_evict(arc_mfu, NULL, delta, FALSE, ARC_BUFC_DATA);
-		adjustment -= delta;
-	}
+		/*
+		 * If we couldn't evict our target number of bytes from
+		 * metadata, we try to get the rest from data.
+		 */
+		target -= bytes;
 
-	if (adjustment > 0 && arc_mfu->arcs_lsize[ARC_BUFC_METADATA] > 0) {
-		int64_t delta = MIN(adjustment,
-		    arc_mfu->arcs_lsize[ARC_BUFC_METADATA]);
-		(void) arc_evict(arc_mfu, NULL, delta, FALSE,
-		    ARC_BUFC_METADATA);
+		total_evicted +=
+		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
+	} else {
+		bytes = arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_DATA);
+		total_evicted += bytes;
+
+		/*
+		 * If we couldn't evict our target number of bytes from
+		 * data, we try to get the rest from data.
+		 */
+		target -= bytes;
+
+		total_evicted +=
+		    arc_adjust_impl(arc_mfu, 0, target, ARC_BUFC_METADATA);
 	}
 
 	/*
 	 * Adjust ghost lists
+	 *
+	 * In addition to the above, the ARC also defines target values
+	 * for the ghost lists. The sum of the mru list and mru ghost
+	 * list should never exceed the target size of the cache, and
+	 * the sum of the mru list, mfu list, mru ghost list, and mfu
+	 * ghost list should never exceed twice the target size of the
+	 * cache. The following logic enforces these limits on the ghost
+	 * caches, and evicts from them as needed.
 	 */
+	target = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c;
 
-	adjustment = arc_mru->arcs_size + arc_mru_ghost->arcs_size - arc_c;
+	bytes = arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_DATA);
+	total_evicted += bytes;
 
-	if (adjustment > 0 && arc_mru_ghost->arcs_size > 0) {
-		delta = MIN(arc_mru_ghost->arcs_size, adjustment);
-		arc_evict_ghost(arc_mru_ghost, NULL, delta);
-	}
+	target -= bytes;
 
-	adjustment =
-	    arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c;
+	total_evicted +=
+	    arc_adjust_impl(arc_mru_ghost, 0, target, ARC_BUFC_METADATA);
 
-	if (adjustment > 0 && arc_mfu_ghost->arcs_size > 0) {
-		delta = MIN(arc_mfu_ghost->arcs_size, adjustment);
-		arc_evict_ghost(arc_mfu_ghost, NULL, delta);
-	}
+	/*
+	 * We assume the sum of the mru list and mfu list is less than
+	 * or equal to arc_c (we enforced this above), which means we
+	 * can use the simpler of the two equations below:
+	 *
+	 *	mru + mfu + mru ghost + mfu ghost <= 2 * arc_c
+	 *		    mru ghost + mfu ghost <= arc_c
+	 */
+	target = arc_mru_ghost->arcs_size + arc_mfu_ghost->arcs_size - arc_c;
+
+	bytes = arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_DATA);
+	total_evicted += bytes;
+
+	target -= bytes;
+
+	total_evicted +=
+	    arc_adjust_impl(arc_mfu_ghost, 0, target, ARC_BUFC_METADATA);
+
+	return (total_evicted);
 }
 
 static void
 arc_do_user_evicts(void)
 {
-	mutex_enter(&arc_eviction_mtx);
+	mutex_enter(&arc_user_evicts_lock);
 	while (arc_eviction_list != NULL) {
 		arc_buf_t *buf = arc_eviction_list;
 		arc_eviction_list = buf->b_next;
 		mutex_enter(&buf->b_evict_lock);
 		buf->b_hdr = NULL;
 		mutex_exit(&buf->b_evict_lock);
-		mutex_exit(&arc_eviction_mtx);
+		mutex_exit(&arc_user_evicts_lock);
 
 		if (buf->b_efunc != NULL)
 			VERIFY0(buf->b_efunc(buf->b_private));
@@ -2580,50 +2937,39 @@ arc_do_user_evicts(void)
 		buf->b_efunc = NULL;
 		buf->b_private = NULL;
 		kmem_cache_free(buf_cache, buf);
-		mutex_enter(&arc_eviction_mtx);
+		mutex_enter(&arc_user_evicts_lock);
 	}
-	mutex_exit(&arc_eviction_mtx);
+	mutex_exit(&arc_user_evicts_lock);
 }
 
-/*
- * Flush all *evictable* data from the cache for the given spa.
- * NOTE: this will not touch "active" (i.e. referenced) data.
- */
 void
-arc_flush(spa_t *spa)
+arc_flush(spa_t *spa, boolean_t retry)
 {
 	uint64_t guid = 0;
 
+	/*
+	 * If retry is TRUE, a spa must not be specified since we have
+	 * no good way to determine if all of a spa's buffers have been
+	 * evicted from an arc state.
+	 */
+	ASSERT(!retry || spa == 0);
+
 	if (spa != NULL)
 		guid = spa_load_guid(spa);
 
-	while (list_head(&arc_mru->arcs_list[ARC_BUFC_DATA])) {
-		(void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_DATA);
-		if (spa != NULL)
-			break;
-	}
-	while (list_head(&arc_mru->arcs_list[ARC_BUFC_METADATA])) {
-		(void) arc_evict(arc_mru, guid, -1, FALSE, ARC_BUFC_METADATA);
-		if (spa != NULL)
-			break;
-	}
-	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_DATA])) {
-		(void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_DATA);
-		if (spa != NULL)
-			break;
-	}
-	while (list_head(&arc_mfu->arcs_list[ARC_BUFC_METADATA])) {
-		(void) arc_evict(arc_mfu, guid, -1, FALSE, ARC_BUFC_METADATA);
-		if (spa != NULL)
-			break;
-	}
+	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_DATA, retry);
+	(void) arc_flush_state(arc_mru, guid, ARC_BUFC_METADATA, retry);
 
-	arc_evict_ghost(arc_mru_ghost, guid, -1);
-	arc_evict_ghost(arc_mfu_ghost, guid, -1);
+	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_DATA, retry);
+	(void) arc_flush_state(arc_mfu, guid, ARC_BUFC_METADATA, retry);
+
+	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_DATA, retry);
+	(void) arc_flush_state(arc_mru_ghost, guid, ARC_BUFC_METADATA, retry);
+
+	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_DATA, retry);
+	(void) arc_flush_state(arc_mfu_ghost, guid, ARC_BUFC_METADATA, retry);
 
-	mutex_enter(&arc_reclaim_thr_lock);
 	arc_do_user_evicts();
-	mutex_exit(&arc_reclaim_thr_lock);
 	ASSERT(spa || arc_eviction_list == NULL);
 }
 
@@ -2647,7 +2993,7 @@ arc_shrink(int64_t to_free)
 	}
 
 	if (arc_size > arc_c)
-		arc_adjust();
+		(void) arc_adjust();
 }
 
 typedef enum free_memory_reason_t {
@@ -2847,17 +3193,37 @@ arc_kmem_reap_now(void)
 	}
 }
 
+/*
+ * Threads can block in arc_get_data_buf() waiting for this thread to evict
+ * enough data and signal them to proceed. When this happens, the threads in
+ * arc_get_data_buf() are sleeping while holding the hash lock for their
+ * particular arc header. Thus, we must be careful to never sleep on a
+ * hash lock in this thread. This is to prevent the following deadlock:
+ *
+ *  - Thread A sleeps on CV in arc_get_data_buf() holding hash lock "L",
+ *    waiting for the reclaim thread to signal it.
+ *
+ *  - arc_reclaim_thread() tries to acquire hash lock "L" using mutex_enter,
+ *    fails, and goes to sleep forever.
+ *
+ * This possible deadlock is avoided by always acquiring a hash lock
+ * using mutex_tryenter() from arc_reclaim_thread().
+ */
 static void
 arc_reclaim_thread(void)
 {
 	clock_t			growtime = 0;
 	callb_cpr_t		cpr;
 
-	CALLB_CPR_INIT(&cpr, &arc_reclaim_thr_lock, callb_generic_cpr, FTAG);
+	CALLB_CPR_INIT(&cpr, &arc_reclaim_lock, callb_generic_cpr, FTAG);
 
-	mutex_enter(&arc_reclaim_thr_lock);
-	while (arc_thread_exit == 0) {
+	mutex_enter(&arc_reclaim_lock);
+	while (!arc_reclaim_thread_exit) {
 		int64_t free_memory = arc_available_memory();
+		uint64_t evicted = 0;
+
+		mutex_exit(&arc_reclaim_lock);
+
 		if (free_memory < 0) {
 
 			arc_no_grow = B_TRUE;
@@ -2891,10 +3257,57 @@ arc_reclaim_thread(void)
 			arc_no_grow = B_FALSE;
 		}
 
-		arc_adjust();
+		evicted = arc_adjust();
 
-		if (arc_eviction_list != NULL)
-			arc_do_user_evicts();
+		mutex_enter(&arc_reclaim_lock);
+
+		/*
+		 * If evicted is zero, we couldn't evict anything via
+		 * arc_adjust(). This could be due to hash lock
+		 * collisions, but more likely due to the majority of
+		 * arc buffers being unevictable. Therefore, even if
+		 * arc_size is above arc_c, another pass is unlikely to
+		 * be helpful and could potentially cause us to enter an
+		 * infinite loop.
+		 */
+		if (arc_size <= arc_c || evicted == 0) {
+			/*
+			 * We're either no longer overflowing, or we
+			 * can't evict anything more, so we should wake
+			 * up any threads before we go to sleep.
+			 */
+			cv_broadcast(&arc_reclaim_waiters_cv);
+
+			/*
+			 * Block until signaled, or after one second (we
+			 * might need to perform arc_kmem_reap_now()
+			 * even if we aren't being signalled)
+			 */
+			CALLB_CPR_SAFE_BEGIN(&cpr);
+			(void) cv_timedwait(&arc_reclaim_thread_cv,
+			    &arc_reclaim_lock, ddi_get_lbolt() + hz);
+			CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_lock);
+		}
+	}
+
+	arc_reclaim_thread_exit = FALSE;
+	cv_broadcast(&arc_reclaim_thread_cv);
+	CALLB_CPR_EXIT(&cpr);		/* drops arc_reclaim_lock */
+	thread_exit();
+}
+
+static void
+arc_user_evicts_thread(void)
+{
+	callb_cpr_t cpr;
+
+	CALLB_CPR_INIT(&cpr, &arc_user_evicts_lock, callb_generic_cpr, FTAG);
+
+	mutex_enter(&arc_user_evicts_lock);
+	while (!arc_user_evicts_thread_exit) {
+		mutex_exit(&arc_user_evicts_lock);
+
+		arc_do_user_evicts();
 
 		/*
 		 * This is necessary in order for the mdb ::arc dcmd to
@@ -2910,16 +3323,21 @@ arc_reclaim_thread(void)
 		if (arc_ksp != NULL)
 			arc_ksp->ks_update(arc_ksp, KSTAT_READ);
 
-		/* block until needed, or one second, whichever is shorter */
+		mutex_enter(&arc_user_evicts_lock);
+
+		/*
+		 * Block until signaled, or after one second (we need to
+		 * call the arc's kstat update function regularly).
+		 */
 		CALLB_CPR_SAFE_BEGIN(&cpr);
-		(void) cv_timedwait(&arc_reclaim_thr_cv,
-		    &arc_reclaim_thr_lock, (ddi_get_lbolt() + hz));
-		CALLB_CPR_SAFE_END(&cpr, &arc_reclaim_thr_lock);
+		(void) cv_timedwait(&arc_user_evicts_cv,
+		    &arc_user_evicts_lock, ddi_get_lbolt() + hz);
+		CALLB_CPR_SAFE_END(&cpr, &arc_user_evicts_lock);
 	}
 
-	arc_thread_exit = 0;
-	cv_broadcast(&arc_reclaim_thr_cv);
-	CALLB_CPR_EXIT(&cpr);		/* drops arc_reclaim_thr_lock */
+	arc_user_evicts_thread_exit = FALSE;
+	cv_broadcast(&arc_user_evicts_cv);
+	CALLB_CPR_EXIT(&cpr);		/* drops arc_user_evicts_lock */
 	thread_exit();
 }
 
@@ -2965,7 +3383,7 @@ arc_adapt(int bytes, arc_state_t *state)
 	ASSERT((int64_t)arc_p >= 0);
 
 	if (arc_reclaim_needed()) {
-		cv_signal(&arc_reclaim_thr_cv);
+		cv_signal(&arc_reclaim_thread_cv);
 		return;
 	}
 
@@ -2992,43 +3410,25 @@ arc_adapt(int bytes, arc_state_t *state)
 }
 
 /*
- * Check if the cache has reached its limits and eviction is required
- * prior to insert.
+ * Check if arc_size has grown past our upper threshold, determined by
+ * zfs_arc_overflow_shift.
  */
-static int
-arc_evict_needed(arc_buf_contents_t type)
+static boolean_t
+arc_is_overflowing(void)
 {
-	if (type == ARC_BUFC_METADATA && arc_meta_used >= arc_meta_limit)
-		return (1);
+	/* Always allow at least one block of overflow */
+	uint64_t overflow = MAX(SPA_MAXBLOCKSIZE,
+	    arc_c >> zfs_arc_overflow_shift);
 
-	if (arc_reclaim_needed())
-		return (1);
-
-	return (arc_size > arc_c);
+	return (arc_size >= arc_c + overflow);
 }
 
 /*
- * The buffer, supplied as the first argument, needs a data block.
- * So, if we are at cache max, determine which cache should be victimized.
- * We have the following cases:
- *
- * 1. Insert for MRU, p > sizeof(arc_anon + arc_mru) ->
- * In this situation if we're out of space, but the resident size of the MFU is
- * under the limit, victimize the MFU cache to satisfy this insertion request.
- *
- * 2. Insert for MRU, p <= sizeof(arc_anon + arc_mru) ->
- * Here, we've used up all of the available space for the MRU, so we need to
- * evict from our own cache instead.  Evict from the set of resident MRU
- * entries.
- *
- * 3. Insert for MFU (c - p) > sizeof(arc_mfu) ->
- * c minus p represents the MFU space in the cache, since p is the size of the
- * cache that is dedicated to the MRU.  In this situation there's still space on
- * the MFU side, so the MRU side needs to be victimized.
- *
- * 4. Insert for MFU (c - p) < sizeof(arc_mfu) ->
- * MFU's resident set is consuming more space than it has been allotted.  In
- * this situation, we must victimize our own cache, the MFU, for this insertion.
+ * The buffer, supplied as the first argument, needs a data block. If we
+ * are hitting the hard limit for the cache size, we must sleep, waiting
+ * for the eviction thread to catch up. If we're past the target size
+ * but below the hard limit, we'll only signal the reclaim thread and
+ * continue on.
  */
 static void
 arc_get_data_buf(arc_buf_t *buf)
@@ -3040,53 +3440,50 @@ arc_get_data_buf(arc_buf_t *buf)
 	arc_adapt(size, state);
 
 	/*
-	 * We have not yet reached cache maximum size,
-	 * just allocate a new buffer.
+	 * If arc_size is currently overflowing, and has grown past our
+	 * upper limit, we must be adding data faster than the evict
+	 * thread can evict. Thus, to ensure we don't compound the
+	 * problem by adding more data and forcing arc_size to grow even
+	 * further past it's target size, we halt and wait for the
+	 * eviction thread to catch up.
+	 *
+	 * It's also possible that the reclaim thread is unable to evict
+	 * enough buffers to get arc_size below the overflow limit (e.g.
+	 * due to buffers being un-evictable, or hash lock collisions).
+	 * In this case, we want to proceed regardless if we're
+	 * overflowing; thus we don't use a while loop here.
 	 */
-	if (!arc_evict_needed(type)) {
-		if (type == ARC_BUFC_METADATA) {
-			buf->b_data = zio_buf_alloc(size);
-			arc_space_consume(size, ARC_SPACE_META);
-		} else {
-			ASSERT(type == ARC_BUFC_DATA);
-			buf->b_data = zio_data_buf_alloc(size);
-			arc_space_consume(size, ARC_SPACE_DATA);
+	if (arc_is_overflowing()) {
+		mutex_enter(&arc_reclaim_lock);
+
+		/*
+		 * Now that we've acquired the lock, we may no longer be
+		 * over the overflow limit, lets check.
+		 *
+		 * We're ignoring the case of spurious wake ups. If that
+		 * were to happen, it'd let this thread consume an ARC
+		 * buffer before it should have (i.e. before we're under
+		 * the overflow limit and were signalled by the reclaim
+		 * thread). As long as that is a rare occurrence, it
+		 * shouldn't cause any harm.
+		 */
+		if (arc_is_overflowing()) {
+			cv_signal(&arc_reclaim_thread_cv);
+			cv_wait(&arc_reclaim_waiters_cv, &arc_reclaim_lock);
 		}
-		goto out;
+
+		mutex_exit(&arc_reclaim_lock);
 	}
 
-	/*
-	 * If we are prefetching from the mfu ghost list, this buffer
-	 * will end up on the mru list; so steal space from there.
-	 */
-	if (state == arc_mfu_ghost)
-		state = HDR_PREFETCH(buf->b_hdr) ? arc_mru : arc_mfu;
-	else if (state == arc_mru_ghost)
-		state = arc_mru;
-
-	if (state == arc_mru || state == arc_anon) {
-		uint64_t mru_used = arc_anon->arcs_size + arc_mru->arcs_size;
-		state = (arc_mfu->arcs_lsize[type] >= size &&
-		    arc_p > mru_used) ? arc_mfu : arc_mru;
+	if (type == ARC_BUFC_METADATA) {
+		buf->b_data = zio_buf_alloc(size);
+		arc_space_consume(size, ARC_SPACE_META);
 	} else {
-		/* MFU cases */
-		uint64_t mfu_space = arc_c - arc_p;
-		state =  (arc_mru->arcs_lsize[type] >= size &&
-		    mfu_space > arc_mfu->arcs_size) ? arc_mru : arc_mfu;
+		ASSERT(type == ARC_BUFC_DATA);
+		buf->b_data = zio_data_buf_alloc(size);
+		arc_space_consume(size, ARC_SPACE_DATA);
 	}
-	if ((buf->b_data = arc_evict(state, NULL, size, TRUE, type)) == NULL) {
-		if (type == ARC_BUFC_METADATA) {
-			buf->b_data = zio_buf_alloc(size);
-			arc_space_consume(size, ARC_SPACE_META);
-		} else {
-			ASSERT(type == ARC_BUFC_DATA);
-			buf->b_data = zio_data_buf_alloc(size);
-			arc_space_consume(size, ARC_SPACE_DATA);
-		}
-		ARCSTAT_BUMP(arcstat_recycle_miss);
-	}
-	ASSERT(buf->b_data != NULL);
-out:
+
 	/*
 	 * Update the state size.  Note that ghost states have a
 	 * "ghost size" and so don't need to be updated.
@@ -3095,7 +3492,17 @@ out:
 		arc_buf_hdr_t *hdr = buf->b_hdr;
 
 		atomic_add_64(&hdr->b_l1hdr.b_state->arcs_size, size);
-		if (list_link_active(&hdr->b_l1hdr.b_arc_node)) {
+
+		/*
+		 * If this is reached via arc_read, the link is
+		 * protected by the hash lock. If reached via
+		 * arc_buf_alloc, the header should not be accessed by
+		 * any other thread. And, if reached via arc_read_done,
+		 * the hash lock will protect it if it's found in the
+		 * hash table; otherwise no other thread should be
+		 * trying to [add|remove]_reference it.
+		 */
+		if (multilist_link_active(&hdr->b_l1hdr.b_arc_node)) {
 			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
 			atomic_add_64(&hdr->b_l1hdr.b_state->arcs_lsize[type],
 			    size);
@@ -3147,7 +3554,8 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 		 */
 		if (HDR_PREFETCH(hdr)) {
 			if (refcount_count(&hdr->b_l1hdr.b_refcnt) == 0) {
-				ASSERT(list_link_active(
+				/* link protected by hash lock */
+				ASSERT(multilist_link_active(
 				    &hdr->b_l1hdr.b_arc_node));
 			} else {
 				hdr->b_flags &= ~ARC_FLAG_PREFETCH;
@@ -3207,7 +3615,8 @@ arc_access(arc_buf_hdr_t *hdr, kmutex_t *hash_lock)
 		 */
 		if ((HDR_PREFETCH(hdr)) != 0) {
 			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
-			ASSERT(list_link_active(&hdr->b_l1hdr.b_arc_node));
+			/* link protected by hash_lock */
+			ASSERT(multilist_link_active(&hdr->b_l1hdr.b_arc_node));
 		}
 		ARCSTAT_BUMP(arcstat_mfu_hits);
 		hdr->b_l1hdr.b_arc_access = ddi_get_lbolt();
@@ -3577,7 +3986,7 @@ top:
 			ASSERT(GHOST_STATE(hdr->b_l1hdr.b_state));
 			ASSERT(!HDR_IO_IN_PROGRESS(hdr));
 			ASSERT(refcount_is_zero(&hdr->b_l1hdr.b_refcnt));
-			ASSERT(hdr->b_l1hdr.b_buf == NULL);
+			ASSERT3P(hdr->b_l1hdr.b_buf, ==, NULL);
 
 			/* if this is a prefetch, we don't have a reference */
 			if (*arc_flags & ARC_FLAG_PREFETCH)
@@ -3836,7 +4245,7 @@ arc_clear_callback(arc_buf_t *buf)
 
 	if (hdr->b_l1hdr.b_datacnt > 1) {
 		mutex_exit(&buf->b_evict_lock);
-		arc_buf_destroy(buf, FALSE, TRUE);
+		arc_buf_destroy(buf, TRUE);
 	} else {
 		ASSERT(buf == hdr->b_l1hdr.b_buf);
 		hdr->b_flags |= ARC_FLAG_BUF_AVAILABLE;
@@ -3867,6 +4276,8 @@ arc_release(arc_buf_t *buf, void *tag)
 
 	mutex_enter(&buf->b_evict_lock);
 
+	ASSERT(HDR_HAS_L1HDR(hdr));
+
 	/*
 	 * We don't grab the hash lock prior to this check, because if
 	 * the buffer's header is in the arc_anon state, it won't be
@@ -3913,6 +4324,13 @@ arc_release(arc_buf_t *buf, void *tag)
 
 		mutex_enter(&hdr->b_l2hdr.b_dev->l2ad_mtx);
 		list_remove(&hdr->b_l2hdr.b_dev->l2ad_buflist, hdr);
+
+		/*
+		 * We don't want to leak the b_tmp_cdata buffer that was
+		 * allocated in l2arc_write_buffers()
+		 */
+		arc_buf_l2_cdata_free(hdr);
+
 		mutex_exit(&hdr->b_l2hdr.b_dev->l2ad_mtx);
 
 		hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
@@ -3978,6 +4396,7 @@ arc_release(arc_buf_t *buf, void *tag)
 		nhdr->b_l1hdr.b_datacnt = 1;
 		nhdr->b_l1hdr.b_state = arc_anon;
 		nhdr->b_l1hdr.b_arc_access = 0;
+		nhdr->b_l1hdr.b_tmp_cdata = NULL;
 		nhdr->b_freeze_cksum = NULL;
 
 		(void) refcount_add(&nhdr->b_l1hdr.b_refcnt, tag);
@@ -3987,8 +4406,8 @@ arc_release(arc_buf_t *buf, void *tag)
 	} else {
 		mutex_exit(&buf->b_evict_lock);
 		ASSERT(refcount_count(&hdr->b_l1hdr.b_refcnt) == 1);
-		/* protected by hash lock */
-		ASSERT(!list_link_active(&hdr->b_l1hdr.b_arc_node));
+		/* protected by hash lock, or hdr is on arc_anon */
+		ASSERT(!multilist_link_active(&hdr->b_l1hdr.b_arc_node));
 		ASSERT(!HDR_IO_IN_PROGRESS(hdr));
 		arc_change_state(arc_anon, hdr, hash_lock);
 		hdr->b_l1hdr.b_arc_access = 0;
@@ -4323,6 +4742,41 @@ arc_kstat_update(kstat_t *ksp, int rw)
 	return (0);
 }
 
+/*
+ * This function *must* return indices evenly distributed between all
+ * sublists of the multilist. This is needed due to how the ARC eviction
+ * code is laid out; arc_evict_state() assumes ARC buffers are evenly
+ * distributed between all sublists and uses this assumption when
+ * deciding which sublist to evict from and how much to evict from it.
+ */
+unsigned int
+arc_state_multilist_index_func(multilist_t *ml, void *obj)
+{
+	arc_buf_hdr_t *hdr = obj;
+
+	/*
+	 * We rely on b_dva to generate evenly distributed index
+	 * numbers using buf_hash below. So, as an added precaution,
+	 * let's make sure we never add empty buffers to the arc lists.
+	 */
+	ASSERT(!BUF_EMPTY(hdr));
+
+	/*
+	 * The assumption here, is the hash value for a given
+	 * arc_buf_hdr_t will remain constant throughout it's lifetime
+	 * (i.e. it's b_spa, b_dva, and b_birth fields don't change).
+	 * Thus, we don't need to store the header's sublist index
+	 * on insertion, as this index can be recalculated on removal.
+	 *
+	 * Also, the low order bits of the hash value are thought to be
+	 * distributed evenly. Otherwise, in the case that the multilist
+	 * has a power of two number of sublists, each sublists' usage
+	 * would not be evenly distributed.
+	 */
+	return (buf_hash(hdr->b_spa, &hdr->b_dva, hdr->b_birth) %
+	    multilist_get_num_sublists(ml));
+}
+
 void
 arc_init(void)
 {
@@ -4335,8 +4789,12 @@ arc_init(void)
 	uint64_t allmem = (physmem * PAGESIZE) / 2;
 #endif
 
-	mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL);
-	cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL);
+	mutex_init(&arc_reclaim_lock, NULL, MUTEX_DEFAULT, NULL);
+	cv_init(&arc_reclaim_thread_cv, NULL, CV_DEFAULT, NULL);
+	cv_init(&arc_reclaim_waiters_cv, NULL, CV_DEFAULT, NULL);
+
+	mutex_init(&arc_user_evicts_lock, NULL, MUTEX_DEFAULT, NULL);
+	cv_init(&arc_user_evicts_cv, NULL, CV_DEFAULT, NULL);
 
 	/* Convert seconds to clock ticks */
 	arc_min_prefetch_lifespan = 1 * hz;
@@ -4405,6 +4863,9 @@ arc_init(void)
 	if (zfs_arc_p_min_shift > 0)
 		arc_p_min_shift = zfs_arc_p_min_shift;
 
+	if (zfs_arc_num_sublists_per_state < 1)
+		zfs_arc_num_sublists_per_state = MAX(boot_ncpus, 1);
+
 	/* if kmem_flags are set, lets try to use less memory */
 	if (kmem_debugging())
 		arc_c = arc_c / 2;
@@ -4419,49 +4880,52 @@ arc_init(void)
 	arc_l2c_only = &ARC_l2c_only;
 	arc_size = 0;
 
-	mutex_init(&arc_anon->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
-	mutex_init(&arc_mru->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
-	mutex_init(&arc_mru_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
-	mutex_init(&arc_mfu->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
-	mutex_init(&arc_mfu_ghost->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
-	mutex_init(&arc_l2c_only->arcs_mtx, NULL, MUTEX_DEFAULT, NULL);
-
-	list_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
+	multilist_create(&arc_mru->arcs_list[ARC_BUFC_METADATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_mru->arcs_list[ARC_BUFC_DATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_mfu->arcs_list[ARC_BUFC_METADATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_mfu->arcs_list[ARC_BUFC_DATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_METADATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
-	list_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
+	multilist_create(&arc_l2c_only->arcs_list[ARC_BUFC_DATA],
 	    sizeof (arc_buf_hdr_t),
-	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node));
+	    offsetof(arc_buf_hdr_t, b_l1hdr.b_arc_node),
+	    zfs_arc_num_sublists_per_state, arc_state_multilist_index_func);
 
 	buf_init();
 
-	arc_thread_exit = 0;
+	arc_reclaim_thread_exit = FALSE;
+	arc_user_evicts_thread_exit = FALSE;
 	arc_eviction_list = NULL;
-	mutex_init(&arc_eviction_mtx, NULL, MUTEX_DEFAULT, NULL);
 	bzero(&arc_eviction_hdr, sizeof (arc_buf_hdr_t));
 
 	arc_ksp = kstat_create("zfs", 0, "arcstats", "misc", KSTAT_TYPE_NAMED,
@@ -4476,6 +4940,9 @@ arc_init(void)
 	(void) thread_create(NULL, 0, arc_reclaim_thread, NULL, 0, &p0,
 	    TS_RUN, minclsyspri);
 
+	(void) thread_create(NULL, 0, arc_user_evicts_thread, NULL, 0, &p0,
+	    TS_RUN, minclsyspri);
+
 	arc_dead = FALSE;
 	arc_warm = B_FALSE;
 
@@ -4498,13 +4965,32 @@ arc_init(void)
 void
 arc_fini(void)
 {
-	mutex_enter(&arc_reclaim_thr_lock);
-	arc_thread_exit = 1;
-	while (arc_thread_exit != 0)
-		cv_wait(&arc_reclaim_thr_cv, &arc_reclaim_thr_lock);
-	mutex_exit(&arc_reclaim_thr_lock);
+	mutex_enter(&arc_reclaim_lock);
+	arc_reclaim_thread_exit = TRUE;
+	/*
+	 * The reclaim thread will set arc_reclaim_thread_exit back to
+	 * FALSE when it is finished exiting; we're waiting for that.
+	 */
+	while (arc_reclaim_thread_exit) {
+		cv_signal(&arc_reclaim_thread_cv);
+		cv_wait(&arc_reclaim_thread_cv, &arc_reclaim_lock);
+	}
+	mutex_exit(&arc_reclaim_lock);
 
-	arc_flush(NULL);
+	mutex_enter(&arc_user_evicts_lock);
+	arc_user_evicts_thread_exit = TRUE;
+	/*
+	 * The user evicts thread will set arc_user_evicts_thread_exit
+	 * to FALSE when it is finished exiting; we're waiting for that.
+	 */
+	while (arc_user_evicts_thread_exit) {
+		cv_signal(&arc_user_evicts_cv);
+		cv_wait(&arc_user_evicts_cv, &arc_user_evicts_lock);
+	}
+	mutex_exit(&arc_user_evicts_lock);
+
+	/* Use TRUE to ensure *all* buffers are evicted */
+	arc_flush(NULL, TRUE);
 
 	arc_dead = TRUE;
 
@@ -4513,25 +4999,21 @@ arc_fini(void)
 		arc_ksp = NULL;
 	}
 
-	mutex_destroy(&arc_eviction_mtx);
-	mutex_destroy(&arc_reclaim_thr_lock);
-	cv_destroy(&arc_reclaim_thr_cv);
+	mutex_destroy(&arc_reclaim_lock);
+	cv_destroy(&arc_reclaim_thread_cv);
+	cv_destroy(&arc_reclaim_waiters_cv);
 
-	list_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
-	list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
-	list_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
-	list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
-	list_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
-	list_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
-	list_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
-	list_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
+	mutex_destroy(&arc_user_evicts_lock);
+	cv_destroy(&arc_user_evicts_cv);
 
-	mutex_destroy(&arc_anon->arcs_mtx);
-	mutex_destroy(&arc_mru->arcs_mtx);
-	mutex_destroy(&arc_mru_ghost->arcs_mtx);
-	mutex_destroy(&arc_mfu->arcs_mtx);
-	mutex_destroy(&arc_mfu_ghost->arcs_mtx);
-	mutex_destroy(&arc_l2c_only->arcs_mtx);
+	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_METADATA]);
+	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_METADATA]);
+	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_METADATA]);
+	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_METADATA]);
+	multilist_destroy(&arc_mru->arcs_list[ARC_BUFC_DATA]);
+	multilist_destroy(&arc_mru_ghost->arcs_list[ARC_BUFC_DATA]);
+	multilist_destroy(&arc_mfu->arcs_list[ARC_BUFC_DATA]);
+	multilist_destroy(&arc_mfu_ghost->arcs_list[ARC_BUFC_DATA]);
 
 	buf_fini();
 
@@ -4859,34 +5341,62 @@ l2arc_write_done(zio_t *zio)
 	if (zio->io_error != 0)
 		ARCSTAT_BUMP(arcstat_l2_writes_error);
 
-	mutex_enter(&dev->l2ad_mtx);
-
 	/*
 	 * All writes completed, or an error was hit.
 	 */
+top:
+	mutex_enter(&dev->l2ad_mtx);
 	for (hdr = list_prev(buflist, head); hdr; hdr = hdr_prev) {
 		hdr_prev = list_prev(buflist, hdr);
 
 		hash_lock = HDR_LOCK(hdr);
+
+		/*
+		 * We cannot use mutex_enter or else we can deadlock
+		 * with l2arc_write_buffers (due to swapping the order
+		 * the hash lock and l2ad_mtx are taken).
+		 */
 		if (!mutex_tryenter(hash_lock)) {
 			/*
-			 * This buffer misses out.  It may be in a stage
-			 * of eviction.  Its ARC_FLAG_L2_WRITING flag will be
-			 * left set, denying reads to this buffer.
+			 * Missed the hash lock. We must retry so we
+			 * don't leave the ARC_FLAG_L2_WRITING bit set.
 			 */
-			ARCSTAT_BUMP(arcstat_l2_writes_hdr_miss);
-			continue;
+			ARCSTAT_BUMP(arcstat_l2_writes_lock_retry);
+
+			/*
+			 * We don't want to rescan the headers we've
+			 * already marked as having been written out, so
+			 * we reinsert the head node so we can pick up
+			 * where we left off.
+			 */
+			list_remove(buflist, head);
+			list_insert_after(buflist, hdr, head);
+
+			mutex_exit(&dev->l2ad_mtx);
+
+			/*
+			 * We wait for the hash lock to become available
+			 * to try and prevent busy waiting, and increase
+			 * the chance we'll be able to acquire the lock
+			 * the next time around.
+			 */
+			mutex_enter(hash_lock);
+			mutex_exit(hash_lock);
+			goto top;
 		}
 
 		/*
-		 * It's possible that this buffer got evicted from the L1 cache
-		 * before we grabbed the vdev + hash locks, in which case
-		 * arc_hdr_realloc freed b_tmp_cdata for us if it was allocated.
-		 * Only free the buffer if we still have an L1 hdr.
+		 * We could not have been moved into the arc_l2c_only
+		 * state while in-flight due to our ARC_FLAG_L2_WRITING
+		 * bit being set. Let's just ensure that's being enforced.
 		 */
-		if (HDR_HAS_L1HDR(hdr) && hdr->b_l1hdr.b_tmp_cdata != NULL &&
-		    HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_OFF)
-			l2arc_release_cdata_buf(hdr);
+		ASSERT(HDR_HAS_L1HDR(hdr));
+
+		/*
+		 * We may have allocated a buffer for L2ARC compression,
+		 * we must release it to avoid leaking this data.
+		 */
+		l2arc_release_cdata_buf(hdr);
 
 		if (zio->io_error != 0) {
 			/*
@@ -4900,7 +5410,8 @@ l2arc_write_done(zio_t *zio)
 		}
 
 		/*
-		 * Allow ARC to begin reads to this L2ARC entry.
+		 * Allow ARC to begin reads and ghost list evictions to
+		 * this L2ARC entry.
 		 */
 		hdr->b_flags &= ~ARC_FLAG_L2_WRITING;
 
@@ -5008,35 +5519,37 @@ l2arc_read_done(zio_t *zio)
  * the data lists.  This function returns a locked list, and also returns
  * the lock pointer.
  */
-static list_t *
-l2arc_list_locked(int list_num, kmutex_t **lock)
+static multilist_sublist_t *
+l2arc_sublist_lock(int list_num)
 {
-	list_t *list = NULL;
+	multilist_t *ml = NULL;
+	unsigned int idx;
 
 	ASSERT(list_num >= 0 && list_num <= 3);
 
 	switch (list_num) {
 	case 0:
-		list = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
-		*lock = &arc_mfu->arcs_mtx;
+		ml = &arc_mfu->arcs_list[ARC_BUFC_METADATA];
 		break;
 	case 1:
-		list = &arc_mru->arcs_list[ARC_BUFC_METADATA];
-		*lock = &arc_mru->arcs_mtx;
+		ml = &arc_mru->arcs_list[ARC_BUFC_METADATA];
 		break;
 	case 2:
-		list = &arc_mfu->arcs_list[ARC_BUFC_DATA];
-		*lock = &arc_mfu->arcs_mtx;
+		ml = &arc_mfu->arcs_list[ARC_BUFC_DATA];
 		break;
 	case 3:
-		list = &arc_mru->arcs_list[ARC_BUFC_DATA];
-		*lock = &arc_mru->arcs_mtx;
+		ml = &arc_mru->arcs_list[ARC_BUFC_DATA];
 		break;
 	}
 
-	ASSERT(!(MUTEX_HELD(*lock)));
-	mutex_enter(*lock);
-	return (list);
+	/*
+	 * Return a randomly-selected sublist. This is acceptable
+	 * because the caller feeds only a little bit of data for each
+	 * call (8MB). Subsequent calls will result in different
+	 * sublists being selected.
+	 */
+	idx = multilist_get_random_index(ml);
+	return (multilist_sublist_lock(ml, idx));
 }
 
 /*
@@ -5082,6 +5595,12 @@ top:
 		hdr_prev = list_prev(buflist, hdr);
 
 		hash_lock = HDR_LOCK(hdr);
+
+		/*
+		 * We cannot use mutex_enter or else we can deadlock
+		 * with l2arc_write_buffers (due to swapping the order
+		 * the hash lock and l2ad_mtx are taken).
+		 */
 		if (!mutex_tryenter(hash_lock)) {
 			/*
 			 * Missed the hash lock.  Retry.
@@ -5143,8 +5662,9 @@ top:
 			hdr->b_flags &= ~ARC_FLAG_HAS_L2HDR;
 			list_remove(buflist, hdr);
 
-			/* This may have been leftover after a failed write. */
-			hdr->b_flags &= ~ARC_FLAG_L2_WRITING;
+			/* Ensure this header has finished being written */
+			ASSERT(!HDR_L2_WRITING(hdr));
+			ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
 		}
 		mutex_exit(hash_lock);
 	}
@@ -5170,11 +5690,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
     boolean_t *headroom_boost)
 {
 	arc_buf_hdr_t *hdr, *hdr_prev, *head;
-	list_t *list;
 	uint64_t write_asize, write_psize, write_sz, headroom,
 	    buf_compress_minsz;
 	void *buf_data;
-	kmutex_t *list_lock;
 	boolean_t full;
 	l2arc_write_callback_t *cb;
 	zio_t *pio, *wzio;
@@ -5202,12 +5720,10 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	/*
 	 * Copy buffers for L2ARC writing.
 	 */
-	mutex_enter(&dev->l2ad_mtx);
 	for (int try = 0; try <= 3; try++) {
+		multilist_sublist_t *mls = l2arc_sublist_lock(try);
 		uint64_t passed_sz = 0;
 
-		list = l2arc_list_locked(try, &list_lock);
-
 		/*
 		 * L2ARC fast warmup.
 		 *
@@ -5215,9 +5731,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 		 * head of the ARC lists rather than the tail.
 		 */
 		if (arc_warm == B_FALSE)
-			hdr = list_head(list);
+			hdr = multilist_sublist_head(mls);
 		else
-			hdr = list_tail(list);
+			hdr = multilist_sublist_tail(mls);
 
 		headroom = target_sz * l2arc_headroom;
 		if (do_headroom_boost)
@@ -5228,9 +5744,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			uint64_t buf_sz;
 
 			if (arc_warm == B_FALSE)
-				hdr_prev = list_next(list, hdr);
+				hdr_prev = multilist_sublist_next(mls, hdr);
 			else
-				hdr_prev = list_prev(list, hdr);
+				hdr_prev = multilist_sublist_prev(mls, hdr);
 
 			hash_lock = HDR_LOCK(hdr);
 			if (!mutex_tryenter(hash_lock)) {
@@ -5266,7 +5782,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 				 * l2arc_write_done() can find where the
 				 * write buffers begin without searching.
 				 */
+				mutex_enter(&dev->l2ad_mtx);
 				list_insert_head(&dev->l2ad_buflist, head);
+				mutex_exit(&dev->l2ad_mtx);
 
 				cb = kmem_alloc(
 				    sizeof (l2arc_write_callback_t), KM_SLEEP);
@@ -5296,7 +5814,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			buf_sz = hdr->b_size;
 			hdr->b_flags |= ARC_FLAG_HAS_L2HDR;
 
+			mutex_enter(&dev->l2ad_mtx);
 			list_insert_head(&dev->l2ad_buflist, hdr);
+			mutex_exit(&dev->l2ad_mtx);
 
 			/*
 			 * Compute and store the buffer cksum before
@@ -5310,7 +5830,7 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			write_sz += buf_sz;
 		}
 
-		mutex_exit(list_lock);
+		multilist_sublist_unlock(mls);
 
 		if (full == B_TRUE)
 			break;
@@ -5319,12 +5839,13 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	/* No buffers selected for writing? */
 	if (pio == NULL) {
 		ASSERT0(write_sz);
-		mutex_exit(&dev->l2ad_mtx);
 		ASSERT(!HDR_HAS_L1HDR(head));
 		kmem_cache_free(hdr_l2only_cache, head);
 		return (0);
 	}
 
+	mutex_enter(&dev->l2ad_mtx);
+
 	/*
 	 * Now start writing the buffers. We're starting at the write head
 	 * and work backwards, retracing the course of the buffer selector
@@ -5334,6 +5855,14 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 	    hdr = list_prev(&dev->l2ad_buflist, hdr)) {
 		uint64_t buf_sz;
 
+		/*
+		 * We rely on the L1 portion of the header below, so
+		 * it's invalid for this header to have been evicted out
+		 * of the ghost cache, prior to being written out. The
+		 * ARC_FLAG_L2_WRITING bit ensures this won't happen.
+		 */
+		ASSERT(HDR_HAS_L1HDR(hdr));
+
 		/*
 		 * We shouldn't need to lock the buffer here, since we flagged
 		 * it as ARC_FLAG_L2_WRITING in the previous step, but we must
@@ -5553,8 +6082,26 @@ l2arc_decompress_zio(zio_t *zio, arc_buf_hdr_t *hdr, enum zio_compress c)
 static void
 l2arc_release_cdata_buf(arc_buf_hdr_t *hdr)
 {
+	enum zio_compress comp = HDR_GET_COMPRESS(hdr);
+
 	ASSERT(HDR_HAS_L1HDR(hdr));
-	if (HDR_GET_COMPRESS(hdr) != ZIO_COMPRESS_EMPTY) {
+	ASSERT(comp == ZIO_COMPRESS_OFF || L2ARC_IS_VALID_COMPRESS(comp));
+
+	if (comp == ZIO_COMPRESS_OFF) {
+		/*
+		 * In this case, b_tmp_cdata points to the same buffer
+		 * as the arc_buf_t's b_data field. We don't want to
+		 * free it, since the arc_buf_t will handle that.
+		 */
+		hdr->b_l1hdr.b_tmp_cdata = NULL;
+	} else if (comp == ZIO_COMPRESS_EMPTY) {
+		/*
+		 * In this case, b_tmp_cdata was compressed to an empty
+		 * buffer, thus there's nothing to free and b_tmp_cdata
+		 * should have been set to NULL in l2arc_write_buffers().
+		 */
+		ASSERT3P(hdr->b_l1hdr.b_tmp_cdata, ==, NULL);
+	} else {
 		/*
 		 * If the data was compressed, then we've allocated a
 		 * temporary buffer for it, so now we need to release it.
@@ -5562,8 +6109,9 @@ l2arc_release_cdata_buf(arc_buf_hdr_t *hdr)
 		ASSERT(hdr->b_l1hdr.b_tmp_cdata != NULL);
 		zio_data_buf_free(hdr->b_l1hdr.b_tmp_cdata,
 		    hdr->b_size);
+		hdr->b_l1hdr.b_tmp_cdata = NULL;
 	}
-	hdr->b_l1hdr.b_tmp_cdata = NULL;
+
 }
 
 /*
diff --git a/uts/common/fs/zfs/dsl_pool.c b/uts/common/fs/zfs/dsl_pool.c
index 30b7b4c8f02..2e7f798485d 100644
--- a/uts/common/fs/zfs/dsl_pool.c
+++ b/uts/common/fs/zfs/dsl_pool.c
@@ -315,7 +315,14 @@ dsl_pool_close(dsl_pool_t *dp)
 	txg_list_destroy(&dp->dp_sync_tasks);
 	txg_list_destroy(&dp->dp_dirty_dirs);
 
-	arc_flush(dp->dp_spa);
+	/*
+	 * We can't set retry to TRUE since we're explicitly specifying
+	 * a spa to flush. This is good enough; any missed buffers for
+	 * this spa won't cause trouble, and they'll eventually fall
+	 * out of the ARC just like any other unused buffer.
+	 */
+	arc_flush(dp->dp_spa, FALSE);
+
 	txg_fini(dp);
 	dsl_scan_fini(dp);
 	dmu_buf_user_evict_wait();
diff --git a/uts/common/fs/zfs/multilist.c b/uts/common/fs/zfs/multilist.c
new file mode 100644
index 00000000000..82960571bcb
--- /dev/null
+++ b/uts/common/fs/zfs/multilist.c
@@ -0,0 +1,366 @@
+/*
+ * CDDL HEADER START
+ *
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
+ */
+
+#include <sys/zfs_context.h>
+#include <sys/multilist.h>
+
+/* needed for spa_get_random() */
+#include <sys/spa.h>
+
+/*
+ * Given the object contained on the list, return a pointer to the
+ * object's multilist_node_t structure it contains.
+ */
+static multilist_node_t *
+multilist_d2l(multilist_t *ml, void *obj)
+{
+	return ((multilist_node_t *)((char *)obj + ml->ml_offset));
+}
+
+/*
+ * Initialize a new mutlilist using the parameters specified.
+ *
+ *  - 'size' denotes the size of the structure containing the
+ *     multilist_node_t.
+ *  - 'offset' denotes the byte offset of the mutlilist_node_t within
+ *     the structure that contains it.
+ *  - 'num' specifies the number of internal sublists to create.
+ *  - 'index_func' is used to determine which sublist to insert into
+ *     when the multilist_insert() function is called; as well as which
+ *     sublist to remove from when multilist_remove() is called. The
+ *     requirements this function must meet, are the following:
+ *
+ *      - It must always return the same value when called on the same
+ *        object (to ensure the object is removed from the list it was
+ *        inserted into).
+ *
+ *      - It must return a value in the range [0, number of sublists).
+ *        The multilist_get_num_sublists() function may be used to
+ *        determine the number of sublists in the multilist.
+ *
+ *     Also, in order to reduce internal contention between the sublists
+ *     during insertion and removal, this function should choose evenly
+ *     between all available sublists when inserting. This isn't a hard
+ *     requirement, but a general rule of thumb in order to garner the
+ *     best multi-threaded performance out of the data structure.
+ */
+void
+multilist_create(multilist_t *ml, size_t size, size_t offset, unsigned int num,
+    multilist_sublist_index_func_t *index_func)
+{
+	ASSERT3P(ml, !=, NULL);
+	ASSERT3U(size, >, 0);
+	ASSERT3U(size, >=, offset + sizeof (multilist_node_t));
+	ASSERT3U(num, >, 0);
+	ASSERT3P(index_func, !=, NULL);
+
+	ml->ml_offset = offset;
+	ml->ml_num_sublists = num;
+	ml->ml_index_func = index_func;
+
+	ml->ml_sublists = kmem_zalloc(sizeof (multilist_sublist_t) *
+	    ml->ml_num_sublists, KM_SLEEP);
+
+	ASSERT3P(ml->ml_sublists, !=, NULL);
+
+	for (int i = 0; i < ml->ml_num_sublists; i++) {
+		multilist_sublist_t *mls = &ml->ml_sublists[i];
+		mutex_init(&mls->mls_lock, NULL, MUTEX_DEFAULT, NULL);
+		list_create(&mls->mls_list, size, offset);
+	}
+}
+
+/*
+ * Destroy the given multilist object, and free up any memory it holds.
+ */
+void
+multilist_destroy(multilist_t *ml)
+{
+	ASSERT(multilist_is_empty(ml));
+
+	for (int i = 0; i < ml->ml_num_sublists; i++) {
+		multilist_sublist_t *mls = &ml->ml_sublists[i];
+
+		ASSERT(list_is_empty(&mls->mls_list));
+
+		list_destroy(&mls->mls_list);
+		mutex_destroy(&mls->mls_lock);
+	}
+
+	ASSERT3P(ml->ml_sublists, !=, NULL);
+	kmem_free(ml->ml_sublists,
+	    sizeof (multilist_sublist_t) * ml->ml_num_sublists);
+
+	ml->ml_num_sublists = 0;
+	ml->ml_offset = 0;
+}
+
+/*
+ * Insert the given object into the multilist.
+ *
+ * This function will insert the object specified into the sublist
+ * determined using the function given at multilist creation time.
+ *
+ * The sublist locks are automatically acquired if not already held, to
+ * ensure consistency when inserting and removing from multiple threads.
+ */
+void
+multilist_insert(multilist_t *ml, void *obj)
+{
+	unsigned int sublist_idx = ml->ml_index_func(ml, obj);
+	multilist_sublist_t *mls;
+	boolean_t need_lock;
+
+	DTRACE_PROBE3(multilist__insert, multilist_t *, ml,
+	    unsigned int, sublist_idx, void *, obj);
+
+	ASSERT3U(sublist_idx, <, ml->ml_num_sublists);
+
+	mls = &ml->ml_sublists[sublist_idx];
+
+	/*
+	 * Note: Callers may already hold the sublist lock by calling
+	 * multilist_sublist_lock().  Here we rely on MUTEX_HELD()
+	 * returning TRUE if and only if the current thread holds the
+	 * lock.  While it's a little ugly to make the lock recursive in
+	 * this way, it works and allows the calling code to be much
+	 * simpler -- otherwise it would have to pass around a flag
+	 * indicating that it already has the lock.
+	 */
+	need_lock = !MUTEX_HELD(&mls->mls_lock);
+
+	if (need_lock)
+		mutex_enter(&mls->mls_lock);
+
+	ASSERT(!multilist_link_active(multilist_d2l(ml, obj)));
+
+	multilist_sublist_insert_head(mls, obj);
+
+	if (need_lock)
+		mutex_exit(&mls->mls_lock);
+}
+
+/*
+ * Remove the given object from the multilist.
+ *
+ * This function will remove the object specified from the sublist
+ * determined using the function given at multilist creation time.
+ *
+ * The necessary sublist locks are automatically acquired, to ensure
+ * consistency when inserting and removing from multiple threads.
+ */
+void
+multilist_remove(multilist_t *ml, void *obj)
+{
+	unsigned int sublist_idx = ml->ml_index_func(ml, obj);
+	multilist_sublist_t *mls;
+	boolean_t need_lock;
+
+	DTRACE_PROBE3(multilist__remove, multilist_t *, ml,
+	    unsigned int, sublist_idx, void *, obj);
+
+	ASSERT3U(sublist_idx, <, ml->ml_num_sublists);
+
+	mls = &ml->ml_sublists[sublist_idx];
+	/* See comment in multilist_insert(). */
+	need_lock = !MUTEX_HELD(&mls->mls_lock);
+
+	if (need_lock)
+		mutex_enter(&mls->mls_lock);
+
+	ASSERT(multilist_link_active(multilist_d2l(ml, obj)));
+
+	multilist_sublist_remove(mls, obj);
+
+	if (need_lock)
+		mutex_exit(&mls->mls_lock);
+}
+
+/*
+ * Check to see if this multilist object is empty.
+ *
+ * This will return TRUE if it finds all of the sublists of this
+ * multilist to be empty, and FALSE otherwise. Each sublist lock will be
+ * automatically acquired as necessary.
+ *
+ * If concurrent insertions and removals are occurring, the semantics
+ * of this function become a little fuzzy. Instead of locking all
+ * sublists for the entire call time of the function, each sublist is
+ * only locked as it is individually checked for emptiness. Thus, it's
+ * possible for this function to return TRUE with non-empty sublists at
+ * the time the function returns. This would be due to another thread
+ * inserting into a given sublist, after that specific sublist was check
+ * and deemed empty, but before all sublists have been checked.
+ */
+int
+multilist_is_empty(multilist_t *ml)
+{
+	for (int i = 0; i < ml->ml_num_sublists; i++) {
+		multilist_sublist_t *mls = &ml->ml_sublists[i];
+		/* See comment in multilist_insert(). */
+		boolean_t need_lock = !MUTEX_HELD(&mls->mls_lock);
+
+		if (need_lock)
+			mutex_enter(&mls->mls_lock);
+
+		if (!list_is_empty(&mls->mls_list)) {
+			if (need_lock)
+				mutex_exit(&mls->mls_lock);
+
+			return (FALSE);
+		}
+
+		if (need_lock)
+			mutex_exit(&mls->mls_lock);
+	}
+
+	return (TRUE);
+}
+
+/* Return the number of sublists composing this multilist */
+unsigned int
+multilist_get_num_sublists(multilist_t *ml)
+{
+	return (ml->ml_num_sublists);
+}
+
+/* Return a randomly selected, valid sublist index for this multilist */
+unsigned int
+multilist_get_random_index(multilist_t *ml)
+{
+	return (spa_get_random(ml->ml_num_sublists));
+}
+
+/* Lock and return the sublist specified at the given index */
+multilist_sublist_t *
+multilist_sublist_lock(multilist_t *ml, unsigned int sublist_idx)
+{
+	multilist_sublist_t *mls;
+
+	ASSERT3U(sublist_idx, <, ml->ml_num_sublists);
+	mls = &ml->ml_sublists[sublist_idx];
+	mutex_enter(&mls->mls_lock);
+
+	return (mls);
+}
+
+void
+multilist_sublist_unlock(multilist_sublist_t *mls)
+{
+	mutex_exit(&mls->mls_lock);
+}
+
+/*
+ * We're allowing any object to be inserted into this specific sublist,
+ * but this can lead to trouble if multilist_remove() is called to
+ * remove this object. Specifically, if calling ml_index_func on this
+ * object returns an index for sublist different than what is passed as
+ * a parameter here, any call to multilist_remove() with this newly
+ * inserted object is undefined! (the call to multilist_remove() will
+ * remove the object from a list that it isn't contained in)
+ */
+void
+multilist_sublist_insert_head(multilist_sublist_t *mls, void *obj)
+{
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	list_insert_head(&mls->mls_list, obj);
+}
+
+/* please see comment above multilist_sublist_insert_head */
+void
+multilist_sublist_insert_tail(multilist_sublist_t *mls, void *obj)
+{
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	list_insert_tail(&mls->mls_list, obj);
+}
+
+/*
+ * Move the object one element forward in the list.
+ *
+ * This function will move the given object forward in the list (towards
+ * the head) by one object. So, in essence, it will swap its position in
+ * the list with its "prev" pointer. If the given object is already at the
+ * head of the list, it cannot be moved forward any more than it already
+ * is, so no action is taken.
+ *
+ * NOTE: This function **must not** remove any object from the list other
+ *       than the object given as the parameter. This is relied upon in
+ *       arc_evict_state_impl().
+ */
+void
+multilist_sublist_move_forward(multilist_sublist_t *mls, void *obj)
+{
+	void *prev = list_prev(&mls->mls_list, obj);
+
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	ASSERT(!list_is_empty(&mls->mls_list));
+
+	/* 'obj' must be at the head of the list, nothing to do */
+	if (prev == NULL)
+		return;
+
+	list_remove(&mls->mls_list, obj);
+	list_insert_before(&mls->mls_list, prev, obj);
+}
+
+void
+multilist_sublist_remove(multilist_sublist_t *mls, void *obj)
+{
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	list_remove(&mls->mls_list, obj);
+}
+
+void *
+multilist_sublist_head(multilist_sublist_t *mls)
+{
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	return (list_head(&mls->mls_list));
+}
+
+void *
+multilist_sublist_tail(multilist_sublist_t *mls)
+{
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	return (list_tail(&mls->mls_list));
+}
+
+void *
+multilist_sublist_next(multilist_sublist_t *mls, void *obj)
+{
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	return (list_next(&mls->mls_list, obj));
+}
+
+void *
+multilist_sublist_prev(multilist_sublist_t *mls, void *obj)
+{
+	ASSERT(MUTEX_HELD(&mls->mls_lock));
+	return (list_prev(&mls->mls_list, obj));
+}
+
+void
+multilist_link_init(multilist_node_t *link)
+{
+	list_link_init(link);
+}
+
+int
+multilist_link_active(multilist_node_t *link)
+{
+	return (list_link_active(link));
+}
diff --git a/uts/common/fs/zfs/sys/arc.h b/uts/common/fs/zfs/sys/arc.h
index 60112baaba5..843acfdab44 100644
--- a/uts/common/fs/zfs/sys/arc.h
+++ b/uts/common/fs/zfs/sys/arc.h
@@ -37,6 +37,12 @@ extern "C" {
 #include <sys/dmu.h>
 #include <sys/spa.h>
 
+/*
+ * Used by arc_flush() to inform arc_evict_state() that it should evict
+ * all available buffers from the arc state being passed in.
+ */
+#define	ARC_EVICT_ALL	-1ULL
+
 typedef struct arc_buf_hdr arc_buf_hdr_t;
 typedef struct arc_buf arc_buf_t;
 typedef void arc_done_func_t(zio_t *zio, arc_buf_t *buf, void *private);
@@ -154,7 +160,7 @@ void arc_freed(spa_t *spa, const blkptr_t *bp);
 void arc_set_callback(arc_buf_t *buf, arc_evict_func_t *func, void *private);
 boolean_t arc_clear_callback(arc_buf_t *buf);
 
-void arc_flush(spa_t *spa);
+void arc_flush(spa_t *spa, boolean_t retry);
 void arc_tempreserve_clear(uint64_t reserve);
 int arc_tempreserve_space(uint64_t reserve, uint64_t txg);
 
diff --git a/uts/common/fs/zfs/sys/multilist.h b/uts/common/fs/zfs/sys/multilist.h
new file mode 100644
index 00000000000..5ebb7fe1a1d
--- /dev/null
+++ b/uts/common/fs/zfs/sys/multilist.h
@@ -0,0 +1,106 @@
+/*
+ * CDDL HEADER START
+ *
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright (c) 2013, 2014 by Delphix. All rights reserved.
+ */
+
+#ifndef	_SYS_MULTILIST_H
+#define	_SYS_MULTILIST_H
+
+#include <sys/zfs_context.h>
+
+#ifdef	__cplusplus
+extern "C" {
+#endif
+
+typedef list_node_t multilist_node_t;
+typedef struct multilist multilist_t;
+typedef struct multilist_sublist multilist_sublist_t;
+typedef unsigned int multilist_sublist_index_func_t(multilist_t *, void *);
+
+struct multilist_sublist {
+	/*
+	 * The mutex used internally to implement thread safe insertions
+	 * and removals to this individual sublist. It can also be locked
+	 * by a consumer using multilist_sublist_{lock,unlock}, which is
+	 * useful if a consumer needs to traverse the list in a thread
+	 * safe manner.
+	 */
+	kmutex_t	mls_lock;
+	/*
+	 * The actual list object containing all objects in this sublist.
+	 */
+	list_t		mls_list;
+	/*
+	 * Pad to cache line (64 bytes), in an effort to try and prevent
+	 * cache line contention.
+	 */
+	uint8_t		mls_pad[24];
+};
+
+struct multilist {
+	/*
+	 * This is used to get to the multilist_node_t structure given
+	 * the void *object contained on the list.
+	 */
+	size_t				ml_offset;
+	/*
+	 * The number of sublists used internally by this multilist.
+	 */
+	uint64_t			ml_num_sublists;
+	/*
+	 * The array of pointers to the actual sublists.
+	 */
+	multilist_sublist_t		*ml_sublists;
+	/*
+	 * Pointer to function which determines the sublist to use
+	 * when inserting and removing objects from this multilist.
+	 * Please see the comment above multilist_create for details.
+	 */
+	multilist_sublist_index_func_t	*ml_index_func;
+};
+
+void multilist_destroy(multilist_t *);
+void multilist_create(multilist_t *, size_t, size_t, unsigned int,
+    multilist_sublist_index_func_t *);
+
+void multilist_insert(multilist_t *, void *);
+void multilist_remove(multilist_t *, void *);
+int  multilist_is_empty(multilist_t *);
+
+unsigned int multilist_get_num_sublists(multilist_t *);
+unsigned int multilist_get_random_index(multilist_t *);
+
+multilist_sublist_t *multilist_sublist_lock(multilist_t *, unsigned int);
+void multilist_sublist_unlock(multilist_sublist_t *);
+
+void multilist_sublist_insert_head(multilist_sublist_t *, void *);
+void multilist_sublist_insert_tail(multilist_sublist_t *, void *);
+void multilist_sublist_move_forward(multilist_sublist_t *mls, void *obj);
+void multilist_sublist_remove(multilist_sublist_t *, void *);
+
+void *multilist_sublist_head(multilist_sublist_t *);
+void *multilist_sublist_tail(multilist_sublist_t *);
+void *multilist_sublist_next(multilist_sublist_t *, void *);
+void *multilist_sublist_prev(multilist_sublist_t *, void *);
+
+void multilist_link_init(multilist_node_t *);
+int  multilist_link_active(multilist_node_t *);
+
+#ifdef	__cplusplus
+}
+#endif
+
+#endif /* _SYS_MULTILIST_H */
diff --git a/uts/common/fs/zfs/zio_inject.c b/uts/common/fs/zfs/zio_inject.c
index 991a0a34ff6..0a7f4e4b3b1 100644
--- a/uts/common/fs/zfs/zio_inject.c
+++ b/uts/common/fs/zfs/zio_inject.c
@@ -438,7 +438,11 @@ zio_inject_fault(char *name, int flags, int *id, zinject_record_t *record)
 	 * fault injection isn't a performance critical path.
 	 */
 	if (flags & ZINJECT_FLUSH_ARC)
-		arc_flush(NULL);
+		/*
+		 * We must use FALSE to ensure arc_flush returns, since
+		 * we're not preventing concurrent ARC insertions.
+		 */
+		arc_flush(NULL, FALSE);
 
 	return (0);
 }

From 360534a6256ec979f05bba873ba2b50049362645 Mon Sep 17 00:00:00 2001
From: Xin LI <delphij@FreeBSD.org>
Date: Tue, 20 Jan 2015 20:18:59 +0000
Subject: [PATCH 116/279] 5518 Memory leaks in libzfs import implementation
 Reviewed by: Dan Fields <dan.fields@nexenta.com> Reviewed by: Matthew Ahrens
 <mahrens@delphix.com> Reviewed by: Serghei Samsi <sscdvp@gmail.com> Approved
 by: Dan McDonald <danmcd@omniti.com> Author: Marcel Telka
 <marcel.telka@nexenta.com>

illumos/illumos-gate@078266a5aafa880521ea55488ef3d676f35e908e
---
 lib/libzfs/common/libzfs_import.c | 55 ++++++++++++++++++-------------
 lib/libzfs/common/libzfs_pool.c   | 25 +++++++-------
 2 files changed, 47 insertions(+), 33 deletions(-)

diff --git a/lib/libzfs/common/libzfs_import.c b/lib/libzfs/common/libzfs_import.c
index ae33cd058e0..19f2fbc57e5 100644
--- a/lib/libzfs/common/libzfs_import.c
+++ b/lib/libzfs/common/libzfs_import.c
@@ -18,10 +18,11 @@
  *
  * CDDL HEADER END
  */
+
 /*
+ * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
- * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
  */
 
 /*
@@ -196,8 +197,10 @@ fix_paths(nvlist_t *nv, name_entry_t *names)
 	if ((devid = get_devid(best->ne_name)) == NULL) {
 		(void) nvlist_remove_all(nv, ZPOOL_CONFIG_DEVID);
 	} else {
-		if (nvlist_add_string(nv, ZPOOL_CONFIG_DEVID, devid) != 0)
+		if (nvlist_add_string(nv, ZPOOL_CONFIG_DEVID, devid) != 0) {
+			devid_str_free(devid);
 			return (-1);
+		}
 		devid_str_free(devid);
 	}
 
@@ -663,8 +666,10 @@ get_configs(libzfs_handle_t *hdl, pool_list_t *pl, boolean_t active_ok)
 				    nvlist_add_uint64(holey,
 				    ZPOOL_CONFIG_ID, c) != 0 ||
 				    nvlist_add_uint64(holey,
-				    ZPOOL_CONFIG_GUID, 0ULL) != 0)
+				    ZPOOL_CONFIG_GUID, 0ULL) != 0) {
+					nvlist_free(holey);
 					goto nomem;
+				}
 				child[c] = holey;
 			}
 		}
@@ -1103,8 +1108,10 @@ zpool_clear_label(int fd)
 
 	for (l = 0; l < VDEV_LABELS; l++) {
 		if (pwrite64(fd, label, sizeof (vdev_label_t),
-		    label_offset(size, l)) != sizeof (vdev_label_t))
+		    label_offset(size, l)) != sizeof (vdev_label_t)) {
+			free(label);
 			return (-1);
+		}
 	}
 
 	free(label);
@@ -1122,7 +1129,6 @@ static nvlist_t *
 zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t *iarg)
 {
 	int i, dirs = iarg->paths;
-	DIR *dirp = NULL;
 	struct dirent64 *dp;
 	char path[MAXPATHLEN];
 	char *end, **dir = iarg->path;
@@ -1152,6 +1158,8 @@ zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t *iarg)
 		tpool_t *t;
 		char *rdsk;
 		int dfd;
+		boolean_t config_failed = B_FALSE;
+		DIR *dirp;
 
 		/* use realpath to normalize the path */
 		if (realpath(dir[i], path) == 0) {
@@ -1176,6 +1184,8 @@ zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t *iarg)
 
 		if ((dfd = open64(rdsk, O_RDONLY)) < 0 ||
 		    (dirp = fdopendir(dfd)) == NULL) {
+			if (dfd >= 0)
+				(void) close(dfd);
 			zfs_error_aux(hdl, strerror(errno));
 			(void) zfs_error_fmt(hdl, EZFS_BADPATH,
 			    dgettext(TEXT_DOMAIN, "cannot open '%s'"),
@@ -1223,7 +1233,7 @@ zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t *iarg)
 		cookie = NULL;
 		while ((slice = avl_destroy_nodes(&slice_cache,
 		    &cookie)) != NULL) {
-			if (slice->rn_config != NULL) {
+			if (slice->rn_config != NULL && !config_failed) {
 				nvlist_t *config = slice->rn_config;
 				boolean_t matched = B_TRUE;
 
@@ -1244,13 +1254,16 @@ zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t *iarg)
 				}
 				if (!matched) {
 					nvlist_free(config);
-					config = NULL;
-					continue;
+				} else {
+					/*
+					 * use the non-raw path for the config
+					 */
+					(void) strlcpy(end, slice->rn_name,
+					    pathleft);
+					if (add_config(hdl, &pools, path,
+					    config) != 0)
+						config_failed = B_TRUE;
 				}
-				/* use the non-raw path for the config */
-				(void) strlcpy(end, slice->rn_name, pathleft);
-				if (add_config(hdl, &pools, path, config) != 0)
-					goto error;
 			}
 			free(slice->rn_name);
 			free(slice);
@@ -1258,7 +1271,9 @@ zpool_find_import_impl(libzfs_handle_t *hdl, importargs_t *iarg)
 		avl_destroy(&slice_cache);
 
 		(void) closedir(dirp);
-		dirp = NULL;
+
+		if (config_failed)
+			goto error;
 	}
 
 	ret = get_configs(hdl, &pools, iarg->can_be_active);
@@ -1281,14 +1296,10 @@ error:
 
 	for (ne = pools.names; ne != NULL; ne = nenext) {
 		nenext = ne->ne_next;
-		if (ne->ne_name)
-			free(ne->ne_name);
+		free(ne->ne_name);
 		free(ne);
 	}
 
-	if (dirp)
-		(void) closedir(dirp);
-
 	return (ret);
 }
 
@@ -1646,9 +1657,9 @@ zpool_in_use(libzfs_handle_t *hdl, int fd, pool_state_t *state, char **namestr,
 		cb.cb_type = ZPOOL_CONFIG_SPARES;
 		if (zpool_iter(hdl, find_aux, &cb) == 1) {
 			name = (char *)zpool_get_name(cb.cb_zhp);
-			ret = TRUE;
+			ret = B_TRUE;
 		} else {
-			ret = FALSE;
+			ret = B_FALSE;
 		}
 		break;
 
@@ -1662,9 +1673,9 @@ zpool_in_use(libzfs_handle_t *hdl, int fd, pool_state_t *state, char **namestr,
 		cb.cb_type = ZPOOL_CONFIG_L2CACHE;
 		if (zpool_iter(hdl, find_aux, &cb) == 1) {
 			name = (char *)zpool_get_name(cb.cb_zhp);
-			ret = TRUE;
+			ret = B_TRUE;
 		} else {
-			ret = FALSE;
+			ret = B_FALSE;
 		}
 		break;
 
diff --git a/lib/libzfs/common/libzfs_pool.c b/lib/libzfs/common/libzfs_pool.c
index 675286c756b..4f8d0b61488 100644
--- a/lib/libzfs/common/libzfs_pool.c
+++ b/lib/libzfs/common/libzfs_pool.c
@@ -20,8 +20,8 @@
  */
 
 /*
+ * Copyright 2015 Nexenta Systems, Inc.  All rights reserved.
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
- * Copyright 2011 Nexenta Systems, Inc. All rights reserved.
  * Copyright (c) 2011, 2014 by Delphix. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
  */
@@ -1682,7 +1682,7 @@ zpool_import_props(libzfs_handle_t *hdl, nvlist_t *config, const char *newname,
 		thename = origname;
 	}
 
-	if (props) {
+	if (props != NULL) {
 		uint64_t version;
 		prop_flags_t flags = { .create = B_FALSE, .import = B_TRUE };
 
@@ -1690,12 +1690,13 @@ zpool_import_props(libzfs_handle_t *hdl, nvlist_t *config, const char *newname,
 		    &version) == 0);
 
 		if ((props = zpool_valid_proplist(hdl, origname,
-		    props, version, flags, errbuf)) == NULL) {
+		    props, version, flags, errbuf)) == NULL)
 			return (-1);
-		} else if (zcmd_write_src_nvlist(hdl, &zc, props) != 0) {
+		if (zcmd_write_src_nvlist(hdl, &zc, props) != 0) {
 			nvlist_free(props);
 			return (-1);
 		}
+		nvlist_free(props);
 	}
 
 	(void) strlcpy(zc.zc_name, thename, sizeof (zc.zc_name));
@@ -1704,11 +1705,11 @@ zpool_import_props(libzfs_handle_t *hdl, nvlist_t *config, const char *newname,
 	    &zc.zc_guid) == 0);
 
 	if (zcmd_write_conf_nvlist(hdl, &zc, config) != 0) {
-		nvlist_free(props);
+		zcmd_free_nvlists(&zc);
 		return (-1);
 	}
 	if (zcmd_alloc_dst_nvlist(hdl, &zc, zc.zc_nvlist_conf_size * 2) != 0) {
-		nvlist_free(props);
+		zcmd_free_nvlists(&zc);
 		return (-1);
 	}
 
@@ -1724,6 +1725,9 @@ zpool_import_props(libzfs_handle_t *hdl, nvlist_t *config, const char *newname,
 		error = errno;
 
 	(void) zcmd_read_dst_nvlist(hdl, &zc, &nv);
+
+	zcmd_free_nvlists(&zc);
+
 	zpool_get_rewind_policy(config, &policy);
 
 	if (error) {
@@ -1829,9 +1833,6 @@ zpool_import_props(libzfs_handle_t *hdl, nvlist_t *config, const char *newname,
 		return (0);
 	}
 
-	zcmd_free_nvlists(&zc);
-	nvlist_free(props);
-
 	return (ret);
 }
 
@@ -3281,8 +3282,10 @@ devid_to_path(char *devid_str)
 	if (ret != 0)
 		return (NULL);
 
-	if ((path = strdup(list[0].devname)) == NULL)
-		return (NULL);
+	/*
+	 * In a case the strdup() fails, we will just return NULL below.
+	 */
+	path = strdup(list[0].devname);
 
 	devid_free_nmlist(list);
 

From 636870ff015610812027ce05ed8b5c40d5869e91 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Tue, 20 Jan 2015 21:15:33 +0000
Subject: [PATCH 117/279] Restore the CAM XPT peripheral generation counter,
 and export it via sysctl.

Define it as an atomic uint32_t.  These increments happen infrequently
enough for the atomic overhead to be a problem, and since they're now
independent atomics, they won't contend with xpt_lock_buses().

This counter is useful as a means of cheaply identifying whether any changes
have been made to the CAM peripheral list.  Userland programs have no guarantee
that the counter won't change on them while being returned or while processing
the information, so they must be written accordingly.

Discussed with:	ken, mav (in general)
MFC after:	1 week
Sponsored by:	Spectra Logic
---
 sys/cam/cam_xpt.c | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c
index f37bc0a890c..33bc2ac17ca 100644
--- a/sys/cam/cam_xpt.c
+++ b/sys/cam/cam_xpt.c
@@ -93,6 +93,8 @@ struct xpt_task {
 };
 
 struct xpt_softc {
+	uint32_t		xpt_generation;
+
 	/* number of high powered commands that can go through right now */
 	struct mtx		xpt_highpower_lock;
 	STAILQ_HEAD(highpowerlist, cam_ed)	highpowerq;
@@ -151,6 +153,8 @@ static struct xpt_softc xsoftc;
 
 SYSCTL_INT(_kern_cam, OID_AUTO, boot_delay, CTLFLAG_RDTUN,
            &xsoftc.boot_delay, 0, "Bus registration wait time");
+SYSCTL_UINT(_kern_cam, OID_AUTO, xpt_generation, CTLFLAG_RD,
+	    &xsoftc.xpt_generation, 0, "CAM peripheral generation count");
 
 struct cam_doneq {
 	struct mtx_padalign	cam_doneq_mtx;
@@ -976,6 +980,7 @@ xpt_add_periph(struct cam_periph *periph)
 		device->generation++;
 		SLIST_INSERT_HEAD(&device->periphs, periph, periph_links);
 		mtx_unlock(&device->target->bus->eb_mtx);
+		atomic_add_32(&xsoftc.xpt_generation, 1);
 	}
 
 	return (status);
@@ -992,6 +997,7 @@ xpt_remove_periph(struct cam_periph *periph)
 		device->generation++;
 		SLIST_REMOVE(&device->periphs, periph, cam_periph, periph_links);
 		mtx_unlock(&device->target->bus->eb_mtx);
+		atomic_add_32(&xsoftc.xpt_generation, 1);
 	}
 }
 

From d142f0da9fd3e0175fbc63566e8e5ca263734696 Mon Sep 17 00:00:00 2001
From: Jan Beich <jbeich@FreeBSD.org>
Date: Tue, 20 Jan 2015 21:49:50 +0000
Subject: [PATCH 118/279] - Add jbeich as a ports committer - List bapt and flo
 as his mentors

Differential Revision:	https://reviews.freebsd.org/D1562
Approved by:		flo (mentor)
---
 share/misc/committers-ports.dot | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/share/misc/committers-ports.dot b/share/misc/committers-ports.dot
index e97acba9584..00e85f755bf 100644
--- a/share/misc/committers-ports.dot
+++ b/share/misc/committers-ports.dot
@@ -116,6 +116,7 @@ itetcu [label="Ion-Mihai Tetcu\nitetcu@FreeBSD.org\n2006/06/07"]
 jacula [label="Giuseppe Pilichi\njacula@FreeBSD.org\n2010/04/05"]
 jadawin [label="Philippe Audeoud\njadawin@FreeBSD.org\n2008/03/02"]
 jase [label="Jase Thew\njase@FreeBSD.org\n2012/05/30"]
+jbeich [label="Jan Beich\njbeich@FreeBSD.org\n2015/01/19"]
 jgh [label="Jason Helfman\njgh@FreeBSD.org\n2011/12/16"]
 jhale [label="Jason E. Hale\njhale@FreeBSD.org\n2012/09/10"]
 jkim [label="Jung-uk Kim\njkim@FreeBSD.org\n2007/09/12"]
@@ -267,6 +268,7 @@ bapt -> bdrewery
 bapt -> bofh
 bapt -> eadler
 bapt -> grembo
+bapt -> jbeich
 bapt -> jlaffaye
 bapt -> marius
 bapt -> marino
@@ -350,6 +352,7 @@ fjoe -> osa
 
 flo -> bar
 flo -> jase
+flo -> jbeich
 flo -> grembo
 
 flz -> garga

From bf5e6a8751762dbf6d8858d6ac1151cee7be64b1 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Tue, 20 Jan 2015 21:58:49 +0000
Subject: [PATCH 119/279] Garbage collect a prove test wrapper

MFC after: 3 days
Sponsored by: EMC / Isilon Storage Division
---
 tools/regression/lib/libc/stdio/test-fmemopen.t | 15 ---------------
 1 file changed, 15 deletions(-)
 delete mode 100644 tools/regression/lib/libc/stdio/test-fmemopen.t

diff --git a/tools/regression/lib/libc/stdio/test-fmemopen.t b/tools/regression/lib/libc/stdio/test-fmemopen.t
deleted file mode 100644
index bd5157b5d8f..00000000000
--- a/tools/regression/lib/libc/stdio/test-fmemopen.t
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/bin/sh
-# $FreeBSD$
-
-cd `dirname $0`
-
-executable=`basename $0 .t`
-
-make $executable 2>&1 > /dev/null
-
-echo 1..1
-if ./$executable; then
-	echo ok 1 - $executable successful
-else
-	echo not ok 1 - $executable failed
-fi

From c9c5e04711ab6de60d398b2fbe50e375f683b3f6 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Tue, 20 Jan 2015 22:27:45 +0000
Subject: [PATCH 120/279] Remove unused strdup() #define.

---
 sys/cddl/compat/opensolaris/sys/sunddi.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/sys/cddl/compat/opensolaris/sys/sunddi.h b/sys/cddl/compat/opensolaris/sys/sunddi.h
index 63b8a8983df..3a493b7f00d 100644
--- a/sys/cddl/compat/opensolaris/sys/sunddi.h
+++ b/sys/cddl/compat/opensolaris/sys/sunddi.h
@@ -35,7 +35,6 @@
 #include <sys/libkern.h>
 #include <sys/sysevent.h>
 
-#define	strdup(ptr)				strdup((ptr), M_SOLARIS)
 #define	ddi_driver_major(zfs_dip)		(0)
 #define	ddi_copyin(from, to, size, flag)				\
 	(copyin((from), (to), (size)), 0)

From 34abed55f33125f51816a7653cdd7612341d83eb Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Tue, 20 Jan 2015 22:29:27 +0000
Subject: [PATCH 121/279] NSEC_TO_TICK(usec) -> NSEC_TO_TICK(nsec)

---
 sys/cddl/compat/opensolaris/sys/time.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/cddl/compat/opensolaris/sys/time.h b/sys/cddl/compat/opensolaris/sys/time.h
index fe4857ad1f9..b54780c45f3 100644
--- a/sys/cddl/compat/opensolaris/sys/time.h
+++ b/sys/cddl/compat/opensolaris/sys/time.h
@@ -51,7 +51,7 @@ typedef longlong_t	hrtime_t;
 #endif
 
 #define	SEC_TO_TICK(sec)	((sec) * hz)
-#define	NSEC_TO_TICK(usec)	((usec) / (NANOSEC / hz))
+#define	NSEC_TO_TICK(nsec)	((nsec) / (NANOSEC / hz))
 
 #ifdef _KERNEL
 static __inline hrtime_t

From fe20fb9fb07092cd23ed958eb75495e00555f407 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Tue, 20 Jan 2015 22:31:26 +0000
Subject: [PATCH 122/279] Use the "zfs_gfs" tag for GFS vnodes to make them
 easier to identify.

MFC after:	1 week
Sponsored by:	Spectra Logic
---
 sys/cddl/contrib/opensolaris/uts/common/fs/gfs.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/gfs.c b/sys/cddl/contrib/opensolaris/uts/common/fs/gfs.c
index 2ff8df6ae89..89882f403f7 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/gfs.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/gfs.c
@@ -489,7 +489,7 @@ gfs_file_create(size_t size, vnode_t *pvp, vfs_t *vfsp, vnodeops_t *ops)
 	 * Allocate vnode and internal data structure
 	 */
 	fp = kmem_zalloc(size, KM_SLEEP);
-	error = getnewvnode("zfs", vfsp, ops, &vp);
+	error = getnewvnode("zfs_gfs", vfsp, ops, &vp);
 	ASSERT(error == 0);
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
 	vp->v_data = (caddr_t)fp;

From 798cbb7523454dd622a718d9952805b0a775d001 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Tue, 20 Jan 2015 22:39:10 +0000
Subject: [PATCH 123/279] Fix arc__shrink DTrace probe's to_free argument.

Remove the unnecessary #ifdef _KERNEL, which did not differ in the true or
false cases.  Actually set the value of to_free before using it.

MFC after:	1 week
Sponsored by:	Spectra Logic
---
 sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
index 29e9eca5670..e180850a936 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
@@ -2540,13 +2540,9 @@ arc_shrink(void)
 	if (arc_c > arc_c_min) {
 		uint64_t to_free;
 
+		to_free = arc_c >> arc_shrink_shift;
 		DTRACE_PROBE4(arc__shrink, uint64_t, arc_c, uint64_t,
 			arc_c_min, uint64_t, arc_p, uint64_t, to_free);
-#ifdef _KERNEL
-		to_free = arc_c >> arc_shrink_shift;
-#else
-		to_free = arc_c >> arc_shrink_shift;
-#endif
 		if (arc_c > arc_c_min + to_free)
 			atomic_add_64(&arc_c, -to_free);
 		else

From 16e2dc663f63ec78bf4a2a6e8a2d648ba201c34f Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Tue, 20 Jan 2015 22:56:59 +0000
Subject: [PATCH 124/279] Add inline implementations of arm
 bus_space_read/write_N().

Reviewed by:	cognet
---
 sys/arm/arm/bus_space-v6.c | 32 ++++++++++-----------
 sys/arm/include/bus.h      | 58 +++++++++++++++++++++++++-------------
 2 files changed, 54 insertions(+), 36 deletions(-)

diff --git a/sys/arm/arm/bus_space-v6.c b/sys/arm/arm/bus_space-v6.c
index d91306e1a1d..738e4c6fc36 100644
--- a/sys/arm/arm/bus_space-v6.c
+++ b/sys/arm/arm/bus_space-v6.c
@@ -65,10 +65,10 @@ static struct bus_space _base_tag = {
 	generic_bs_barrier,
 
 	/* read (single) */
-	generic_bs_r_1,
-	generic_armv4_bs_r_2,
-	generic_bs_r_4,
-	NULL,
+	NULL,	/* bs_r_1, Use inline code in bus.h */
+	NULL,	/* bs_r_2, Use inline code in bus.h */
+	NULL,	/* bs_r_4, Use inline code in bus.h */
+	NULL,	/* bs_r_8, Use inline code in bus.h */
 
 	/* read multiple */
 	generic_bs_rm_1,
@@ -83,10 +83,10 @@ static struct bus_space _base_tag = {
 	NULL,
 
 	/* write (single) */
-	generic_bs_w_1,
-	generic_armv4_bs_w_2,
-	generic_bs_w_4,
-	NULL,
+	NULL,	/* bs_w_1, Use inline code in bus.h */
+	NULL,	/* bs_w_2, Use inline code in bus.h */
+	NULL,	/* bs_w_4, Use inline code in bus.h */
+	NULL,	/* bs_w_8, Use inline code in bus.h */
 
 	/* write multiple */
 	generic_bs_wm_1,
@@ -119,10 +119,10 @@ static struct bus_space _base_tag = {
 	NULL,
 
 	/* read stream (single) */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
+	NULL,   /* bs_r_1_s, Use inline code in bus.h */ 
+	NULL,   /* bs_r_2_s, Use inline code in bus.h */ 
+	NULL,   /* bs_r_4_s, Use inline code in bus.h */ 
+	NULL,   /* bs_r_8_s, Use inline code in bus.h */ 
 
 	/* read multiple stream */
 	NULL,
@@ -137,10 +137,10 @@ static struct bus_space _base_tag = {
 	NULL,
 
 	/* write stream (single) */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
+	NULL,   /* bs_w_1_s, Use inline code in bus.h */ 
+	NULL,   /* bs_w_2_s, Use inline code in bus.h */ 
+	NULL,   /* bs_w_4_s, Use inline code in bus.h */ 
+	NULL,   /* bs_w_8_s, Use inline code in bus.h */ 
 
 	/* write multiple stream */
 	NULL,
diff --git a/sys/arm/include/bus.h b/sys/arm/include/bus.h
index 0749838df8e..f044bec4a0b 100644
--- a/sys/arm/include/bus.h
+++ b/sys/arm/include/bus.h
@@ -252,10 +252,6 @@ struct bus_space {
 #define	__bs_c(a,b)		__CONCAT(a,b)
 #define	__bs_opname(op,size)	__bs_c(__bs_c(__bs_c(bs_,op),_),size)
 
-#define	__bs_rs(sz, t, h, o)						\
-	(*(t)->__bs_opname(r,sz))((t)->bs_cookie, h, o)
-#define	__bs_ws(sz, t, h, o, v)						\
-	(*(t)->__bs_opname(w,sz))((t)->bs_cookie, h, o, v)
 #define	__bs_nonsingle(type, sz, t, h, o, a, c)				\
 	(*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, a, c)
 #define	__bs_set(type, sz, t, h, o, v, c)				\
@@ -272,6 +268,28 @@ struct bus_space {
 	(*(t)->__bs_opname_s(type,sz))((t)->bs_cookie, h, o, a, c)
 
 
+#define __generate_inline_bs_rs(IFN, MBR, TYP)					\
+	static inline TYP						\
+	IFN(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)	\
+	{								\
+									\
+		if (__predict_true(t->MBR == NULL))			\
+			return (*(volatile TYP *)(h + o));		\
+		else							\
+			return (t->MBR(t->bs_cookie, h, o));		\
+	}
+
+#define __generate_inline_bs_ws(IFN, MBR, TYP)					\
+	static inline void						\
+	IFN(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, TYP v)\
+	{								\
+									\
+		if (__predict_true(t->MBR == NULL))			\
+			*(volatile TYP *)(h + o) = v;			\
+		else							\
+			t->MBR(t->bs_cookie, h, o, v);			\
+	}
+
 /*
  * Mapping and unmapping operations.
  */
@@ -304,15 +322,15 @@ struct bus_space {
 /*
  * Bus read (single) operations.
  */
-#define	bus_space_read_1(t, h, o)	__bs_rs(1,(t),(h),(o))
-#define	bus_space_read_2(t, h, o)	__bs_rs(2,(t),(h),(o))
-#define	bus_space_read_4(t, h, o)	__bs_rs(4,(t),(h),(o))
-#define	bus_space_read_8(t, h, o)	__bs_rs(8,(t),(h),(o))
+__generate_inline_bs_rs(bus_space_read_1, bs_r_1, uint8_t);
+__generate_inline_bs_rs(bus_space_read_2, bs_r_2, uint16_t);
+__generate_inline_bs_rs(bus_space_read_4, bs_r_4, uint32_t);
+__generate_inline_bs_rs(bus_space_read_8, bs_r_8, uint64_t);
 
-#define bus_space_read_stream_1(t, h, o)        __bs_rs_s(1,(t), (h), (o))
-#define bus_space_read_stream_2(t, h, o)        __bs_rs_s(2,(t), (h), (o))
-#define bus_space_read_stream_4(t, h, o)        __bs_rs_s(4,(t), (h), (o))
-#define	bus_space_read_stream_8(t, h, o)	__bs_rs_s(8,8,(t),(h),(o))
+__generate_inline_bs_rs(bus_space_read_stream_1, bs_r_1_s, uint8_t);           
+__generate_inline_bs_rs(bus_space_read_stream_2, bs_r_2_s, uint16_t);          
+__generate_inline_bs_rs(bus_space_read_stream_4, bs_r_4_s, uint32_t);          
+__generate_inline_bs_rs(bus_space_read_stream_8, bs_r_8_s, uint64_t);          
 
 /*
  * Bus read multiple operations.
@@ -361,15 +379,15 @@ struct bus_space {
 /*
  * Bus write (single) operations.
  */
-#define	bus_space_write_1(t, h, o, v)	__bs_ws(1,(t),(h),(o),(v))
-#define	bus_space_write_2(t, h, o, v)	__bs_ws(2,(t),(h),(o),(v))
-#define	bus_space_write_4(t, h, o, v)	__bs_ws(4,(t),(h),(o),(v))
-#define	bus_space_write_8(t, h, o, v)	__bs_ws(8,(t),(h),(o),(v))
+__generate_inline_bs_ws(bus_space_write_1, bs_w_1, uint8_t);
+__generate_inline_bs_ws(bus_space_write_2, bs_w_2, uint16_t);
+__generate_inline_bs_ws(bus_space_write_4, bs_w_4, uint32_t);
+__generate_inline_bs_ws(bus_space_write_8, bs_w_8, uint64_t);
 
-#define	bus_space_write_stream_1(t, h, o, v)	__bs_ws_s(1,(t),(h),(o),(v))
-#define	bus_space_write_stream_2(t, h, o, v)	__bs_ws_s(2,(t),(h),(o),(v))
-#define	bus_space_write_stream_4(t, h, o, v)	__bs_ws_s(4,(t),(h),(o),(v))
-#define	bus_space_write_stream_8(t, h, o, v)	__bs_ws_s(8,(t),(h),(o),(v))
+__generate_inline_bs_ws(bus_space_write_stream_1, bs_w_1_s, uint8_t);           
+__generate_inline_bs_ws(bus_space_write_stream_2, bs_w_2_s, uint16_t);          
+__generate_inline_bs_ws(bus_space_write_stream_4, bs_w_4_s, uint32_t);          
+__generate_inline_bs_ws(bus_space_write_stream_8, bs_w_8_s, uint64_t);          
 
 
 /*

From 4098ccafa45d5a8504ec27ce53112c261f7fd014 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 01:06:08 +0000
Subject: [PATCH 125/279] Revise the arm bus_space implementation to avoid
 dereferencing the tag on every operation to retrieve the bs_cookie value
 almost nothing actually uses.

The bus_space struct contains a private data pointer (poorly named bs_cookie,
now renamed to bs_privdata) which is used only by a few old armv4 xscale
implementations.  The bus_space functions were all defined to take this
value as the first parameter instead of the bus_space_tag_t, requiring all
the inline macro and function expansions to dereference the tag to pass it
to another function, which never uses it.  Now all the functions take the tag
as the first parameter and retrieve the privdata if they need it.

Also fix a couple bus_space_unmap() implementations that were calling
kva_free() instead of pmap_unmapdev().

Discussed with:	   cognet
---
 sys/arm/arm/bus_space_generic.c           |  12 +-
 sys/arm/at91/at91.c                       |  19 +-
 sys/arm/include/bus.h                     | 416 +++++++++++-----------
 sys/arm/versatile/bus_space.c             |   2 +-
 sys/arm/xilinx/zy7_bus_space.c            |   2 +-
 sys/arm/xscale/i80321/i80321_space.c      |  40 +--
 sys/arm/xscale/i8134x/i81342_space.c      |  30 +-
 sys/arm/xscale/ixp425/avila_ata.c         |  38 +-
 sys/arm/xscale/ixp425/cambria_exp_space.c |  34 +-
 sys/arm/xscale/ixp425/ixp425_a4x_space.c  |   2 +-
 sys/arm/xscale/ixp425/ixp425_pci_space.c  | 151 ++++----
 sys/arm/xscale/ixp425/ixp425_space.c      |   2 +-
 sys/arm/xscale/pxa/pxa_space.c            |  20 +-
 sys/dev/usb/controller/ehci_ixp4xx.c      |  34 +-
 14 files changed, 388 insertions(+), 414 deletions(-)

diff --git a/sys/arm/arm/bus_space_generic.c b/sys/arm/arm/bus_space_generic.c
index 35052ecf990..2df18dc1510 100644
--- a/sys/arm/arm/bus_space_generic.c
+++ b/sys/arm/arm/bus_space_generic.c
@@ -57,7 +57,7 @@ __FBSDID("$FreeBSD$");
 bs_protos(generic);
 
 int
-generic_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flags,
+generic_bs_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size, int flags,
     bus_space_handle_t *bshp)
 {
 	void *va;
@@ -74,7 +74,7 @@ generic_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flags,
 }
 
 int
-generic_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend, bus_size_t size,
+generic_bs_alloc(bus_space_tag_t t, bus_addr_t rstart, bus_addr_t rend, bus_size_t size,
     bus_size_t alignment, bus_size_t boundary, int flags, bus_addr_t *bpap,
     bus_space_handle_t *bshp)
 {
@@ -84,21 +84,21 @@ generic_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend, bus_size_t size,
 
 
 void
-generic_bs_unmap(void *t, bus_space_handle_t h, bus_size_t size)
+generic_bs_unmap(bus_space_tag_t t, bus_space_handle_t h, bus_size_t size)
 {
 
 	pmap_unmapdev((vm_offset_t)h, size);
 }
 
 void
-generic_bs_free(void *t, bus_space_handle_t bsh, bus_size_t size)
+generic_bs_free(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size)
 {
 
 	panic("generic_bs_free(): not implemented");
 }
 
 int
-generic_bs_subregion(void *t, bus_space_handle_t bsh, bus_size_t offset,
+generic_bs_subregion(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t offset,
     bus_size_t size, bus_space_handle_t *nbshp)
 {
 
@@ -107,7 +107,7 @@ generic_bs_subregion(void *t, bus_space_handle_t bsh, bus_size_t offset,
 }
 
 void
-generic_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
+generic_bs_barrier(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t offset,
     bus_size_t len, int flags)
 {
 
diff --git a/sys/arm/at91/at91.c b/sys/arm/at91/at91.c
index 1942389738b..a8a1f5e4af3 100644
--- a/sys/arm/at91/at91.c
+++ b/sys/arm/at91/at91.c
@@ -55,7 +55,7 @@ __FBSDID("$FreeBSD$");
 uint32_t at91_master_clock;
 
 static int
-at91_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flags,
+at91_bs_map(bus_space_tag_t tag, bus_addr_t bpa, bus_size_t size, int flags,
     bus_space_handle_t *bshp)
 {
 	vm_paddr_t pa, endpa;
@@ -77,23 +77,18 @@ at91_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flags,
 }
 
 static void
-at91_bs_unmap(void *t, bus_space_handle_t h, bus_size_t size)
+at91_bs_unmap(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t size)
 {
-	vm_offset_t va, endva;
+	vm_offset_t va;
 
-	if (t == 0)
-		return;
-	va = trunc_page((vm_offset_t)t);
+	va = (vm_offset_t)h;
 	if (va >= AT91_BASE && va <= AT91_BASE + 0xff00000)
 		return;
-	endva = round_page((vm_offset_t)t + size);
-
-	/* Free the kernel virtual mapping. */
-	kva_free(va, endva - va);
+	pmap_unmapdev(va, size);
 }
 
 static int
-at91_bs_subregion(void *t, bus_space_handle_t bsh, bus_size_t offset,
+at91_bs_subregion(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset,
     bus_size_t size, bus_space_handle_t *nbshp)
 {
 
@@ -102,7 +97,7 @@ at91_bs_subregion(void *t, bus_space_handle_t bsh, bus_size_t offset,
 }
 
 static void
-at91_barrier(void *t, bus_space_handle_t bsh, bus_size_t size, bus_size_t b,
+at91_barrier(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t size, bus_size_t b,
     int a)
 {
 }
diff --git a/sys/arm/include/bus.h b/sys/arm/include/bus.h
index f044bec4a0b..f9a677e4f26 100644
--- a/sys/arm/include/bus.h
+++ b/sys/arm/include/bus.h
@@ -81,168 +81,168 @@
 
 struct bus_space {
 	/* cookie */
-	void		*bs_cookie;
+	void		*bs_privdata;
 
 	/* mapping/unmapping */
-	int		(*bs_map) (void *, bus_addr_t, bus_size_t,
+	int		(*bs_map) (bus_space_tag_t, bus_addr_t, bus_size_t,
 			    int, bus_space_handle_t *);
-	void		(*bs_unmap) (void *, bus_space_handle_t, bus_size_t);
-	int		(*bs_subregion) (void *, bus_space_handle_t,
+	void		(*bs_unmap) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	int		(*bs_subregion) (bus_space_tag_t, bus_space_handle_t,
 			    bus_size_t, bus_size_t, bus_space_handle_t *);
 
 	/* allocation/deallocation */
-	int		(*bs_alloc) (void *, bus_addr_t, bus_addr_t,
+	int		(*bs_alloc) (bus_space_tag_t, bus_addr_t, bus_addr_t,
 			    bus_size_t, bus_size_t, bus_size_t, int,
 			    bus_addr_t *, bus_space_handle_t *);
-	void		(*bs_free) (void *, bus_space_handle_t,
+	void		(*bs_free) (bus_space_tag_t, bus_space_handle_t,
 			    bus_size_t);
 
 	/* get kernel virtual address */
 	/* barrier */
-	void		(*bs_barrier) (void *, bus_space_handle_t,
+	void		(*bs_barrier) (bus_space_tag_t, bus_space_handle_t,
 			    bus_size_t, bus_size_t, int);
 
 	/* read (single) */
-	u_int8_t	(*bs_r_1) (void *, bus_space_handle_t, bus_size_t);
-	u_int16_t	(*bs_r_2) (void *, bus_space_handle_t, bus_size_t);
-	u_int32_t	(*bs_r_4) (void *, bus_space_handle_t, bus_size_t);
-	u_int64_t	(*bs_r_8) (void *, bus_space_handle_t, bus_size_t);
+	uint8_t		(*bs_r_1) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	uint16_t	(*bs_r_2) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	uint32_t	(*bs_r_4) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	uint64_t	(*bs_r_8) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
 
 	/* read multiple */
-	void		(*bs_rm_1) (void *, bus_space_handle_t, bus_size_t,
-	    u_int8_t *, bus_size_t);
-	void		(*bs_rm_2) (void *, bus_space_handle_t, bus_size_t,
-	    u_int16_t *, bus_size_t);
-	void		(*bs_rm_4) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t *, bus_size_t);
-	void		(*bs_rm_8) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t *, bus_size_t);
+	void		(*bs_rm_1) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
+	    uint8_t *, bus_size_t);
+	void		(*bs_rm_2) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
+	    uint16_t *, bus_size_t);
+	void		(*bs_rm_4) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t *, bus_size_t);
+	void		(*bs_rm_8) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t *, bus_size_t);
 					
 	/* read region */
-	void		(*bs_rr_1) (void *, bus_space_handle_t,
-			    bus_size_t, u_int8_t *, bus_size_t);
-	void		(*bs_rr_2) (void *, bus_space_handle_t,
-			    bus_size_t, u_int16_t *, bus_size_t);
-	void		(*bs_rr_4) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t *, bus_size_t);
-	void		(*bs_rr_8) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t *, bus_size_t);
+	void		(*bs_rr_1) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint8_t *, bus_size_t);
+	void		(*bs_rr_2) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint16_t *, bus_size_t);
+	void		(*bs_rr_4) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t *, bus_size_t);
+	void		(*bs_rr_8) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t *, bus_size_t);
 					
 	/* write (single) */
-	void		(*bs_w_1) (void *, bus_space_handle_t,
-			    bus_size_t, u_int8_t);
-	void		(*bs_w_2) (void *, bus_space_handle_t,
-			    bus_size_t, u_int16_t);
-	void		(*bs_w_4) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t);
-	void		(*bs_w_8) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t);
+	void		(*bs_w_1) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint8_t);
+	void		(*bs_w_2) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint16_t);
+	void		(*bs_w_4) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t);
+	void		(*bs_w_8) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t);
 
 	/* write multiple */
-	void		(*bs_wm_1) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int8_t *, bus_size_t);
-	void		(*bs_wm_2) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int16_t *, bus_size_t);
-	void		(*bs_wm_4) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int32_t *, bus_size_t);
-	void		(*bs_wm_8) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int64_t *, bus_size_t);
+	void		(*bs_wm_1) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint8_t *, bus_size_t);
+	void		(*bs_wm_2) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint16_t *, bus_size_t);
+	void		(*bs_wm_4) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint32_t *, bus_size_t);
+	void		(*bs_wm_8) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint64_t *, bus_size_t);
 					
 	/* write region */
-	void		(*bs_wr_1) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int8_t *, bus_size_t);
-	void		(*bs_wr_2) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int16_t *, bus_size_t);
-	void		(*bs_wr_4) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int32_t *, bus_size_t);
-	void		(*bs_wr_8) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int64_t *, bus_size_t);
+	void		(*bs_wr_1) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint8_t *, bus_size_t);
+	void		(*bs_wr_2) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint16_t *, bus_size_t);
+	void		(*bs_wr_4) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint32_t *, bus_size_t);
+	void		(*bs_wr_8) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint64_t *, bus_size_t);
 
 	/* set multiple */
-	void		(*bs_sm_1) (void *, bus_space_handle_t,
-			    bus_size_t, u_int8_t, bus_size_t);
-	void		(*bs_sm_2) (void *, bus_space_handle_t,
-			    bus_size_t, u_int16_t, bus_size_t);
-	void		(*bs_sm_4) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t, bus_size_t);
-	void		(*bs_sm_8) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t, bus_size_t);
+	void		(*bs_sm_1) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint8_t, bus_size_t);
+	void		(*bs_sm_2) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint16_t, bus_size_t);
+	void		(*bs_sm_4) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t, bus_size_t);
+	void		(*bs_sm_8) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t, bus_size_t);
 
 	/* set region */
-	void		(*bs_sr_1) (void *, bus_space_handle_t,
-			    bus_size_t, u_int8_t, bus_size_t);
-	void		(*bs_sr_2) (void *, bus_space_handle_t,
-			    bus_size_t, u_int16_t, bus_size_t);
-	void		(*bs_sr_4) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t, bus_size_t);
-	void		(*bs_sr_8) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t, bus_size_t);
+	void		(*bs_sr_1) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint8_t, bus_size_t);
+	void		(*bs_sr_2) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint16_t, bus_size_t);
+	void		(*bs_sr_4) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t, bus_size_t);
+	void		(*bs_sr_8) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t, bus_size_t);
 
 	/* copy */
-	void		(*bs_c_1) (void *, bus_space_handle_t, bus_size_t,
+	void		(*bs_c_1) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
 			    bus_space_handle_t, bus_size_t, bus_size_t);
-	void		(*bs_c_2) (void *, bus_space_handle_t, bus_size_t,
+	void		(*bs_c_2) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
 			    bus_space_handle_t, bus_size_t, bus_size_t);
-	void		(*bs_c_4) (void *, bus_space_handle_t, bus_size_t,
+	void		(*bs_c_4) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
 			    bus_space_handle_t, bus_size_t, bus_size_t);
-	void		(*bs_c_8) (void *, bus_space_handle_t, bus_size_t,
+	void		(*bs_c_8) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
 			    bus_space_handle_t, bus_size_t, bus_size_t);
 
 	/* read stream (single) */
-	u_int8_t	(*bs_r_1_s) (void *, bus_space_handle_t, bus_size_t);
-	u_int16_t	(*bs_r_2_s) (void *, bus_space_handle_t, bus_size_t);
-	u_int32_t	(*bs_r_4_s) (void *, bus_space_handle_t, bus_size_t);
-	u_int64_t	(*bs_r_8_s) (void *, bus_space_handle_t, bus_size_t);
+	uint8_t	(*bs_r_1_s) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	uint16_t	(*bs_r_2_s) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	uint32_t	(*bs_r_4_s) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	uint64_t	(*bs_r_8_s) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
 
 	/* read multiple stream */
-	void		(*bs_rm_1_s) (void *, bus_space_handle_t, bus_size_t,
-	    u_int8_t *, bus_size_t);
-	void		(*bs_rm_2_s) (void *, bus_space_handle_t, bus_size_t,
-	    u_int16_t *, bus_size_t);
-	void		(*bs_rm_4_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t *, bus_size_t);
-	void		(*bs_rm_8_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t *, bus_size_t);
+	void		(*bs_rm_1_s) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
+	    uint8_t *, bus_size_t);
+	void		(*bs_rm_2_s) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
+	    uint16_t *, bus_size_t);
+	void		(*bs_rm_4_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t *, bus_size_t);
+	void		(*bs_rm_8_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t *, bus_size_t);
 					
 	/* read region stream */
-	void		(*bs_rr_1_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int8_t *, bus_size_t);
-	void		(*bs_rr_2_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int16_t *, bus_size_t);
-	void		(*bs_rr_4_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t *, bus_size_t);
-	void		(*bs_rr_8_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t *, bus_size_t);
+	void		(*bs_rr_1_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint8_t *, bus_size_t);
+	void		(*bs_rr_2_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint16_t *, bus_size_t);
+	void		(*bs_rr_4_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t *, bus_size_t);
+	void		(*bs_rr_8_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t *, bus_size_t);
 					
 	/* write stream (single) */
-	void		(*bs_w_1_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int8_t);
-	void		(*bs_w_2_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int16_t);
-	void		(*bs_w_4_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t);
-	void		(*bs_w_8_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t);
+	void		(*bs_w_1_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint8_t);
+	void		(*bs_w_2_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint16_t);
+	void		(*bs_w_4_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t);
+	void		(*bs_w_8_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t);
 
 	/* write multiple stream */
-	void		(*bs_wm_1_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int8_t *, bus_size_t);
-	void		(*bs_wm_2_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int16_t *, bus_size_t);
-	void		(*bs_wm_4_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int32_t *, bus_size_t);
-	void		(*bs_wm_8_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int64_t *, bus_size_t);
+	void		(*bs_wm_1_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint8_t *, bus_size_t);
+	void		(*bs_wm_2_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint16_t *, bus_size_t);
+	void		(*bs_wm_4_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint32_t *, bus_size_t);
+	void		(*bs_wm_8_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint64_t *, bus_size_t);
 					
 	/* write region stream */
-	void		(*bs_wr_1_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int8_t *, bus_size_t);
-	void		(*bs_wr_2_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int16_t *, bus_size_t);
-	void		(*bs_wr_4_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int32_t *, bus_size_t);
-	void		(*bs_wr_8_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int64_t *, bus_size_t);
+	void		(*bs_wr_1_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint8_t *, bus_size_t);
+	void		(*bs_wr_2_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint16_t *, bus_size_t);
+	void		(*bs_wr_4_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint32_t *, bus_size_t);
+	void		(*bs_wr_8_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint64_t *, bus_size_t);
 };
 
 
@@ -253,19 +253,19 @@ struct bus_space {
 #define	__bs_opname(op,size)	__bs_c(__bs_c(__bs_c(bs_,op),_),size)
 
 #define	__bs_nonsingle(type, sz, t, h, o, a, c)				\
-	(*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, a, c)
+	(*(t)->__bs_opname(type,sz))((t), h, o, a, c)
 #define	__bs_set(type, sz, t, h, o, v, c)				\
-	(*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, v, c)
+	(*(t)->__bs_opname(type,sz))((t), h, o, v, c)
 #define	__bs_copy(sz, t, h1, o1, h2, o2, cnt)				\
-	(*(t)->__bs_opname(c,sz))((t)->bs_cookie, h1, o1, h2, o2, cnt)
+	(*(t)->__bs_opname(c,sz))((t), h1, o1, h2, o2, cnt)
 
 #define	__bs_opname_s(op,size)	__bs_c(__bs_c(__bs_c(__bs_c(bs_,op),_),size),_s)
 #define	__bs_rs_s(sz, t, h, o)						\
-	(*(t)->__bs_opname_s(r,sz))((t)->bs_cookie, h, o)
+	(*(t)->__bs_opname_s(r,sz))((t), h, o)
 #define	__bs_ws_s(sz, t, h, o, v)					\
-	(*(t)->__bs_opname_s(w,sz))((t)->bs_cookie, h, o, v)
+	(*(t)->__bs_opname_s(w,sz))((t), h, o, v)
 #define	__bs_nonsingle_s(type, sz, t, h, o, a, c)			\
-	(*(t)->__bs_opname_s(type,sz))((t)->bs_cookie, h, o, a, c)
+	(*(t)->__bs_opname_s(type,sz))((t), h, o, a, c)
 
 
 #define __generate_inline_bs_rs(IFN, MBR, TYP)					\
@@ -276,7 +276,7 @@ struct bus_space {
 		if (__predict_true(t->MBR == NULL))			\
 			return (*(volatile TYP *)(h + o));		\
 		else							\
-			return (t->MBR(t->bs_cookie, h, o));		\
+			return (t->MBR(t, h, o));		\
 	}
 
 #define __generate_inline_bs_ws(IFN, MBR, TYP)					\
@@ -287,34 +287,34 @@ struct bus_space {
 		if (__predict_true(t->MBR == NULL))			\
 			*(volatile TYP *)(h + o) = v;			\
 		else							\
-			t->MBR(t->bs_cookie, h, o, v);			\
+			t->MBR(t, h, o, v);			\
 	}
 
 /*
  * Mapping and unmapping operations.
  */
 #define	bus_space_map(t, a, s, c, hp)					\
-	(*(t)->bs_map)((t)->bs_cookie, (a), (s), (c), (hp))
+	(*(t)->bs_map)((t), (a), (s), (c), (hp))
 #define	bus_space_unmap(t, h, s)					\
-	(*(t)->bs_unmap)((t)->bs_cookie, (h), (s))
+	(*(t)->bs_unmap)((t), (h), (s))
 #define	bus_space_subregion(t, h, o, s, hp)				\
-	(*(t)->bs_subregion)((t)->bs_cookie, (h), (o), (s), (hp))
+	(*(t)->bs_subregion)((t), (h), (o), (s), (hp))
 
 
 /*
  * Allocation and deallocation operations.
  */
 #define	bus_space_alloc(t, rs, re, s, a, b, c, ap, hp)			\
-	(*(t)->bs_alloc)((t)->bs_cookie, (rs), (re), (s), (a), (b),	\
+	(*(t)->bs_alloc)((t), (rs), (re), (s), (a), (b),	\
 	    (c), (ap), (hp))
 #define	bus_space_free(t, h, s)						\
-	(*(t)->bs_free)((t)->bs_cookie, (h), (s))
+	(*(t)->bs_free)((t), (h), (s))
 
 /*
  * Bus barrier operations.
  */
 #define	bus_space_barrier(t, h, o, l, f)				\
-	(*(t)->bs_barrier)((t)->bs_cookie, (h), (o), (l), (f))
+	(*(t)->bs_barrier)((t), (h), (o), (l), (f))
 
 #define	BUS_SPACE_BARRIER_READ	0x01
 #define	BUS_SPACE_BARRIER_WRITE	0x02
@@ -478,204 +478,204 @@ __generate_inline_bs_ws(bus_space_write_stream_8, bs_w_8_s, uint64_t);
  */
 
 #define bs_map_proto(f)							\
-int	__bs_c(f,_bs_map) (void *t, bus_addr_t addr,		\
+int	__bs_c(f,_bs_map) (bus_space_tag_t t, bus_addr_t addr,		\
 	    bus_size_t size, int cacheable, bus_space_handle_t *bshp);
 
 #define bs_unmap_proto(f)						\
-void	__bs_c(f,_bs_unmap) (void *t, bus_space_handle_t bsh,		\
+void	__bs_c(f,_bs_unmap) (bus_space_tag_t t, bus_space_handle_t bsh,		\
 	    bus_size_t size);
 
 #define bs_subregion_proto(f)						\
-int	__bs_c(f,_bs_subregion) (void *t, bus_space_handle_t bsh,	\
+int	__bs_c(f,_bs_subregion) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 	    bus_size_t offset, bus_size_t size, 			\
 	    bus_space_handle_t *nbshp);
 
 #define bs_alloc_proto(f)						\
-int	__bs_c(f,_bs_alloc) (void *t, bus_addr_t rstart,		\
+int	__bs_c(f,_bs_alloc) (bus_space_tag_t t, bus_addr_t rstart,		\
 	    bus_addr_t rend, bus_size_t size, bus_size_t align,		\
 	    bus_size_t boundary, int cacheable, bus_addr_t *addrp,	\
 	    bus_space_handle_t *bshp);
 
 #define bs_free_proto(f)						\
-void	__bs_c(f,_bs_free) (void *t, bus_space_handle_t bsh,	\
+void	__bs_c(f,_bs_free) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 	    bus_size_t size);
 
 #define bs_mmap_proto(f)						\
 int	__bs_c(f,_bs_mmap) (struct cdev *, vm_offset_t, vm_paddr_t *, int);
 
 #define bs_barrier_proto(f)						\
-void	__bs_c(f,_bs_barrier) (void *t, bus_space_handle_t bsh,	\
+void	__bs_c(f,_bs_barrier) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 	    bus_size_t offset, bus_size_t len, int flags);
 
 #define	bs_r_1_proto(f)							\
-u_int8_t	__bs_c(f,_bs_r_1) (void *t, bus_space_handle_t bsh,	\
+uint8_t	__bs_c(f,_bs_r_1) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 		    bus_size_t offset);
 
 #define	bs_r_2_proto(f)							\
-u_int16_t	__bs_c(f,_bs_r_2) (void *t, bus_space_handle_t bsh,	\
+uint16_t	__bs_c(f,_bs_r_2) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 		    bus_size_t offset);
 
 #define	bs_r_4_proto(f)							\
-u_int32_t	__bs_c(f,_bs_r_4) (void *t, bus_space_handle_t bsh,	\
+uint32_t	__bs_c(f,_bs_r_4) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 		    bus_size_t offset);
 
 #define	bs_r_8_proto(f)							\
-u_int64_t	__bs_c(f,_bs_r_8) (void *t, bus_space_handle_t bsh,	\
+uint64_t	__bs_c(f,_bs_r_8) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 		    bus_size_t offset);
 
 #define	bs_r_1_s_proto(f)						\
-u_int8_t	__bs_c(f,_bs_r_1_s) (void *t, bus_space_handle_t bsh,	\
+uint8_t	__bs_c(f,_bs_r_1_s) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 		    bus_size_t offset);
 
 #define	bs_r_2_s_proto(f)						\
-u_int16_t	__bs_c(f,_bs_r_2_s) (void *t, bus_space_handle_t bsh,	\
+uint16_t	__bs_c(f,_bs_r_2_s) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 		    bus_size_t offset);
 
 #define	bs_r_4_s_proto(f)						\
-u_int32_t	__bs_c(f,_bs_r_4_s) (void *t, bus_space_handle_t bsh,	\
+uint32_t	__bs_c(f,_bs_r_4_s) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 		    bus_size_t offset);
 
 #define	bs_w_1_proto(f)							\
-void	__bs_c(f,_bs_w_1) (void *t, bus_space_handle_t bsh,		\
-	    bus_size_t offset, u_int8_t value);
+void	__bs_c(f,_bs_w_1) (bus_space_tag_t t, bus_space_handle_t bsh,		\
+	    bus_size_t offset, uint8_t value);
 
 #define	bs_w_2_proto(f)							\
-void	__bs_c(f,_bs_w_2) (void *t, bus_space_handle_t bsh,		\
-	    bus_size_t offset, u_int16_t value);
+void	__bs_c(f,_bs_w_2) (bus_space_tag_t t, bus_space_handle_t bsh,		\
+	    bus_size_t offset, uint16_t value);
 
 #define	bs_w_4_proto(f)							\
-void	__bs_c(f,_bs_w_4) (void *t, bus_space_handle_t bsh,		\
-	    bus_size_t offset, u_int32_t value);
+void	__bs_c(f,_bs_w_4) (bus_space_tag_t t, bus_space_handle_t bsh,		\
+	    bus_size_t offset, uint32_t value);
 
 #define	bs_w_8_proto(f)							\
-void	__bs_c(f,_bs_w_8) (void *t, bus_space_handle_t bsh,		\
-	    bus_size_t offset, u_int64_t value);
+void	__bs_c(f,_bs_w_8) (bus_space_tag_t t, bus_space_handle_t bsh,		\
+	    bus_size_t offset, uint64_t value);
 
 #define	bs_w_1_s_proto(f)						\
-void	__bs_c(f,_bs_w_1_s) (void *t, bus_space_handle_t bsh,		\
-	    bus_size_t offset, u_int8_t value);
+void	__bs_c(f,_bs_w_1_s) (bus_space_tag_t t, bus_space_handle_t bsh,		\
+	    bus_size_t offset, uint8_t value);
 
 #define	bs_w_2_s_proto(f)						\
-void	__bs_c(f,_bs_w_2_s) (void *t, bus_space_handle_t bsh,		\
-	    bus_size_t offset, u_int16_t value);
+void	__bs_c(f,_bs_w_2_s) (bus_space_tag_t t, bus_space_handle_t bsh,		\
+	    bus_size_t offset, uint16_t value);
 
 #define	bs_w_4_s_proto(f)						\
-void	__bs_c(f,_bs_w_4_s) (void *t, bus_space_handle_t bsh,		\
-	    bus_size_t offset, u_int32_t value);
+void	__bs_c(f,_bs_w_4_s) (bus_space_tag_t t, bus_space_handle_t bsh,		\
+	    bus_size_t offset, uint32_t value);
 
 #define	bs_rm_1_proto(f)						\
-void	__bs_c(f,_bs_rm_1) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int8_t *addr, bus_size_t count);
+void	__bs_c(f,_bs_rm_1) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint8_t *addr, bus_size_t count);
 
 #define	bs_rm_2_proto(f)						\
-void	__bs_c(f,_bs_rm_2) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int16_t *addr, bus_size_t count);
+void	__bs_c(f,_bs_rm_2) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint16_t *addr, bus_size_t count);
 
 #define	bs_rm_4_proto(f)						\
-void	__bs_c(f,_bs_rm_4) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int32_t *addr, bus_size_t count);		
+void	__bs_c(f,_bs_rm_4) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint32_t *addr, bus_size_t count);		
 
 #define	bs_rm_8_proto(f)						\
-void	__bs_c(f,_bs_rm_8) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int64_t *addr, bus_size_t count);
+void	__bs_c(f,_bs_rm_8) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint64_t *addr, bus_size_t count);
 
 #define	bs_wm_1_proto(f)						\
-void	__bs_c(f,_bs_wm_1) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int8_t *addr, bus_size_t count);
+void	__bs_c(f,_bs_wm_1) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint8_t *addr, bus_size_t count);
 
 #define	bs_wm_2_proto(f)						\
-void	__bs_c(f,_bs_wm_2) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int16_t *addr, bus_size_t count);
+void	__bs_c(f,_bs_wm_2) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint16_t *addr, bus_size_t count);
 
 #define	bs_wm_4_proto(f)						\
-void	__bs_c(f,_bs_wm_4) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int32_t *addr, bus_size_t count);
+void	__bs_c(f,_bs_wm_4) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint32_t *addr, bus_size_t count);
 
 #define	bs_wm_8_proto(f)						\
-void	__bs_c(f,_bs_wm_8) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int64_t *addr, bus_size_t count);
+void	__bs_c(f,_bs_wm_8) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint64_t *addr, bus_size_t count);
 
 #define	bs_rr_1_proto(f)						\
-void	__bs_c(f, _bs_rr_1) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int8_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_rr_1) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint8_t *addr, bus_size_t count);
 
 #define	bs_rr_2_proto(f)						\
-void	__bs_c(f, _bs_rr_2) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int16_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_rr_2) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint16_t *addr, bus_size_t count);
 
 #define	bs_rr_4_proto(f)						\
-void	__bs_c(f, _bs_rr_4) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int32_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_rr_4) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint32_t *addr, bus_size_t count);
 
 #define	bs_rr_8_proto(f)						\
-void	__bs_c(f, _bs_rr_8) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int64_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_rr_8) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint64_t *addr, bus_size_t count);
 
 #define	bs_wr_1_proto(f)						\
-void	__bs_c(f, _bs_wr_1) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int8_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_wr_1) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint8_t *addr, bus_size_t count);
 
 #define	bs_wr_2_proto(f)						\
-void	__bs_c(f, _bs_wr_2) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int16_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_wr_2) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint16_t *addr, bus_size_t count);
 
 #define	bs_wr_4_proto(f)						\
-void	__bs_c(f, _bs_wr_4) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int32_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_wr_4) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint32_t *addr, bus_size_t count);
 
 #define	bs_wr_8_proto(f)						\
-void	__bs_c(f, _bs_wr_8) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int64_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_wr_8) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint64_t *addr, bus_size_t count);
 
 #define	bs_sm_1_proto(f)						\
-void	__bs_c(f,_bs_sm_1) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int8_t value, bus_size_t count);
+void	__bs_c(f,_bs_sm_1) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint8_t value, bus_size_t count);
 
 #define	bs_sm_2_proto(f)						\
-void	__bs_c(f,_bs_sm_2) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int16_t value, bus_size_t count);
+void	__bs_c(f,_bs_sm_2) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint16_t value, bus_size_t count);
 
 #define	bs_sm_4_proto(f)						\
-void	__bs_c(f,_bs_sm_4) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int32_t value, bus_size_t count);
+void	__bs_c(f,_bs_sm_4) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint32_t value, bus_size_t count);
 
 #define	bs_sm_8_proto(f)						\
-void	__bs_c(f,_bs_sm_8) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int64_t value, bus_size_t count);
+void	__bs_c(f,_bs_sm_8) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint64_t value, bus_size_t count);
 
 #define	bs_sr_1_proto(f)						\
-void	__bs_c(f,_bs_sr_1) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int8_t value, bus_size_t count);
+void	__bs_c(f,_bs_sr_1) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint8_t value, bus_size_t count);
 
 #define	bs_sr_2_proto(f)						\
-void	__bs_c(f,_bs_sr_2) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int16_t value, bus_size_t count);
+void	__bs_c(f,_bs_sr_2) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint16_t value, bus_size_t count);
 
 #define	bs_sr_4_proto(f)						\
-void	__bs_c(f,_bs_sr_4) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int32_t value, bus_size_t count);
+void	__bs_c(f,_bs_sr_4) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint32_t value, bus_size_t count);
 
 #define	bs_sr_8_proto(f)						\
-void	__bs_c(f,_bs_sr_8) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int64_t value, bus_size_t count);
+void	__bs_c(f,_bs_sr_8) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint64_t value, bus_size_t count);
 
 #define	bs_c_1_proto(f)							\
-void	__bs_c(f,_bs_c_1) (void *t, bus_space_handle_t bsh1,	\
+void	__bs_c(f,_bs_c_1) (bus_space_tag_t t, bus_space_handle_t bsh1,	\
 	    bus_size_t offset1, bus_space_handle_t bsh2,		\
 	    bus_size_t offset2, bus_size_t count);
 
 #define	bs_c_2_proto(f)							\
-void	__bs_c(f,_bs_c_2) (void *t, bus_space_handle_t bsh1,	\
+void	__bs_c(f,_bs_c_2) (bus_space_tag_t t, bus_space_handle_t bsh1,	\
 	    bus_size_t offset1, bus_space_handle_t bsh2,		\
 	    bus_size_t offset2, bus_size_t count);
 
 #define	bs_c_4_proto(f)							\
-void	__bs_c(f,_bs_c_4) (void *t, bus_space_handle_t bsh1,	\
+void	__bs_c(f,_bs_c_4) (bus_space_tag_t t, bus_space_handle_t bsh1,	\
 	    bus_size_t offset1, bus_space_handle_t bsh2,		\
 	    bus_size_t offset2, bus_size_t count);
 
 #define	bs_c_8_proto(f)							\
-void	__bs_c(f,_bs_c_8) (void *t, bus_space_handle_t bsh1,	\
+void	__bs_c(f,_bs_c_8) (bus_space_tag_t t, bus_space_handle_t bsh1,	\
 	    bus_size_t offset1, bus_space_handle_t bsh2,		\
 	    bus_size_t offset2, bus_size_t count);
 
diff --git a/sys/arm/versatile/bus_space.c b/sys/arm/versatile/bus_space.c
index 375c5514c40..e7ffc09f021 100644
--- a/sys/arm/versatile/bus_space.c
+++ b/sys/arm/versatile/bus_space.c
@@ -46,7 +46,7 @@ bs_protos(generic_armv4);
 
 struct bus_space _base_tag = {
 	/* cookie */
-	.bs_cookie	= (void *) 0,
+	.bs_privdata	= (void *) 0,
 	
 	/* mapping/unmapping */
 	.bs_map		= generic_bs_map,
diff --git a/sys/arm/xilinx/zy7_bus_space.c b/sys/arm/xilinx/zy7_bus_space.c
index 375c5514c40..e7ffc09f021 100644
--- a/sys/arm/xilinx/zy7_bus_space.c
+++ b/sys/arm/xilinx/zy7_bus_space.c
@@ -46,7 +46,7 @@ bs_protos(generic_armv4);
 
 struct bus_space _base_tag = {
 	/* cookie */
-	.bs_cookie	= (void *) 0,
+	.bs_privdata	= (void *) 0,
 	
 	/* mapping/unmapping */
 	.bs_map		= generic_bs_map,
diff --git a/sys/arm/xscale/i80321/i80321_space.c b/sys/arm/xscale/i80321/i80321_space.c
index ce1db83bfd4..44a2ca80021 100644
--- a/sys/arm/xscale/i80321/i80321_space.c
+++ b/sys/arm/xscale/i80321/i80321_space.c
@@ -182,7 +182,7 @@ i80321_bs_init(bus_space_tag_t bs, void *cookie)
 {
 
 	*bs = i80321_bs_tag_template;
-	bs->bs_cookie = cookie;
+	bs->bs_privdata = cookie;
 }
 
 void
@@ -190,7 +190,7 @@ i80321_io_bs_init(bus_space_tag_t bs, void *cookie)
 {
 
 	*bs = i80321_bs_tag_template;
-	bs->bs_cookie = cookie;
+	bs->bs_privdata = cookie;
 
 	bs->bs_map = i80321_io_bs_map;
 	bs->bs_unmap = i80321_io_bs_unmap;
@@ -204,7 +204,7 @@ i80321_mem_bs_init(bus_space_tag_t bs, void *cookie)
 {
 
 	*bs = i80321_bs_tag_template;
-	bs->bs_cookie = cookie;
+	bs->bs_privdata = cookie;
 
 	bs->bs_map = i80321_mem_bs_map;
 	bs->bs_unmap = i80321_mem_bs_unmap;
@@ -216,7 +216,7 @@ i80321_mem_bs_init(bus_space_tag_t bs, void *cookie)
 /* *** Routines shared by i80321, PCI IO, and PCI MEM. *** */
 
 int
-i80321_bs_subregion(void *t, bus_space_handle_t bsh, bus_size_t offset,
+i80321_bs_subregion(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset,
     bus_size_t size, bus_space_handle_t *nbshp)
 {
 
@@ -225,7 +225,7 @@ i80321_bs_subregion(void *t, bus_space_handle_t bsh, bus_size_t offset,
 }
 
 void
-i80321_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
+i80321_bs_barrier(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset,
     bus_size_t len, int flags)
 {
 
@@ -236,7 +236,7 @@ i80321_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
 
 extern struct i80321_softc *i80321_softc;
 int
-i80321_io_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flags,
+i80321_io_bs_map(bus_space_tag_t tag, bus_addr_t bpa, bus_size_t size, int flags,
     bus_space_handle_t *bshp)
 {
 	struct i80321_softc *sc = i80321_softc;
@@ -264,14 +264,14 @@ i80321_io_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flags,
 }
 
 void
-i80321_io_bs_unmap(void *t, bus_space_handle_t h, bus_size_t size)
+i80321_io_bs_unmap(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t size)
 {
 
 	/* Nothing to do. */
 }
 
 int
-i80321_io_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend,
+i80321_io_bs_alloc(bus_space_tag_t tag, bus_addr_t rstart, bus_addr_t rend,
     bus_size_t size, bus_size_t alignment, bus_size_t boundary, int flags,
     bus_addr_t *bpap, bus_space_handle_t *bshp)
 {
@@ -280,7 +280,7 @@ i80321_io_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend,
 }
 
 void
-i80321_io_bs_free(void *t, bus_space_handle_t bsh, bus_size_t size)
+i80321_io_bs_free(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t size)
 {
 
 	panic("i80321_io_bs_free(): not implemented");
@@ -290,33 +290,23 @@ i80321_io_bs_free(void *t, bus_space_handle_t bsh, bus_size_t size)
 /* *** Routines for PCI MEM. *** */
 extern int badaddr_read(void *, int, void *);
 int
-i80321_mem_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flags,
+i80321_mem_bs_map(bus_space_tag_t tag, bus_addr_t bpa, bus_size_t size, int flags,
     bus_space_handle_t *bshp)
 {
-	vm_paddr_t pa, endpa;
 
-	pa = trunc_page(bpa);
-	endpa = round_page(bpa + size);
-
-	*bshp = (vm_offset_t)pmap_mapdev(pa, endpa - pa);
-		
+	*bshp = (vm_offset_t)pmap_mapdev(bpa, size);
 	return (0);
 }
 
 void
-i80321_mem_bs_unmap(void *t, bus_space_handle_t h, bus_size_t size)
+i80321_mem_bs_unmap(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t size)
 {
-	vm_offset_t va, endva;
 
-	va = trunc_page((vm_offset_t)t);
-	endva = va + round_page(size);
-
-	/* Free the kernel virtual mapping. */
-	kva_free(va, endva - va);
+	pmap_unmapdev((vm_offset_t)h, size);
 }
 
 int
-i80321_mem_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend,
+i80321_mem_bs_alloc(bus_space_tag_t tag, bus_addr_t rstart, bus_addr_t rend,
     bus_size_t size, bus_size_t alignment, bus_size_t boundary, int flags,
     bus_addr_t *bpap, bus_space_handle_t *bshp)
 {
@@ -325,7 +315,7 @@ i80321_mem_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend,
 }
 
 void
-i80321_mem_bs_free(void *t, bus_space_handle_t bsh, bus_size_t size)
+i80321_mem_bs_free(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t size)
 {
 
 	panic("i80321_mem_bs_free(): not implemented");
diff --git a/sys/arm/xscale/i8134x/i81342_space.c b/sys/arm/xscale/i8134x/i81342_space.c
index bd19a77c541..e19447472d7 100644
--- a/sys/arm/xscale/i8134x/i81342_space.c
+++ b/sys/arm/xscale/i8134x/i81342_space.c
@@ -183,7 +183,7 @@ i81342_bs_init(bus_space_tag_t bs, void *cookie)
 {
 
 	*bs = i81342_bs_tag_template;
-	bs->bs_cookie = cookie;
+	bs->bs_privdata = cookie;
 }
 
 void
@@ -191,7 +191,7 @@ i81342_io_bs_init(bus_space_tag_t bs, void *cookie)
 {
 
 	*bs = i81342_bs_tag_template;
-	bs->bs_cookie = cookie;
+	bs->bs_privdata = cookie;
 
 	bs->bs_map = i81342_io_bs_map;
 	bs->bs_unmap = i81342_io_bs_unmap;
@@ -205,7 +205,7 @@ i81342_mem_bs_init(bus_space_tag_t bs, void *cookie)
 {
 
 	*bs = i81342_bs_tag_template;
-	bs->bs_cookie = cookie;
+	bs->bs_privdata = cookie;
 
 	bs->bs_map = i81342_mem_bs_map;
 	bs->bs_unmap = i81342_mem_bs_unmap;
@@ -217,7 +217,7 @@ i81342_mem_bs_init(bus_space_tag_t bs, void *cookie)
 /* *** Routines shared by i81342, PCI IO, and PCI MEM. *** */
 
 int
-i81342_bs_subregion(void *t, bus_space_handle_t bsh, bus_size_t offset,
+i81342_bs_subregion(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset,
     bus_size_t size, bus_space_handle_t *nbshp)
 {
 
@@ -226,7 +226,7 @@ i81342_bs_subregion(void *t, bus_space_handle_t bsh, bus_size_t offset,
 }
 
 void
-i81342_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
+i81342_bs_barrier(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset,
     bus_size_t len, int flags)
 {
 
@@ -236,7 +236,7 @@ i81342_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
 /* *** Routines for PCI IO. *** */
 
 int
-i81342_io_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flags,
+i81342_io_bs_map(bus_space_tag_t tag, bus_addr_t bpa, bus_size_t size, int flags,
     bus_space_handle_t *bshp)
 {
 
@@ -245,14 +245,14 @@ i81342_io_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flags,
 }
 
 void
-i81342_io_bs_unmap(void *t, bus_space_handle_t h, bus_size_t size)
+i81342_io_bs_unmap(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t size)
 {
 
 	/* Nothing to do. */
 }
 
 int
-i81342_io_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend,
+i81342_io_bs_alloc(bus_space_tag_t tag, bus_addr_t rstart, bus_addr_t rend,
     bus_size_t size, bus_size_t alignment, bus_size_t boundary, int flags,
     bus_addr_t *bpap, bus_space_handle_t *bshp)
 {
@@ -261,7 +261,7 @@ i81342_io_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend,
 }
 
 void
-i81342_io_bs_free(void *t, bus_space_handle_t bsh, bus_size_t size)
+i81342_io_bs_free(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t size)
 {
 
 	panic("i81342_io_bs_free(): not implemented");
@@ -272,10 +272,10 @@ i81342_io_bs_free(void *t, bus_space_handle_t bsh, bus_size_t size)
 extern int badaddr_read(void *, int, void *);
 static vm_offset_t allocable = 0xe1000000;
 int
-i81342_mem_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flags,
+i81342_mem_bs_map(bus_space_tag_t tag, bus_addr_t bpa, bus_size_t size, int flags,
     bus_space_handle_t *bshp)
 {
-	struct i81342_pci_softc *sc = (struct i81342_pci_softc *)t;
+	struct i81342_pci_softc *sc = (struct i81342_pci_softc *)tag->bs_privdata;
 	struct i81342_pci_map *tmp;
 	vm_offset_t addr, endaddr;
 	vm_paddr_t paddr;
@@ -315,12 +315,12 @@ i81342_mem_bs_map(void *t, bus_addr_t bpa, bus_size_t size, int flags,
 }
 
 void
-i81342_mem_bs_unmap(void *t, bus_space_handle_t h, bus_size_t size)
+i81342_mem_bs_unmap(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t size)
 {
 #if 0
 	vm_offset_t va, endva;
 
-	va = trunc_page((vm_offset_t)t);
+	va = trunc_page((vm_offset_t)h);
 	endva = va + round_page(size);
 
 	/* Free the kernel virtual mapping. */
@@ -329,7 +329,7 @@ i81342_mem_bs_unmap(void *t, bus_space_handle_t h, bus_size_t size)
 }
 
 int
-i81342_mem_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend,
+i81342_mem_bs_alloc(bus_space_tag_t tag, bus_addr_t rstart, bus_addr_t rend,
     bus_size_t size, bus_size_t alignment, bus_size_t boundary, int flags,
     bus_addr_t *bpap, bus_space_handle_t *bshp)
 {
@@ -338,7 +338,7 @@ i81342_mem_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend,
 }
 
 void
-i81342_mem_bs_free(void *t, bus_space_handle_t bsh, bus_size_t size)
+i81342_mem_bs_free(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t size)
 {
 
 	panic("i81342_mem_bs_free(): not implemented");
diff --git a/sys/arm/xscale/ixp425/avila_ata.c b/sys/arm/xscale/ixp425/avila_ata.c
index e6307142051..217cfc8d405 100644
--- a/sys/arm/xscale/ixp425/avila_ata.c
+++ b/sys/arm/xscale/ixp425/avila_ata.c
@@ -147,9 +147,9 @@ struct ata_avila_softc {
 
 static void ata_avila_intr(void *);
 bs_protos(ata);
-static	void ata_bs_rm_2_s(void *, bus_space_handle_t, bus_size_t,
+static	void ata_bs_rm_2_s(bus_space_tag_t tag, bus_space_handle_t, bus_size_t,
 		u_int16_t *, bus_size_t);
-static	void ata_bs_wm_2_s(void *, bus_space_handle_t, bus_size_t,
+static	void ata_bs_wm_2_s(bus_space_tag_t tag, bus_space_handle_t, bus_size_t,
 		const u_int16_t *, bus_size_t);
 
 static int
@@ -200,7 +200,7 @@ ata_avila_attach(device_t dev)
 		 * XXX probably should just make this generic for
 		 * accessing the expansion bus.
 		 */
-		sc->sc_expbus_tag.bs_cookie = sc;	/* NB: backpointer */
+		sc->sc_expbus_tag.bs_privdata = sc;	/* NB: backpointer */
 		/* read single */
 		sc->sc_expbus_tag.bs_r_1	= ata_bs_r_1,
 		sc->sc_expbus_tag.bs_r_2	= ata_bs_r_2,
@@ -355,25 +355,25 @@ disable_16(struct ata_avila_softc *sc)
 }
 
 uint8_t
-ata_bs_r_1(void *t, bus_space_handle_t h, bus_size_t o)
+ata_bs_r_1(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o)
 {
-	struct ata_avila_softc *sc = t;
+	struct ata_avila_softc *sc = tag->bs_privdata;
 
 	return bus_space_read_1(sc->sc_iot, h, o);
 }
 
 void
-ata_bs_w_1(void *t, bus_space_handle_t h, bus_size_t o, u_int8_t v)
+ata_bs_w_1(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o, u_int8_t v)
 {
-	struct ata_avila_softc *sc = t;
+	struct ata_avila_softc *sc = tag->bs_privdata;
 
 	bus_space_write_1(sc->sc_iot, h, o, v);
 }
 
 uint16_t
-ata_bs_r_2(void *t, bus_space_handle_t h, bus_size_t o)
+ata_bs_r_2(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o)
 {
-	struct ata_avila_softc *sc = t;
+	struct ata_avila_softc *sc = tag->bs_privdata;
 	uint16_t v;
 
 	enable_16(sc);
@@ -383,9 +383,9 @@ ata_bs_r_2(void *t, bus_space_handle_t h, bus_size_t o)
 }
 
 void
-ata_bs_w_2(void *t, bus_space_handle_t h, bus_size_t o, uint16_t v)
+ata_bs_w_2(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o, uint16_t v)
 {
-	struct ata_avila_softc *sc = t;
+	struct ata_avila_softc *sc = tag->bs_privdata;
 
 	enable_16(sc);
 	bus_space_write_2(sc->sc_iot, h, o, v);
@@ -393,10 +393,10 @@ ata_bs_w_2(void *t, bus_space_handle_t h, bus_size_t o, uint16_t v)
 }
 
 void
-ata_bs_rm_2(void *t, bus_space_handle_t h, bus_size_t o,
+ata_bs_rm_2(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o,
 	u_int16_t *d, bus_size_t c)
 {
-	struct ata_avila_softc *sc = t;
+	struct ata_avila_softc *sc = tag->bs_privdata;
 
 	enable_16(sc);
 	bus_space_read_multi_2(sc->sc_iot, h, o, d, c);
@@ -404,10 +404,10 @@ ata_bs_rm_2(void *t, bus_space_handle_t h, bus_size_t o,
 }
 
 void
-ata_bs_wm_2(void *t, bus_space_handle_t h, bus_size_t o,
+ata_bs_wm_2(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o,
 	const u_int16_t *d, bus_size_t c)
 {
-	struct ata_avila_softc *sc = t;
+	struct ata_avila_softc *sc = tag->bs_privdata;
 
 	enable_16(sc);
 	bus_space_write_multi_2(sc->sc_iot, h, o, d, c);
@@ -417,10 +417,10 @@ ata_bs_wm_2(void *t, bus_space_handle_t h, bus_size_t o,
 /* XXX workaround ata driver by (incorrectly) byte swapping stream cases */
 
 void
-ata_bs_rm_2_s(void *t, bus_space_handle_t h, bus_size_t o,
+ata_bs_rm_2_s(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o,
 	u_int16_t *d, bus_size_t c)
 {
-	struct ata_avila_softc *sc = t;
+	struct ata_avila_softc *sc = tag->bs_privdata;
 	uint16_t v;
 	bus_size_t i;
 
@@ -437,10 +437,10 @@ ata_bs_rm_2_s(void *t, bus_space_handle_t h, bus_size_t o,
 }
 
 void
-ata_bs_wm_2_s(void *t, bus_space_handle_t h, bus_size_t o,
+ata_bs_wm_2_s(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o,
 	const u_int16_t *d, bus_size_t c)
 {
-	struct ata_avila_softc *sc = t;
+	struct ata_avila_softc *sc = tag->bs_privdata;
 	bus_size_t i;
 
 	enable_16(sc);
diff --git a/sys/arm/xscale/ixp425/cambria_exp_space.c b/sys/arm/xscale/ixp425/cambria_exp_space.c
index 30dfac5582a..9dce5778b8e 100644
--- a/sys/arm/xscale/ixp425/cambria_exp_space.c
+++ b/sys/arm/xscale/ixp425/cambria_exp_space.c
@@ -83,9 +83,9 @@ disable_16(struct ixp425_softc *sc, bus_size_t cs)
 }
 
 static uint8_t
-cambria_bs_r_1(void *t, bus_space_handle_t h, bus_size_t o)
+cambria_bs_r_1(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o)
 {
-	struct expbus_softc *exp = t;
+	struct expbus_softc *exp = tag->bs_privdata;
 	struct ixp425_softc *sc = exp->sc;
 	uint8_t v;
 
@@ -96,9 +96,9 @@ cambria_bs_r_1(void *t, bus_space_handle_t h, bus_size_t o)
 }
 
 static void
-cambria_bs_w_1(void *t, bus_space_handle_t h, bus_size_t o, u_int8_t v)
+cambria_bs_w_1(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o, u_int8_t v)
 {
-	struct expbus_softc *exp = t;
+	struct expbus_softc *exp = tag->bs_privdata;
 	struct ixp425_softc *sc = exp->sc;
 
 	EXP_LOCK(exp);
@@ -107,9 +107,9 @@ cambria_bs_w_1(void *t, bus_space_handle_t h, bus_size_t o, u_int8_t v)
 }
 
 static uint16_t
-cambria_bs_r_2(void *t, bus_space_handle_t h, bus_size_t o)
+cambria_bs_r_2(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o)
 {
-	struct expbus_softc *exp = t;
+	struct expbus_softc *exp = tag->bs_privdata;
 	struct ixp425_softc *sc = exp->sc;
 	uint16_t v;
 
@@ -122,9 +122,9 @@ cambria_bs_r_2(void *t, bus_space_handle_t h, bus_size_t o)
 }
 
 static void
-cambria_bs_w_2(void *t, bus_space_handle_t h, bus_size_t o, uint16_t v)
+cambria_bs_w_2(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o, uint16_t v)
 {
-	struct expbus_softc *exp = t;
+	struct expbus_softc *exp = tag->bs_privdata;
 	struct ixp425_softc *sc = exp->sc;
 
 	EXP_LOCK(exp);
@@ -135,10 +135,10 @@ cambria_bs_w_2(void *t, bus_space_handle_t h, bus_size_t o, uint16_t v)
 }
 
 static void
-cambria_bs_rm_2(void *t, bus_space_handle_t h, bus_size_t o,
+cambria_bs_rm_2(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o,
 	u_int16_t *d, bus_size_t c)
 {
-	struct expbus_softc *exp = t;
+	struct expbus_softc *exp = tag->bs_privdata;
 	struct ixp425_softc *sc = exp->sc;
 
 	EXP_LOCK(exp);
@@ -149,10 +149,10 @@ cambria_bs_rm_2(void *t, bus_space_handle_t h, bus_size_t o,
 }
 
 static void
-cambria_bs_wm_2(void *t, bus_space_handle_t h, bus_size_t o,
+cambria_bs_wm_2(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o,
 	const u_int16_t *d, bus_size_t c)
 {
-	struct expbus_softc *exp = t;
+	struct expbus_softc *exp = tag->bs_privdata;
 	struct ixp425_softc *sc = exp->sc;
 
 	EXP_LOCK(exp);
@@ -165,10 +165,10 @@ cambria_bs_wm_2(void *t, bus_space_handle_t h, bus_size_t o,
 /* XXX workaround ata driver by (incorrectly) byte swapping stream cases */
 
 static void
-cambria_bs_rm_2_s(void *t, bus_space_handle_t h, bus_size_t o,
+cambria_bs_rm_2_s(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o,
 	u_int16_t *d, bus_size_t c)
 {
-	struct expbus_softc *exp = t;
+	struct expbus_softc *exp = tag->bs_privdata;
 	struct ixp425_softc *sc = exp->sc;
 	uint16_t v;
 	bus_size_t i;
@@ -188,10 +188,10 @@ cambria_bs_rm_2_s(void *t, bus_space_handle_t h, bus_size_t o,
 }
 
 static void
-cambria_bs_wm_2_s(void *t, bus_space_handle_t h, bus_size_t o,
+cambria_bs_wm_2_s(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o,
 	const u_int16_t *d, bus_size_t c)
 {
-	struct expbus_softc *exp = t;
+	struct expbus_softc *exp = tag->bs_privdata;
 	struct ixp425_softc *sc = exp->sc;
 	bus_size_t i;
 
@@ -244,7 +244,7 @@ cambria_exp_bus_init(struct ixp425_softc *sc)
 	c3.sc = sc;
 	c3.csoff = EXP_TIMING_CS3_OFFSET;
 	EXP_LOCK_INIT(&c3);
-	cambria_exp_bs_tag.bs_cookie = &c3;
+	cambria_exp_bs_tag.bs_privdata = &c3;
 
 	cs3 = EXP_BUS_READ_4(sc, EXP_TIMING_CS3_OFFSET);
 	/* XXX force slowest possible timings and byte mode */
diff --git a/sys/arm/xscale/ixp425/ixp425_a4x_space.c b/sys/arm/xscale/ixp425/ixp425_a4x_space.c
index 74239db6f5f..23e48e8bb10 100644
--- a/sys/arm/xscale/ixp425/ixp425_a4x_space.c
+++ b/sys/arm/xscale/ixp425/ixp425_a4x_space.c
@@ -66,7 +66,7 @@ bs_protos(generic_armv4);
 
 struct bus_space ixp425_a4x_bs_tag = {
 	/* cookie */
-	.bs_cookie	= (void *) 0,
+	.bs_privdata	= (void *) 0,
 
 	/* mapping/unmapping */
 	.bs_map		= generic_bs_map,
diff --git a/sys/arm/xscale/ixp425/ixp425_pci_space.c b/sys/arm/xscale/ixp425/ixp425_pci_space.c
index 8617e8e1be1..bec3d9c9a98 100644
--- a/sys/arm/xscale/ixp425/ixp425_pci_space.c
+++ b/sys/arm/xscale/ixp425/ixp425_pci_space.c
@@ -72,30 +72,30 @@ bs_protos(ixp425_pci_io);
 bs_protos(ixp425_pci_mem);
 
 /* special I/O functions */
-static u_int8_t  _pci_io_bs_r_1(void *, bus_space_handle_t, bus_size_t);
-static u_int16_t _pci_io_bs_r_2(void *, bus_space_handle_t, bus_size_t);
-static u_int32_t _pci_io_bs_r_4(void *, bus_space_handle_t, bus_size_t);
+static u_int8_t  _pci_io_bs_r_1(bus_space_tag_t tag, bus_space_handle_t, bus_size_t);
+static u_int16_t _pci_io_bs_r_2(bus_space_tag_t tag, bus_space_handle_t, bus_size_t);
+static u_int32_t _pci_io_bs_r_4(bus_space_tag_t tag, bus_space_handle_t, bus_size_t);
 
-static void _pci_io_bs_w_1(void *, bus_space_handle_t, bus_size_t, u_int8_t);
-static void _pci_io_bs_w_2(void *, bus_space_handle_t, bus_size_t, u_int16_t);
-static void _pci_io_bs_w_4(void *, bus_space_handle_t, bus_size_t, u_int32_t);
+static void _pci_io_bs_w_1(bus_space_tag_t tag, bus_space_handle_t, bus_size_t, u_int8_t);
+static void _pci_io_bs_w_2(bus_space_tag_t tag, bus_space_handle_t, bus_size_t, u_int16_t);
+static void _pci_io_bs_w_4(bus_space_tag_t tag, bus_space_handle_t, bus_size_t, u_int32_t);
 
 #ifdef __ARMEB__
-static u_int8_t  _pci_io_bs_r_1_s(void *, bus_space_handle_t, bus_size_t);
-static u_int16_t _pci_io_bs_r_2_s(void *, bus_space_handle_t, bus_size_t);
-static u_int32_t _pci_io_bs_r_4_s(void *, bus_space_handle_t, bus_size_t);
+static u_int8_t  _pci_io_bs_r_1_s(bus_space_tag_t tag, bus_space_handle_t, bus_size_t);
+static u_int16_t _pci_io_bs_r_2_s(bus_space_tag_t tag, bus_space_handle_t, bus_size_t);
+static u_int32_t _pci_io_bs_r_4_s(bus_space_tag_t tag, bus_space_handle_t, bus_size_t);
 
-static void _pci_io_bs_w_1_s(void *, bus_space_handle_t, bus_size_t, u_int8_t);
-static void _pci_io_bs_w_2_s(void *, bus_space_handle_t, bus_size_t, u_int16_t);
-static void _pci_io_bs_w_4_s(void *, bus_space_handle_t, bus_size_t, u_int32_t);
+static void _pci_io_bs_w_1_s(bus_space_tag_t tag, bus_space_handle_t, bus_size_t, u_int8_t);
+static void _pci_io_bs_w_2_s(bus_space_tag_t tag, bus_space_handle_t, bus_size_t, u_int16_t);
+static void _pci_io_bs_w_4_s(bus_space_tag_t tag, bus_space_handle_t, bus_size_t, u_int32_t);
 
-static u_int8_t  _pci_mem_bs_r_1(void *, bus_space_handle_t, bus_size_t);
-static u_int16_t _pci_mem_bs_r_2(void *, bus_space_handle_t, bus_size_t);
-static u_int32_t _pci_mem_bs_r_4(void *, bus_space_handle_t, bus_size_t);
+static u_int8_t  _pci_mem_bs_r_1(bus_space_tag_t tag, bus_space_handle_t, bus_size_t);
+static u_int16_t _pci_mem_bs_r_2(bus_space_tag_t tag, bus_space_handle_t, bus_size_t);
+static u_int32_t _pci_mem_bs_r_4(bus_space_tag_t tag, bus_space_handle_t, bus_size_t);
 
-static void _pci_mem_bs_w_1(void *, bus_space_handle_t, bus_size_t, u_int8_t);
-static void _pci_mem_bs_w_2(void *, bus_space_handle_t, bus_size_t, u_int16_t);
-static void _pci_mem_bs_w_4(void *, bus_space_handle_t, bus_size_t, u_int32_t);
+static void _pci_mem_bs_w_1(bus_space_tag_t tag, bus_space_handle_t, bus_size_t, u_int8_t);
+static void _pci_mem_bs_w_2(bus_space_tag_t tag, bus_space_handle_t, bus_size_t, u_int16_t);
+static void _pci_mem_bs_w_4(bus_space_tag_t tag, bus_space_handle_t, bus_size_t, u_int32_t);
 #endif
 
 struct bus_space ixp425_pci_io_bs_tag_template = {
@@ -146,7 +146,7 @@ void
 ixp425_io_bs_init(bus_space_tag_t bs, void *cookie)
 {
 	*bs = ixp425_pci_io_bs_tag_template;
-	bs->bs_cookie = cookie;
+	bs->bs_privdata = cookie;
 }
 
 struct bus_space ixp425_pci_mem_bs_tag_template = {
@@ -202,12 +202,12 @@ void
 ixp425_mem_bs_init(bus_space_tag_t bs, void *cookie)
 {
 	*bs = ixp425_pci_mem_bs_tag_template;
-	bs->bs_cookie = cookie;
+	bs->bs_privdata = cookie;
 }
 
 /* common routine */
 int
-ixp425_pci_bs_subregion(void *t, bus_space_handle_t bsh, bus_size_t offset,
+ixp425_pci_bs_subregion(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset,
 	bus_size_t size, bus_space_handle_t *nbshp)
 {
 	*nbshp = bsh + offset;
@@ -215,7 +215,7 @@ ixp425_pci_bs_subregion(void *t, bus_space_handle_t bsh, bus_size_t offset,
 }
 
 void
-ixp425_pci_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
+ixp425_pci_bs_barrier(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset,
     bus_size_t len, int flags)
 {
 	/* NULL */
@@ -223,7 +223,7 @@ ixp425_pci_bs_barrier(void *t, bus_space_handle_t bsh, bus_size_t offset,
 
 /* io bs */
 int
-ixp425_pci_io_bs_map(void *t, bus_addr_t bpa, bus_size_t size,
+ixp425_pci_io_bs_map(bus_space_tag_t tag, bus_addr_t bpa, bus_size_t size,
 	int cacheable, bus_space_handle_t *bshp)
 {
 	*bshp = bpa;
@@ -231,13 +231,13 @@ ixp425_pci_io_bs_map(void *t, bus_addr_t bpa, bus_size_t size,
 }
 
 void
-ixp425_pci_io_bs_unmap(void *t, bus_space_handle_t h, bus_size_t size)
+ixp425_pci_io_bs_unmap(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t size)
 {
 	/* Nothing to do. */
 }
 
 int
-ixp425_pci_io_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend,
+ixp425_pci_io_bs_alloc(bus_space_tag_t tag, bus_addr_t rstart, bus_addr_t rend,
 	bus_size_t size, bus_size_t alignment, bus_size_t boundary, int cacheable,
 	bus_addr_t *bpap, bus_space_handle_t *bshp)
 {
@@ -245,14 +245,14 @@ ixp425_pci_io_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend,
 }
 
 void
-ixp425_pci_io_bs_free(void *t, bus_space_handle_t bsh, bus_size_t size)
+ixp425_pci_io_bs_free(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t size)
 {
 	panic("ixp425_pci_io_bs_free(): not implemented\n");
 }
 
 /* special I/O functions */
 static __inline u_int32_t
-_bs_r(void *v, bus_space_handle_t ioh, bus_size_t off, u_int32_t be)
+_bs_r(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off, u_int32_t be)
 {
 	u_int32_t data;
 
@@ -266,75 +266,75 @@ _bs_r(void *v, bus_space_handle_t ioh, bus_size_t off, u_int32_t be)
 }
 
 static u_int8_t
-_pci_io_bs_r_1(void *v, bus_space_handle_t ioh, bus_size_t off)
+_pci_io_bs_r_1(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off)
 {
 	u_int32_t data, n, be;
 
 	n = (ioh + off) % 4;
 	be = (0xf & ~(1U << n)) << NP_CBE_SHIFT;
-	data = _bs_r(v, ioh, off, be);
+	data = _bs_r(tag, ioh, off, be);
 
 	return data >> (8 * n);
 }
 
 static u_int16_t
-_pci_io_bs_r_2(void *v, bus_space_handle_t ioh, bus_size_t off)
+_pci_io_bs_r_2(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off)
 {
 	u_int32_t data, n, be;
 
 	n = (ioh + off) % 4;
 	be = (0xf & ~((1U << n) | (1U << (n + 1)))) << NP_CBE_SHIFT;
-	data = _bs_r(v, ioh, off, be);
+	data = _bs_r(tag, ioh, off, be);
 
 	return data >> (8 * n);
 }
 
 static u_int32_t
-_pci_io_bs_r_4(void *v, bus_space_handle_t ioh, bus_size_t off)
+_pci_io_bs_r_4(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off)
 {
 	u_int32_t data;
 
-	data = _bs_r(v, ioh, off, 0);
+	data = _bs_r(tag, ioh, off, 0);
 	return data;
 }
 
 #ifdef __ARMEB__
 static u_int8_t
-_pci_io_bs_r_1_s(void *v, bus_space_handle_t ioh, bus_size_t off)
+_pci_io_bs_r_1_s(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off)
 {
 	u_int32_t data, n, be;
 
 	n = (ioh + off) % 4;
 	be = (0xf & ~(1U << n)) << NP_CBE_SHIFT;
-	data = _bs_r(v, ioh, off, be);
+	data = _bs_r(tag, ioh, off, be);
 
 	return data >> (8 * n);
 }
 
 static u_int16_t
-_pci_io_bs_r_2_s(void *v, bus_space_handle_t ioh, bus_size_t off)
+_pci_io_bs_r_2_s(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off)
 {
 	u_int32_t data, n, be;
 
 	n = (ioh + off) % 4;
 	be = (0xf & ~((1U << n) | (1U << (n + 1)))) << NP_CBE_SHIFT;
-	data = _bs_r(v, ioh, off, be);
+	data = _bs_r(tag, ioh, off, be);
 
 	return data >> (8 * n);
 }
 
 static u_int32_t
-_pci_io_bs_r_4_s(void *v, bus_space_handle_t ioh, bus_size_t off)
+_pci_io_bs_r_4_s(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off)
 {
 	u_int32_t data;
 
-	data = _bs_r(v, ioh, off, 0);
+	data = _bs_r(tag, ioh, off, 0);
 	return le32toh(data);
 }
 #endif /* __ARMEB__ */
 
 static __inline void
-_bs_w(void *v, bus_space_handle_t ioh, bus_size_t off,
+_bs_w(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off,
 	u_int32_t be, u_int32_t data)
 {
 	CSR_WRITE_4(PCI_NP_AD, (ioh + off) & ~3);
@@ -345,7 +345,7 @@ _bs_w(void *v, bus_space_handle_t ioh, bus_size_t off,
 }
 
 static void
-_pci_io_bs_w_1(void *v, bus_space_handle_t ioh, bus_size_t off,
+_pci_io_bs_w_1(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off,
 	u_int8_t val)
 {
 	u_int32_t data, n, be;
@@ -353,11 +353,11 @@ _pci_io_bs_w_1(void *v, bus_space_handle_t ioh, bus_size_t off,
 	n = (ioh + off) % 4;
 	be = (0xf & ~(1U << n)) << NP_CBE_SHIFT;
 	data = val << (8 * n);
-	_bs_w(v, ioh, off, be, data);
+	_bs_w(tag, ioh, off, be, data);
 }
 
 static void
-_pci_io_bs_w_2(void *v, bus_space_handle_t ioh, bus_size_t off,
+_pci_io_bs_w_2(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off,
 	u_int16_t val)
 {
 	u_int32_t data, n, be;
@@ -365,19 +365,19 @@ _pci_io_bs_w_2(void *v, bus_space_handle_t ioh, bus_size_t off,
 	n = (ioh + off) % 4;
 	be = (0xf & ~((1U << n) | (1U << (n + 1)))) << NP_CBE_SHIFT;
 	data = val << (8 * n);
-	_bs_w(v, ioh, off, be, data);
+	_bs_w(tag, ioh, off, be, data);
 }
 
 static void
-_pci_io_bs_w_4(void *v, bus_space_handle_t ioh, bus_size_t off,
+_pci_io_bs_w_4(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off,
 	u_int32_t val)
 {
-	_bs_w(v, ioh, off, 0, val);
+	_bs_w(tag, ioh, off, 0, val);
 }
 
 #ifdef __ARMEB__
 static void
-_pci_io_bs_w_1_s(void *v, bus_space_handle_t ioh, bus_size_t off,
+_pci_io_bs_w_1_s(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off,
 	u_int8_t val)
 {
 	u_int32_t data, n, be;
@@ -385,11 +385,11 @@ _pci_io_bs_w_1_s(void *v, bus_space_handle_t ioh, bus_size_t off,
 	n = (ioh + off) % 4;
 	be = (0xf & ~(1U << n)) << NP_CBE_SHIFT;
 	data = val << (8 * n);
-	_bs_w(v, ioh, off, be, data);
+	_bs_w(tag, ioh, off, be, data);
 }
 
 static void
-_pci_io_bs_w_2_s(void *v, bus_space_handle_t ioh, bus_size_t off,
+_pci_io_bs_w_2_s(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off,
 	u_int16_t val)
 {
 	u_int32_t data, n, be;
@@ -397,46 +397,35 @@ _pci_io_bs_w_2_s(void *v, bus_space_handle_t ioh, bus_size_t off,
 	n = (ioh + off) % 4;
 	be = (0xf & ~((1U << n) | (1U << (n + 1)))) << NP_CBE_SHIFT;
 	data = val << (8 * n);
-	_bs_w(v, ioh, off, be, data);
+	_bs_w(tag, ioh, off, be, data);
 }
 
 static void
-_pci_io_bs_w_4_s(void *v, bus_space_handle_t ioh, bus_size_t off,
+_pci_io_bs_w_4_s(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off,
 	u_int32_t val)
 {
-	_bs_w(v, ioh, off, 0, htole32(val));
+	_bs_w(tag, ioh, off, 0, htole32(val));
 }
 #endif /* __ARMEB__ */
 
 /* mem bs */
 int
-ixp425_pci_mem_bs_map(void *t, bus_addr_t bpa, bus_size_t size,
+ixp425_pci_mem_bs_map(bus_space_tag_t tag, bus_addr_t bpa, bus_size_t size,
 	      int cacheable, bus_space_handle_t *bshp)
 {
-	vm_paddr_t pa, endpa;
-
-	pa = trunc_page(bpa);
-	endpa = round_page(bpa + size);
-
-	*bshp = (vm_offset_t)pmap_mapdev(pa, endpa - pa);
-
+	*bshp = (vm_offset_t)pmap_mapdev(bpa, size);
 	return (0);
 }
 
 void
-ixp425_pci_mem_bs_unmap(void *t, bus_space_handle_t h, bus_size_t size)
+ixp425_pci_mem_bs_unmap(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t size)
 {
-	vm_offset_t va, endva;
 
-	va = trunc_page((vm_offset_t)t);
-	endva = va + round_page(size);
-
-	/* Free the kernel virtual mapping. */
-	kva_free(va, endva - va);
+	pmap_unmapdev((vm_offset_t)h, size);
 }
 
 int
-ixp425_pci_mem_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend,
+ixp425_pci_mem_bs_alloc(bus_space_tag_t tag, bus_addr_t rstart, bus_addr_t rend,
 	bus_size_t size, bus_size_t alignment, bus_size_t boundary, int cacheable,
 	bus_addr_t *bpap, bus_space_handle_t *bshp)
 {
@@ -444,52 +433,52 @@ ixp425_pci_mem_bs_alloc(void *t, bus_addr_t rstart, bus_addr_t rend,
 }
 
 void
-ixp425_pci_mem_bs_free(void *t, bus_space_handle_t bsh, bus_size_t size)
+ixp425_pci_mem_bs_free(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t size)
 {
 	panic("ixp425_mem_bs_free(): not implemented\n");
 }
 
 #ifdef __ARMEB__
 static u_int8_t
-_pci_mem_bs_r_1(void *v, bus_space_handle_t ioh, bus_size_t off)
+_pci_mem_bs_r_1(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off)
 {
-	return ixp425_pci_mem_bs_r_1(v, ioh, off);
+	return ixp425_pci_mem_bs_r_1(tag, ioh, off);
 }
 
 static u_int16_t
-_pci_mem_bs_r_2(void *v, bus_space_handle_t ioh, bus_size_t off)
+_pci_mem_bs_r_2(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off)
 {
-	return (ixp425_pci_mem_bs_r_2(v, ioh, off));
+	return (ixp425_pci_mem_bs_r_2(tag, ioh, off));
 }
 
 static u_int32_t
-_pci_mem_bs_r_4(void *v, bus_space_handle_t ioh, bus_size_t off)
+_pci_mem_bs_r_4(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off)
 {
 	u_int32_t data;
 
-	data = ixp425_pci_mem_bs_r_4(v, ioh, off);
+	data = ixp425_pci_mem_bs_r_4(tag, ioh, off);
 	return (le32toh(data));
 }
 
 static void
-_pci_mem_bs_w_1(void *v, bus_space_handle_t ioh, bus_size_t off,
+_pci_mem_bs_w_1(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off,
 	u_int8_t val)
 {
-	ixp425_pci_mem_bs_w_1(v, ioh, off, val);
+	ixp425_pci_mem_bs_w_1(tag, ioh, off, val);
 }
 
 static void
-_pci_mem_bs_w_2(void *v, bus_space_handle_t ioh, bus_size_t off,
+_pci_mem_bs_w_2(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off,
 	u_int16_t val)
 {
-	ixp425_pci_mem_bs_w_2(v, ioh, off, val);
+	ixp425_pci_mem_bs_w_2(tag, ioh, off, val);
 }
 
 static void
-_pci_mem_bs_w_4(void *v, bus_space_handle_t ioh, bus_size_t off,
+_pci_mem_bs_w_4(bus_space_tag_t tag, bus_space_handle_t ioh, bus_size_t off,
 	u_int32_t val)
 {
-	ixp425_pci_mem_bs_w_4(v, ioh, off, htole32(val));
+	ixp425_pci_mem_bs_w_4(tag, ioh, off, htole32(val));
 }
 #endif /* __ARMEB__ */
 
diff --git a/sys/arm/xscale/ixp425/ixp425_space.c b/sys/arm/xscale/ixp425/ixp425_space.c
index 982ad4b1fed..102bcd2acc0 100644
--- a/sys/arm/xscale/ixp425/ixp425_space.c
+++ b/sys/arm/xscale/ixp425/ixp425_space.c
@@ -63,7 +63,7 @@ bs_protos(generic_armv4);
 
 struct bus_space ixp425_bs_tag = {
 	/* cookie */
-	.bs_cookie	= (void *) 0,
+	.bs_privdata	= (void *) 0,
 
 	/* mapping/unmapping */
 	.bs_map		= generic_bs_map,
diff --git a/sys/arm/xscale/pxa/pxa_space.c b/sys/arm/xscale/pxa/pxa_space.c
index f728e1d5200..cfaac37f7da 100644
--- a/sys/arm/xscale/pxa/pxa_space.c
+++ b/sys/arm/xscale/pxa/pxa_space.c
@@ -145,7 +145,7 @@ pxa_obio_tag_init()
 {
 
 	bcopy(&_base_tag, &_obio_tag, sizeof(struct bus_space));
-	_obio_tag.bs_cookie = (void *)PXA2X0_PERIPH_OFFSET;
+	_obio_tag.bs_privdata = (void *)PXA2X0_PERIPH_OFFSET;
 	obio_tag = &_obio_tag;
 }
 
@@ -161,7 +161,7 @@ pxa_bus_tag_alloc(bus_addr_t offset)
 	}
 
 	bcopy(&_base_tag, tag, sizeof(struct bus_space));
-	tag->bs_cookie = (void *)offset;
+	tag->bs_privdata = (void *)offset;
 
 	return ((bus_space_tag_t)tag);
 }
@@ -169,11 +169,11 @@ pxa_bus_tag_alloc(bus_addr_t offset)
 
 #define	READ_SINGLE(type, proto, base)					\
 	type								\
-	proto(void *cookie, bus_space_handle_t bsh, bus_size_t offset)	\
+	proto(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset)	\
 	{								\
 		bus_addr_t	tag_offset;				\
 		type		value;					\
-		tag_offset = (bus_addr_t)cookie;			\
+		tag_offset = (bus_addr_t)tag->bs_privdata;		\
 		value = base(NULL, bsh + tag_offset, offset);		\
 		return (value);						\
 	}
@@ -186,11 +186,11 @@ READ_SINGLE(u_int32_t, pxa_bs_r_4, generic_bs_r_4)
 
 #define	WRITE_SINGLE(type, proto, base)					\
 	void								\
-	proto(void *cookie, bus_space_handle_t bsh, bus_size_t offset,	\
+	proto(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset,	\
 	    type value)							\
 	{								\
 		bus_addr_t	tag_offset;				\
-		tag_offset = (bus_addr_t)cookie;			\
+		tag_offset = (bus_addr_t)tag->bs_privdata;		\
 		base(NULL, bsh + tag_offset, offset, value);		\
 	}
 
@@ -202,11 +202,11 @@ WRITE_SINGLE(u_int32_t, pxa_bs_w_4, generic_bs_w_4)
 
 #define	READ_MULTI(type, proto, base)					\
 	void								\
-	proto(void *cookie, bus_space_handle_t bsh, bus_size_t offset,	\
+	proto(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset,	\
 	    type *dest, bus_size_t count)				\
 	{								\
 		bus_addr_t	tag_offset;				\
-		tag_offset = (bus_addr_t)cookie;			\
+		tag_offset = (bus_addr_t)tag->bs_privdata;		\
 		base(NULL, bsh + tag_offset, offset, dest, count);	\
 	}
 
@@ -219,11 +219,11 @@ READ_MULTI(u_int8_t,  pxa_bs_rr_1, generic_bs_rr_1)
 
 #define	WRITE_MULTI(type, proto, base)					\
 	void								\
-	proto(void *cookie, bus_space_handle_t bsh, bus_size_t offset,	\
+	proto(bus_space_tag_t tag, bus_space_handle_t bsh, bus_size_t offset,	\
 	    const type *src, bus_size_t count)				\
 	{								\
 		bus_addr_t	tag_offset;				\
-		tag_offset = (bus_addr_t)cookie;			\
+		tag_offset = (bus_addr_t)tag->bs_privdata;		\
 		base(NULL, bsh + tag_offset, offset, src, count);	\
 	}
 
diff --git a/sys/dev/usb/controller/ehci_ixp4xx.c b/sys/dev/usb/controller/ehci_ixp4xx.c
index 956f73951a8..301032ed026 100644
--- a/sys/dev/usb/controller/ehci_ixp4xx.c
+++ b/sys/dev/usb/controller/ehci_ixp4xx.c
@@ -79,12 +79,12 @@ struct ixp_ehci_softc {
 static device_attach_t ehci_ixp_attach;
 static device_detach_t ehci_ixp_detach;
 
-static uint8_t ehci_bs_r_1(void *, bus_space_handle_t, bus_size_t);
-static void ehci_bs_w_1(void *, bus_space_handle_t, bus_size_t, u_int8_t);
-static uint16_t ehci_bs_r_2(void *, bus_space_handle_t, bus_size_t);
-static void ehci_bs_w_2(void *, bus_space_handle_t, bus_size_t, uint16_t);
-static uint32_t ehci_bs_r_4(void *, bus_space_handle_t, bus_size_t);
-static void ehci_bs_w_4(void *, bus_space_handle_t, bus_size_t, uint32_t);
+static uint8_t ehci_bs_r_1(bus_space_tag_t tag, bus_space_handle_t, bus_size_t);
+static void ehci_bs_w_1(bus_space_tag_t tag, bus_space_handle_t, bus_size_t, u_int8_t);
+static uint16_t ehci_bs_r_2(bus_space_tag_t tag, bus_space_handle_t, bus_size_t);
+static void ehci_bs_w_2(bus_space_tag_t tag, bus_space_handle_t, bus_size_t, uint16_t);
+static uint32_t ehci_bs_r_4(bus_space_tag_t tag, bus_space_handle_t, bus_size_t);
+static void ehci_bs_w_4(bus_space_tag_t tag, bus_space_handle_t, bus_size_t, uint32_t);
 
 static int
 ehci_ixp_probe(device_t self)
@@ -132,7 +132,7 @@ ehci_ixp_attach(device_t self)
 	 * done with bus_space_subregion.
 	 */
 	isc->iot = rman_get_bustag(sc->sc_io_res);
-	isc->tag.bs_cookie = isc->iot;
+	isc->tag.bs_privdata = isc->iot;
 	/* read single */
 	isc->tag.bs_r_1	= ehci_bs_r_1,
 	isc->tag.bs_r_2	= ehci_bs_r_2,
@@ -252,41 +252,41 @@ ehci_ixp_detach(device_t self)
  */
 
 static uint8_t
-ehci_bs_r_1(void *t, bus_space_handle_t h, bus_size_t o)
+ehci_bs_r_1(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o)
 {
-	return bus_space_read_1((bus_space_tag_t) t, h,
+	return bus_space_read_1((bus_space_tag_t)tag->bs_privdata, h,
 	    0x100 + (o &~ 3) + (3 - (o & 3)));
 }
 
 static void
-ehci_bs_w_1(void *t, bus_space_handle_t h, bus_size_t o, u_int8_t v)
+ehci_bs_w_1(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o, u_int8_t v)
 {
 	panic("%s", __func__);
 }
 
 static uint16_t
-ehci_bs_r_2(void *t, bus_space_handle_t h, bus_size_t o)
+ehci_bs_r_2(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o)
 {
-	return bus_space_read_2((bus_space_tag_t) t, h,
+	return bus_space_read_2((bus_space_tag_t)tag->bs_privdata, h,
 	    0x100 + (o &~ 3) + (2 - (o & 3)));
 }
 
 static void
-ehci_bs_w_2(void *t, bus_space_handle_t h, bus_size_t o, uint16_t v)
+ehci_bs_w_2(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o, uint16_t v)
 {
 	panic("%s", __func__);
 }
 
 static uint32_t
-ehci_bs_r_4(void *t, bus_space_handle_t h, bus_size_t o)
+ehci_bs_r_4(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o)
 {
-	return bus_space_read_4((bus_space_tag_t) t, h, 0x100 + o);
+	return bus_space_read_4((bus_space_tag_t) tag->bs_privdata, h, 0x100 + o);
 }
 
 static void
-ehci_bs_w_4(void *t, bus_space_handle_t h, bus_size_t o, uint32_t v)
+ehci_bs_w_4(bus_space_tag_t tag, bus_space_handle_t h, bus_size_t o, uint32_t v)
 {
-	bus_space_write_4((bus_space_tag_t) t, h, 0x100 + o, v);
+	bus_space_write_4((bus_space_tag_t) tag->bs_privdata, h, 0x100 + o, v);
 }
 
 static device_method_t ehci_methods[] = {

From c31c0579574cfc7b7d3054a762f78441a2c73d0f Mon Sep 17 00:00:00 2001
From: Mateusz Guzik <mjg@FreeBSD.org>
Date: Wed, 21 Jan 2015 01:06:14 +0000
Subject: [PATCH 126/279] filedesc: plug a test for impossible condition in
 _fget

---
 sys/kern/kern_descrip.c | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index 82ce56ddcaf..4b4af296769 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -2439,8 +2439,7 @@ _fget(struct thread *td, int fd, struct file **fpp, int flags,
 	int error;
 
 	*fpp = NULL;
-	if (td == NULL || (fdp = td->td_proc->p_fd) == NULL)
-		return (EBADF);
+	fdp = td->td_proc->p_fd;
 	if (needrightsp != NULL)
 		needrights = *needrightsp;
 	else

From 57736250480e254798e7869eb82a8268bca03a0d Mon Sep 17 00:00:00 2001
From: Ed Maste <emaste@FreeBSD.org>
Date: Wed, 21 Jan 2015 01:07:58 +0000
Subject: [PATCH 127/279] redelf: Add missing R_X86_64_ relocation types

PR:		196918
Reviewed by:	dim
Sponsored by:	The FreeBSD Foundation
Differential Revision:	https://reviews.freebsd.org/D1570
---
 contrib/elftoolchain/common/elfdefinitions.h |  8 +++++++-
 contrib/elftoolchain/readelf/readelf.c       | 14 ++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/contrib/elftoolchain/common/elfdefinitions.h b/contrib/elftoolchain/common/elfdefinitions.h
index f63dc7f2178..8b28aeb5eac 100644
--- a/contrib/elftoolchain/common/elfdefinitions.h
+++ b/contrib/elftoolchain/common/elfdefinitions.h
@@ -1948,11 +1948,17 @@ _ELF_DEFINE_RELOC(R_X86_64_TPOFF32,	23)	\
 _ELF_DEFINE_RELOC(R_X86_64_PC64,	24)	\
 _ELF_DEFINE_RELOC(R_X86_64_GOTOFF64,	25)	\
 _ELF_DEFINE_RELOC(R_X86_64_GOTPC32,	26)	\
+_ELF_DEFINE_RELOC(R_X86_64_GOT64,	27)	\
+_ELF_DEFINE_RELOC(R_X86_64_GOTPCREL64,	28)	\
+_ELF_DEFINE_RELOC(R_X86_64_GOTPC64,	29)	\
+_ELF_DEFINE_RELOC(R_X86_64_GOTPLT64,	30)	\
+_ELF_DEFINE_RELOC(R_X86_64_PLTOFF64,	31)	\
 _ELF_DEFINE_RELOC(R_X86_64_SIZE32,	32)	\
 _ELF_DEFINE_RELOC(R_X86_64_SIZE64,	33)	\
 _ELF_DEFINE_RELOC(R_X86_64_GOTPC32_TLSDESC, 34)	\
 _ELF_DEFINE_RELOC(R_X86_64_TLSDESC_CALL, 35)	\
-_ELF_DEFINE_RELOC(R_X86_64_TLSDESC,	36)
+_ELF_DEFINE_RELOC(R_X86_64_TLSDESC,	36)	\
+_ELF_DEFINE_RELOC(R_X86_64_IRELATIVE,	37)
 
 #define	_ELF_DEFINE_RELOCATIONS()		\
 _ELF_DEFINE_386_RELOCATIONS()			\
diff --git a/contrib/elftoolchain/readelf/readelf.c b/contrib/elftoolchain/readelf/readelf.c
index 6e1e52abfc5..a8c15c465e5 100644
--- a/contrib/elftoolchain/readelf/readelf.c
+++ b/contrib/elftoolchain/readelf/readelf.c
@@ -1480,6 +1480,20 @@ r_type(unsigned int mach, unsigned int type)
 		case 21: return "R_X86_64_DTPOFF32";
 		case 22: return "R_X86_64_GOTTPOFF";
 		case 23: return "R_X86_64_TPOFF32";
+		case 24: return "R_X86_64_PC64";
+		case 25: return "R_X86_64_GOTOFF64";
+		case 26: return "R_X86_64_GOTPC32";
+		case 27: return "R_X86_64_GOT64";
+		case 28: return "R_X86_64_GOTPCREL64";
+		case 29: return "R_X86_64_GOTPC64";
+		case 30: return "R_X86_64_GOTPLT64";
+		case 31: return "R_X86_64_PLTOFF64";
+		case 32: return "R_X86_64_SIZE32";
+		case 33: return "R_X86_64_SIZE64";
+		case 34: return "R_X86_64_GOTPC32_TLSDESC";
+		case 35: return "R_X86_64_TLSDESC_CALL";
+		case 36: return "R_X86_64_TLSDESC";
+		case 37: return "R_X86_64_IRELATIVE";
 		default: return "";
 		}
 	default: return "";

From a5e43ff69ead26f9d07cbb450c2f3b6ff5cdae70 Mon Sep 17 00:00:00 2001
From: Xin LI <delphij@FreeBSD.org>
Date: Wed, 21 Jan 2015 01:11:37 +0000
Subject: [PATCH 128/279] Fix xz handling for files larger than 32K.

Submitted by:	Stefan Ehmann <shoesoft gmx net>
PR:		bin/186861
MFC after:	2 weeks
---
 usr.bin/grep/file.c | 65 +++++++++++++++++++++++++++++----------------
 1 file changed, 42 insertions(+), 23 deletions(-)

diff --git a/usr.bin/grep/file.c b/usr.bin/grep/file.c
index 6bcaa52a7ec..81f227d8073 100644
--- a/usr.bin/grep/file.c
+++ b/usr.bin/grep/file.c
@@ -65,6 +65,8 @@ __FBSDID("$FreeBSD$");
 static gzFile gzbufdesc;
 #ifndef WITHOUT_LZMA
 static lzma_stream lstrm = LZMA_STREAM_INIT;
+static lzma_action laction;
+static uint8_t lin_buf[MAXBUFSIZ];
 #endif
 #ifndef WITHOUT_BZIP2
 static BZFILE* bzbufdesc;
@@ -123,34 +125,34 @@ grep_refill(struct file *f)
 #endif
 #ifndef WITHOUT_LZMA
 	} else if ((filebehave == FILE_XZ) || (filebehave == FILE_LZMA)) {
-		lzma_action action = LZMA_RUN;
-		uint8_t in_buf[MAXBUFSIZ];
 		lzma_ret ret;
-
-		ret = (filebehave == FILE_XZ) ?
-		    lzma_stream_decoder(&lstrm, UINT64_MAX,
-		    LZMA_CONCATENATED) :
-		    lzma_alone_decoder(&lstrm, UINT64_MAX);
-
-		if (ret != LZMA_OK)
-			return (-1);
-
 		lstrm.next_out = buffer;
-		lstrm.avail_out = MAXBUFSIZ;
-		lstrm.next_in = in_buf;
-		nr = read(f->fd, in_buf, MAXBUFSIZ);
 
-		if (nr < 0)
-			return (-1);
-		else if (nr == 0)
-			action = LZMA_FINISH;
+		do {
+			if (lstrm.avail_in == 0) {
+				lstrm.next_in = lin_buf;
+				nr = read(f->fd, lin_buf, MAXBUFSIZ);
 
-		lstrm.avail_in = nr;
-		ret = lzma_code(&lstrm, action);
+				if (nr < 0)
+					return (-1);
+				else if (nr == 0)
+					laction = LZMA_FINISH;
+
+				lstrm.avail_in = nr;
+			}
+
+			ret = lzma_code(&lstrm, laction);
+
+			if (ret != LZMA_OK && ret != LZMA_STREAM_END)
+				return (-1);
+
+			if (lstrm.avail_out == 0 || ret == LZMA_STREAM_END) {
+				bufrem = MAXBUFSIZ - lstrm.avail_out;
+				lstrm.next_out = buffer;
+				lstrm.avail_out = MAXBUFSIZ;
+			}
+		} while (bufrem == 0 && ret != LZMA_STREAM_END);
 
-		if (ret != LZMA_OK && ret != LZMA_STREAM_END)
-			return (-1);
-		bufrem = MAXBUFSIZ - lstrm.avail_out;
 		return (0);
 #endif	/* WIHTOUT_LZMA */
 	} else
@@ -291,6 +293,23 @@ grep_open(const char *path)
 	    (bzbufdesc = BZ2_bzdopen(f->fd, "r")) == NULL)
 		goto error2;
 #endif
+#ifndef WITHOUT_LZMA
+	else if ((filebehave == FILE_XZ) || (filebehave == FILE_LZMA)) {
+		lzma_ret ret;
+
+		ret = (filebehave == FILE_XZ) ?
+			lzma_stream_decoder(&lstrm, UINT64_MAX,
+					LZMA_CONCATENATED) :
+			lzma_alone_decoder(&lstrm, UINT64_MAX);
+
+		if (ret != LZMA_OK)
+			goto error2;
+
+		lstrm.avail_in = 0;
+		lstrm.avail_out = MAXBUFSIZ;
+		laction = LZMA_RUN;
+	}
+#endif
 
 	/* Fill read buffer, also catches errors early */
 	if (bufrem == 0 && grep_refill(f) != 0)

From 008991975783e756fef78ac436e3e3b4f6a938c2 Mon Sep 17 00:00:00 2001
From: Ed Maste <emaste@FreeBSD.org>
Date: Wed, 21 Jan 2015 01:12:21 +0000
Subject: [PATCH 129/279] Add missing R_X86_64_ constants to elf_common.h

PR:		196918
MFC after:	2 weeks
Sponsored by:	The FreeBSD Foundation
---
 sys/sys/elf_common.h | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/sys/sys/elf_common.h b/sys/sys/elf_common.h
index ebe2ee23288..c496a901425 100644
--- a/sys/sys/elf_common.h
+++ b/sys/sys/elf_common.h
@@ -1223,6 +1223,19 @@ typedef struct {
 #define	R_X86_64_DTPOFF32	21	/* Offset in TLS block */
 #define	R_X86_64_GOTTPOFF	22	/* PC relative offset to IE GOT entry */
 #define	R_X86_64_TPOFF32	23	/* Offset in static TLS block */
+#define	R_X86_64_PC64		24	/* PC-relative 64 bit signed sym value. */
+#define	R_X86_64_GOTOFF64	25
+#define	R_X86_64_GOTPC32	26
+#define	R_X86_64_GOT64		27
+#define	R_X86_64_GOTPCREL64	28
+#define	R_X86_64_GOTPC64	29
+#define	R_X86_64_GOTPLT64	30
+#define	R_X86_64_PLTOFF64	31
+#define	R_X86_64_SIZE32		32
+#define	R_X86_64_SIZE64		33
+#define	R_X86_64_GOTPC32_TLSDESC 34
+#define	R_X86_64_TLSDESC_CALL	35
+#define	R_X86_64_TLSDESC	36
 #define	R_X86_64_IRELATIVE	37
 
 

From 942dfd1f3fe3465f7776c4a9d00d13c0e23f14c6 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 02:35:04 +0000
Subject: [PATCH 130/279] Use the explicit member initializer style to init the
 bus_space struct. Fill in some formerly NULL members where the implementation
 function exists.  Add a dummy function that panics and use it as a
 placeholder for thigns that are still unimplemented.  Remove a few unused
 includes.

---
 sys/arm/arm/bus_space-v6.c | 149 +++++++++++++++++++------------------
 1 file changed, 77 insertions(+), 72 deletions(-)

diff --git a/sys/arm/arm/bus_space-v6.c b/sys/arm/arm/bus_space-v6.c
index 738e4c6fc36..3414c5009cc 100644
--- a/sys/arm/arm/bus_space-v6.c
+++ b/sys/arm/arm/bus_space-v6.c
@@ -34,125 +34,130 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
-#include <sys/bus.h>
-#include <sys/kernel.h>
-#include <sys/malloc.h>
-
 #include <machine/bus.h>
 
 /* Prototypes for all the bus_space structure functions */
 bs_protos(generic);
 bs_protos(generic_armv4);
 
+static void
+bs_unimplemented(void)
+{
+
+	panic("unimplemented bus_space function called");
+}
+
+#define	BS_UNIMPLEMENTED	(void *)bs_unimplemented
+
 /*
  * The bus space tag.  This is constant for all instances, so
  * we never have to explicitly "create" it.
  */
 static struct bus_space _base_tag = {
-	/* cookie */
-	(void *) 0,
+	/* privdata is whatever the implementer wants; unused in base tag */
+	.bs_privdata	= NULL,
 
 	/* mapping/unmapping */
-	generic_bs_map,
-	generic_bs_unmap,
-	generic_bs_subregion,
+	.bs_map		= generic_bs_map,
+	.bs_unmap	= generic_bs_unmap,
+	.bs_subregion	= generic_bs_subregion,
 
 	/* allocation/deallocation */
-	generic_bs_alloc,
-	generic_bs_free,
+	.bs_alloc	= generic_bs_alloc,
+	.bs_free	= generic_bs_free,
 
 	/* barrier */
-	generic_bs_barrier,
+	.bs_barrier	= generic_bs_barrier,
 
 	/* read (single) */
-	NULL,	/* bs_r_1, Use inline code in bus.h */
-	NULL,	/* bs_r_2, Use inline code in bus.h */
-	NULL,	/* bs_r_4, Use inline code in bus.h */
-	NULL,	/* bs_r_8, Use inline code in bus.h */
+	.bs_r_1		= NULL,	/* Use inline code in bus.h */
+	.bs_r_2		= NULL,	/* Use inline code in bus.h */
+	.bs_r_4		= NULL,	/* Use inline code in bus.h */
+	.bs_r_8		= NULL,	/* Use inline code in bus.h */
 
 	/* read multiple */
-	generic_bs_rm_1,
-	generic_armv4_bs_rm_2,
-	generic_bs_rm_4,
-	NULL,
+	.bs_rm_1	= generic_bs_rm_1,
+	.bs_rm_2	= generic_armv4_bs_rm_2,
+	.bs_rm_4	= generic_bs_rm_4,
+	.bs_rm_8	= BS_UNIMPLEMENTED,
 
 	/* read region */
-	generic_bs_rr_1,
-	generic_armv4_bs_rr_2,
-	generic_bs_rr_4,
-	NULL,
+	.bs_rr_1	= generic_bs_rr_1,
+	.bs_rr_2	= generic_armv4_bs_rr_2,
+	.bs_rr_4	= generic_bs_rr_4,
+	.bs_rr_8	= BS_UNIMPLEMENTED,
 
 	/* write (single) */
-	NULL,	/* bs_w_1, Use inline code in bus.h */
-	NULL,	/* bs_w_2, Use inline code in bus.h */
-	NULL,	/* bs_w_4, Use inline code in bus.h */
-	NULL,	/* bs_w_8, Use inline code in bus.h */
+	.bs_w_1		= NULL,	/* Use inline code in bus.h */
+	.bs_w_2		= NULL,	/* Use inline code in bus.h */
+	.bs_w_4		= NULL,	/* Use inline code in bus.h */
+	.bs_w_8		= NULL,	/* Use inline code in bus.h */
 
 	/* write multiple */
-	generic_bs_wm_1,
-	generic_armv4_bs_wm_2,
-	generic_bs_wm_4,
-	NULL,
+	.bs_wm_1	= generic_bs_wm_1,
+	.bs_wm_2	= generic_armv4_bs_wm_2,
+	.bs_wm_4	= generic_bs_wm_4,
+	.bs_wm_8	= BS_UNIMPLEMENTED,
 
 	/* write region */
-	generic_bs_wr_1,
-	generic_armv4_bs_wr_2,
-	generic_bs_wr_4,
-	NULL,
+	.bs_wr_1	= generic_bs_wr_1,
+	.bs_wr_2	= generic_armv4_bs_wr_2,
+	.bs_wr_4	= generic_bs_wr_4,
+	.bs_wr_8	= BS_UNIMPLEMENTED,
 
 	/* set multiple */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
+	.bs_sm_1	= BS_UNIMPLEMENTED,
+	.bs_sm_2	= BS_UNIMPLEMENTED,
+	.bs_sm_4	= BS_UNIMPLEMENTED,
+	.bs_sm_8	= BS_UNIMPLEMENTED,
 
 	/* set region */
-	generic_bs_sr_1,
-	generic_armv4_bs_sr_2,
-	generic_bs_sr_4,
-	NULL,
+	.bs_sr_1	= generic_bs_sr_1,
+	.bs_sr_2	= generic_armv4_bs_sr_2,
+	.bs_sr_4	= generic_bs_sr_4,
+	.bs_sr_8	= BS_UNIMPLEMENTED,
 
 	/* copy */
-	NULL,
-	generic_armv4_bs_c_2,
-	NULL,
-	NULL,
+	.bs_c_1		= BS_UNIMPLEMENTED,
+	.bs_c_2		= generic_armv4_bs_c_2,
+	.bs_c_4		= BS_UNIMPLEMENTED,
+	.bs_c_8		= BS_UNIMPLEMENTED,
 
 	/* read stream (single) */
-	NULL,   /* bs_r_1_s, Use inline code in bus.h */ 
-	NULL,   /* bs_r_2_s, Use inline code in bus.h */ 
-	NULL,   /* bs_r_4_s, Use inline code in bus.h */ 
-	NULL,   /* bs_r_8_s, Use inline code in bus.h */ 
+	.bs_r_1_s	= NULL,   /* Use inline code in bus.h */ 
+	.bs_r_2_s	= NULL,   /* Use inline code in bus.h */ 
+	.bs_r_4_s	= NULL,   /* Use inline code in bus.h */ 
+	.bs_r_8_s	= NULL,   /* Use inline code in bus.h */ 
 
 	/* read multiple stream */
-	NULL,
-	generic_armv4_bs_rm_2,		/* bus_space_read_multi_stream_2 */
-	NULL,
-	NULL,
+	.bs_rm_1_s	= generic_bs_rm_1,
+	.bs_rm_2_s	= generic_armv4_bs_rm_2,
+	.bs_rm_4_s	= generic_bs_rm_4,
+	.bs_rm_8_s	= BS_UNIMPLEMENTED,
 
 	/* read region stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
+	.bs_rr_1_s	= generic_bs_rr_1,
+	.bs_rr_2_s	= generic_bs_rr_2,
+	.bs_rr_4_s	= generic_bs_rr_4,
+	.bs_rr_8_s	= BS_UNIMPLEMENTED,
 
 	/* write stream (single) */
-	NULL,   /* bs_w_1_s, Use inline code in bus.h */ 
-	NULL,   /* bs_w_2_s, Use inline code in bus.h */ 
-	NULL,   /* bs_w_4_s, Use inline code in bus.h */ 
-	NULL,   /* bs_w_8_s, Use inline code in bus.h */ 
+	.bs_w_1_s	= NULL,   /* Use inline code in bus.h */ 
+	.bs_w_2_s	= NULL,   /* Use inline code in bus.h */ 
+	.bs_w_4_s	= NULL,   /* Use inline code in bus.h */ 
+	.bs_w_8_s	= NULL,   /* Use inline code in bus.h */ 
 
 	/* write multiple stream */
-	NULL,
-	generic_armv4_bs_wm_2,		/* bus_space_write_multi_stream_2 */
-	NULL,
-	NULL,
+	.bs_wm_1_s	= generic_bs_wm_1,
+	.bs_wm_2_s	= generic_armv4_bs_wm_2,
+	.bs_wm_4_s	= generic_bs_wm_4,
+	.bs_wm_8_s	= BS_UNIMPLEMENTED,
 
 	/* write region stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL
+	.bs_wr_1_s	= generic_bs_wr_1,
+	.bs_wr_2_s	= generic_bs_wr_4,
+	.bs_wr_4_s	= generic_bs_wr_8,
+	.bs_wr_8_s	= BS_UNIMPLEMENTED,
 };
 
 bus_space_tag_t fdtbus_bs_tag = &_base_tag;

From 36cc12c9dabe37115510bd7115c2a9433cd8520c Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 02:49:19 +0000
Subject: [PATCH 131/279] Use arm/bus_space-v6.c for all armv6 systems, the
 essentially identical files for lpc and xilinx aren't needed.  Also, fix a
 couple paste-os.

---
 sys/arm/arm/bus_space-v6.c     |   6 +-
 sys/arm/lpc/files.lpc          |   2 +-
 sys/arm/lpc/lpc_space.c        | 147 ---------------------------------
 sys/arm/xilinx/files.zynq7     |   2 +-
 sys/arm/xilinx/zy7_bus_space.c | 115 --------------------------
 5 files changed, 5 insertions(+), 267 deletions(-)
 delete mode 100644 sys/arm/lpc/lpc_space.c
 delete mode 100644 sys/arm/xilinx/zy7_bus_space.c

diff --git a/sys/arm/arm/bus_space-v6.c b/sys/arm/arm/bus_space-v6.c
index 3414c5009cc..6b6d12004ab 100644
--- a/sys/arm/arm/bus_space-v6.c
+++ b/sys/arm/arm/bus_space-v6.c
@@ -137,7 +137,7 @@ static struct bus_space _base_tag = {
 
 	/* read region stream */
 	.bs_rr_1_s	= generic_bs_rr_1,
-	.bs_rr_2_s	= generic_bs_rr_2,
+	.bs_rr_2_s	= generic_armv4_bs_rr_2,
 	.bs_rr_4_s	= generic_bs_rr_4,
 	.bs_rr_8_s	= BS_UNIMPLEMENTED,
 
@@ -155,8 +155,8 @@ static struct bus_space _base_tag = {
 
 	/* write region stream */
 	.bs_wr_1_s	= generic_bs_wr_1,
-	.bs_wr_2_s	= generic_bs_wr_4,
-	.bs_wr_4_s	= generic_bs_wr_8,
+	.bs_wr_2_s	= generic_armv4_bs_wr_2,
+	.bs_wr_4_s	= generic_bs_wr_4,
 	.bs_wr_8_s	= BS_UNIMPLEMENTED,
 };
 
diff --git a/sys/arm/lpc/files.lpc b/sys/arm/lpc/files.lpc
index 8b24d3b0a9b..8e196ad5512 100644
--- a/sys/arm/lpc/files.lpc
+++ b/sys/arm/lpc/files.lpc
@@ -1,9 +1,9 @@
 # $FreeBSD$
+arm/arm/bus_space-v6.c			standard
 arm/arm/bus_space_generic.c		standard
 arm/arm/cpufunc_asm_arm9.S		standard
 arm/arm/cpufunc_asm_armv5.S		standard
 arm/lpc/lpc_machdep.c			standard
-arm/lpc/lpc_space.c			standard
 arm/lpc/lpc_pwr.c			standard
 arm/lpc/lpc_intc.c			standard
 arm/lpc/lpc_timer.c			standard
diff --git a/sys/arm/lpc/lpc_space.c b/sys/arm/lpc/lpc_space.c
deleted file mode 100644
index 194e469f222..00000000000
--- a/sys/arm/lpc/lpc_space.c
+++ /dev/null
@@ -1,147 +0,0 @@
-/*-
- * Copyright (c) 2011 Jakub Wojciech Klama <jceel@FreeBSD.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-#include <sys/kernel.h>
-#include <sys/malloc.h>
-#include <machine/bus.h>
-
-bs_protos(generic);
-bs_protos(generic_armv4);
-
-static struct bus_space _base_tag = {
-	/* cookie */
-	NULL,
-
-	/* mapping/unmapping */
-	generic_bs_map,
-	generic_bs_unmap,
-	generic_bs_subregion,
-
-	/* allocation/deallocation */
-	generic_bs_alloc,
-	generic_bs_free,
-
-	/* barrier */
-	generic_bs_barrier,
-
-	/* read (single) */
-	generic_bs_r_1,
-	generic_armv4_bs_r_2,
-	generic_bs_r_4,
-	NULL,
-
-	/* read (multiple) */
-	generic_bs_rm_1,
-	generic_armv4_bs_rm_2,
-	generic_bs_rm_4,
-	NULL,
-
-	/* read region */
-	generic_bs_rr_1,
-	generic_armv4_bs_rr_2,
-	generic_bs_rr_4,
-	NULL,
-
-	/* write (single) */
-	generic_bs_w_1,
-	generic_armv4_bs_w_2,
-	generic_bs_w_4,
-	NULL,
-
-	/* write multiple */
-	generic_bs_wm_1,
-	generic_armv4_bs_wm_2,
-	generic_bs_wm_4,
-	NULL,
-
-	/* write region */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* set multiple */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* set region */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* copy */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* read stream (single) */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* read multiple stream */
-	NULL,
-	generic_armv4_bs_rm_2,
-	NULL,
-	NULL,
-
-	/* read region stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* write stream (single) */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* write multiple stream */
-	NULL,
-	generic_armv4_bs_wm_2,
-	NULL,
-	NULL,
-
-	/* write region stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-};
-
-bus_space_tag_t fdtbus_bs_tag = &_base_tag;
diff --git a/sys/arm/xilinx/files.zynq7 b/sys/arm/xilinx/files.zynq7
index 4caf90afbec..9682904d91a 100644
--- a/sys/arm/xilinx/files.zynq7
+++ b/sys/arm/xilinx/files.zynq7
@@ -5,6 +5,7 @@
 
 kern/kern_clocksource.c				standard
 
+arm/arm/bus_space-v6.c				standard
 arm/arm/bus_space_generic.c			standard
 arm/arm/bus_space_asm_generic.S			standard
 arm/arm/cpufunc_asm_armv5.S			standard
@@ -18,7 +19,6 @@ arm/arm/pl310.c					standard
 
 arm/xilinx/zy7_machdep.c			standard
 arm/xilinx/zy7_l2cache.c			standard
-arm/xilinx/zy7_bus_space.c			standard
 arm/xilinx/zy7_slcr.c				standard
 arm/xilinx/zy7_devcfg.c				standard
 arm/xilinx/zy7_mp.c				optional smp
diff --git a/sys/arm/xilinx/zy7_bus_space.c b/sys/arm/xilinx/zy7_bus_space.c
deleted file mode 100644
index e7ffc09f021..00000000000
--- a/sys/arm/xilinx/zy7_bus_space.c
+++ /dev/null
@@ -1,115 +0,0 @@
-/*-
- * Copyright (C) 2008 MARVELL INTERNATIONAL LTD.
- * All rights reserved.
- *
- * Developed by Semihalf.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of MARVELL nor the names of contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-#include <sys/kernel.h>
-#include <sys/malloc.h>
-
-#include <machine/bus.h>
-
-/* Prototypes for all the bus_space structure functions */
-bs_protos(generic);
-bs_protos(generic_armv4);
-
-struct bus_space _base_tag = {
-	/* cookie */
-	.bs_privdata	= (void *) 0,
-	
-	/* mapping/unmapping */
-	.bs_map		= generic_bs_map,
-	.bs_unmap	= generic_bs_unmap,
-	.bs_subregion	= generic_bs_subregion,
-	
-	/* allocation/deallocation */
-	.bs_alloc	= generic_bs_alloc,
-	.bs_free	= generic_bs_free,
-	
-	/* barrier */
-	.bs_barrier	= generic_bs_barrier,
-	
-	/* read (single) */
-	.bs_r_1		= generic_bs_r_1,
-	.bs_r_2		= generic_armv4_bs_r_2,
-	.bs_r_4		= generic_bs_r_4,
-	.bs_r_8		= NULL,
-	
-	/* read multiple */
-	.bs_rm_1	= generic_bs_rm_1,
-	.bs_rm_2	= generic_armv4_bs_rm_2,
-	.bs_rm_4	= generic_bs_rm_4,
-	.bs_rm_8	= NULL,
-	
-	/* read region */
-	.bs_rr_1	= generic_bs_rr_1,
-	.bs_rr_2	= generic_armv4_bs_rr_2,
-	.bs_rr_4	= generic_bs_rr_4,
-	.bs_rr_8	= NULL,
-	
-	/* write (single) */
-	.bs_w_1		= generic_bs_w_1,
-	.bs_w_2		= generic_armv4_bs_w_2,
-	.bs_w_4		= generic_bs_w_4,
-	.bs_w_8		= NULL,
-	
-	/* write multiple */
-	.bs_wm_1	= generic_bs_wm_1,
-	.bs_wm_2	= generic_armv4_bs_wm_2,
-	.bs_wm_4	= generic_bs_wm_4,
-	.bs_wm_8	= NULL,
-	
-	/* write region */
-	.bs_wr_1	= generic_bs_wr_1,
-	.bs_wr_2	= generic_armv4_bs_wr_2,
-	.bs_wr_4	= generic_bs_wr_4,
-	.bs_wr_8	= NULL,
-	
-	/* set multiple */
-	/* XXX not implemented */
-	
-	/* set region */
-	.bs_sr_1	= NULL,
-	.bs_sr_2	= generic_armv4_bs_sr_2,
-	.bs_sr_4	= generic_bs_sr_4,
-	.bs_sr_8	= NULL,
-	
-	/* copy */
-	.bs_c_1		= NULL,
-	.bs_c_2		= generic_armv4_bs_c_2,
-	.bs_c_4		= NULL,
-	.bs_c_8		= NULL,
-};
-
-bus_space_tag_t fdtbus_bs_tag = &_base_tag;

From 110e70ba1dc0a8c761a6d1ac656e16a876d6e3b5 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 02:56:13 +0000
Subject: [PATCH 132/279] For some reason, all the arm bus_space functions that
 work with uint16 values have armv4 in the name.  There's nothing
 armv4-special about them, so just use the same sort of names as all the other
 functions.

---
 sys/arm/arm/bus_space-v6.c                  | 21 +++++++-------
 sys/arm/arm/bus_space_asm_generic.S         | 32 ++++++++++-----------
 sys/arm/at91/at91.c                         | 29 +++++++++----------
 sys/arm/cavium/cns11xx/econa.c              | 15 +++++-----
 sys/arm/mv/bus_space.c                      | 15 +++++-----
 sys/arm/samsung/s3c2xx0/s3c2xx0_space.c     | 17 ++++++-----
 sys/arm/versatile/bus_space.c               | 17 ++++++-----
 sys/arm/versatile/versatile_pci_bus_space.c | 13 ++++-----
 sys/arm/xscale/i80321/i80321_space.c        | 29 +++++++++----------
 sys/arm/xscale/i80321/obio_space.c          |  5 ++--
 sys/arm/xscale/i8134x/i81342_space.c        | 29 +++++++++----------
 sys/arm/xscale/i8134x/obio_space.c          |  5 ++--
 sys/arm/xscale/ixp425/ixp425_a4x_space.c    |  1 -
 sys/arm/xscale/ixp425/ixp425_space.c        | 17 ++++++-----
 sys/arm/xscale/pxa/pxa_space.c              |  9 +++---
 15 files changed, 120 insertions(+), 134 deletions(-)

diff --git a/sys/arm/arm/bus_space-v6.c b/sys/arm/arm/bus_space-v6.c
index 6b6d12004ab..ce26c46171e 100644
--- a/sys/arm/arm/bus_space-v6.c
+++ b/sys/arm/arm/bus_space-v6.c
@@ -38,7 +38,6 @@ __FBSDID("$FreeBSD$");
 
 /* Prototypes for all the bus_space structure functions */
 bs_protos(generic);
-bs_protos(generic_armv4);
 
 static void
 bs_unimplemented(void)
@@ -77,13 +76,13 @@ static struct bus_space _base_tag = {
 
 	/* read multiple */
 	.bs_rm_1	= generic_bs_rm_1,
-	.bs_rm_2	= generic_armv4_bs_rm_2,
+	.bs_rm_2	= generic_bs_rm_2,
 	.bs_rm_4	= generic_bs_rm_4,
 	.bs_rm_8	= BS_UNIMPLEMENTED,
 
 	/* read region */
 	.bs_rr_1	= generic_bs_rr_1,
-	.bs_rr_2	= generic_armv4_bs_rr_2,
+	.bs_rr_2	= generic_bs_rr_2,
 	.bs_rr_4	= generic_bs_rr_4,
 	.bs_rr_8	= BS_UNIMPLEMENTED,
 
@@ -95,13 +94,13 @@ static struct bus_space _base_tag = {
 
 	/* write multiple */
 	.bs_wm_1	= generic_bs_wm_1,
-	.bs_wm_2	= generic_armv4_bs_wm_2,
+	.bs_wm_2	= generic_bs_wm_2,
 	.bs_wm_4	= generic_bs_wm_4,
 	.bs_wm_8	= BS_UNIMPLEMENTED,
 
 	/* write region */
 	.bs_wr_1	= generic_bs_wr_1,
-	.bs_wr_2	= generic_armv4_bs_wr_2,
+	.bs_wr_2	= generic_bs_wr_2,
 	.bs_wr_4	= generic_bs_wr_4,
 	.bs_wr_8	= BS_UNIMPLEMENTED,
 
@@ -113,13 +112,13 @@ static struct bus_space _base_tag = {
 
 	/* set region */
 	.bs_sr_1	= generic_bs_sr_1,
-	.bs_sr_2	= generic_armv4_bs_sr_2,
+	.bs_sr_2	= generic_bs_sr_2,
 	.bs_sr_4	= generic_bs_sr_4,
 	.bs_sr_8	= BS_UNIMPLEMENTED,
 
 	/* copy */
 	.bs_c_1		= BS_UNIMPLEMENTED,
-	.bs_c_2		= generic_armv4_bs_c_2,
+	.bs_c_2		= generic_bs_c_2,
 	.bs_c_4		= BS_UNIMPLEMENTED,
 	.bs_c_8		= BS_UNIMPLEMENTED,
 
@@ -131,13 +130,13 @@ static struct bus_space _base_tag = {
 
 	/* read multiple stream */
 	.bs_rm_1_s	= generic_bs_rm_1,
-	.bs_rm_2_s	= generic_armv4_bs_rm_2,
+	.bs_rm_2_s	= generic_bs_rm_2,
 	.bs_rm_4_s	= generic_bs_rm_4,
 	.bs_rm_8_s	= BS_UNIMPLEMENTED,
 
 	/* read region stream */
 	.bs_rr_1_s	= generic_bs_rr_1,
-	.bs_rr_2_s	= generic_armv4_bs_rr_2,
+	.bs_rr_2_s	= generic_bs_rr_2,
 	.bs_rr_4_s	= generic_bs_rr_4,
 	.bs_rr_8_s	= BS_UNIMPLEMENTED,
 
@@ -149,13 +148,13 @@ static struct bus_space _base_tag = {
 
 	/* write multiple stream */
 	.bs_wm_1_s	= generic_bs_wm_1,
-	.bs_wm_2_s	= generic_armv4_bs_wm_2,
+	.bs_wm_2_s	= generic_bs_wm_2,
 	.bs_wm_4_s	= generic_bs_wm_4,
 	.bs_wm_8_s	= BS_UNIMPLEMENTED,
 
 	/* write region stream */
 	.bs_wr_1_s	= generic_bs_wr_1,
-	.bs_wr_2_s	= generic_armv4_bs_wr_2,
+	.bs_wr_2_s	= generic_bs_wr_2,
 	.bs_wr_4_s	= generic_bs_wr_4,
 	.bs_wr_8_s	= BS_UNIMPLEMENTED,
 };
diff --git a/sys/arm/arm/bus_space_asm_generic.S b/sys/arm/arm/bus_space_asm_generic.S
index 4aa719760a5..9d2b11df2fe 100644
--- a/sys/arm/arm/bus_space_asm_generic.S
+++ b/sys/arm/arm/bus_space_asm_generic.S
@@ -52,10 +52,10 @@ ENTRY(generic_bs_r_1)
 	RET
 END(generic_bs_r_1)
 
-ENTRY(generic_armv4_bs_r_2)
+ENTRY(generic_bs_r_2)
 	ldrh	r0, [r1, r2]
 	RET
-END(generic_armv4_bs_r_2)
+END(generic_bs_r_2)
 
 ENTRY(generic_bs_r_4)
 	ldr	r0, [r1, r2]
@@ -71,10 +71,10 @@ ENTRY(generic_bs_w_1)
 	RET
 END(generic_bs_w_1)
 
-ENTRY(generic_armv4_bs_w_2)
+ENTRY(generic_bs_w_2)
 	strh	r3, [r1, r2]
 	RET
-END(generic_armv4_bs_w_2)
+END(generic_bs_w_2)
 
 ENTRY(generic_bs_w_4)
 	str	r3, [r1, r2]
@@ -100,7 +100,7 @@ ENTRY(generic_bs_rm_1)
 	RET
 END(generic_bs_rm_1)
 
-ENTRY(generic_armv4_bs_rm_2)
+ENTRY(generic_bs_rm_2)
 	add	r0, r1, r2
 	mov	r1, r3
 	ldr	r2, [sp, #0]
@@ -113,7 +113,7 @@ ENTRY(generic_armv4_bs_rm_2)
 	bne	1b
 
 	RET
-END(generic_armv4_bs_rm_2)
+END(generic_bs_rm_2)
 
 ENTRY(generic_bs_rm_4)
 	add	r0, r1, r2
@@ -149,7 +149,7 @@ ENTRY(generic_bs_wm_1)
 	RET
 END(generic_bs_wm_1)
 
-ENTRY(generic_armv4_bs_wm_2)
+ENTRY(generic_bs_wm_2)
 	add	r0, r1, r2
 	mov	r1, r3
 	ldr	r2, [sp, #0]
@@ -162,7 +162,7 @@ ENTRY(generic_armv4_bs_wm_2)
 	bne	1b
 
 	RET
-END(generic_armv4_bs_wm_2)
+END(generic_bs_wm_2)
 
 ENTRY(generic_bs_wm_4)
 	add	r0, r1, r2
@@ -198,7 +198,7 @@ ENTRY(generic_bs_rr_1)
 	RET
 END(generic_bs_rr_1)
 
-ENTRY(generic_armv4_bs_rr_2)
+ENTRY(generic_bs_rr_2)
 	add	r0, r1, r2
 	mov	r1, r3
 	ldr	r2, [sp, #0]
@@ -211,7 +211,7 @@ ENTRY(generic_armv4_bs_rr_2)
 	bne	1b
 
 	RET
-END(generic_armv4_bs_rr_2)
+END(generic_bs_rr_2)
 
 ENTRY(generic_bs_rr_4)
 	add	r0, r1, r2
@@ -247,7 +247,7 @@ ENTRY(generic_bs_wr_1)
 	RET
 END(generic_bs_wr_1)
 
-ENTRY(generic_armv4_bs_wr_2)
+ENTRY(generic_bs_wr_2)
 	add	r0, r1, r2
 	mov	r1, r3
 	ldr	r2, [sp, #0]
@@ -260,7 +260,7 @@ ENTRY(generic_armv4_bs_wr_2)
 	bne	1b
 
 	RET
-END(generic_armv4_bs_wr_2)
+END(generic_bs_wr_2)
 
 ENTRY(generic_bs_wr_4)
 	add	r0, r1, r2
@@ -295,7 +295,7 @@ ENTRY(generic_bs_sr_1)
 	RET
 END(generic_bs_sr_1)
 
-ENTRY(generic_armv4_bs_sr_2)
+ENTRY(generic_bs_sr_2)
 	add	r0, r1, r2
 	mov	r1, r3
 	ldr	r2, [sp, #0]
@@ -307,7 +307,7 @@ ENTRY(generic_armv4_bs_sr_2)
 	bne	1b
 
 	RET
-END(generic_armv4_bs_sr_2)
+END(generic_bs_sr_2)
 
 ENTRY(generic_bs_sr_4)
 	add	r0, r1, r2
@@ -327,7 +327,7 @@ END(generic_bs_sr_4)
  * copy region
  */
 
-ENTRY(generic_armv4_bs_c_2)
+ENTRY(generic_bs_c_2)
 	add	r0, r1, r2
 	ldr	r2, [sp, #0]
 	add	r1, r2, r3
@@ -356,5 +356,5 @@ ENTRY(generic_armv4_bs_c_2)
 	bne	3b
 
 	RET
-END(generic_armv4_bs_c_2)
+END(generic_bs_c_2)
 
diff --git a/sys/arm/at91/at91.c b/sys/arm/at91/at91.c
index a8a1f5e4af3..8c5c86c39c2 100644
--- a/sys/arm/at91/at91.c
+++ b/sys/arm/at91/at91.c
@@ -116,7 +116,6 @@ bus_dma_get_range_nb(void)
 }
 
 bs_protos(generic);
-bs_protos(generic_armv4);
 
 struct bus_space at91_bs_tag = {
 	/* cookie */
@@ -136,37 +135,37 @@ struct bus_space at91_bs_tag = {
 
 	/* read (single) */
 	generic_bs_r_1,
-	generic_armv4_bs_r_2,
+	generic_bs_r_2,
 	generic_bs_r_4,
 	NULL,
 
 	/* read multiple */
 	generic_bs_rm_1,
-	generic_armv4_bs_rm_2,
+	generic_bs_rm_2,
 	generic_bs_rm_4,
 	NULL,
 
 	/* read region */
 	generic_bs_rr_1,
-	generic_armv4_bs_rr_2,
+	generic_bs_rr_2,
 	generic_bs_rr_4,
 	NULL,
 
 	/* write (single) */
 	generic_bs_w_1,
-	generic_armv4_bs_w_2,
+	generic_bs_w_2,
 	generic_bs_w_4,
 	NULL,
 
 	/* write multiple */
 	generic_bs_wm_1,
-	generic_armv4_bs_wm_2,
+	generic_bs_wm_2,
 	generic_bs_wm_4,
 	NULL,
 
 	/* write region */
 	NULL,
-	generic_armv4_bs_wr_2,
+	generic_bs_wr_2,
 	generic_bs_wr_4,
 	NULL,
 
@@ -178,49 +177,49 @@ struct bus_space at91_bs_tag = {
 
 	/* set region */
 	NULL,
-	generic_armv4_bs_sr_2,
+	generic_bs_sr_2,
 	generic_bs_sr_4,
 	NULL,
 
 	/* copy */
 	NULL,
-	generic_armv4_bs_c_2,
+	generic_bs_c_2,
 	NULL,
 	NULL,
 
 	/* read (single) stream */
 	generic_bs_r_1,
-	generic_armv4_bs_r_2,
+	generic_bs_r_2,
 	generic_bs_r_4,
 	NULL,
 
 	/* read multiple stream */
 	generic_bs_rm_1,
-	generic_armv4_bs_rm_2,
+	generic_bs_rm_2,
 	generic_bs_rm_4,
 	NULL,
 
 	/* read region stream */
 	generic_bs_rr_1,
-	generic_armv4_bs_rr_2,
+	generic_bs_rr_2,
 	generic_bs_rr_4,
 	NULL,
 
 	/* write (single) stream */
 	generic_bs_w_1,
-	generic_armv4_bs_w_2,
+	generic_bs_w_2,
 	generic_bs_w_4,
 	NULL,
 
 	/* write multiple stream */
 	generic_bs_wm_1,
-	generic_armv4_bs_wm_2,
+	generic_bs_wm_2,
 	generic_bs_wm_4,
 	NULL,
 
 	/* write region stream */
 	NULL,
-	generic_armv4_bs_wr_2,
+	generic_bs_wr_2,
 	generic_bs_wr_4,
 	NULL,
 };
diff --git a/sys/arm/cavium/cns11xx/econa.c b/sys/arm/cavium/cns11xx/econa.c
index c331e20d926..e79e6e8d810 100644
--- a/sys/arm/cavium/cns11xx/econa.c
+++ b/sys/arm/cavium/cns11xx/econa.c
@@ -57,7 +57,6 @@ unsigned int AHB_clock;
 unsigned int APB_clock;
 
 bs_protos(generic);
-bs_protos(generic_armv4);
 
 struct bus_space econa_bs_tag = {
 	/* cookie */
@@ -77,31 +76,31 @@ struct bus_space econa_bs_tag = {
 
 	/* read (single) */
 	generic_bs_r_1,
-	generic_armv4_bs_r_2,
+	generic_bs_r_2,
 	generic_bs_r_4,
 	NULL,
 
 	/* read multiple */
 	generic_bs_rm_1,
-	generic_armv4_bs_rm_2,
+	generic_bs_rm_2,
 	generic_bs_rm_4,
 	NULL,
 
 	/* read region */
 	generic_bs_rr_1,
-	generic_armv4_bs_rr_2,
+	generic_bs_rr_2,
 	generic_bs_rr_4,
 	NULL,
 
 	/* write (single) */
 	generic_bs_w_1,
-	generic_armv4_bs_w_2,
+	generic_bs_w_2,
 	generic_bs_w_4,
 	NULL,
 
 	/* write multiple */
 	generic_bs_wm_1,
-	generic_armv4_bs_wm_2,
+	generic_bs_wm_2,
 	generic_bs_wm_4,
 	NULL,
 
@@ -137,7 +136,7 @@ struct bus_space econa_bs_tag = {
 
 	/* read multiple stream */
 	NULL,
-	generic_armv4_bs_rm_2,
+	generic_bs_rm_2,
 	NULL,
 	NULL,
 
@@ -155,7 +154,7 @@ struct bus_space econa_bs_tag = {
 
 	/* write multiple stream */
 	NULL,
-	generic_armv4_bs_wm_2,
+	generic_bs_wm_2,
 	NULL,
 	NULL,
 
diff --git a/sys/arm/mv/bus_space.c b/sys/arm/mv/bus_space.c
index 356f3e3db1d..6bd4f3486df 100644
--- a/sys/arm/mv/bus_space.c
+++ b/sys/arm/mv/bus_space.c
@@ -46,7 +46,6 @@ __FBSDID("$FreeBSD$");
 
 /* Prototypes for all the bus_space structure functions */
 bs_protos(generic);
-bs_protos(generic_armv4);
 
 /*
  * The bus space tag.  This is constant for all instances, so
@@ -70,31 +69,31 @@ static struct bus_space _base_tag = {
 
 	/* read (single) */
 	generic_bs_r_1,
-	generic_armv4_bs_r_2,
+	generic_bs_r_2,
 	generic_bs_r_4,
 	NULL,
 
 	/* read multiple */
 	generic_bs_rm_1,
-	generic_armv4_bs_rm_2,
+	generic_bs_rm_2,
 	generic_bs_rm_4,
 	NULL,
 
 	/* read region */
 	generic_bs_rr_1,
-	generic_armv4_bs_rr_2,
+	generic_bs_rr_2,
 	generic_bs_rr_4,
 	NULL,
 
 	/* write (single) */
 	generic_bs_w_1,
-	generic_armv4_bs_w_2,
+	generic_bs_w_2,
 	generic_bs_w_4,
 	NULL,
 
 	/* write multiple */
 	generic_bs_wm_1,
-	generic_armv4_bs_wm_2,
+	generic_bs_wm_2,
 	generic_bs_wm_4,
 	NULL,
 
@@ -130,7 +129,7 @@ static struct bus_space _base_tag = {
 
 	/* read multiple stream */
 	NULL,
-	generic_armv4_bs_rm_2,		/* bus_space_read_multi_stream_2 */
+	generic_bs_rm_2,		/* bus_space_read_multi_stream_2 */
 	NULL,
 	NULL,
 
@@ -148,7 +147,7 @@ static struct bus_space _base_tag = {
 
 	/* write multiple stream */
 	NULL,
-	generic_armv4_bs_wm_2,		/* bus_space_write_multi_stream_2 */
+	generic_bs_wm_2,		/* bus_space_write_multi_stream_2 */
 	NULL,
 	NULL,
 
diff --git a/sys/arm/samsung/s3c2xx0/s3c2xx0_space.c b/sys/arm/samsung/s3c2xx0/s3c2xx0_space.c
index fe01d2b389c..61c00eb6841 100644
--- a/sys/arm/samsung/s3c2xx0/s3c2xx0_space.c
+++ b/sys/arm/samsung/s3c2xx0/s3c2xx0_space.c
@@ -90,7 +90,6 @@ __FBSDID("$FreeBSD$");
 
 /* Prototypes for all the bus_space structure functions */
 bs_protos(generic);
-bs_protos(generic_armv4);
 
 struct bus_space s3c2xx0_bs_tag = {
 	/* cookie */
@@ -110,37 +109,37 @@ struct bus_space s3c2xx0_bs_tag = {
 
 	/* read (single) */
 	generic_bs_r_1,
-	generic_armv4_bs_r_2,
+	generic_bs_r_2,
 	generic_bs_r_4,
 	NULL,
 
 	/* read multiple */
 	generic_bs_rm_1,
-	generic_armv4_bs_rm_2,
+	generic_bs_rm_2,
 	generic_bs_rm_4,
 	NULL,
 
 	/* read region */
 	generic_bs_rr_1,
-	generic_armv4_bs_rr_2,
+	generic_bs_rr_2,
 	generic_bs_rr_4,
 	NULL,
 
 	/* write (single) */
 	generic_bs_w_1,
-	generic_armv4_bs_w_2,
+	generic_bs_w_2,
 	generic_bs_w_4,
 	NULL,
 
 	/* write multiple */
 	generic_bs_wm_1,
-	generic_armv4_bs_wm_2,
+	generic_bs_wm_2,
 	generic_bs_wm_4,
 	NULL,
 
 	/* write region */
 	generic_bs_wr_1,
-	generic_armv4_bs_wr_2,
+	generic_bs_wr_2,
 	generic_bs_wr_4,
 	NULL,
 
@@ -152,13 +151,13 @@ struct bus_space s3c2xx0_bs_tag = {
 
 	/* set region */
 	generic_bs_sr_1,
-	generic_armv4_bs_sr_2,
+	generic_bs_sr_2,
 	NULL,
 	NULL,
 
 	/* copy */
 	NULL,
-	generic_armv4_bs_c_2,
+	generic_bs_c_2,
 	NULL,
 	NULL,
 };
diff --git a/sys/arm/versatile/bus_space.c b/sys/arm/versatile/bus_space.c
index e7ffc09f021..850feefb1af 100644
--- a/sys/arm/versatile/bus_space.c
+++ b/sys/arm/versatile/bus_space.c
@@ -42,7 +42,6 @@ __FBSDID("$FreeBSD$");
 
 /* Prototypes for all the bus_space structure functions */
 bs_protos(generic);
-bs_protos(generic_armv4);
 
 struct bus_space _base_tag = {
 	/* cookie */
@@ -62,37 +61,37 @@ struct bus_space _base_tag = {
 	
 	/* read (single) */
 	.bs_r_1		= generic_bs_r_1,
-	.bs_r_2		= generic_armv4_bs_r_2,
+	.bs_r_2		= generic_bs_r_2,
 	.bs_r_4		= generic_bs_r_4,
 	.bs_r_8		= NULL,
 	
 	/* read multiple */
 	.bs_rm_1	= generic_bs_rm_1,
-	.bs_rm_2	= generic_armv4_bs_rm_2,
+	.bs_rm_2	= generic_bs_rm_2,
 	.bs_rm_4	= generic_bs_rm_4,
 	.bs_rm_8	= NULL,
 	
 	/* read region */
 	.bs_rr_1	= generic_bs_rr_1,
-	.bs_rr_2	= generic_armv4_bs_rr_2,
+	.bs_rr_2	= generic_bs_rr_2,
 	.bs_rr_4	= generic_bs_rr_4,
 	.bs_rr_8	= NULL,
 	
 	/* write (single) */
 	.bs_w_1		= generic_bs_w_1,
-	.bs_w_2		= generic_armv4_bs_w_2,
+	.bs_w_2		= generic_bs_w_2,
 	.bs_w_4		= generic_bs_w_4,
 	.bs_w_8		= NULL,
 	
 	/* write multiple */
 	.bs_wm_1	= generic_bs_wm_1,
-	.bs_wm_2	= generic_armv4_bs_wm_2,
+	.bs_wm_2	= generic_bs_wm_2,
 	.bs_wm_4	= generic_bs_wm_4,
 	.bs_wm_8	= NULL,
 	
 	/* write region */
 	.bs_wr_1	= generic_bs_wr_1,
-	.bs_wr_2	= generic_armv4_bs_wr_2,
+	.bs_wr_2	= generic_bs_wr_2,
 	.bs_wr_4	= generic_bs_wr_4,
 	.bs_wr_8	= NULL,
 	
@@ -101,13 +100,13 @@ struct bus_space _base_tag = {
 	
 	/* set region */
 	.bs_sr_1	= NULL,
-	.bs_sr_2	= generic_armv4_bs_sr_2,
+	.bs_sr_2	= generic_bs_sr_2,
 	.bs_sr_4	= generic_bs_sr_4,
 	.bs_sr_8	= NULL,
 	
 	/* copy */
 	.bs_c_1		= NULL,
-	.bs_c_2		= generic_armv4_bs_c_2,
+	.bs_c_2		= generic_bs_c_2,
 	.bs_c_4		= NULL,
 	.bs_c_8		= NULL,
 };
diff --git a/sys/arm/versatile/versatile_pci_bus_space.c b/sys/arm/versatile/versatile_pci_bus_space.c
index 2c2e0b537c9..89454401697 100644
--- a/sys/arm/versatile/versatile_pci_bus_space.c
+++ b/sys/arm/versatile/versatile_pci_bus_space.c
@@ -37,7 +37,6 @@ __FBSDID("$FreeBSD$");
 
 /* Prototypes for all the bus_space structure functions */
 bs_protos(generic);
-bs_protos(generic_armv4);
 
 /*
  * Bus space that handles offsets in word for 1/2 bytes read/write access.
@@ -61,37 +60,37 @@ static struct bus_space bus_space_pcimem = {
 
 	/* read (single) */
 	generic_bs_r_1,
-	generic_armv4_bs_r_2,
+	generic_bs_r_2,
 	generic_bs_r_4,
 	NULL,
 
 	/* read multiple */
 	generic_bs_rm_1,
-	generic_armv4_bs_rm_2,
+	generic_bs_rm_2,
 	generic_bs_rm_4,
 	NULL,
 
 	/* read region */
 	generic_bs_rr_1,
-	generic_armv4_bs_rr_2,
+	generic_bs_rr_2,
 	generic_bs_rr_4,
 	NULL,
 
 	/* write (single) */
 	generic_bs_w_1,
-	generic_armv4_bs_w_2,
+	generic_bs_w_2,
 	generic_bs_w_4,
 	NULL,
 
 	/* write multiple */
 	generic_bs_wm_1,
-	generic_armv4_bs_wm_2,
+	generic_bs_wm_2,
 	generic_bs_wm_4,
 	NULL,
 
 	/* write region */
 	generic_bs_wr_1,
-	generic_armv4_bs_wr_2,
+	generic_bs_wr_2,
 	generic_bs_wr_4,
 	NULL,
 
diff --git a/sys/arm/xscale/i80321/i80321_space.c b/sys/arm/xscale/i80321/i80321_space.c
index 44a2ca80021..da4ad2185a8 100644
--- a/sys/arm/xscale/i80321/i80321_space.c
+++ b/sys/arm/xscale/i80321/i80321_space.c
@@ -64,7 +64,6 @@ bs_protos(i80321);
 bs_protos(i80321_io);
 bs_protos(i80321_mem);
 bs_protos(generic);
-bs_protos(generic_armv4);
 
 /*
  * Template bus_space -- copied, and the bits that are NULL are
@@ -88,37 +87,37 @@ const struct bus_space i80321_bs_tag_template = {
 
 	/* read (single) */
 	generic_bs_r_1,
-	generic_armv4_bs_r_2,
+	generic_bs_r_2,
 	generic_bs_r_4,
 	NULL,
 
 	/* read multiple */
 	generic_bs_rm_1,
-	generic_armv4_bs_rm_2,
+	generic_bs_rm_2,
 	generic_bs_rm_4,
 	NULL,
 
 	/* read region */
 	generic_bs_rr_1,
-	generic_armv4_bs_rr_2,
+	generic_bs_rr_2,
 	generic_bs_rr_4,
 	NULL,
 
 	/* write (single) */
 	generic_bs_w_1,
-	generic_armv4_bs_w_2,
+	generic_bs_w_2,
 	generic_bs_w_4,
 	NULL,
 
 	/* write multiple */
 	generic_bs_wm_1,
-	generic_armv4_bs_wm_2,
+	generic_bs_wm_2,
 	generic_bs_wm_4,
 	NULL,
 
 	/* write region */
 	NULL,
-	generic_armv4_bs_wr_2,
+	generic_bs_wr_2,
 	generic_bs_wr_4,
 	NULL,
 
@@ -130,49 +129,49 @@ const struct bus_space i80321_bs_tag_template = {
 
 	/* set region */
 	NULL,
-	generic_armv4_bs_sr_2,
+	generic_bs_sr_2,
 	generic_bs_sr_4,
 	NULL,
 
 	/* copy */
 	NULL,
-	generic_armv4_bs_c_2,
+	generic_bs_c_2,
 	NULL,
 	NULL,
 
 	/* read (single) stream */
 	generic_bs_r_1,
-	generic_armv4_bs_r_2,
+	generic_bs_r_2,
 	generic_bs_r_4,
 	NULL,
 
 	/* read multiple stream */
 	generic_bs_rm_1,
-	generic_armv4_bs_rm_2,
+	generic_bs_rm_2,
 	generic_bs_rm_4,
 	NULL,
 
 	/* read region stream */
 	generic_bs_rr_1,
-	generic_armv4_bs_rr_2,
+	generic_bs_rr_2,
 	generic_bs_rr_4,
 	NULL,
 
 	/* write (single) stream */
 	generic_bs_w_1,
-	generic_armv4_bs_w_2,
+	generic_bs_w_2,
 	generic_bs_w_4,
 	NULL,
 
 	/* write multiple stream */
 	generic_bs_wm_1,
-	generic_armv4_bs_wm_2,
+	generic_bs_wm_2,
 	generic_bs_wm_4,
 	NULL,
 
 	/* write region stream */
 	NULL,
-	generic_armv4_bs_wr_2,
+	generic_bs_wr_2,
 	generic_bs_wr_4,
 	NULL,
 };
diff --git a/sys/arm/xscale/i80321/obio_space.c b/sys/arm/xscale/i80321/obio_space.c
index 8b7b35e71e2..4eba7000dbc 100644
--- a/sys/arm/xscale/i80321/obio_space.c
+++ b/sys/arm/xscale/i80321/obio_space.c
@@ -50,7 +50,6 @@ __FBSDID("$FreeBSD$");
 
 /* Prototypes for all the bus_space structure functions */
 bs_protos(generic);
-bs_protos(generic_armv4);
 
 /*
  * The obio bus space tag.  This is constant for all instances, so
@@ -74,7 +73,7 @@ struct bus_space obio_bs_tag = {
 
 	/* read (single) */
 	generic_bs_r_1,
-	generic_armv4_bs_r_2,
+	generic_bs_r_2,
 	generic_bs_r_4,
 	NULL,
 
@@ -92,7 +91,7 @@ struct bus_space obio_bs_tag = {
 
 	/* write (single) */
 	generic_bs_w_1,
-	generic_armv4_bs_w_2,
+	generic_bs_w_2,
 	generic_bs_w_4,
 	NULL,
 
diff --git a/sys/arm/xscale/i8134x/i81342_space.c b/sys/arm/xscale/i8134x/i81342_space.c
index e19447472d7..0f7912c83ca 100644
--- a/sys/arm/xscale/i8134x/i81342_space.c
+++ b/sys/arm/xscale/i8134x/i81342_space.c
@@ -65,7 +65,6 @@ bs_protos(i81342);
 bs_protos(i81342_io);
 bs_protos(i81342_mem);
 bs_protos(generic);
-bs_protos(generic_armv4);
 
 /*
  * Template bus_space -- copied, and the bits that are NULL are
@@ -89,37 +88,37 @@ const struct bus_space i81342_bs_tag_template = {
 
 	/* read (single) */
 	generic_bs_r_1,
-	generic_armv4_bs_r_2,
+	generic_bs_r_2,
 	generic_bs_r_4,
 	NULL,
 
 	/* read multiple */
 	generic_bs_rm_1,
-	generic_armv4_bs_rm_2,
+	generic_bs_rm_2,
 	generic_bs_rm_4,
 	NULL,
 
 	/* read region */
 	generic_bs_rr_1,
-	generic_armv4_bs_rr_2,
+	generic_bs_rr_2,
 	generic_bs_rr_4,
 	NULL,
 
 	/* write (single) */
 	generic_bs_w_1,
-	generic_armv4_bs_w_2,
+	generic_bs_w_2,
 	generic_bs_w_4,
 	NULL,
 
 	/* write multiple */
 	generic_bs_wm_1,
-	generic_armv4_bs_wm_2,
+	generic_bs_wm_2,
 	generic_bs_wm_4,
 	NULL,
 
 	/* write region */
 	NULL,
-	generic_armv4_bs_wr_2,
+	generic_bs_wr_2,
 	generic_bs_wr_4,
 	NULL,
 
@@ -131,49 +130,49 @@ const struct bus_space i81342_bs_tag_template = {
 
 	/* set region */
 	NULL,
-	generic_armv4_bs_sr_2,
+	generic_bs_sr_2,
 	generic_bs_sr_4,
 	NULL,
 
 	/* copy */
 	NULL,
-	generic_armv4_bs_c_2,
+	generic_bs_c_2,
 	NULL,
 	NULL,
 
 	/* read (single) stream */
 	generic_bs_r_1,
-	generic_armv4_bs_r_2,
+	generic_bs_r_2,
 	generic_bs_r_4,
 	NULL,
 
 	/* read multiple stream */
 	generic_bs_rm_1,
-	generic_armv4_bs_rm_2,
+	generic_bs_rm_2,
 	generic_bs_rm_4,
 	NULL,
 
 	/* read region stream */
 	generic_bs_rr_1,
-	generic_armv4_bs_rr_2,
+	generic_bs_rr_2,
 	generic_bs_rr_4,
 	NULL,
 
 	/* write (single) stream */
 	generic_bs_w_1,
-	generic_armv4_bs_w_2,
+	generic_bs_w_2,
 	generic_bs_w_4,
 	NULL,
 
 	/* write multiple stream */
 	generic_bs_wm_1,
-	generic_armv4_bs_wm_2,
+	generic_bs_wm_2,
 	generic_bs_wm_4,
 	NULL,
 
 	/* write region stream */
 	NULL,
-	generic_armv4_bs_wr_2,
+	generic_bs_wr_2,
 	generic_bs_wr_4,
 	NULL,
 };
diff --git a/sys/arm/xscale/i8134x/obio_space.c b/sys/arm/xscale/i8134x/obio_space.c
index 8b7b35e71e2..4eba7000dbc 100644
--- a/sys/arm/xscale/i8134x/obio_space.c
+++ b/sys/arm/xscale/i8134x/obio_space.c
@@ -50,7 +50,6 @@ __FBSDID("$FreeBSD$");
 
 /* Prototypes for all the bus_space structure functions */
 bs_protos(generic);
-bs_protos(generic_armv4);
 
 /*
  * The obio bus space tag.  This is constant for all instances, so
@@ -74,7 +73,7 @@ struct bus_space obio_bs_tag = {
 
 	/* read (single) */
 	generic_bs_r_1,
-	generic_armv4_bs_r_2,
+	generic_bs_r_2,
 	generic_bs_r_4,
 	NULL,
 
@@ -92,7 +91,7 @@ struct bus_space obio_bs_tag = {
 
 	/* write (single) */
 	generic_bs_w_1,
-	generic_armv4_bs_w_2,
+	generic_bs_w_2,
 	generic_bs_w_4,
 	NULL,
 
diff --git a/sys/arm/xscale/ixp425/ixp425_a4x_space.c b/sys/arm/xscale/ixp425/ixp425_a4x_space.c
index 23e48e8bb10..1b128bbf0e2 100644
--- a/sys/arm/xscale/ixp425/ixp425_a4x_space.c
+++ b/sys/arm/xscale/ixp425/ixp425_a4x_space.c
@@ -62,7 +62,6 @@ __FBSDID("$FreeBSD$");
 /* Prototypes for all the bus_space structure functions */
 bs_protos(a4x);
 bs_protos(generic);
-bs_protos(generic_armv4);
 
 struct bus_space ixp425_a4x_bs_tag = {
 	/* cookie */
diff --git a/sys/arm/xscale/ixp425/ixp425_space.c b/sys/arm/xscale/ixp425/ixp425_space.c
index 102bcd2acc0..932c71bf539 100644
--- a/sys/arm/xscale/ixp425/ixp425_space.c
+++ b/sys/arm/xscale/ixp425/ixp425_space.c
@@ -59,7 +59,6 @@ __FBSDID("$FreeBSD$");
 
 /* Proto types for all the bus_space structure functions */
 bs_protos(generic);
-bs_protos(generic_armv4);
 
 struct bus_space ixp425_bs_tag = {
 	/* cookie */
@@ -79,37 +78,37 @@ struct bus_space ixp425_bs_tag = {
 
 	/* read (single) */
 	.bs_r_1		= generic_bs_r_1,
-	.bs_r_2		= generic_armv4_bs_r_2,
+	.bs_r_2		= generic_bs_r_2,
 	.bs_r_4		= generic_bs_r_4,
 	.bs_r_8		= NULL,
 
 	/* read multiple */
 	.bs_rm_1	= generic_bs_rm_1,
-	.bs_rm_2	= generic_armv4_bs_rm_2,
+	.bs_rm_2	= generic_bs_rm_2,
 	.bs_rm_4	= generic_bs_rm_4,
 	.bs_rm_8	= NULL,
 
 	/* read region */
 	.bs_rr_1	= generic_bs_rr_1,
-	.bs_rr_2	= generic_armv4_bs_rr_2,
+	.bs_rr_2	= generic_bs_rr_2,
 	.bs_rr_4	= generic_bs_rr_4,
 	.bs_rr_8	= NULL,
 
 	/* write (single) */
 	.bs_w_1		= generic_bs_w_1,
-	.bs_w_2		= generic_armv4_bs_w_2,
+	.bs_w_2		= generic_bs_w_2,
 	.bs_w_4		= generic_bs_w_4,
 	.bs_w_8		= NULL,
 
 	/* write multiple */
 	.bs_wm_1	= generic_bs_wm_1,
-	.bs_wm_2	= generic_armv4_bs_wm_2,
+	.bs_wm_2	= generic_bs_wm_2,
 	.bs_wm_4	= generic_bs_wm_4,
 	.bs_wm_8	= NULL,
 
 	/* write region */
 	.bs_wr_1	= generic_bs_wr_1,
-	.bs_wr_2	= generic_armv4_bs_wr_2,
+	.bs_wr_2	= generic_bs_wr_2,
 	.bs_wr_4	= generic_bs_wr_4,
 	.bs_wr_8	= NULL,
 
@@ -118,13 +117,13 @@ struct bus_space ixp425_bs_tag = {
 
 	/* set region */
 	.bs_sr_1	= NULL,
-	.bs_sr_2	= generic_armv4_bs_sr_2,
+	.bs_sr_2	= generic_bs_sr_2,
 	.bs_sr_4	= generic_bs_sr_4,
 	.bs_sr_8	= NULL,
 
 	/* copy */
 	.bs_c_1		= NULL,
-	.bs_c_2		= generic_armv4_bs_c_2,
+	.bs_c_2		= generic_bs_c_2,
 	.bs_c_4		= NULL,
 	.bs_c_8		= NULL,
 };
diff --git a/sys/arm/xscale/pxa/pxa_space.c b/sys/arm/xscale/pxa/pxa_space.c
index cfaac37f7da..7a9160a8974 100644
--- a/sys/arm/xscale/pxa/pxa_space.c
+++ b/sys/arm/xscale/pxa/pxa_space.c
@@ -57,7 +57,6 @@ static MALLOC_DEFINE(M_PXATAG, "PXA bus_space tags", "Bus_space tags for PXA");
 
 /* Prototypes for all the bus_space structure functions */
 bs_protos(generic);
-bs_protos(generic_armv4);
 bs_protos(pxa);
 
 /*
@@ -179,7 +178,7 @@ pxa_bus_tag_alloc(bus_addr_t offset)
 	}
 
 READ_SINGLE(u_int8_t,  pxa_bs_r_1, generic_bs_r_1)
-READ_SINGLE(u_int16_t, pxa_bs_r_2, generic_armv4_bs_r_2)
+READ_SINGLE(u_int16_t, pxa_bs_r_2, generic_bs_r_2)
 READ_SINGLE(u_int32_t, pxa_bs_r_4, generic_bs_r_4)
 
 #undef READ_SINGLE
@@ -195,7 +194,7 @@ READ_SINGLE(u_int32_t, pxa_bs_r_4, generic_bs_r_4)
 	}
 
 WRITE_SINGLE(u_int8_t,  pxa_bs_w_1, generic_bs_w_1)
-WRITE_SINGLE(u_int16_t, pxa_bs_w_2, generic_armv4_bs_w_2)
+WRITE_SINGLE(u_int16_t, pxa_bs_w_2, generic_bs_w_2)
 WRITE_SINGLE(u_int32_t, pxa_bs_w_4, generic_bs_w_4)
 
 #undef WRITE_SINGLE
@@ -211,7 +210,7 @@ WRITE_SINGLE(u_int32_t, pxa_bs_w_4, generic_bs_w_4)
 	}
 
 READ_MULTI(u_int8_t,  pxa_bs_rm_1, generic_bs_rm_1)
-READ_MULTI(u_int16_t, pxa_bs_rm_2, generic_armv4_bs_rm_2)
+READ_MULTI(u_int16_t, pxa_bs_rm_2, generic_bs_rm_2)
 
 READ_MULTI(u_int8_t,  pxa_bs_rr_1, generic_bs_rr_1)
 
@@ -228,6 +227,6 @@ READ_MULTI(u_int8_t,  pxa_bs_rr_1, generic_bs_rr_1)
 	}
 
 WRITE_MULTI(u_int8_t,  pxa_bs_wm_1, generic_bs_wm_1)
-WRITE_MULTI(u_int16_t, pxa_bs_wm_2, generic_armv4_bs_wm_2)
+WRITE_MULTI(u_int16_t, pxa_bs_wm_2, generic_bs_wm_2)
 
 #undef WRITE_MULTI

From e5fadf2a31160229a7e323d5b0a03a608a259471 Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Wed, 21 Jan 2015 02:57:54 +0000
Subject: [PATCH 133/279] On 64-bit PowerPC, use more native forms of the PPC
 970 HID restore sequences, like are used to read the HIDs. This is both
 easier to read and avoids a miscompilation by GCC in certain circumstances.
 Also avoid double restoration of HID4 and HID5.

MFC after:	2 weeks
---
 sys/powerpc/aim/mp_cpudep.c | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/sys/powerpc/aim/mp_cpudep.c b/sys/powerpc/aim/mp_cpudep.c
index 3bc21a42f5d..59a24e22e62 100644
--- a/sys/powerpc/aim/mp_cpudep.c
+++ b/sys/powerpc/aim/mp_cpudep.c
@@ -58,7 +58,9 @@ SYSINIT(cpu_save_config, SI_SUB_CPU, SI_ORDER_ANY, cpudep_save_config, NULL);
 void
 cpudep_ap_early_bootstrap(void)
 {
+#ifndef __powerpc64__
 	register_t reg;
+#endif
 
 	__asm __volatile("mtsprg 0, %0" :: "r"(ap_pcpu));
 	powerpc_sync();
@@ -69,12 +71,17 @@ cpudep_ap_early_bootstrap(void)
 	case IBM970MP:
 		/* Restore HID4 and HID5, which are necessary for the MMU */
 
+#ifdef __powerpc64__
+		mtspr(SPR_HID4, bsp_state[2]); powerpc_sync(); isync();
+		mtspr(SPR_HID5, bsp_state[3]); powerpc_sync(); isync();
+#else
 		__asm __volatile("ld %0, 16(%2); sync; isync;	\
 		    mtspr %1, %0; sync; isync;"
 		    : "=r"(reg) : "K"(SPR_HID4), "r"(bsp_state));
 		__asm __volatile("ld %0, 24(%2); sync; isync;	\
 		    mtspr %1, %0; sync; isync;"
 		    : "=r"(reg) : "K"(SPR_HID5), "r"(bsp_state));
+#endif
 		powerpc_sync();
 		break;
 	}
@@ -292,9 +299,24 @@ cpudep_ap_setup()
 		/*
 		 * The 970 has strange rules about how to update HID registers.
 		 * See Table 2-3, 970MP manual
+		 *
+		 * Note: HID4 and HID5 restored already in
+		 * cpudep_ap_early_bootstrap()
 		 */
 
 		__asm __volatile("mtasr %0; sync" :: "r"(0));
+	#ifdef __powerpc64__
+		__asm __volatile(" \
+			sync; isync;					\
+			mtspr	%1, %0;					\
+			mfspr	%0, %1;	mfspr	%0, %1;	mfspr	%0, %1;	\
+			mfspr	%0, %1;	mfspr	%0, %1;	mfspr	%0, %1; \
+			sync; isync" 
+		    :: "r"(bsp_state[0]), "K"(SPR_HID0));
+		__asm __volatile("sync; isync;	\
+		    mtspr %1, %0; mtspr %1, %0; sync; isync"
+		    :: "r"(bsp_state[1]), "K"(SPR_HID1));
+	#else
 		__asm __volatile(" \
 			ld	%0,0(%2);				\
 			sync; isync;					\
@@ -306,12 +328,7 @@ cpudep_ap_setup()
 		__asm __volatile("ld %0, 8(%2); sync; isync;	\
 		    mtspr %1, %0; mtspr %1, %0; sync; isync"
 		    : "=r"(reg) : "K"(SPR_HID1), "r"(bsp_state));
-		__asm __volatile("ld %0, 16(%2); sync; isync;	\
-		    mtspr %1, %0; sync; isync;"
-		    : "=r"(reg) : "K"(SPR_HID4), "r"(bsp_state));
-		__asm __volatile("ld %0, 24(%2); sync; isync;	\
-		    mtspr %1, %0; sync; isync;"
-		    : "=r"(reg) : "K"(SPR_HID5), "r"(bsp_state));
+	#endif
 
 		powerpc_sync();
 		break;

From a23e40a7615ef7c9104c5c812b63cf5a23a98354 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 03:22:37 +0000
Subject: [PATCH 134/279] The mv/bus_space.c file is essentially identical to
 arm/bus_space-v6.c, so just use it.

---
 sys/arm/mv/bus_space.c | 161 -----------------------------------------
 sys/arm/mv/files.mv    |   2 +-
 2 files changed, 1 insertion(+), 162 deletions(-)
 delete mode 100644 sys/arm/mv/bus_space.c

diff --git a/sys/arm/mv/bus_space.c b/sys/arm/mv/bus_space.c
deleted file mode 100644
index 6bd4f3486df..00000000000
--- a/sys/arm/mv/bus_space.c
+++ /dev/null
@@ -1,161 +0,0 @@
-/*-
- * Copyright (C) 2008 MARVELL INTERNATIONAL LTD.
- * All rights reserved.
- *
- * Developed by Semihalf.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of MARVELL nor the names of contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-#include <sys/kernel.h>
-#include <sys/malloc.h>
-
-#include <machine/bus.h>
-
-/*
- * Bus space functions for Marvell SoC family
- */
-
-/* Prototypes for all the bus_space structure functions */
-bs_protos(generic);
-
-/*
- * The bus space tag.  This is constant for all instances, so
- * we never have to explicitly "create" it.
- */
-static struct bus_space _base_tag = {
-	/* cookie */
-	(void *) 0,
-
-	/* mapping/unmapping */
-	generic_bs_map,
-	generic_bs_unmap,
-	generic_bs_subregion,
-
-	/* allocation/deallocation */
-	generic_bs_alloc,
-	generic_bs_free,
-
-	/* barrier */
-	generic_bs_barrier,
-
-	/* read (single) */
-	generic_bs_r_1,
-	generic_bs_r_2,
-	generic_bs_r_4,
-	NULL,
-
-	/* read multiple */
-	generic_bs_rm_1,
-	generic_bs_rm_2,
-	generic_bs_rm_4,
-	NULL,
-
-	/* read region */
-	generic_bs_rr_1,
-	generic_bs_rr_2,
-	generic_bs_rr_4,
-	NULL,
-
-	/* write (single) */
-	generic_bs_w_1,
-	generic_bs_w_2,
-	generic_bs_w_4,
-	NULL,
-
-	/* write multiple */
-	generic_bs_wm_1,
-	generic_bs_wm_2,
-	generic_bs_wm_4,
-	NULL,
-
-	/* write region */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* set multiple */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* set region */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* copy */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* read stream (single) */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* read multiple stream */
-	NULL,
-	generic_bs_rm_2,		/* bus_space_read_multi_stream_2 */
-	NULL,
-	NULL,
-
-	/* read region stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* write stream (single) */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* write multiple stream */
-	NULL,
-	generic_bs_wm_2,		/* bus_space_write_multi_stream_2 */
-	NULL,
-	NULL,
-
-	/* write region stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL
-};
-
-bus_space_tag_t fdtbus_bs_tag = &_base_tag;
diff --git a/sys/arm/mv/files.mv b/sys/arm/mv/files.mv
index 0d14e4d92f6..15622ae0d13 100644
--- a/sys/arm/mv/files.mv
+++ b/sys/arm/mv/files.mv
@@ -12,6 +12,7 @@
 # - JTAG/ICE
 # - Vector Floating Point (VFP) unit
 #
+arm/arm/bus_space-v6.c		standard
 arm/arm/bus_space_generic.c	standard
 arm/arm/cpufunc_asm_arm10.S	standard
 arm/arm/cpufunc_asm_arm11.S	standard
@@ -21,7 +22,6 @@ arm/arm/cpufunc_asm_armv7.S	standard
 arm/arm/cpufunc_asm_sheeva.S	standard
 arm/arm/cpufunc_asm_pj4b.S	standard
 
-arm/mv/bus_space.c		standard
 arm/mv/gpio.c			standard
 arm/mv/mv_common.c		standard
 arm/mv/mv_localbus.c		standard

From 2737497b8398ee7f0a3de051b5b0ffac8995fdfe Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 03:24:18 +0000
Subject: [PATCH 135/279] Move bs_unimplemented() to bus_space_generic.c so it
 can be shared.

---
 sys/arm/arm/bus_space-v6.c      | 9 ---------
 sys/arm/arm/bus_space_generic.c | 7 +++++++
 sys/arm/include/bus.h           | 3 +++
 3 files changed, 10 insertions(+), 9 deletions(-)

diff --git a/sys/arm/arm/bus_space-v6.c b/sys/arm/arm/bus_space-v6.c
index ce26c46171e..4997c02bf88 100644
--- a/sys/arm/arm/bus_space-v6.c
+++ b/sys/arm/arm/bus_space-v6.c
@@ -39,15 +39,6 @@ __FBSDID("$FreeBSD$");
 /* Prototypes for all the bus_space structure functions */
 bs_protos(generic);
 
-static void
-bs_unimplemented(void)
-{
-
-	panic("unimplemented bus_space function called");
-}
-
-#define	BS_UNIMPLEMENTED	(void *)bs_unimplemented
-
 /*
  * The bus space tag.  This is constant for all instances, so
  * we never have to explicitly "create" it.
diff --git a/sys/arm/arm/bus_space_generic.c b/sys/arm/arm/bus_space_generic.c
index 2df18dc1510..7beda42ab0e 100644
--- a/sys/arm/arm/bus_space_generic.c
+++ b/sys/arm/arm/bus_space_generic.c
@@ -53,6 +53,13 @@ __FBSDID("$FreeBSD$");
 #include <machine/cpufunc.h>
 #include <machine/devmap.h>
 
+void
+generic_bs_unimplemented(void)
+{
+
+	panic("unimplemented bus_space function called");
+}
+
 /* Prototypes for all the bus_space structure functions */
 bs_protos(generic);
 
diff --git a/sys/arm/include/bus.h b/sys/arm/include/bus.h
index f9a677e4f26..88458d799ea 100644
--- a/sys/arm/include/bus.h
+++ b/sys/arm/include/bus.h
@@ -730,6 +730,9 @@ bs_c_2_proto(f);		\
 bs_c_4_proto(f);		\
 bs_c_8_proto(f);
 
+void generic_bs_unimplemented(void);
+#define	BS_UNIMPLEMENTED	(void *)generic_bs_unimplemented
+
 #define BUS_SPACE_ALIGNED_POINTER(p, t) ALIGNED_POINTER(p, t)
 
 #define BUS_SPACE_MAXADDR_24BIT	0xFFFFFF

From 05ca38474ecb02091cc10ee354a9b7bde8154fbb Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 03:28:07 +0000
Subject: [PATCH 136/279] Use explicit initializer style, fill in missing
 functions.

---
 sys/arm/at91/at91.c     | 140 ++++++++++++++++++++--------------------
 sys/arm/at91/files.at91 |   1 +
 2 files changed, 71 insertions(+), 70 deletions(-)

diff --git a/sys/arm/at91/at91.c b/sys/arm/at91/at91.c
index 8c5c86c39c2..209a11997a1 100644
--- a/sys/arm/at91/at91.c
+++ b/sys/arm/at91/at91.c
@@ -118,110 +118,110 @@ bus_dma_get_range_nb(void)
 bs_protos(generic);
 
 struct bus_space at91_bs_tag = {
-	/* cookie */
-	(void *) 0,
+	/* privdata is whatever the implementer wants; unused in base tag */
+	.bs_privdata	= NULL,
 
 	/* mapping/unmapping */
-	at91_bs_map,
-	at91_bs_unmap,
-	at91_bs_subregion,
+	.bs_map		= at91_bs_map,
+	.bs_unmap	= at91_bs_unmap,
+	.bs_subregion	= at91_bs_subregion,
 
 	/* allocation/deallocation */
-	NULL,
-	NULL,
+	.bs_alloc	= generic_bs_alloc,
+	.bs_free	= generic_bs_free,
 
 	/* barrier */
-	at91_barrier,
+	.bs_barrier	= at91_barrier,
 
 	/* read (single) */
-	generic_bs_r_1,
-	generic_bs_r_2,
-	generic_bs_r_4,
-	NULL,
+	.bs_r_1		= NULL,	/* Use inline code in bus.h */
+	.bs_r_2		= NULL,	/* Use inline code in bus.h */
+	.bs_r_4		= NULL,	/* Use inline code in bus.h */
+	.bs_r_8		= NULL,	/* Use inline code in bus.h */
 
 	/* read multiple */
-	generic_bs_rm_1,
-	generic_bs_rm_2,
-	generic_bs_rm_4,
-	NULL,
+	.bs_rm_1	= generic_bs_rm_1,
+	.bs_rm_2	= generic_bs_rm_2,
+	.bs_rm_4	= generic_bs_rm_4,
+	.bs_rm_8	= BS_UNIMPLEMENTED,
 
 	/* read region */
-	generic_bs_rr_1,
-	generic_bs_rr_2,
-	generic_bs_rr_4,
-	NULL,
+	.bs_rr_1	= generic_bs_rr_1,
+	.bs_rr_2	= generic_bs_rr_2,
+	.bs_rr_4	= generic_bs_rr_4,
+	.bs_rr_8	= BS_UNIMPLEMENTED,
 
 	/* write (single) */
-	generic_bs_w_1,
-	generic_bs_w_2,
-	generic_bs_w_4,
-	NULL,
+	.bs_w_1		= NULL,	/* Use inline code in bus.h */
+	.bs_w_2		= NULL,	/* Use inline code in bus.h */
+	.bs_w_4		= NULL,	/* Use inline code in bus.h */
+	.bs_w_8		= NULL,	/* Use inline code in bus.h */
 
 	/* write multiple */
-	generic_bs_wm_1,
-	generic_bs_wm_2,
-	generic_bs_wm_4,
-	NULL,
+	.bs_wm_1	= generic_bs_wm_1,
+	.bs_wm_2	= generic_bs_wm_2,
+	.bs_wm_4	= generic_bs_wm_4,
+	.bs_wm_8	= BS_UNIMPLEMENTED,
 
 	/* write region */
-	NULL,
-	generic_bs_wr_2,
-	generic_bs_wr_4,
-	NULL,
+	.bs_wr_1	= generic_bs_wr_1,
+	.bs_wr_2	= generic_bs_wr_2,
+	.bs_wr_4	= generic_bs_wr_4,
+	.bs_wr_8	= BS_UNIMPLEMENTED,
 
 	/* set multiple */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
+	.bs_sm_1	= BS_UNIMPLEMENTED,
+	.bs_sm_2	= BS_UNIMPLEMENTED,
+	.bs_sm_4	= BS_UNIMPLEMENTED,
+	.bs_sm_8	= BS_UNIMPLEMENTED,
 
 	/* set region */
-	NULL,
-	generic_bs_sr_2,
-	generic_bs_sr_4,
-	NULL,
+	.bs_sr_1	= generic_bs_sr_1,
+	.bs_sr_2	= generic_bs_sr_2,
+	.bs_sr_4	= generic_bs_sr_4,
+	.bs_sr_8	= BS_UNIMPLEMENTED,
 
 	/* copy */
-	NULL,
-	generic_bs_c_2,
-	NULL,
-	NULL,
+	.bs_c_1		= BS_UNIMPLEMENTED,
+	.bs_c_2		= generic_bs_c_2,
+	.bs_c_4		= BS_UNIMPLEMENTED,
+	.bs_c_8		= BS_UNIMPLEMENTED,
 
-	/* read (single) stream */
-	generic_bs_r_1,
-	generic_bs_r_2,
-	generic_bs_r_4,
-	NULL,
+	/* read stream (single) */
+	.bs_r_1_s	= NULL,   /* Use inline code in bus.h */ 
+	.bs_r_2_s	= NULL,   /* Use inline code in bus.h */ 
+	.bs_r_4_s	= NULL,   /* Use inline code in bus.h */ 
+	.bs_r_8_s	= NULL,   /* Use inline code in bus.h */ 
 
 	/* read multiple stream */
-	generic_bs_rm_1,
-	generic_bs_rm_2,
-	generic_bs_rm_4,
-	NULL,
+	.bs_rm_1_s	= generic_bs_rm_1,
+	.bs_rm_2_s	= generic_bs_rm_2,
+	.bs_rm_4_s	= generic_bs_rm_4,
+	.bs_rm_8_s	= BS_UNIMPLEMENTED,
 
 	/* read region stream */
-	generic_bs_rr_1,
-	generic_bs_rr_2,
-	generic_bs_rr_4,
-	NULL,
+	.bs_rr_1_s	= generic_bs_rr_1,
+	.bs_rr_2_s	= generic_bs_rr_2,
+	.bs_rr_4_s	= generic_bs_rr_4,
+	.bs_rr_8_s	= BS_UNIMPLEMENTED,
 
-	/* write (single) stream */
-	generic_bs_w_1,
-	generic_bs_w_2,
-	generic_bs_w_4,
-	NULL,
+	/* write stream (single) */
+	.bs_w_1_s	= NULL,   /* Use inline code in bus.h */ 
+	.bs_w_2_s	= NULL,   /* Use inline code in bus.h */ 
+	.bs_w_4_s	= NULL,   /* Use inline code in bus.h */ 
+	.bs_w_8_s	= NULL,   /* Use inline code in bus.h */ 
 
 	/* write multiple stream */
-	generic_bs_wm_1,
-	generic_bs_wm_2,
-	generic_bs_wm_4,
-	NULL,
+	.bs_wm_1_s	= generic_bs_wm_1,
+	.bs_wm_2_s	= generic_bs_wm_2,
+	.bs_wm_4_s	= generic_bs_wm_4,
+	.bs_wm_8_s	= BS_UNIMPLEMENTED,
 
 	/* write region stream */
-	NULL,
-	generic_bs_wr_2,
-	generic_bs_wr_4,
-	NULL,
+	.bs_wr_1_s	= generic_bs_wr_1,
+	.bs_wr_2_s	= generic_bs_wr_2,
+	.bs_wr_4_s	= generic_bs_wr_4,
+	.bs_wr_8_s	= BS_UNIMPLEMENTED,
 };
 
 #ifndef FDT
diff --git a/sys/arm/at91/files.at91 b/sys/arm/at91/files.at91
index 29ada7d26c1..e3061e11db0 100644
--- a/sys/arm/at91/files.at91
+++ b/sys/arm/at91/files.at91
@@ -1,4 +1,5 @@
 # $FreeBSD$
+arm/arm/bus_space_generic.c	standard
 arm/arm/cpufunc_asm_arm9.S	standard
 arm/at91/at91_machdep.c		standard
 arm/at91/at91_aic.c		standard

From eb8711ce163a61a26d712fe9b69112e8ca3a5edb Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 03:44:29 +0000
Subject: [PATCH 137/279] Rename bus_space-v6.c to bus_space_base.c, because
 it's not v6-specific and now some v5 Marvell systems are using it.  Only
 define fdt_bus_tag if option FDT is defined.

---
 sys/arm/allwinner/a20/files.a20                  | 4 ++--
 sys/arm/allwinner/files.a10                      | 4 ++--
 sys/arm/altera/socfpga/files.socfpga             | 2 +-
 sys/arm/arm/{bus_space-v6.c => bus_space_base.c} | 8 ++++++--
 sys/arm/broadcom/bcm2835/files.bcm2835           | 2 +-
 sys/arm/freescale/imx/files.imx51                | 2 +-
 sys/arm/freescale/imx/files.imx53                | 2 +-
 sys/arm/freescale/imx/files.imx6                 | 2 +-
 sys/arm/freescale/vybrid/files.vybrid            | 2 +-
 sys/arm/lpc/files.lpc                            | 2 +-
 sys/arm/mv/files.mv                              | 2 +-
 sys/arm/rockchip/files.rk30xx                    | 2 +-
 sys/arm/samsung/exynos/files.exynos5             | 2 +-
 sys/arm/ti/files.ti                              | 2 +-
 sys/arm/xilinx/files.zynq7                       | 2 +-
 15 files changed, 22 insertions(+), 18 deletions(-)
 rename sys/arm/arm/{bus_space-v6.c => bus_space_base.c} (97%)

diff --git a/sys/arm/allwinner/a20/files.a20 b/sys/arm/allwinner/a20/files.a20
index 1c02e90a6ed..04f23794309 100644
--- a/sys/arm/allwinner/a20/files.a20
+++ b/sys/arm/allwinner/a20/files.a20
@@ -17,7 +17,7 @@ arm/allwinner/a10_ehci.c		optional	ehci
 arm/allwinner/if_emac.c			optional	emac
 arm/allwinner/a10_wdog.c		standard
 arm/allwinner/timer.c			standard
-arm/arm/bus_space-v6.c			standard
-arm/allwinner/a10_common.c			standard
+arm/arm/bus_space_base.c		standard
+arm/allwinner/a10_common.c		standard
 arm/allwinner/a10_machdep.c		standard
 arm/allwinner/a20/a20_mp.c		optional	smp
diff --git a/sys/arm/allwinner/files.a10 b/sys/arm/allwinner/files.a10
index c14ef2e4b64..eb5154eb7f6 100644
--- a/sys/arm/allwinner/files.a10
+++ b/sys/arm/allwinner/files.a10
@@ -19,5 +19,5 @@ arm/allwinner/a20/a20_cpu_cfg.c 	standard
 arm/allwinner/aintc.c			standard
 arm/allwinner/if_emac.c			optional	emac
 arm/allwinner/timer.c			standard
-arm/arm/bus_space-v6.c			standard
-#arm/allwinner/console.c			standard
+arm/arm/bus_space_base.c		standard
+#arm/allwinner/console.c		standard
diff --git a/sys/arm/altera/socfpga/files.socfpga b/sys/arm/altera/socfpga/files.socfpga
index 652ab9801ea..22d7c28aebb 100644
--- a/sys/arm/altera/socfpga/files.socfpga
+++ b/sys/arm/altera/socfpga/files.socfpga
@@ -9,7 +9,7 @@ arm/arm/cpufunc_asm_arm10.S			standard
 arm/arm/cpufunc_asm_arm11.S			standard
 arm/arm/cpufunc_asm_armv7.S			standard
 
-arm/arm/bus_space-v6.c				standard
+arm/arm/bus_space_base.c			standard
 arm/arm/gic.c					standard
 arm/arm/mpcore_timer.c				standard
 
diff --git a/sys/arm/arm/bus_space-v6.c b/sys/arm/arm/bus_space_base.c
similarity index 97%
rename from sys/arm/arm/bus_space-v6.c
rename to sys/arm/arm/bus_space_base.c
index 4997c02bf88..8ddfcb0d3a0 100644
--- a/sys/arm/arm/bus_space-v6.c
+++ b/sys/arm/arm/bus_space_base.c
@@ -36,6 +36,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <machine/bus.h>
 
+#include "opt_platform.h"
+
 /* Prototypes for all the bus_space structure functions */
 bs_protos(generic);
 
@@ -43,7 +45,7 @@ bs_protos(generic);
  * The bus space tag.  This is constant for all instances, so
  * we never have to explicitly "create" it.
  */
-static struct bus_space _base_tag = {
+static struct bus_space arm_base_tag = {
 	/* privdata is whatever the implementer wants; unused in base tag */
 	.bs_privdata	= NULL,
 
@@ -150,4 +152,6 @@ static struct bus_space _base_tag = {
 	.bs_wr_8_s	= BS_UNIMPLEMENTED,
 };
 
-bus_space_tag_t fdtbus_bs_tag = &_base_tag;
+#ifdef FDT
+bus_space_tag_t fdtbus_bs_tag = &arm_base_tag;
+#endif
diff --git a/sys/arm/broadcom/bcm2835/files.bcm2835 b/sys/arm/broadcom/bcm2835/files.bcm2835
index 11158cb822a..53fd12f5c3d 100644
--- a/sys/arm/broadcom/bcm2835/files.bcm2835
+++ b/sys/arm/broadcom/bcm2835/files.bcm2835
@@ -15,7 +15,7 @@ arm/broadcom/bcm2835/bcm2835_spi.c		optional bcm2835_spi
 arm/broadcom/bcm2835/bcm2835_systimer.c		standard
 arm/broadcom/bcm2835/bcm2835_wdog.c		standard
 
-arm/arm/bus_space-v6.c				standard
+arm/arm/bus_space_base.c			standard
 arm/arm/bus_space_generic.c                     standard
 arm/arm/bus_space_asm_generic.S                 standard
 arm/arm/cpufunc_asm_arm11.S                     standard
diff --git a/sys/arm/freescale/imx/files.imx51 b/sys/arm/freescale/imx/files.imx51
index 951cb4d148b..986fc2c0af2 100644
--- a/sys/arm/freescale/imx/files.imx51
+++ b/sys/arm/freescale/imx/files.imx51
@@ -10,7 +10,7 @@ kern/kern_clocksource.c			standard
 arm/freescale/imx/imx_common.c		standard
 arm/freescale/imx/imx_machdep.c		standard
 arm/freescale/imx/imx51_machdep.c	standard
-arm/arm/bus_space-v6.c			standard
+arm/arm/bus_space_base.c		standard
 
 # Dummy serial console
 #arm/freescale/imx/console.c		standard
diff --git a/sys/arm/freescale/imx/files.imx53 b/sys/arm/freescale/imx/files.imx53
index 3918400edab..c5dd30bf15b 100644
--- a/sys/arm/freescale/imx/files.imx53
+++ b/sys/arm/freescale/imx/files.imx53
@@ -10,7 +10,7 @@ kern/kern_clocksource.c			standard
 arm/freescale/imx/imx_common.c		standard
 arm/freescale/imx/imx_machdep.c		standard
 arm/freescale/imx/imx53_machdep.c	standard
-arm/arm/bus_space-v6.c			standard
+arm/arm/bus_space_base.c		standard
 
 # Special serial console for debuging early boot code
 #arm/freescale/imx/console.c		standard
diff --git a/sys/arm/freescale/imx/files.imx6 b/sys/arm/freescale/imx/files.imx6
index 2fc6d3fb7df..df86ef9d596 100644
--- a/sys/arm/freescale/imx/files.imx6
+++ b/sys/arm/freescale/imx/files.imx6
@@ -15,7 +15,7 @@ kern/kern_clocksource.c			standard
 #
 arm/arm/gic.c				standard
 arm/arm/pl310.c				standard
-arm/arm/bus_space-v6.c			standard
+arm/arm/bus_space_base.c		standard
 arm/arm/mpcore_timer.c			standard
 arm/freescale/fsl_ocotp.c		standard
 arm/freescale/imx/imx6_anatop.c		standard
diff --git a/sys/arm/freescale/vybrid/files.vybrid b/sys/arm/freescale/vybrid/files.vybrid
index 572a3ef1c4b..1d301f23c49 100644
--- a/sys/arm/freescale/vybrid/files.vybrid
+++ b/sys/arm/freescale/vybrid/files.vybrid
@@ -9,7 +9,7 @@ arm/arm/cpufunc_asm_arm10.S			standard
 arm/arm/cpufunc_asm_arm11.S			standard
 arm/arm/cpufunc_asm_armv7.S			standard
 
-arm/arm/bus_space-v6.c				standard
+arm/arm/bus_space_base.c			standard
 arm/arm/gic.c					standard
 arm/arm/mpcore_timer.c				standard
 
diff --git a/sys/arm/lpc/files.lpc b/sys/arm/lpc/files.lpc
index 8e196ad5512..1f8d768b7a6 100644
--- a/sys/arm/lpc/files.lpc
+++ b/sys/arm/lpc/files.lpc
@@ -1,5 +1,5 @@
 # $FreeBSD$
-arm/arm/bus_space-v6.c			standard
+arm/arm/bus_space_base.c		standard
 arm/arm/bus_space_generic.c		standard
 arm/arm/cpufunc_asm_arm9.S		standard
 arm/arm/cpufunc_asm_armv5.S		standard
diff --git a/sys/arm/mv/files.mv b/sys/arm/mv/files.mv
index 15622ae0d13..d730c01d644 100644
--- a/sys/arm/mv/files.mv
+++ b/sys/arm/mv/files.mv
@@ -12,7 +12,7 @@
 # - JTAG/ICE
 # - Vector Floating Point (VFP) unit
 #
-arm/arm/bus_space-v6.c		standard
+arm/arm/bus_space_base.c	standard
 arm/arm/bus_space_generic.c	standard
 arm/arm/cpufunc_asm_arm10.S	standard
 arm/arm/cpufunc_asm_arm11.S	standard
diff --git a/sys/arm/rockchip/files.rk30xx b/sys/arm/rockchip/files.rk30xx
index 0bcc34c3657..4160c1e684d 100644
--- a/sys/arm/rockchip/files.rk30xx
+++ b/sys/arm/rockchip/files.rk30xx
@@ -11,7 +11,7 @@ arm/arm/cpufunc_asm_armv7.S		standard
 arm/arm/gic.c				standard
 arm/arm/mpcore_timer.c			standard
 
-arm/arm/bus_space-v6.c			standard
+arm/arm/bus_space_base.c		standard
 arm/rockchip/rk30xx_common.c		standard
 arm/rockchip/rk30xx_machdep.c		standard
 arm/rockchip/rk30xx_pmu.c		standard
diff --git a/sys/arm/samsung/exynos/files.exynos5 b/sys/arm/samsung/exynos/files.exynos5
index 3883b2848a9..f1174dcb720 100644
--- a/sys/arm/samsung/exynos/files.exynos5
+++ b/sys/arm/samsung/exynos/files.exynos5
@@ -9,7 +9,7 @@ arm/arm/cpufunc_asm_arm10.S			standard
 arm/arm/cpufunc_asm_arm11.S			standard
 arm/arm/cpufunc_asm_armv7.S			standard
 
-arm/arm/bus_space-v6.c				standard
+arm/arm/bus_space_base.c			standard
 arm/arm/gic.c					standard
 arm/arm/generic_timer.c				standard
 
diff --git a/sys/arm/ti/files.ti b/sys/arm/ti/files.ti
index ef77cb99dfe..52ee22b05d5 100644
--- a/sys/arm/ti/files.ti
+++ b/sys/arm/ti/files.ti
@@ -2,7 +2,7 @@
 
 kern/kern_clocksource.c				standard
 
-arm/arm/bus_space-v6.c				standard
+arm/arm/bus_space_base.c			standard
 arm/arm/bus_space_generic.c			standard
 arm/arm/bus_space_asm_generic.S			standard
 arm/arm/cpufunc_asm_armv5.S			standard
diff --git a/sys/arm/xilinx/files.zynq7 b/sys/arm/xilinx/files.zynq7
index 9682904d91a..19759ac0e58 100644
--- a/sys/arm/xilinx/files.zynq7
+++ b/sys/arm/xilinx/files.zynq7
@@ -5,7 +5,7 @@
 
 kern/kern_clocksource.c				standard
 
-arm/arm/bus_space-v6.c				standard
+arm/arm/bus_space_base.c			standard
 arm/arm/bus_space_generic.c			standard
 arm/arm/bus_space_asm_generic.S			standard
 arm/arm/cpufunc_asm_armv5.S			standard

From afe43c7b5fad28feb67aa078ecf8f07315cd7448 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 04:06:36 +0000
Subject: [PATCH 138/279] The versatile platform had two copies of a bus_space
 that are essentially duplicates of the standard arm base bus_space, so just
 use it.

---
 sys/arm/arm/bus_space_base.c                |   6 +-
 sys/arm/include/bus.h                       |   1 +
 sys/arm/versatile/bus_space.c               | 114 ---------------
 sys/arm/versatile/files.versatile           |   3 +-
 sys/arm/versatile/versatile_pci.c           |   2 +-
 sys/arm/versatile/versatile_pci_bus_space.c | 152 --------------------
 sys/arm/versatile/versatile_pci_bus_space.h |  35 -----
 7 files changed, 7 insertions(+), 306 deletions(-)
 delete mode 100644 sys/arm/versatile/bus_space.c
 delete mode 100644 sys/arm/versatile/versatile_pci_bus_space.c
 delete mode 100644 sys/arm/versatile/versatile_pci_bus_space.h

diff --git a/sys/arm/arm/bus_space_base.c b/sys/arm/arm/bus_space_base.c
index 8ddfcb0d3a0..a80b06466cc 100644
--- a/sys/arm/arm/bus_space_base.c
+++ b/sys/arm/arm/bus_space_base.c
@@ -45,7 +45,7 @@ bs_protos(generic);
  * The bus space tag.  This is constant for all instances, so
  * we never have to explicitly "create" it.
  */
-static struct bus_space arm_base_tag = {
+static struct bus_space arm_base_bus_space = {
 	/* privdata is whatever the implementer wants; unused in base tag */
 	.bs_privdata	= NULL,
 
@@ -153,5 +153,7 @@ static struct bus_space arm_base_tag = {
 };
 
 #ifdef FDT
-bus_space_tag_t fdtbus_bs_tag = &arm_base_tag;
+bus_space_tag_t fdtbus_bs_tag = &arm_base_bus_space;
 #endif
+
+bus_space_tag_t arm_base_bs_tag = &arm_base_bus_space;
diff --git a/sys/arm/include/bus.h b/sys/arm/include/bus.h
index 88458d799ea..757919974a1 100644
--- a/sys/arm/include/bus.h
+++ b/sys/arm/include/bus.h
@@ -245,6 +245,7 @@ struct bus_space {
 			    bus_size_t, const uint64_t *, bus_size_t);
 };
 
+extern bus_space_tag_t arm_base_bs_tag;
 
 /*
  * Utility macros; INTERNAL USE ONLY.
diff --git a/sys/arm/versatile/bus_space.c b/sys/arm/versatile/bus_space.c
deleted file mode 100644
index 850feefb1af..00000000000
--- a/sys/arm/versatile/bus_space.c
+++ /dev/null
@@ -1,114 +0,0 @@
-/*-
- * Copyright (C) 2008 MARVELL INTERNATIONAL LTD.
- * All rights reserved.
- *
- * Developed by Semihalf.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of MARVELL nor the names of contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-#include <sys/kernel.h>
-#include <sys/malloc.h>
-
-#include <machine/bus.h>
-
-/* Prototypes for all the bus_space structure functions */
-bs_protos(generic);
-
-struct bus_space _base_tag = {
-	/* cookie */
-	.bs_privdata	= (void *) 0,
-	
-	/* mapping/unmapping */
-	.bs_map		= generic_bs_map,
-	.bs_unmap	= generic_bs_unmap,
-	.bs_subregion	= generic_bs_subregion,
-	
-	/* allocation/deallocation */
-	.bs_alloc	= generic_bs_alloc,
-	.bs_free	= generic_bs_free,
-	
-	/* barrier */
-	.bs_barrier	= generic_bs_barrier,
-	
-	/* read (single) */
-	.bs_r_1		= generic_bs_r_1,
-	.bs_r_2		= generic_bs_r_2,
-	.bs_r_4		= generic_bs_r_4,
-	.bs_r_8		= NULL,
-	
-	/* read multiple */
-	.bs_rm_1	= generic_bs_rm_1,
-	.bs_rm_2	= generic_bs_rm_2,
-	.bs_rm_4	= generic_bs_rm_4,
-	.bs_rm_8	= NULL,
-	
-	/* read region */
-	.bs_rr_1	= generic_bs_rr_1,
-	.bs_rr_2	= generic_bs_rr_2,
-	.bs_rr_4	= generic_bs_rr_4,
-	.bs_rr_8	= NULL,
-	
-	/* write (single) */
-	.bs_w_1		= generic_bs_w_1,
-	.bs_w_2		= generic_bs_w_2,
-	.bs_w_4		= generic_bs_w_4,
-	.bs_w_8		= NULL,
-	
-	/* write multiple */
-	.bs_wm_1	= generic_bs_wm_1,
-	.bs_wm_2	= generic_bs_wm_2,
-	.bs_wm_4	= generic_bs_wm_4,
-	.bs_wm_8	= NULL,
-	
-	/* write region */
-	.bs_wr_1	= generic_bs_wr_1,
-	.bs_wr_2	= generic_bs_wr_2,
-	.bs_wr_4	= generic_bs_wr_4,
-	.bs_wr_8	= NULL,
-	
-	/* set multiple */
-	/* XXX not implemented */
-	
-	/* set region */
-	.bs_sr_1	= NULL,
-	.bs_sr_2	= generic_bs_sr_2,
-	.bs_sr_4	= generic_bs_sr_4,
-	.bs_sr_8	= NULL,
-	
-	/* copy */
-	.bs_c_1		= NULL,
-	.bs_c_2		= generic_bs_c_2,
-	.bs_c_4		= NULL,
-	.bs_c_8		= NULL,
-};
-
-bus_space_tag_t fdtbus_bs_tag = &_base_tag;
diff --git a/sys/arm/versatile/files.versatile b/sys/arm/versatile/files.versatile
index 199fecbb566..217109f03ce 100644
--- a/sys/arm/versatile/files.versatile
+++ b/sys/arm/versatile/files.versatile
@@ -1,5 +1,6 @@
 # $FreeBSD$
 
+arm/arm/bus_space_base.c 			standard
 arm/arm/bus_space_asm_generic.S			standard
 arm/arm/bus_space_generic.c                     standard
 arm/arm/cpufunc_asm_arm11.S                     standard
@@ -7,14 +8,12 @@ arm/arm/cpufunc_asm_arm11x6.S			standard
 arm/arm/cpufunc_asm_armv5.S                     standard
 arm/arm/cpufunc_asm_armv6.S                     standard
 
-arm/versatile/bus_space.c			standard
 arm/versatile/pl050.c				optional sc
 arm/versatile/sp804.c				standard
 arm/versatile/versatile_machdep.c		standard
 arm/versatile/versatile_clcd.c			optional sc
 arm/versatile/versatile_common.c		standard
 arm/versatile/versatile_pci.c			optional pci
-arm/versatile/versatile_pci_bus_space.c		optional pci
 arm/versatile/versatile_sic.c			standard
 arm/versatile/versatile_timer.c			standard
 
diff --git a/sys/arm/versatile/versatile_pci.c b/sys/arm/versatile/versatile_pci.c
index 0e484968287..40a120a5779 100644
--- a/sys/arm/versatile/versatile_pci.c
+++ b/sys/arm/versatile/versatile_pci.c
@@ -355,7 +355,7 @@ versatile_pci_activate_resource(device_t bus, device_t child, int type, int rid,
 		vaddr = (vm_offset_t)pmap_mapdev(rman_get_start(r),
 				rman_get_size(r));
 		rman_set_bushandle(r, vaddr);
-		rman_set_bustag(r, versatile_bus_space_pcimem);
+		rman_set_bustag(r, arm_base_bs_tag);
 		res = rman_activate_resource(r);
 		break;
 	case SYS_RES_IRQ:
diff --git a/sys/arm/versatile/versatile_pci_bus_space.c b/sys/arm/versatile/versatile_pci_bus_space.c
deleted file mode 100644
index 89454401697..00000000000
--- a/sys/arm/versatile/versatile_pci_bus_space.c
+++ /dev/null
@@ -1,152 +0,0 @@
-/*-
- * Copyright (c) 2009, Oleksandr Tymoshenko <gonzo@FreeBSD.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice unmodified, this list of conditions, and the following
- *    disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-#include <sys/endian.h>
-
-#include <machine/bus.h>
-#include <arm/versatile/versatile_pci_bus_space.h>
-
-/* Prototypes for all the bus_space structure functions */
-bs_protos(generic);
-
-/*
- * Bus space that handles offsets in word for 1/2 bytes read/write access.
- * Byte order of values is handled by device drivers itself. 
- */
-static struct bus_space bus_space_pcimem = {
-	/* cookie */
-	(void *) 0,
-
-	/* mapping/unmapping */
-	generic_bs_map,
-	generic_bs_unmap,
-	generic_bs_subregion,
-
-	/* allocation/deallocation */
-	NULL,
-	NULL,
-
-	/* barrier */
-	generic_bs_barrier,
-
-	/* read (single) */
-	generic_bs_r_1,
-	generic_bs_r_2,
-	generic_bs_r_4,
-	NULL,
-
-	/* read multiple */
-	generic_bs_rm_1,
-	generic_bs_rm_2,
-	generic_bs_rm_4,
-	NULL,
-
-	/* read region */
-	generic_bs_rr_1,
-	generic_bs_rr_2,
-	generic_bs_rr_4,
-	NULL,
-
-	/* write (single) */
-	generic_bs_w_1,
-	generic_bs_w_2,
-	generic_bs_w_4,
-	NULL,
-
-	/* write multiple */
-	generic_bs_wm_1,
-	generic_bs_wm_2,
-	generic_bs_wm_4,
-	NULL,
-
-	/* write region */
-	generic_bs_wr_1,
-	generic_bs_wr_2,
-	generic_bs_wr_4,
-	NULL,
-
-	/* set multiple */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* set region */
-	NULL,
-	NULL,
-	generic_bs_sr_4,
-	NULL,
-
-	/* copy */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* read (single) stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* read multiple stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* read region stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* write (single) stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* write multiple stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* write region stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-};
-
-bus_space_tag_t versatile_bus_space_pcimem = &bus_space_pcimem;
diff --git a/sys/arm/versatile/versatile_pci_bus_space.h b/sys/arm/versatile/versatile_pci_bus_space.h
deleted file mode 100644
index d5b54efdcc4..00000000000
--- a/sys/arm/versatile/versatile_pci_bus_space.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*-
- * Copyright (c) 2009, Oleksandr Tymoshenko <gonzo@FreeBSD.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice unmodified, this list of conditions, and the following
- *    disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#ifndef __VERSATILE_PCI_BUS_SPACEH__
-#define __VERSATILE_PCI_BUS_SPACEH__
-
-extern bus_space_tag_t versatile_bus_space_pcimem;
-
-#endif /* __VERSATILE_PCI_BUS_SPACEH__ */

From f8812377e391a114ba059731e6d72267734dbe60 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 04:19:54 +0000
Subject: [PATCH 139/279] Remove a no-longer-used include.

---
 sys/arm/versatile/versatile_pci.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/sys/arm/versatile/versatile_pci.c b/sys/arm/versatile/versatile_pci.c
index 40a120a5779..7b198ca8f86 100644
--- a/sys/arm/versatile/versatile_pci.c
+++ b/sys/arm/versatile/versatile_pci.c
@@ -53,8 +53,6 @@ __FBSDID("$FreeBSD$");
 #include <machine/bus.h>
 #include <machine/fdt.h>
 
-#include <arm/versatile/versatile_pci_bus_space.h>
-
 #define	MEM_SYS		0
 #define	MEM_CORE	1
 #define	MEM_BASE	2

From 7a4a35ab52476ab6e95465572ea4511986887f2d Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 04:22:20 +0000
Subject: [PATCH 140/279] Use the base arm bus_space instead of an identical
 local copy.

---
 sys/arm/cavium/cns11xx/econa.c     | 117 ++---------------------------
 sys/arm/cavium/cns11xx/files.econa |   1 +
 2 files changed, 6 insertions(+), 112 deletions(-)

diff --git a/sys/arm/cavium/cns11xx/econa.c b/sys/arm/cavium/cns11xx/econa.c
index e79e6e8d810..b9446f09606 100644
--- a/sys/arm/cavium/cns11xx/econa.c
+++ b/sys/arm/cavium/cns11xx/econa.c
@@ -56,116 +56,7 @@ unsigned int CPU_clock = 200000000;
 unsigned int AHB_clock;
 unsigned int APB_clock;
 
-bs_protos(generic);
-
-struct bus_space econa_bs_tag = {
-	/* cookie */
-	(void *) 0,
-
-	/* mapping/unmapping */
-	generic_bs_map,
-	generic_bs_unmap,
-	generic_bs_subregion,
-
-	/* allocation/deallocation */
-	generic_bs_alloc,
-	generic_bs_free,
-
-	/* barrier */
-	generic_bs_barrier,
-
-	/* read (single) */
-	generic_bs_r_1,
-	generic_bs_r_2,
-	generic_bs_r_4,
-	NULL,
-
-	/* read multiple */
-	generic_bs_rm_1,
-	generic_bs_rm_2,
-	generic_bs_rm_4,
-	NULL,
-
-	/* read region */
-	generic_bs_rr_1,
-	generic_bs_rr_2,
-	generic_bs_rr_4,
-	NULL,
-
-	/* write (single) */
-	generic_bs_w_1,
-	generic_bs_w_2,
-	generic_bs_w_4,
-	NULL,
-
-	/* write multiple */
-	generic_bs_wm_1,
-	generic_bs_wm_2,
-	generic_bs_wm_4,
-	NULL,
-
-	/* write region */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* set multiple */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* set region */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* copy */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* read (single) stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* read multiple stream */
-	NULL,
-	generic_bs_rm_2,
-	NULL,
-	NULL,
-
-	/* read region stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* write (single) stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* write multiple stream */
-	NULL,
-	generic_bs_wm_2,
-	NULL,
-	NULL,
-
-	/* write region stream */
-	NULL,
-	NULL,
-	NULL,
-	NULL
-};
-
-bus_space_tag_t obio_tag = &econa_bs_tag;
+bus_space_tag_t obio_tag;
 
 static int
 econa_probe(device_t dev)
@@ -464,8 +355,10 @@ econa_attach(device_t dev)
 	struct econa_softc *sc = device_get_softc(dev);
 	int i;
 
+	obio_tag = arm_base_bs_tag;
+
 	econa_softc = sc;
-	sc->ec_st = &econa_bs_tag;
+	sc->ec_st = arm_base_bs_tag;
 	sc->ec_sh = ECONA_IO_BASE;
 	sc->dev = dev;
 	if (bus_space_subregion(sc->ec_st, sc->ec_sh, ECONA_PIC_BASE,
@@ -547,7 +440,7 @@ econa_alloc_resource(device_t dev, device_t child, int type, int *rid,
 		rle->res = rman_reserve_resource(&sc->ec_mem_rman,
 		    start, end, count, flags, child);
 		if (rle->res != NULL) {
-			rman_set_bustag(rle->res, &econa_bs_tag);
+			rman_set_bustag(rle->res, arm_base_bs_tag);
 			rman_set_bushandle(rle->res, start);
 		}
 		break;
diff --git a/sys/arm/cavium/cns11xx/files.econa b/sys/arm/cavium/cns11xx/files.econa
index c69895a8390..14d9151cb29 100644
--- a/sys/arm/cavium/cns11xx/files.econa
+++ b/sys/arm/cavium/cns11xx/files.econa
@@ -6,6 +6,7 @@ arm/cavium/cns11xx/timer.c			standard
 arm/cavium/cns11xx/uart_bus_ec.c		optional	uart
 arm/cavium/cns11xx/uart_cpu_ec.c		optional	uart
 dev/uart/uart_dev_ns8250.c	optional	uart
+arm/arm/bus_space_base.c		standard
 arm/arm/bus_space_generic.c		standard
 arm/cavium/cns11xx/ehci_ebus.c	optional	ehci
 arm/cavium/cns11xx/ohci_ec.c	optional	ohci

From c0501ea3fd859208b6c42f107ed82f4a0c2dba75 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 04:28:19 +0000
Subject: [PATCH 141/279] Use the base arm bus_space instead of an identical
 local copy.

---
 sys/arm/samsung/s3c2xx0/files.s3c2xx0      |  1 +
 sys/arm/samsung/s3c2xx0/s3c24x0.c          | 57 ++++++++++++----------
 sys/arm/samsung/s3c2xx0/s3c2xx0var.h       |  2 +-
 sys/arm/samsung/s3c2xx0/uart_cpu_s3c2410.c |  4 +-
 4 files changed, 34 insertions(+), 30 deletions(-)

diff --git a/sys/arm/samsung/s3c2xx0/files.s3c2xx0 b/sys/arm/samsung/s3c2xx0/files.s3c2xx0
index eb02fc52de3..2ab57d5190d 100644
--- a/sys/arm/samsung/s3c2xx0/files.s3c2xx0
+++ b/sys/arm/samsung/s3c2xx0/files.s3c2xx0
@@ -1,4 +1,5 @@
 # $FreeBSD$
+arm/arm/bus_space_base.c	standard
 arm/arm/bus_space_asm_generic.S	standard
 arm/arm/bus_space_generic.c	standard
 arm/arm/cpufunc_asm_arm9.S	standard
diff --git a/sys/arm/samsung/s3c2xx0/s3c24x0.c b/sys/arm/samsung/s3c2xx0/s3c24x0.c
index 49acdc65dbd..845651fd846 100644
--- a/sys/arm/samsung/s3c2xx0/s3c24x0.c
+++ b/sys/arm/samsung/s3c2xx0/s3c24x0.c
@@ -56,6 +56,8 @@ __FBSDID("$FreeBSD$");
 
 #define S3C2XX0_XTAL_CLK 12000000
 
+bus_space_tag_t s3c2xx0_bs_tag;
+
 #define IPL_LEVELS 13
 u_int irqmasks[IPL_LEVELS];
 
@@ -349,7 +351,7 @@ s3c24x0_alloc_resource(device_t bus, device_t child, int type, int *rid,
 			panic("Unable to map address space %#lX-%#lX", start,
 			    end);
 
-		rman_set_bustag(res, &s3c2xx0_bs_tag);
+		rman_set_bustag(res, s3c2xx0_bs_tag);
 		rman_set_bushandle(res, start);
 		if (flags & RF_ACTIVE) {
 			if (bus_activate_resource(child, type, *rid, res)) {
@@ -442,8 +444,9 @@ s3c24x0_attach(device_t dev)
 	unsigned int i, j;
 	u_long irqmax;
 
+	s3c2xx0_bs_tag = arm_base_bs_tag;
 	s3c2xx0_softc = &(sc->sc_sx);
-	sc->sc_sx.sc_iot = iot = &s3c2xx0_bs_tag;
+	sc->sc_sx.sc_iot = iot = s3c2xx0_bs_tag;
 	s3c2xx0_softc->s3c2xx0_irq_rman.rm_type = RMAN_ARRAY;
 	s3c2xx0_softc->s3c2xx0_irq_rman.rm_descr = "S3C24X0 IRQs";
 	s3c2xx0_softc->s3c2xx0_mem_rman.rm_type = RMAN_ARRAY;
@@ -641,7 +644,7 @@ cpu_reset(void)
 {
 	(void) disable_interrupts(PSR_I|PSR_F);
 
-	bus_space_write_4(&s3c2xx0_bs_tag, s3c2xx0_softc->sc_wdt_ioh, WDT_WTCON,
+	bus_space_write_4(s3c2xx0_bs_tag, s3c2xx0_softc->sc_wdt_ioh, WDT_WTCON,
 	    WTCON_ENABLE | WTCON_CLKSEL_16 | WTCON_ENRST);
 	for(;;);
 }
@@ -651,9 +654,9 @@ s3c24x0_sleep(int mode __unused)
 {
 	int reg;
 
-	reg = bus_space_read_4(&s3c2xx0_bs_tag, s3c2xx0_softc->sc_clkman_ioh,
+	reg = bus_space_read_4(s3c2xx0_bs_tag, s3c2xx0_softc->sc_clkman_ioh,
 	    CLKMAN_CLKCON);
-	bus_space_write_4(&s3c2xx0_bs_tag, s3c2xx0_softc->sc_clkman_ioh,
+	bus_space_write_4(s3c2xx0_bs_tag, s3c2xx0_softc->sc_clkman_ioh,
 	    CLKMAN_CLKCON, reg | CLKCON_IDLE);
 }
 
@@ -664,15 +667,15 @@ arm_get_next_irq(int last __unused)
 	uint32_t intpnd;
 	int irq, subirq;
 
-	if ((irq = bus_space_read_4(&s3c2xx0_bs_tag,
+	if ((irq = bus_space_read_4(s3c2xx0_bs_tag,
 	    s3c2xx0_softc->sc_intctl_ioh, INTCTL_INTOFFSET)) != 0) {
 
 		/* Clear the pending bit */
-		intpnd = bus_space_read_4(&s3c2xx0_bs_tag,
+		intpnd = bus_space_read_4(s3c2xx0_bs_tag,
 		    s3c2xx0_softc->sc_intctl_ioh, INTCTL_INTPND);
-		bus_space_write_4(&s3c2xx0_bs_tag, s3c2xx0_softc->sc_intctl_ioh,
+		bus_space_write_4(s3c2xx0_bs_tag, s3c2xx0_softc->sc_intctl_ioh,
 		    INTCTL_SRCPND, intpnd);
-		bus_space_write_4(&s3c2xx0_bs_tag, s3c2xx0_softc->sc_intctl_ioh,
+		bus_space_write_4(s3c2xx0_bs_tag, s3c2xx0_softc->sc_intctl_ioh,
 		    INTCTL_INTPND, intpnd);
 
 		switch (irq) {
@@ -682,9 +685,9 @@ arm_get_next_irq(int last __unused)
 		case S3C24X0_INT_UART2:
 			/* Find the sub IRQ */
 			subirq = 0x7ff;
-			subirq &= bus_space_read_4(&s3c2xx0_bs_tag,
+			subirq &= bus_space_read_4(s3c2xx0_bs_tag,
 			    s3c2xx0_softc->sc_intctl_ioh, INTCTL_SUBSRCPND);
-			subirq &= ~(bus_space_read_4(&s3c2xx0_bs_tag,
+			subirq &= ~(bus_space_read_4(s3c2xx0_bs_tag,
 			    s3c2xx0_softc->sc_intctl_ioh, INTCTL_INTSUBMSK));
 			if (subirq == 0)
 				return (irq);
@@ -692,7 +695,7 @@ arm_get_next_irq(int last __unused)
 			subirq = ffs(subirq) - 1;
 
 			/* Clear the sub irq pending bit */
-			bus_space_write_4(&s3c2xx0_bs_tag,
+			bus_space_write_4(s3c2xx0_bs_tag,
 			    s3c2xx0_softc->sc_intctl_ioh, INTCTL_SUBSRCPND,
 			    (1 << subirq));
 
@@ -716,9 +719,9 @@ arm_get_next_irq(int last __unused)
 		case S3C24X0_INT_8_23:
 			/* Find the external interrupt being called */
 			subirq = 0x7fffff;
-			subirq &= bus_space_read_4(&s3c2xx0_bs_tag,
+			subirq &= bus_space_read_4(s3c2xx0_bs_tag,
 			    s3c2xx0_softc->sc_gpio_ioh, GPIO_EINTPEND);
-			subirq &= ~bus_space_read_4(&s3c2xx0_bs_tag,
+			subirq &= ~bus_space_read_4(s3c2xx0_bs_tag,
 			    s3c2xx0_softc->sc_gpio_ioh, GPIO_EINTMASK);
 			if (subirq == 0)
 				return (irq);
@@ -726,7 +729,7 @@ arm_get_next_irq(int last __unused)
 			subirq = ffs(subirq) - 1;
 
 			/* Clear the external irq pending bit */
-			bus_space_write_4(&s3c2xx0_bs_tag,
+			bus_space_write_4(s3c2xx0_bs_tag,
 			    s3c2xx0_softc->sc_gpio_ioh, GPIO_EINTPEND,
 			    (1 << subirq));
 
@@ -748,22 +751,22 @@ arm_mask_irq(uintptr_t irq)
 		irq -= S3C24X0_EXTIRQ_MIN;
 	}
 	if (irq < S3C24X0_SUBIRQ_MIN) {
-		mask = bus_space_read_4(&s3c2xx0_bs_tag,
+		mask = bus_space_read_4(s3c2xx0_bs_tag,
 		    s3c2xx0_softc->sc_intctl_ioh, INTCTL_INTMSK);
 		mask |= (1 << irq);
-		bus_space_write_4(&s3c2xx0_bs_tag,
+		bus_space_write_4(s3c2xx0_bs_tag,
 		    s3c2xx0_softc->sc_intctl_ioh, INTCTL_INTMSK, mask);
 	} else if (irq < S3C24X0_EXTIRQ_MIN) {
-		mask = bus_space_read_4(&s3c2xx0_bs_tag,
+		mask = bus_space_read_4(s3c2xx0_bs_tag,
 		    s3c2xx0_softc->sc_intctl_ioh, INTCTL_INTSUBMSK);
 		mask |= (1 << (irq - S3C24X0_SUBIRQ_MIN));
-		bus_space_write_4(&s3c2xx0_bs_tag,
+		bus_space_write_4(s3c2xx0_bs_tag,
 		    s3c2xx0_softc->sc_intctl_ioh, INTCTL_INTSUBMSK, mask);
 	} else {
-		mask = bus_space_read_4(&s3c2xx0_bs_tag,
+		mask = bus_space_read_4(s3c2xx0_bs_tag,
 		    s3c2xx0_softc->sc_gpio_ioh, GPIO_EINTMASK);
 		mask |= (1 << (irq - S3C24X0_EXTIRQ_MIN));
-		bus_space_write_4(&s3c2xx0_bs_tag,
+		bus_space_write_4(s3c2xx0_bs_tag,
 		    s3c2xx0_softc->sc_intctl_ioh, GPIO_EINTMASK, mask);
 	}
 }
@@ -778,22 +781,22 @@ arm_unmask_irq(uintptr_t irq)
 		irq -= S3C24X0_EXTIRQ_MIN;
 	}
 	if (irq < S3C24X0_SUBIRQ_MIN) {
-		mask = bus_space_read_4(&s3c2xx0_bs_tag,
+		mask = bus_space_read_4(s3c2xx0_bs_tag,
 		    s3c2xx0_softc->sc_intctl_ioh, INTCTL_INTMSK);
 		mask &= ~(1 << irq);
-		bus_space_write_4(&s3c2xx0_bs_tag,
+		bus_space_write_4(s3c2xx0_bs_tag,
 		    s3c2xx0_softc->sc_intctl_ioh, INTCTL_INTMSK, mask);
 	} else if (irq < S3C24X0_EXTIRQ_MIN) {
-		mask = bus_space_read_4(&s3c2xx0_bs_tag,
+		mask = bus_space_read_4(s3c2xx0_bs_tag,
 		    s3c2xx0_softc->sc_intctl_ioh, INTCTL_INTSUBMSK);
 		mask &= ~(1 << (irq - S3C24X0_SUBIRQ_MIN));
-		bus_space_write_4(&s3c2xx0_bs_tag,
+		bus_space_write_4(s3c2xx0_bs_tag,
 		    s3c2xx0_softc->sc_intctl_ioh, INTCTL_INTSUBMSK, mask);
 	} else {
-		mask = bus_space_read_4(&s3c2xx0_bs_tag,
+		mask = bus_space_read_4(s3c2xx0_bs_tag,
 		    s3c2xx0_softc->sc_gpio_ioh, GPIO_EINTMASK);
 		mask &= ~(1 << (irq - S3C24X0_EXTIRQ_MIN));
-		bus_space_write_4(&s3c2xx0_bs_tag,
+		bus_space_write_4(s3c2xx0_bs_tag,
 		    s3c2xx0_softc->sc_intctl_ioh, GPIO_EINTMASK, mask);
 	}
 }
diff --git a/sys/arm/samsung/s3c2xx0/s3c2xx0var.h b/sys/arm/samsung/s3c2xx0/s3c2xx0var.h
index fea0982e9b0..505c6b0f789 100644
--- a/sys/arm/samsung/s3c2xx0/s3c2xx0var.h
+++ b/sys/arm/samsung/s3c2xx0/s3c2xx0var.h
@@ -74,7 +74,7 @@ struct s3c2xx0_ivar {
 
 typedef void *s3c2xx0_chipset_tag_t;
 
-extern struct bus_space s3c2xx0_bs_tag;
+extern bus_space_tag_t s3c2xx0_bs_tag;
 extern struct s3c2xx0_softc *s3c2xx0_softc;
 extern struct arm32_bus_dma_tag s3c2xx0_bus_dma;
 
diff --git a/sys/arm/samsung/s3c2xx0/uart_cpu_s3c2410.c b/sys/arm/samsung/s3c2xx0/uart_cpu_s3c2410.c
index 37b668e85da..1beb12d7731 100644
--- a/sys/arm/samsung/s3c2xx0/uart_cpu_s3c2410.c
+++ b/sys/arm/samsung/s3c2xx0/uart_cpu_s3c2410.c
@@ -61,7 +61,7 @@ uart_cpu_getdev(int devtype, struct uart_devinfo *di)
 
 	di->ops = uart_getops(&uart_s3c2410_class);
 	di->bas.chan = 0;
-	di->bas.bst = &s3c2xx0_bs_tag;
+	di->bas.bst = s3c2xx0_bs_tag;
 	di->bas.bsh = s3c2410_uart_vaddr;
 	di->bas.regshft = 0;
 	di->bas.rclk = s3c2410_pclk;
@@ -69,7 +69,7 @@ uart_cpu_getdev(int devtype, struct uart_devinfo *di)
 	di->databits = 8;
 	di->stopbits = 1;
 	di->parity = UART_PARITY_NONE;
-	uart_bus_space_io = &s3c2xx0_bs_tag;
+	uart_bus_space_io = s3c2xx0_bs_tag;
 	uart_bus_space_mem = NULL;
 	
 	return (0);

From 41adadc9c03f878849b644af61331fdda6692193 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 05:05:07 +0000
Subject: [PATCH 142/279] Use the base arm bus_space instead of an identical
 local copy.

---
 sys/arm/xscale/i80321/ep80219_machdep.c |   2 +-
 sys/arm/xscale/i80321/files.ep80219     |   1 -
 sys/arm/xscale/i80321/files.i80219      |   1 +
 sys/arm/xscale/i80321/files.i80321      |   1 +
 sys/arm/xscale/i80321/files.iq31244     |   1 -
 sys/arm/xscale/i80321/iq31244_machdep.c |   2 +-
 sys/arm/xscale/i80321/obio.c            |   5 +-
 sys/arm/xscale/i80321/obio_space.c      | 127 ------------------------
 sys/arm/xscale/i80321/obiovar.h         |   2 +-
 sys/arm/xscale/i80321/uart_cpu_i80321.c |   4 +-
 sys/arm/xscale/i8134x/crb_machdep.c     |   2 +-
 sys/arm/xscale/i8134x/files.i81342      |   2 +-
 sys/arm/xscale/i8134x/obio.c            |   4 +-
 sys/arm/xscale/i8134x/obiovar.h         |   2 +-
 sys/arm/xscale/i8134x/uart_cpu_i81342.c |   4 +-
 15 files changed, 19 insertions(+), 141 deletions(-)
 delete mode 100644 sys/arm/xscale/i80321/obio_space.c

diff --git a/sys/arm/xscale/i80321/ep80219_machdep.c b/sys/arm/xscale/i80321/ep80219_machdep.c
index bf3e49ada30..f04f7fdef97 100644
--- a/sys/arm/xscale/i80321/ep80219_machdep.c
+++ b/sys/arm/xscale/i80321/ep80219_machdep.c
@@ -312,7 +312,7 @@ initarm(struct arm_boot_params *abp)
 	 * registers.
 	 */
 	i80321_calibrate_delay();
-	i80321_sdram_bounds(&obio_bs_tag, IQ80321_80321_VBASE + VERDE_MCU_BASE,
+	i80321_sdram_bounds(obio_bs_tag, IQ80321_80321_VBASE + VERDE_MCU_BASE,
 	    &memstart, &memsize);
 	physmem = memsize / PAGE_SIZE;
 	cninit();
diff --git a/sys/arm/xscale/i80321/files.ep80219 b/sys/arm/xscale/i80321/files.ep80219
index cbf2e39ed5a..0eaf9db2872 100644
--- a/sys/arm/xscale/i80321/files.ep80219
+++ b/sys/arm/xscale/i80321/files.ep80219
@@ -6,7 +6,6 @@
 arm/xscale/i80321/iq80321.c		standard
 arm/xscale/i80321/ep80219_machdep.c	standard
 arm/xscale/i80321/obio.c		standard
-arm/xscale/i80321/obio_space.c		standard
 arm/xscale/i80321/uart_cpu_i80321.c	optional	uart
 arm/xscale/i80321/uart_bus_i80321.c	optional	uart
 dev/uart/uart_dev_ns8250.c		optional	uart
diff --git a/sys/arm/xscale/i80321/files.i80219 b/sys/arm/xscale/i80321/files.i80219
index e9d9eab9564..10b7630b3da 100644
--- a/sys/arm/xscale/i80321/files.i80219
+++ b/sys/arm/xscale/i80321/files.i80219
@@ -2,6 +2,7 @@
 #
 # IOP Specific
 # 
+arm/arm/bus_space_base.c		standard
 arm/arm/bus_space_generic.c		standard
 arm/arm/cpufunc_asm_xscale.S		standard
 arm/xscale/i80321/i80321.c		standard
diff --git a/sys/arm/xscale/i80321/files.i80321 b/sys/arm/xscale/i80321/files.i80321
index 48082242e3a..48f9b08a07f 100644
--- a/sys/arm/xscale/i80321/files.i80321
+++ b/sys/arm/xscale/i80321/files.i80321
@@ -1,4 +1,5 @@
 #$FreeBSD$
+arm/arm/bus_space_base.c		standard
 arm/arm/bus_space_generic.c		standard
 arm/arm/cpufunc_asm_xscale.S		standard
 arm/xscale/i80321/i80321.c		standard
diff --git a/sys/arm/xscale/i80321/files.iq31244 b/sys/arm/xscale/i80321/files.iq31244
index 66d89b3db63..d28d49b3a6b 100644
--- a/sys/arm/xscale/i80321/files.iq31244
+++ b/sys/arm/xscale/i80321/files.iq31244
@@ -3,7 +3,6 @@ arm/xscale/i80321/iq80321.c		standard
 arm/xscale/i80321/iq31244_machdep.c	standard
 arm/xscale/i80321/iq31244_7seg.c	optional	iq31244_7seg
 arm/xscale/i80321/obio.c		standard
-arm/xscale/i80321/obio_space.c		standard
 arm/xscale/i80321/uart_cpu_i80321.c	optional	uart
 arm/xscale/i80321/uart_bus_i80321.c	optional	uart
 dev/uart/uart_dev_ns8250.c		optional	uart
diff --git a/sys/arm/xscale/i80321/iq31244_machdep.c b/sys/arm/xscale/i80321/iq31244_machdep.c
index 9c119ed25bc..1a1b9e72fc7 100644
--- a/sys/arm/xscale/i80321/iq31244_machdep.c
+++ b/sys/arm/xscale/i80321/iq31244_machdep.c
@@ -313,7 +313,7 @@ initarm(struct arm_boot_params *abp)
 	 * registers.
 	 */
 	i80321_calibrate_delay();
-	i80321_sdram_bounds(&obio_bs_tag, IQ80321_80321_VBASE + VERDE_MCU_BASE,
+	i80321_sdram_bounds(obio_bs_tag, IQ80321_80321_VBASE + VERDE_MCU_BASE,
 	    &memstart, &memsize);
 	physmem = memsize / PAGE_SIZE;
 	cninit();
diff --git a/sys/arm/xscale/i80321/obio.c b/sys/arm/xscale/i80321/obio.c
index 566d6cd11ce..55218ccd20d 100644
--- a/sys/arm/xscale/i80321/obio.c
+++ b/sys/arm/xscale/i80321/obio.c
@@ -58,6 +58,8 @@ __FBSDID("$FreeBSD$");
 #include <arm/xscale/i80321/iq80321reg.h>
 #include <arm/xscale/i80321/obiovar.h>
 
+bus_space_tag_t obio_bs_tag;
+
 int	obio_probe(device_t);
 int	obio_attach(device_t);
 
@@ -72,7 +74,8 @@ obio_attach(device_t dev)
 {
 	struct obio_softc *sc = device_get_softc(dev);
 
-	sc->oba_st = &obio_bs_tag;
+	obio_bs_tag = arm_base_bs_tag;
+        sc->oba_st = obio_bs_tag;
 	sc->oba_addr = IQ80321_OBIO_BASE;
 	sc->oba_size = IQ80321_OBIO_SIZE;
 	sc->oba_rman.rm_type = RMAN_ARRAY;
diff --git a/sys/arm/xscale/i80321/obio_space.c b/sys/arm/xscale/i80321/obio_space.c
deleted file mode 100644
index 4eba7000dbc..00000000000
--- a/sys/arm/xscale/i80321/obio_space.c
+++ /dev/null
@@ -1,127 +0,0 @@
-/*	$NetBSD: obio_space.c,v 1.6 2003/07/15 00:25:05 lukem Exp $	*/
-
-/*-
- * Copyright (c) 2001, 2002, 2003 Wasabi Systems, Inc.
- * All rights reserved.
- *
- * Written by Jason R. Thorpe for Wasabi Systems, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed for the NetBSD Project by
- *	Wasabi Systems, Inc.
- * 4. The name of Wasabi Systems, Inc. may not be used to endorse
- *    or promote products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
- * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-/*
- * bus_space functions for IQ80321 on-board devices
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-
-#include <machine/bus.h>
-
-/* Prototypes for all the bus_space structure functions */
-bs_protos(generic);
-
-/*
- * The obio bus space tag.  This is constant for all instances, so
- * we never have to explicitly "create" it.
- */
-struct bus_space obio_bs_tag = {
-	/* cookie */
-	(void *) 0,
-
-	/* mapping/unmapping */
-	generic_bs_map,
-	generic_bs_unmap,
-	generic_bs_subregion,
-
-	/* allocation/deallocation */
-	generic_bs_alloc,
-	generic_bs_free,
-
-	/* barrier */
-	generic_bs_barrier,
-
-	/* read (single) */
-	generic_bs_r_1,
-	generic_bs_r_2,
-	generic_bs_r_4,
-	NULL,
-
-	/* read multiple */
-	generic_bs_rm_1,
-	NULL,
-	NULL,
-	NULL,
-
-	/* read region */
-	generic_bs_rr_1,
-	NULL,
-	NULL,
-	NULL,
-
-	/* write (single) */
-	generic_bs_w_1,
-	generic_bs_w_2,
-	generic_bs_w_4,
-	NULL,
-
-	/* write multiple */
-	generic_bs_wm_1,
-	NULL,
-	NULL,
-	NULL,
-
-	/* write region */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* set multiple */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* set region */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* copy */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-};
diff --git a/sys/arm/xscale/i80321/obiovar.h b/sys/arm/xscale/i80321/obiovar.h
index 8e0c1459d70..182b52f57f5 100644
--- a/sys/arm/xscale/i80321/obiovar.h
+++ b/sys/arm/xscale/i80321/obiovar.h
@@ -53,6 +53,6 @@ struct obio_softc {
 	struct rman oba_irq_rman;
 	
 };
-extern struct bus_space obio_bs_tag;
+extern bus_space_tag_t obio_bs_tag;
 
 #endif /* _IQ80321_OBIOVAR_H_ */
diff --git a/sys/arm/xscale/i80321/uart_cpu_i80321.c b/sys/arm/xscale/i80321/uart_cpu_i80321.c
index 32dd463cfb4..a3faec7289d 100644
--- a/sys/arm/xscale/i80321/uart_cpu_i80321.c
+++ b/sys/arm/xscale/i80321/uart_cpu_i80321.c
@@ -53,14 +53,14 @@ uart_cpu_getdev(int devtype, struct uart_devinfo *di)
 {
 	di->ops = uart_getops(&uart_ns8250_class);
 	di->bas.chan = 0;
-	di->bas.bst = &obio_bs_tag;
+	di->bas.bst = obio_bs_tag;
 	di->bas.regshft = 0;
 	di->bas.rclk = 0;
 	di->baudrate = 115200;
 	di->databits = 8;
 	di->stopbits = 1;
 	di->parity = UART_PARITY_NONE;
-	uart_bus_space_io = &obio_bs_tag;
+	uart_bus_space_io = obio_bs_tag;
 	uart_bus_space_mem = NULL;
 	di->bas.bsh = 0xfe800000;
 	return (0);
diff --git a/sys/arm/xscale/i8134x/crb_machdep.c b/sys/arm/xscale/i8134x/crb_machdep.c
index 195304485bf..3e6440d4b01 100644
--- a/sys/arm/xscale/i8134x/crb_machdep.c
+++ b/sys/arm/xscale/i8134x/crb_machdep.c
@@ -293,7 +293,7 @@ initarm(struct arm_boot_params *abp)
 	cpu_setup("");
 
 	i80321_calibrate_delay();
-	i81342_sdram_bounds(&obio_bs_tag, IOP34X_VADDR, &memstart, &memsize);
+	i81342_sdram_bounds(obio_bs_tag, IOP34X_VADDR, &memstart, &memsize);
 	physmem = memsize / PAGE_SIZE;
 	cninit();
 	/* Set stack for exception handlers */
diff --git a/sys/arm/xscale/i8134x/files.i81342 b/sys/arm/xscale/i8134x/files.i81342
index c9bd619ed64..08930168985 100644
--- a/sys/arm/xscale/i8134x/files.i81342
+++ b/sys/arm/xscale/i8134x/files.i81342
@@ -1,4 +1,5 @@
 # $FreeBSD$
+arm/arm/bus_space_base.c		standard
 arm/arm/bus_space_generic.c		standard
 arm/arm/cpufunc_asm_xscale.S		standard
 arm/arm/cpufunc_asm_xscale_c3.S		standard
@@ -9,7 +10,6 @@ arm/xscale/i8134x/i81342_mcu.c		standard
 arm/xscale/i8134x/i81342_pci.c		optional	pci
 arm/xscale/i8134x/i81342_space.c	standard
 arm/xscale/i8134x/obio.c		standard
-arm/xscale/i8134x/obio_space.c		standard
 arm/xscale/i8134x/uart_bus_i81342.c	optional	uart
 arm/xscale/i8134x/uart_cpu_i81342.c	optional	uart
 dev/uart/uart_dev_ns8250.c		optional	uart
diff --git a/sys/arm/xscale/i8134x/obio.c b/sys/arm/xscale/i8134x/obio.c
index b8bccce43a1..5aa3c477d2e 100644
--- a/sys/arm/xscale/i8134x/obio.c
+++ b/sys/arm/xscale/i8134x/obio.c
@@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$");
 #include <arm/xscale/i8134x/i81342reg.h>
 #include <arm/xscale/i8134x/obiovar.h>
 
+bus_space_tag_t obio_bs_tag;
 
 static int
 obio_probe(device_t dev)
@@ -68,7 +69,8 @@ obio_attach(device_t dev)
 {
 	struct obio_softc *sc = device_get_softc(dev);
 
-	sc->oba_st = &obio_bs_tag;
+	obio_bs_tag = arm_base_bs_tag;
+	sc->oba_st = obio_bs_tag;
 	sc->oba_rman.rm_type = RMAN_ARRAY;
 	sc->oba_rman.rm_descr = "OBIO I/O";
 	if (rman_init(&sc->oba_rman) != 0 ||
diff --git a/sys/arm/xscale/i8134x/obiovar.h b/sys/arm/xscale/i8134x/obiovar.h
index 7350d0ade1b..237dd9e7d94 100644
--- a/sys/arm/xscale/i8134x/obiovar.h
+++ b/sys/arm/xscale/i8134x/obiovar.h
@@ -50,6 +50,6 @@ struct obio_softc {
 	struct rman oba_irq_rman;
 	
 };
-extern struct bus_space obio_bs_tag;
+extern bus_space_tag_t obio_bs_tag;
 
 #endif /* _IQ80321_OBIOVAR_H_ */
diff --git a/sys/arm/xscale/i8134x/uart_cpu_i81342.c b/sys/arm/xscale/i8134x/uart_cpu_i81342.c
index 2cb2902a9d1..1d98d78bcfe 100644
--- a/sys/arm/xscale/i8134x/uart_cpu_i81342.c
+++ b/sys/arm/xscale/i8134x/uart_cpu_i81342.c
@@ -54,14 +54,14 @@ uart_cpu_getdev(int devtype, struct uart_devinfo *di)
 
 	di->ops = uart_getops(&uart_ns8250_class);
 	di->bas.chan = 0;
-	di->bas.bst = &obio_bs_tag;
+	di->bas.bst = obio_bs_tag;
 	di->bas.regshft = 2;
 	di->bas.rclk = 33334000;
 	di->baudrate = 115200;
 	di->databits = 8;
 	di->stopbits = 1;
 	di->parity = UART_PARITY_NONE;
-	uart_bus_space_io = &obio_bs_tag;
+	uart_bus_space_io = obio_bs_tag;
 	uart_bus_space_mem = NULL;
 	di->bas.bsh = IOP34X_UART0_VADDR;
 	return (0);

From ccae8a4b0ef0871d083bb3a6fd94e510f18ae818 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 05:10:23 +0000
Subject: [PATCH 143/279] Use the base arm bus_space instead of an identical
 local copy.

---
 sys/arm/xscale/i80321/i80321_space.c | 118 +--------------------------
 sys/arm/xscale/i8134x/i81342_space.c | 118 +--------------------------
 2 files changed, 6 insertions(+), 230 deletions(-)

diff --git a/sys/arm/xscale/i80321/i80321_space.c b/sys/arm/xscale/i80321/i80321_space.c
index da4ad2185a8..64c222522cd 100644
--- a/sys/arm/xscale/i80321/i80321_space.c
+++ b/sys/arm/xscale/i80321/i80321_space.c
@@ -63,124 +63,12 @@ __FBSDID("$FreeBSD$");
 bs_protos(i80321);
 bs_protos(i80321_io);
 bs_protos(i80321_mem);
-bs_protos(generic);
-
-/*
- * Template bus_space -- copied, and the bits that are NULL are
- * filled in.
- */
-const struct bus_space i80321_bs_tag_template = {
-	/* cookie */
-	(void *) 0,
-
-	/* mapping/unmapping */
-	NULL,
-	NULL,
-	i80321_bs_subregion,
-
-	/* allocation/deallocation */
-	NULL,
-	NULL,
-
-	/* barrier */
-	i80321_bs_barrier,
-
-	/* read (single) */
-	generic_bs_r_1,
-	generic_bs_r_2,
-	generic_bs_r_4,
-	NULL,
-
-	/* read multiple */
-	generic_bs_rm_1,
-	generic_bs_rm_2,
-	generic_bs_rm_4,
-	NULL,
-
-	/* read region */
-	generic_bs_rr_1,
-	generic_bs_rr_2,
-	generic_bs_rr_4,
-	NULL,
-
-	/* write (single) */
-	generic_bs_w_1,
-	generic_bs_w_2,
-	generic_bs_w_4,
-	NULL,
-
-	/* write multiple */
-	generic_bs_wm_1,
-	generic_bs_wm_2,
-	generic_bs_wm_4,
-	NULL,
-
-	/* write region */
-	NULL,
-	generic_bs_wr_2,
-	generic_bs_wr_4,
-	NULL,
-
-	/* set multiple */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* set region */
-	NULL,
-	generic_bs_sr_2,
-	generic_bs_sr_4,
-	NULL,
-
-	/* copy */
-	NULL,
-	generic_bs_c_2,
-	NULL,
-	NULL,
-
-	/* read (single) stream */
-	generic_bs_r_1,
-	generic_bs_r_2,
-	generic_bs_r_4,
-	NULL,
-
-	/* read multiple stream */
-	generic_bs_rm_1,
-	generic_bs_rm_2,
-	generic_bs_rm_4,
-	NULL,
-
-	/* read region stream */
-	generic_bs_rr_1,
-	generic_bs_rr_2,
-	generic_bs_rr_4,
-	NULL,
-
-	/* write (single) stream */
-	generic_bs_w_1,
-	generic_bs_w_2,
-	generic_bs_w_4,
-	NULL,
-
-	/* write multiple stream */
-	generic_bs_wm_1,
-	generic_bs_wm_2,
-	generic_bs_wm_4,
-	NULL,
-
-	/* write region stream */
-	NULL,
-	generic_bs_wr_2,
-	generic_bs_wr_4,
-	NULL,
-};
 
 void
 i80321_bs_init(bus_space_tag_t bs, void *cookie)
 {
 
-	*bs = i80321_bs_tag_template;
+	*bs = *arm_base_bs_tag;
 	bs->bs_privdata = cookie;
 }
 
@@ -188,7 +76,7 @@ void
 i80321_io_bs_init(bus_space_tag_t bs, void *cookie)
 {
 
-	*bs = i80321_bs_tag_template;
+	*bs = *arm_base_bs_tag;
 	bs->bs_privdata = cookie;
 
 	bs->bs_map = i80321_io_bs_map;
@@ -202,7 +90,7 @@ void
 i80321_mem_bs_init(bus_space_tag_t bs, void *cookie)
 {
 
-	*bs = i80321_bs_tag_template;
+	*bs = *arm_base_bs_tag;
 	bs->bs_privdata = cookie;
 
 	bs->bs_map = i80321_mem_bs_map;
diff --git a/sys/arm/xscale/i8134x/i81342_space.c b/sys/arm/xscale/i8134x/i81342_space.c
index 0f7912c83ca..d5d9143c70c 100644
--- a/sys/arm/xscale/i8134x/i81342_space.c
+++ b/sys/arm/xscale/i8134x/i81342_space.c
@@ -64,124 +64,12 @@ __FBSDID("$FreeBSD$");
 bs_protos(i81342);
 bs_protos(i81342_io);
 bs_protos(i81342_mem);
-bs_protos(generic);
-
-/*
- * Template bus_space -- copied, and the bits that are NULL are
- * filled in.
- */
-const struct bus_space i81342_bs_tag_template = {
-	/* cookie */
-	(void *) 0,
-
-	/* mapping/unmapping */
-	NULL,
-	NULL,
-	i81342_bs_subregion,
-
-	/* allocation/deallocation */
-	NULL,
-	NULL,
-
-	/* barrier */
-	i81342_bs_barrier,
-
-	/* read (single) */
-	generic_bs_r_1,
-	generic_bs_r_2,
-	generic_bs_r_4,
-	NULL,
-
-	/* read multiple */
-	generic_bs_rm_1,
-	generic_bs_rm_2,
-	generic_bs_rm_4,
-	NULL,
-
-	/* read region */
-	generic_bs_rr_1,
-	generic_bs_rr_2,
-	generic_bs_rr_4,
-	NULL,
-
-	/* write (single) */
-	generic_bs_w_1,
-	generic_bs_w_2,
-	generic_bs_w_4,
-	NULL,
-
-	/* write multiple */
-	generic_bs_wm_1,
-	generic_bs_wm_2,
-	generic_bs_wm_4,
-	NULL,
-
-	/* write region */
-	NULL,
-	generic_bs_wr_2,
-	generic_bs_wr_4,
-	NULL,
-
-	/* set multiple */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* set region */
-	NULL,
-	generic_bs_sr_2,
-	generic_bs_sr_4,
-	NULL,
-
-	/* copy */
-	NULL,
-	generic_bs_c_2,
-	NULL,
-	NULL,
-
-	/* read (single) stream */
-	generic_bs_r_1,
-	generic_bs_r_2,
-	generic_bs_r_4,
-	NULL,
-
-	/* read multiple stream */
-	generic_bs_rm_1,
-	generic_bs_rm_2,
-	generic_bs_rm_4,
-	NULL,
-
-	/* read region stream */
-	generic_bs_rr_1,
-	generic_bs_rr_2,
-	generic_bs_rr_4,
-	NULL,
-
-	/* write (single) stream */
-	generic_bs_w_1,
-	generic_bs_w_2,
-	generic_bs_w_4,
-	NULL,
-
-	/* write multiple stream */
-	generic_bs_wm_1,
-	generic_bs_wm_2,
-	generic_bs_wm_4,
-	NULL,
-
-	/* write region stream */
-	NULL,
-	generic_bs_wr_2,
-	generic_bs_wr_4,
-	NULL,
-};
 
 void
 i81342_bs_init(bus_space_tag_t bs, void *cookie)
 {
 
-	*bs = i81342_bs_tag_template;
+	*bs = *arm_base_bs_tag;
 	bs->bs_privdata = cookie;
 }
 
@@ -189,7 +77,7 @@ void
 i81342_io_bs_init(bus_space_tag_t bs, void *cookie)
 {
 
-	*bs = i81342_bs_tag_template;
+	*bs = *arm_base_bs_tag;
 	bs->bs_privdata = cookie;
 
 	bs->bs_map = i81342_io_bs_map;
@@ -203,7 +91,7 @@ void
 i81342_mem_bs_init(bus_space_tag_t bs, void *cookie)
 {
 
-	*bs = i81342_bs_tag_template;
+	*bs = *arm_base_bs_tag;
 	bs->bs_privdata = cookie;
 
 	bs->bs_map = i81342_mem_bs_map;

From e81002d93aa6f6fc970d69c4202d8471706899f7 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 05:23:09 +0000
Subject: [PATCH 144/279] Use explicit initializer style, fill in missing
 functions as unimplemented.

---
 sys/arm/xscale/pxa/pxa_space.c | 122 +++++++++++++++++++++------------
 1 file changed, 79 insertions(+), 43 deletions(-)

diff --git a/sys/arm/xscale/pxa/pxa_space.c b/sys/arm/xscale/pxa/pxa_space.c
index 7a9160a8974..35b6cd9f422 100644
--- a/sys/arm/xscale/pxa/pxa_space.c
+++ b/sys/arm/xscale/pxa/pxa_space.c
@@ -65,73 +65,109 @@ bs_protos(pxa);
  */
 struct bus_space _base_tag = {
 	/* cookie */
-	(void *) 0,
+	.bs_privdata	= NULL,
 
 	/* mapping/unmapping */
-	generic_bs_map,
-	generic_bs_unmap,
-	generic_bs_subregion,
+	.bs_map		= generic_bs_map,
+	.bs_unmap	= generic_bs_unmap,
+	.bs_subregion	= generic_bs_subregion,
 
 	/* allocation/deallocation */
-	generic_bs_alloc,
-	generic_bs_free,
+	.bs_alloc	= generic_bs_alloc,
+	.bs_free	= generic_bs_free,
 
 	/* barrier */
-	generic_bs_barrier,
+	.bs_barrier	= generic_bs_barrier,
 
 	/* read (single) */
-	pxa_bs_r_1,
-	pxa_bs_r_2,
-	pxa_bs_r_4,
-	NULL,
+	.bs_r_1		= pxa_bs_r_1,
+	.bs_r_2		= pxa_bs_r_2,
+	.bs_r_4		= pxa_bs_r_4,
+	.bs_r_8		= BS_UNIMPLEMENTED,
 
 	/* read multiple */
-	pxa_bs_rm_1,
-	pxa_bs_rm_2,
-	NULL,
-	NULL,
+	.bs_rm_1	= pxa_bs_rm_1,
+	.bs_rm_2	= pxa_bs_rm_2,
+	.bs_rm_4	= BS_UNIMPLEMENTED,
+	.bs_rm_8	= BS_UNIMPLEMENTED,
 
 	/* read region */
-	pxa_bs_rr_1,
-	NULL,
-	NULL,
-	NULL,
+	.bs_rr_1	= pxa_bs_rr_1,
+	.bs_rr_2	= BS_UNIMPLEMENTED,
+	.bs_rr_4	= BS_UNIMPLEMENTED,
+	.bs_rr_8	= BS_UNIMPLEMENTED,
 
 	/* write (single) */
-	pxa_bs_w_1,
-	pxa_bs_w_2,
-	pxa_bs_w_4,
-	NULL,
+	.bs_w_1		= pxa_bs_w_1,
+	.bs_w_2		= pxa_bs_w_2,
+	.bs_w_4		= pxa_bs_w_4,
+	.bs_w_8		= BS_UNIMPLEMENTED,
 
 	/* write multiple */
-	pxa_bs_wm_1,
-	pxa_bs_wm_2,
-	NULL,
-	NULL,
+	.bs_wm_1	= pxa_bs_wm_1,
+	.bs_wm_2	= pxa_bs_wm_2,
+	.bs_wm_4	= BS_UNIMPLEMENTED,
+	.bs_wm_8	= BS_UNIMPLEMENTED,
 
 	/* write region */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
+	.bs_wr_1	= BS_UNIMPLEMENTED,
+	.bs_wr_2	= BS_UNIMPLEMENTED,
+	.bs_wr_4	= BS_UNIMPLEMENTED,
+	.bs_wr_8	= BS_UNIMPLEMENTED,
 
 	/* set multiple */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
+	.bs_sm_1	= BS_UNIMPLEMENTED,
+	.bs_sm_2	= BS_UNIMPLEMENTED,
+	.bs_sm_4	= BS_UNIMPLEMENTED,
+	.bs_sm_8	= BS_UNIMPLEMENTED,
 
 	/* set region */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
+	.bs_sr_1	= BS_UNIMPLEMENTED,
+	.bs_sr_2	= BS_UNIMPLEMENTED,
+	.bs_sr_4	= BS_UNIMPLEMENTED,
+	.bs_sr_8	= BS_UNIMPLEMENTED,
 
 	/* copy */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
+	.bs_c_1		= BS_UNIMPLEMENTED,
+	.bs_c_2		= BS_UNIMPLEMENTED,
+	.bs_c_4		= BS_UNIMPLEMENTED,
+	.bs_c_8		= BS_UNIMPLEMENTED,
+
+	/* read stream (single) */
+	.bs_r_1_s	= BS_UNIMPLEMENTED,
+	.bs_r_2_s	= BS_UNIMPLEMENTED,
+	.bs_r_4_s	= BS_UNIMPLEMENTED,
+	.bs_r_8_s	= BS_UNIMPLEMENTED,
+
+	/* read multiple stream */
+	.bs_rm_1_s	= BS_UNIMPLEMENTED,
+	.bs_rm_2_s	= BS_UNIMPLEMENTED,
+	.bs_rm_4_s	= BS_UNIMPLEMENTED,
+	.bs_rm_8_s	= BS_UNIMPLEMENTED,
+
+	/* read region stream */
+	.bs_rr_1_s	= BS_UNIMPLEMENTED,
+	.bs_rr_2_s	= BS_UNIMPLEMENTED,
+	.bs_rr_4_s	= BS_UNIMPLEMENTED,
+	.bs_rr_8_s	= BS_UNIMPLEMENTED,
+
+	/* write stream (single) */
+	.bs_w_1_s	= BS_UNIMPLEMENTED, 
+	.bs_w_2_s	= BS_UNIMPLEMENTED, 
+	.bs_w_4_s	= BS_UNIMPLEMENTED, 
+	.bs_w_8_s	= BS_UNIMPLEMENTED, 
+
+	/* write multiple stream */
+	.bs_wm_1_s	= BS_UNIMPLEMENTED,
+	.bs_wm_2_s	= BS_UNIMPLEMENTED,
+	.bs_wm_4_s	= BS_UNIMPLEMENTED,
+	.bs_wm_8_s	= BS_UNIMPLEMENTED,
+
+	/* write region stream */
+	.bs_wr_1_s	= BS_UNIMPLEMENTED,
+	.bs_wr_2_s	= BS_UNIMPLEMENTED,
+	.bs_wr_4_s	= BS_UNIMPLEMENTED,
+	.bs_wr_8_s	= BS_UNIMPLEMENTED,
 };
 
 static struct bus_space	_obio_tag;

From 099cce842d8252f364ab18b38eb22ccaed40acf4 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 05:31:54 +0000
Subject: [PATCH 145/279] Remove a couple files that are no longer used
 (functionality take over by arm/bus_space_base.c).

---
 sys/arm/samsung/s3c2xx0/files.s3c2xx0   |   1 -
 sys/arm/samsung/s3c2xx0/s3c2xx0_space.c | 164 ------------------------
 sys/arm/xscale/i8134x/obio_space.c      | 127 ------------------
 3 files changed, 292 deletions(-)
 delete mode 100644 sys/arm/samsung/s3c2xx0/s3c2xx0_space.c
 delete mode 100644 sys/arm/xscale/i8134x/obio_space.c

diff --git a/sys/arm/samsung/s3c2xx0/files.s3c2xx0 b/sys/arm/samsung/s3c2xx0/files.s3c2xx0
index 2ab57d5190d..7b6c7e4e9ec 100644
--- a/sys/arm/samsung/s3c2xx0/files.s3c2xx0
+++ b/sys/arm/samsung/s3c2xx0/files.s3c2xx0
@@ -7,7 +7,6 @@ arm/samsung/s3c2xx0/board_ln2410sbc.c	optional	board_ln2410sbc
 arm/samsung/s3c2xx0/s3c24x0_rtc.c	standard
 arm/samsung/s3c2xx0/s3c24x0_machdep.c	standard
 arm/samsung/s3c2xx0/s3c24x0.c   	standard
-arm/samsung/s3c2xx0/s3c2xx0_space.c	standard
 arm/samsung/s3c2xx0/s3c24x0_clk.c	standard
 arm/samsung/s3c2xx0/uart_bus_s3c2410.c	optional	uart
 arm/samsung/s3c2xx0/uart_cpu_s3c2410.c	optional	uart
diff --git a/sys/arm/samsung/s3c2xx0/s3c2xx0_space.c b/sys/arm/samsung/s3c2xx0/s3c2xx0_space.c
deleted file mode 100644
index 61c00eb6841..00000000000
--- a/sys/arm/samsung/s3c2xx0/s3c2xx0_space.c
+++ /dev/null
@@ -1,164 +0,0 @@
-/*	$NetBSD: s3c2xx0_space.c,v 1.7 2005/11/24 13:08:32 yamt Exp $ */
-
-/*
- * Copyright (c) 2002 Fujitsu Component Limited
- * Copyright (c) 2002 Genetec Corporation
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. Neither the name of The Fujitsu Component Limited nor the name of
- *    Genetec corporation may not be used to endorse or promote products
- *    derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY FUJITSU COMPONENT LIMITED AND GENETEC
- * CORPORATION ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
- * INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
- * DISCLAIMED.  IN NO EVENT SHALL FUJITSU COMPONENT LIMITED OR GENETEC
- * CORPORATION BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
- * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
- * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
- * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
- * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
- * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-/* derived from sa11x0_io.c */
-
-/*
- * Copyright (c) 1997 Mark Brinicombe.
- * Copyright (c) 1997 Causality Limited.
- * All rights reserved.
- *
- * This code is derived from software contributed to The NetBSD Foundation
- * by Ichiro FUKUHARA.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by Mark Brinicombe.
- * 4. The name of the company nor the name of the author may be used to
- *    endorse or promote products derived from this software without specific
- *    prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
- * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
- * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
- * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
- * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- */
-
-/*
- * bus_space functions for Samsung S3C2800/2400/2410.
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-
-#include <vm/vm.h>
-#include <vm/vm_kern.h>
-#include <vm/pmap.h>
-#include <vm/vm_page.h>
-#include <vm/vm_extern.h>
-
-#include <machine/bus.h>
-
-/* Prototypes for all the bus_space structure functions */
-bs_protos(generic);
-
-struct bus_space s3c2xx0_bs_tag = {
-	/* cookie */
-	(void *) 0,
-
-	/* mapping/unmapping */
-	generic_bs_map,
-	generic_bs_unmap,
-	generic_bs_subregion,
-
-	/* allocation/deallocation */
-	generic_bs_alloc,	/* not implemented */
-	generic_bs_free,	/* not implemented */
-
-	/* barrier */
-	generic_bs_barrier,
-
-	/* read (single) */
-	generic_bs_r_1,
-	generic_bs_r_2,
-	generic_bs_r_4,
-	NULL,
-
-	/* read multiple */
-	generic_bs_rm_1,
-	generic_bs_rm_2,
-	generic_bs_rm_4,
-	NULL,
-
-	/* read region */
-	generic_bs_rr_1,
-	generic_bs_rr_2,
-	generic_bs_rr_4,
-	NULL,
-
-	/* write (single) */
-	generic_bs_w_1,
-	generic_bs_w_2,
-	generic_bs_w_4,
-	NULL,
-
-	/* write multiple */
-	generic_bs_wm_1,
-	generic_bs_wm_2,
-	generic_bs_wm_4,
-	NULL,
-
-	/* write region */
-	generic_bs_wr_1,
-	generic_bs_wr_2,
-	generic_bs_wr_4,
-	NULL,
-
-	/* set multiple */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* set region */
-	generic_bs_sr_1,
-	generic_bs_sr_2,
-	NULL,
-	NULL,
-
-	/* copy */
-	NULL,
-	generic_bs_c_2,
-	NULL,
-	NULL,
-};
-
diff --git a/sys/arm/xscale/i8134x/obio_space.c b/sys/arm/xscale/i8134x/obio_space.c
deleted file mode 100644
index 4eba7000dbc..00000000000
--- a/sys/arm/xscale/i8134x/obio_space.c
+++ /dev/null
@@ -1,127 +0,0 @@
-/*	$NetBSD: obio_space.c,v 1.6 2003/07/15 00:25:05 lukem Exp $	*/
-
-/*-
- * Copyright (c) 2001, 2002, 2003 Wasabi Systems, Inc.
- * All rights reserved.
- *
- * Written by Jason R. Thorpe for Wasabi Systems, Inc.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed for the NetBSD Project by
- *	Wasabi Systems, Inc.
- * 4. The name of Wasabi Systems, Inc. may not be used to endorse
- *    or promote products derived from this software without specific prior
- *    written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
- * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
- * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
- * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
- * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
- * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
- * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
- * POSSIBILITY OF SUCH DAMAGE.
- */
-
-/*
- * bus_space functions for IQ80321 on-board devices
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/bus.h>
-
-#include <machine/bus.h>
-
-/* Prototypes for all the bus_space structure functions */
-bs_protos(generic);
-
-/*
- * The obio bus space tag.  This is constant for all instances, so
- * we never have to explicitly "create" it.
- */
-struct bus_space obio_bs_tag = {
-	/* cookie */
-	(void *) 0,
-
-	/* mapping/unmapping */
-	generic_bs_map,
-	generic_bs_unmap,
-	generic_bs_subregion,
-
-	/* allocation/deallocation */
-	generic_bs_alloc,
-	generic_bs_free,
-
-	/* barrier */
-	generic_bs_barrier,
-
-	/* read (single) */
-	generic_bs_r_1,
-	generic_bs_r_2,
-	generic_bs_r_4,
-	NULL,
-
-	/* read multiple */
-	generic_bs_rm_1,
-	NULL,
-	NULL,
-	NULL,
-
-	/* read region */
-	generic_bs_rr_1,
-	NULL,
-	NULL,
-	NULL,
-
-	/* write (single) */
-	generic_bs_w_1,
-	generic_bs_w_2,
-	generic_bs_w_4,
-	NULL,
-
-	/* write multiple */
-	generic_bs_wm_1,
-	NULL,
-	NULL,
-	NULL,
-
-	/* write region */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* set multiple */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* set region */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-
-	/* copy */
-	NULL,
-	NULL,
-	NULL,
-	NULL,
-};

From b1d634e62747ce692c7a8d5beb58a779c4b5f974 Mon Sep 17 00:00:00 2001
From: Kevin Lo <kevlo@FreeBSD.org>
Date: Wed, 21 Jan 2015 09:01:48 +0000
Subject: [PATCH 146/279] Typo: ivalid -> invalid.

---
 sys/dev/mii/mii.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sys/dev/mii/mii.c b/sys/dev/mii/mii.c
index c4ff4ac092c..e20c102e8e5 100644
--- a/sys/dev/mii/mii.c
+++ b/sys/dev/mii/mii.c
@@ -374,7 +374,7 @@ mii_attach(device_t dev, device_t *miibus, if_t ifp,
 	}
 
 	if (offloc != MII_OFFSET_ANY && (offloc < 0 || offloc >= MII_NPHY)) {
-		printf("%s: ivalid offloc %d\n", __func__, offloc);
+		printf("%s: invalid offloc %d\n", __func__, offloc);
 		return (EINVAL);
 	}
 
@@ -383,7 +383,7 @@ mii_attach(device_t dev, device_t *miibus, if_t ifp,
 		phymax = MII_NPHY - 1;
 	} else {
 		if (phyloc < 0 || phyloc >= MII_NPHY) {
-			printf("%s: ivalid phyloc %d\n", __func__, phyloc);
+			printf("%s: invalid phyloc %d\n", __func__, phyloc);
 			return (EINVAL);
 		}
 		phymin = phymax = phyloc;

From 9f02b3ec86169e810aa9bc2004dc5bc08ea62dc5 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Wed, 21 Jan 2015 10:47:28 +0000
Subject: [PATCH 147/279] Follow up to r277449 by fixing the remaining
 NSEC_TO_TICK macro to have the same named parameters

Reported by: Ben Perrault <ben.perrault@gmail.com>, Willem Jan Withagen <wjw@digiware.nl>
---
 cddl/contrib/opensolaris/lib/libzpool/common/sys/zfs_context.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cddl/contrib/opensolaris/lib/libzpool/common/sys/zfs_context.h b/cddl/contrib/opensolaris/lib/libzpool/common/sys/zfs_context.h
index 03027c35dc0..493be334891 100644
--- a/cddl/contrib/opensolaris/lib/libzpool/common/sys/zfs_context.h
+++ b/cddl/contrib/opensolaris/lib/libzpool/common/sys/zfs_context.h
@@ -535,7 +535,7 @@ extern vnode_t *rootdir;
 extern void delay(clock_t ticks);
 
 #define	SEC_TO_TICK(sec)	((sec) * hz)
-#define	NSEC_TO_TICK(usec)	((usec) / (NANOSEC / hz))
+#define	NSEC_TO_TICK(nsec)	((nsec) / (NANOSEC / hz))
 
 #define	gethrestime_sec() time(NULL)
 #define	gethrestime(t) \

From 8a3bbeb234955c5c3f748edcb306a12252bf8edb Mon Sep 17 00:00:00 2001
From: Randall Stewart <rrs@FreeBSD.org>
Date: Wed, 21 Jan 2015 13:03:18 +0000
Subject: [PATCH 148/279] Fix minor errors found by coverity. Thanks Gleb for
 the pointers to the email!

---
 usr.sbin/pmcstudy/pmcstudy.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/usr.sbin/pmcstudy/pmcstudy.c b/usr.sbin/pmcstudy/pmcstudy.c
index 6547489f430..a99d59a8cb3 100644
--- a/usr.sbin/pmcstudy/pmcstudy.c
+++ b/usr.sbin/pmcstudy/pmcstudy.c
@@ -1808,6 +1808,9 @@ process_file(char *filename)
 	if (cnts == NULL) {
 		/* Nothing we can do */
 		printf("Nothing to do -- no counters built\n");
+		if (io) {
+			fclose(io);
+		}
 		return;
 	}
 	lace_cpus_together();
@@ -2044,7 +2047,7 @@ get_cpuid_set(void)
 				printf("No memory3 allocation fails at startup?\n");	
 				exit(-1);
 			}
-			memset(more, sz, 0);
+			memset(more, 0, sz);
 			memcpy(more, valid_pmcs, sz);
 			pmc_allocated_cnt *= 2;
 			free(valid_pmcs);

From 459ea53d0509cec96e012a8a2917b6c2d8a7fc39 Mon Sep 17 00:00:00 2001
From: Gavin Atkinson <gavin@FreeBSD.org>
Date: Wed, 21 Jan 2015 13:48:06 +0000
Subject: [PATCH 149/279] "softc" is short for "software context", use that
 phrase in the device_get_softc(9) man page.

MFC after:	1 week
---
 share/man/man9/device_get_softc.9 | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/share/man/man9/device_get_softc.9 b/share/man/man9/device_get_softc.9
index 76f94d448af..5685773aeb9 100644
--- a/share/man/man9/device_get_softc.9
+++ b/share/man/man9/device_get_softc.9
@@ -28,7 +28,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd August 2, 2005
+.Dd January 21, 2015
 .Dt DEVICE_GET_SOFTC 9
 .Os
 .Sh NAME
@@ -40,7 +40,7 @@
 .Ft void *
 .Fn device_get_softc "device_t dev"
 .Sh DESCRIPTION
-Return the driver-specific state of
+Return the driver-specific software context of
 .Fa dev .
 The softc is automatically allocated and zeroed when the device is
 attached.

From 47671d1babdc480b91b25e8293f8a7b5d3021096 Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Wed, 21 Jan 2015 16:10:37 +0000
Subject: [PATCH 150/279] An update for the i915 GPU driver, which brings the
 code up to Linux commit 4d93914ae3db4a897ead4b.  Some related drm
 infrastructure changes are imported as needed.

Biggest update is the rewrite of the i915 gem io to more closely
follow Linux model, althought the mechanism used by FreeBSD port is
different.

Sponsored by:	The FreeBSD Foundation
MFC after:	2 month
---
 sys/dev/drm2/drm.h                           |    5 +
 sys/dev/drm2/drmP.h                          |    1 +
 sys/dev/drm2/drm_crtc.c                      |  586 ++-
 sys/dev/drm2/drm_crtc.h                      |   53 +-
 sys/dev/drm2/drm_crtc_helper.c               |    2 +-
 sys/dev/drm2/drm_crtc_helper.h               |    2 +-
 sys/dev/drm2/drm_drv.c                       |   62 +-
 sys/dev/drm2/drm_edid.c                      |  224 +-
 sys/dev/drm2/drm_edid.h                      |   26 +-
 sys/dev/drm2/drm_edid_modes.h                |  412 +-
 sys/dev/drm2/drm_fb_helper.c                 |    5 +-
 sys/dev/drm2/drm_ioctl.c                     |    4 +
 sys/dev/drm2/drm_irq.c                       |   18 +-
 sys/dev/drm2/drm_memory.c                    |    8 +
 sys/dev/drm2/drm_mode.h                      |   16 +
 sys/dev/drm2/drm_pciids.h                    |    7 +
 sys/dev/drm2/drm_stub.c                      |    4 +-
 sys/dev/drm2/i915/i915_debug.c               |  352 +-
 sys/dev/drm2/i915/i915_dma.c                 |  980 ++--
 sys/dev/drm2/i915/i915_drm.h                 |   19 +-
 sys/dev/drm2/i915/i915_drv.c                 |  212 +-
 sys/dev/drm2/i915/i915_drv.h                 |  228 +-
 sys/dev/drm2/i915/i915_gem.c                 | 2319 +++++----
 sys/dev/drm2/i915/i915_gem_context.c         |    2 +-
 sys/dev/drm2/i915/i915_gem_evict.c           |   40 +-
 sys/dev/drm2/i915/i915_gem_execbuffer.c      |  203 +-
 sys/dev/drm2/i915/i915_gem_gtt.c             |   60 +-
 sys/dev/drm2/i915/i915_gem_stolen.c          |  205 +
 sys/dev/drm2/i915/i915_gem_tiling.c          |   41 +-
 sys/dev/drm2/i915/i915_irq.c                 | 1813 ++++---
 sys/dev/drm2/i915/i915_reg.h                 |  492 +-
 sys/dev/drm2/i915/i915_suspend.c             |   18 +-
 sys/dev/drm2/i915/intel_bios.c               |   45 +-
 sys/dev/drm2/i915/intel_crt.c                |   78 +-
 sys/dev/drm2/i915/intel_ddi.c                |  761 +++
 sys/dev/drm2/i915/intel_display.c            | 4409 +++++-------------
 sys/dev/drm2/i915/intel_dp.c                 |   53 +-
 sys/dev/drm2/i915/intel_drv.h                |  100 +-
 sys/dev/drm2/i915/intel_fb.c                 |    2 +-
 sys/dev/drm2/i915/intel_hdmi.c               |  320 +-
 sys/dev/drm2/i915/intel_iic.c                |  402 +-
 sys/dev/drm2/i915/intel_lvds.c               |   13 +-
 sys/dev/drm2/i915/intel_modes.c              |    4 +-
 sys/dev/drm2/i915/intel_overlay.c            |  107 +-
 sys/dev/drm2/i915/intel_panel.c              |   20 +-
 sys/dev/drm2/i915/intel_pm.c                 | 3788 +++++++++++++++
 sys/dev/drm2/i915/intel_ringbuffer.c         |  754 ++-
 sys/dev/drm2/i915/intel_ringbuffer.h         |   23 +-
 sys/dev/drm2/i915/intel_sdvo.c               |   98 +-
 sys/dev/drm2/i915/intel_sprite.c             |  102 +-
 sys/dev/drm2/i915/intel_tv.c                 |   17 +-
 sys/dev/drm2/radeon/atombios_encoders.c      |    4 +-
 sys/dev/drm2/radeon/radeon_legacy_encoders.c |    2 +-
 sys/modules/drm2/i915kms/Makefile            |    3 +
 54 files changed, 12383 insertions(+), 7141 deletions(-)
 create mode 100644 sys/dev/drm2/i915/i915_gem_stolen.c
 create mode 100644 sys/dev/drm2/i915/intel_ddi.c
 create mode 100644 sys/dev/drm2/i915/intel_pm.c

diff --git a/sys/dev/drm2/drm.h b/sys/dev/drm2/drm.h
index dff0f94e853..eca588531be 100644
--- a/sys/dev/drm2/drm.h
+++ b/sys/dev/drm2/drm.h
@@ -1018,6 +1018,9 @@ struct drm_event_vblank {
 #define DRM_CAP_PRIME 0x5
 #define DRM_CAP_TIMESTAMP_MONOTONIC 0x6
 
+#define DRM_PRIME_CAP_IMPORT 0x1
+#define DRM_PRIME_CAP_EXPORT 0x2
+
 #include "drm_mode.h"
 
 /**
@@ -1126,6 +1129,8 @@ struct drm_event_vblank {
 #define DRM_IOCTL_MODE_GETPLANE		DRM_IOWR(0xB6, struct drm_mode_get_plane)
 #define DRM_IOCTL_MODE_SETPLANE		DRM_IOWR(0xB7, struct drm_mode_set_plane)
 #define DRM_IOCTL_MODE_ADDFB2		DRM_IOWR(0xB8, struct drm_mode_fb_cmd2)
+#define DRM_IOCTL_MODE_OBJ_GETPROPERTIES	DRM_IOWR(0xB9, struct drm_mode_obj_get_properties)
+#define DRM_IOCTL_MODE_OBJ_SETPROPERTY	DRM_IOWR(0xBA, struct drm_mode_obj_set_property)
 
 #define DRM_IOCTL_MM_INIT               DRM_IOWR(0xc0, struct drm_mm_init_arg)
 #define DRM_IOCTL_MM_TAKEDOWN           DRM_IOWR(0xc1, struct drm_mm_type_arg)
diff --git a/sys/dev/drm2/drmP.h b/sys/dev/drm2/drmP.h
index c724a2dc7d7..e33ab88003b 100644
--- a/sys/dev/drm2/drmP.h
+++ b/sys/dev/drm2/drmP.h
@@ -1246,6 +1246,7 @@ int	drm_ati_pcigart_cleanup(struct drm_device *dev,
 
 /* Cache management (drm_memory.c) */
 void	drm_clflush_pages(vm_page_t *pages, unsigned long num_pages);
+void	drm_clflush_virt_range(char *addr, unsigned long length);
 
 /* Locking IOCTL support (drm_drv.c) */
 int	drm_lock(struct drm_device *dev, void *data,
diff --git a/sys/dev/drm2/drm_crtc.c b/sys/dev/drm2/drm_crtc.c
index 7c6cb5d1b5b..e044ec23241 100644
--- a/sys/dev/drm2/drm_crtc.c
+++ b/sys/dev/drm2/drm_crtc.c
@@ -352,7 +352,7 @@ void drm_framebuffer_cleanup(struct drm_framebuffer *fb)
  * @funcs: callbacks for the new CRTC
  *
  * LOCKING:
- * Caller must hold mode config lock.
+ * Takes mode_config lock.
  *
  * Inits a new object created as base part of an driver crtc object.
  *
@@ -372,8 +372,11 @@ int drm_crtc_init(struct drm_device *dev, struct drm_crtc *crtc,
 	if (ret)
 		goto out;
 
+	crtc->base.properties = &crtc->properties;
+
 	list_add_tail(&crtc->head, &dev->mode_config.crtc_list);
 	dev->mode_config.num_crtc++;
+
 out:
 	sx_xunlock(&dev->mode_config.mutex);
 
@@ -474,6 +477,7 @@ int drm_connector_init(struct drm_device *dev,
 	if (ret)
 		goto out;
 
+	connector->base.properties = &connector->properties;
 	connector->dev = dev;
 	connector->funcs = funcs;
 	connector->connector_type = connector_type;
@@ -582,6 +586,7 @@ int drm_plane_init(struct drm_device *dev, struct drm_plane *plane,
 	if (ret)
 		goto out;
 
+	plane->base.properties = &plane->properties;
 	plane->dev = dev;
 	plane->funcs = funcs;
 	plane->format_types = malloc(sizeof(uint32_t) * format_count,
@@ -1399,11 +1404,7 @@ int drm_mode_getconnector(struct drm_device *dev, void *data,
 	}
 	connector = obj_to_connector(obj);
 
-	for (i = 0; i < DRM_CONNECTOR_MAX_PROPERTY; i++) {
-		if (connector->property_ids[i] != 0) {
-			props_count++;
-		}
-	}
+	props_count = connector->properties.count;
 
 	for (i = 0; i < DRM_CONNECTOR_MAX_ENCODER; i++) {
 		if (connector->encoder_ids[i] != 0) {
@@ -1456,21 +1457,19 @@ int drm_mode_getconnector(struct drm_device *dev, void *data,
 		copied = 0;
 		prop_ptr = (uint32_t *)(uintptr_t)(out_resp->props_ptr);
 		prop_values = (uint64_t *)(uintptr_t)(out_resp->prop_values_ptr);
-		for (i = 0; i < DRM_CONNECTOR_MAX_PROPERTY; i++) {
-			if (connector->property_ids[i] != 0) {
-				if (copyout(&connector->property_ids[i],
-				    prop_ptr + copied, sizeof(uint32_t))) {
-					ret = EFAULT;
-					goto out;
-				}
-
-				if (copyout(&connector->property_values[i],
-				    prop_values + copied, sizeof(uint64_t))) {
-					ret = EFAULT;
-					goto out;
-				}
-				copied++;
+		for (i = 0; i < connector->properties.count; i++) {
+			if (copyout(&connector->properties.ids[i],
+			    prop_ptr + copied, sizeof(uint32_t))) {
+				ret = EFAULT;
+				goto out;
 			}
+
+			if (copyout(&connector->properties.values[i],
+			    prop_values + copied, sizeof(uint64_t))) {
+				ret = EFAULT;
+				goto out;
+			}
+			copied++;
 		}
 	}
 	out_resp->count_props = props_count;
@@ -1808,7 +1807,7 @@ int drm_mode_setcrtc(struct drm_device *dev, void *data,
 	struct drm_display_mode *mode = NULL;
 	struct drm_mode_set set;
 	uint32_t *set_connectors_ptr;
-	int ret = 0;
+	int ret;
 	int i;
 
 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
@@ -2070,7 +2069,7 @@ int drm_mode_addfb(struct drm_device *dev,
 
 	ret = -dev->mode_config.funcs->fb_create(dev, file_priv, &r, &fb);
 	if (ret != 0) {
-		DRM_ERROR("could not create framebuffer, error %d\n", ret);
+		DRM_DEBUG_KMS("could not create framebuffer, error %d\n", ret);
 		goto out;
 	}
 
@@ -2152,6 +2151,47 @@ static int format_check(struct drm_mode_fb_cmd2 *r)
 	}
 }
 
+static int framebuffer_check(struct drm_mode_fb_cmd2 *r)
+{
+	int ret, hsub, vsub, num_planes, i;
+
+	ret = format_check(r);
+	if (ret) {
+		DRM_DEBUG_KMS("bad framebuffer format 0x%08x\n", r->pixel_format);
+		return ret;
+	}
+
+	hsub = drm_format_horz_chroma_subsampling(r->pixel_format);
+	vsub = drm_format_vert_chroma_subsampling(r->pixel_format);
+	num_planes = drm_format_num_planes(r->pixel_format);
+
+	if (r->width == 0 || r->width % hsub) {
+		DRM_DEBUG_KMS("bad framebuffer width %u\n", r->height);
+		return -EINVAL;
+	}
+
+	if (r->height == 0 || r->height % vsub) {
+		DRM_DEBUG_KMS("bad framebuffer height %u\n", r->height);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < num_planes; i++) {
+		unsigned int width = r->width / (i != 0 ? hsub : 1);
+
+		if (!r->handles[i]) {
+			DRM_DEBUG_KMS("no buffer object handle for plane %d\n", i);
+			return -EINVAL;
+		}
+
+		if (r->pitches[i] < drm_format_plane_cpp(r->pixel_format, i) * width) {
+			DRM_DEBUG_KMS("bad pitch %u for plane %d\n", r->pitches[i], i);
+			return -EINVAL;
+		}
+	}
+
+	return 0;
+}
+
 /**
  * drm_mode_addfb2 - add an FB to the graphics configuration
  * @inode: inode from the ioctl
@@ -2181,21 +2221,19 @@ int drm_mode_addfb2(struct drm_device *dev,
 		return (EINVAL);
 
 	if ((config->min_width > r->width) || (r->width > config->max_width)) {
-		DRM_ERROR("bad framebuffer width %d, should be >= %d && <= %d\n",
+		DRM_DEBUG_KMS("bad framebuffer width %d, should be >= %d && <= %d\n",
 			  r->width, config->min_width, config->max_width);
 		return (EINVAL);
 	}
 	if ((config->min_height > r->height) || (r->height > config->max_height)) {
-		DRM_ERROR("bad framebuffer height %d, should be >= %d && <= %d\n",
+		DRM_DEBUG_KMS("bad framebuffer height %d, should be >= %d && <= %d\n",
 			  r->height, config->min_height, config->max_height);
 		return (EINVAL);
 	}
 
-	ret = format_check(r);
-	if (ret) {
-		DRM_ERROR("bad framebuffer format 0x%08x\n", r->pixel_format);
-		return ret;
-	}
+	ret = framebuffer_check(r);
+	if (ret)
+		return -ret;
 
 	sx_xlock(&dev->mode_config.mutex);
 
@@ -2204,7 +2242,7 @@ int drm_mode_addfb2(struct drm_device *dev,
 
 	ret = -dev->mode_config.funcs->fb_create(dev, file_priv, r, &fb);
 	if (ret != 0) {
-		DRM_ERROR("could not create framebuffer, error %d\n", ret);
+		DRM_DEBUG_KMS("could not create framebuffer, error %d\n", ret);
 		goto out;
 	}
 
@@ -2335,7 +2373,7 @@ int drm_mode_dirtyfb_ioctl(struct drm_device *dev,
 	struct drm_framebuffer *fb;
 	unsigned flags;
 	int num_clips;
-	int ret = 0;
+	int ret;
 
 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
 		return (EINVAL);
@@ -2530,7 +2568,7 @@ int drm_mode_attachmode_ioctl(struct drm_device *dev,
 	struct drm_display_mode *mode;
 	struct drm_mode_object *obj;
 	struct drm_mode_modeinfo *umode = &mode_cmd->mode;
-	int ret = 0;
+	int ret;
 
 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
 		return -EINVAL;
@@ -2584,7 +2622,7 @@ int drm_mode_detachmode_ioctl(struct drm_device *dev,
 	struct drm_connector *connector;
 	struct drm_display_mode mode;
 	struct drm_mode_modeinfo *umode = &mode_cmd->mode;
-	int ret = 0;
+	int ret;
 
 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
 		return -EINVAL;
@@ -2672,6 +2710,33 @@ struct drm_property *drm_property_create_enum(struct drm_device *dev, int flags,
 	return property;
 }
 
+struct drm_property *drm_property_create_bitmask(struct drm_device *dev,
+					 int flags, const char *name,
+					 const struct drm_prop_enum_list *props,
+					 int num_values)
+{
+	struct drm_property *property;
+	int i, ret;
+
+	flags |= DRM_MODE_PROP_BITMASK;
+
+	property = drm_property_create(dev, flags, name, num_values);
+	if (!property)
+		return NULL;
+
+	for (i = 0; i < num_values; i++) {
+		ret = drm_property_add_enum(property, i,
+				      props[i].type,
+				      props[i].name);
+		if (ret) {
+			drm_property_destroy(dev, property);
+			return NULL;
+		}
+	}
+
+	return property;
+}
+
 struct drm_property *drm_property_create_range(struct drm_device *dev, int flags,
 					 const char *name,
 					 uint64_t min, uint64_t max)
@@ -2695,7 +2760,14 @@ int drm_property_add_enum(struct drm_property *property, int index,
 {
 	struct drm_property_enum *prop_enum;
 
-	if (!(property->flags & DRM_MODE_PROP_ENUM))
+	if (!(property->flags & (DRM_MODE_PROP_ENUM | DRM_MODE_PROP_BITMASK)))
+		return -EINVAL;
+
+	/*
+	 * Bitmask enum properties have the additional constraint of values
+	 * from 0 to 63
+	 */
+	if ((property->flags & DRM_MODE_PROP_BITMASK) && (value > 63))
 		return -EINVAL;
 
 	if (!list_empty(&property->enum_blob_list)) {
@@ -2736,56 +2808,71 @@ void drm_property_destroy(struct drm_device *dev, struct drm_property *property)
 	free(property, DRM_MEM_KMS);
 }
 
-int drm_connector_attach_property(struct drm_connector *connector,
+void drm_connector_attach_property(struct drm_connector *connector,
 			       struct drm_property *property, uint64_t init_val)
 {
-	int i;
-
-	for (i = 0; i < DRM_CONNECTOR_MAX_PROPERTY; i++) {
-		if (connector->property_ids[i] == 0) {
-			connector->property_ids[i] = property->base.id;
-			connector->property_values[i] = init_val;
-			break;
-		}
-	}
-
-	if (i == DRM_CONNECTOR_MAX_PROPERTY)
-		return -EINVAL;
-	return 0;
+	drm_object_attach_property(&connector->base, property, init_val);
 }
 
 int drm_connector_property_set_value(struct drm_connector *connector,
 				  struct drm_property *property, uint64_t value)
 {
-	int i;
-
-	for (i = 0; i < DRM_CONNECTOR_MAX_PROPERTY; i++) {
-		if (connector->property_ids[i] == property->base.id) {
-			connector->property_values[i] = value;
-			break;
-		}
-	}
-
-	if (i == DRM_CONNECTOR_MAX_PROPERTY)
-		return -EINVAL;
-	return 0;
+	return drm_object_property_set_value(&connector->base, property, value);
 }
 
 int drm_connector_property_get_value(struct drm_connector *connector,
 				  struct drm_property *property, uint64_t *val)
+{
+	return drm_object_property_get_value(&connector->base, property, val);
+}
+
+void drm_object_attach_property(struct drm_mode_object *obj,
+				struct drm_property *property,
+				uint64_t init_val)
+{
+	int count = obj->properties->count;
+
+	if (count == DRM_OBJECT_MAX_PROPERTY) {
+		printf("Failed to attach object property (type: 0x%x). Please "
+			"increase DRM_OBJECT_MAX_PROPERTY by 1 for each time "
+			"you see this message on the same object type.\n",
+			obj->type);
+		return;
+ 	}
+
+	obj->properties->ids[count] = property->base.id;
+	obj->properties->values[count] = init_val;
+	obj->properties->count++;
+}
+
+int drm_object_property_set_value(struct drm_mode_object *obj,
+				  struct drm_property *property, uint64_t val)
 {
 	int i;
 
-	for (i = 0; i < DRM_CONNECTOR_MAX_PROPERTY; i++) {
-		if (connector->property_ids[i] == property->base.id) {
-			*val = connector->property_values[i];
-			break;
+	for (i = 0; i < obj->properties->count; i++) {
+		if (obj->properties->ids[i] == property->base.id) {
+			obj->properties->values[i] = val;
+			return 0;
 		}
 	}
 
-	if (i == DRM_CONNECTOR_MAX_PROPERTY)
-		return -EINVAL;
-	return 0;
+	return -EINVAL;
+}
+
+int drm_object_property_get_value(struct drm_mode_object *obj,
+				  struct drm_property *property, uint64_t *val)
+{
+	int i;
+
+	for (i = 0; i < obj->properties->count; i++) {
+		if (obj->properties->ids[i] == property->base.id) {
+			*val = obj->properties->values[i];
+			return 0;
+		}
+	}
+
+	return -EINVAL;
 }
 
 int drm_mode_getproperty_ioctl(struct drm_device *dev,
@@ -2817,7 +2904,7 @@ int drm_mode_getproperty_ioctl(struct drm_device *dev,
 	}
 	property = obj_to_property(obj);
 
-	if (property->flags & DRM_MODE_PROP_ENUM) {
+	if (property->flags & (DRM_MODE_PROP_ENUM | DRM_MODE_PROP_BITMASK)) {
 		list_for_each_entry(prop_enum, &property->enum_blob_list, head)
 			enum_count++;
 	} else if (property->flags & DRM_MODE_PROP_BLOB) {
@@ -2842,7 +2929,7 @@ int drm_mode_getproperty_ioctl(struct drm_device *dev,
 	}
 	out_resp->count_values = value_count;
 
-	if (property->flags & DRM_MODE_PROP_ENUM) {
+	if (property->flags & (DRM_MODE_PROP_ENUM | DRM_MODE_PROP_BITMASK)) {
 		if ((out_resp->count_enum_blobs >= enum_count) && enum_count) {
 			copied = 0;
 			enum_ptr = (struct drm_mode_property_enum *)(uintptr_t)out_resp->enum_blob_ptr;
@@ -2965,7 +3052,7 @@ int drm_mode_connector_update_edid_property(struct drm_connector *connector,
 					    struct edid *edid)
 {
 	struct drm_device *dev = connector->dev;
-	int ret = 0, size;
+	int ret, size;
 
 	if (connector->edid_blob_ptr)
 		drm_property_destroy_blob(dev, connector->edid_blob_ptr);
@@ -2988,13 +3075,159 @@ int drm_mode_connector_update_edid_property(struct drm_connector *connector,
 	return ret;
 }
 
+static bool drm_property_change_is_valid(struct drm_property *property,
+					 u64 value)
+{
+	if (property->flags & DRM_MODE_PROP_IMMUTABLE)
+		return false;
+	if (property->flags & DRM_MODE_PROP_RANGE) {
+		if (value < property->values[0] || value > property->values[1])
+			return false;
+		return true;
+	} else if (property->flags & DRM_MODE_PROP_BITMASK) {
+		int i;
+		u64 valid_mask = 0;
+		for (i = 0; i < property->num_values; i++)
+			valid_mask |= (1ULL << property->values[i]);
+		return !(value & ~valid_mask);
+	} else {
+		int i;
+		for (i = 0; i < property->num_values; i++)
+			if (property->values[i] == value)
+				return true;
+		return false;
+	}
+}
+
 int drm_mode_connector_property_set_ioctl(struct drm_device *dev,
 				       void *data, struct drm_file *file_priv)
 {
-	struct drm_mode_connector_set_property *out_resp = data;
+	struct drm_mode_connector_set_property *conn_set_prop = data;
+	struct drm_mode_obj_set_property obj_set_prop = {
+		.value = conn_set_prop->value,
+		.prop_id = conn_set_prop->prop_id,
+		.obj_id = conn_set_prop->connector_id,
+		.obj_type = DRM_MODE_OBJECT_CONNECTOR
+	};
+
+	/* It does all the locking and checking we need */
+	return drm_mode_obj_set_property_ioctl(dev, &obj_set_prop, file_priv);
+}
+
+static int drm_mode_connector_set_obj_prop(struct drm_mode_object *obj,
+					   struct drm_property *property,
+					   uint64_t value)
+{
+	int ret = -EINVAL;
+	struct drm_connector *connector = obj_to_connector(obj);
+
+	/* Do DPMS ourselves */
+	if (property == connector->dev->mode_config.dpms_property) {
+		if (connector->funcs->dpms)
+			(*connector->funcs->dpms)(connector, (int)value);
+		ret = 0;
+	} else if (connector->funcs->set_property)
+		ret = connector->funcs->set_property(connector, property, value);
+
+	/* store the property value if successful */
+	if (!ret)
+		drm_connector_property_set_value(connector, property, value);
+	return ret;
+}
+
+static int drm_mode_crtc_set_obj_prop(struct drm_mode_object *obj,
+				      struct drm_property *property,
+				      uint64_t value)
+{
+	int ret = -EINVAL;
+	struct drm_crtc *crtc = obj_to_crtc(obj);
+
+	if (crtc->funcs->set_property)
+		ret = crtc->funcs->set_property(crtc, property, value);
+	if (!ret)
+		drm_object_property_set_value(obj, property, value);
+
+	return ret;
+}
+
+static int drm_mode_plane_set_obj_prop(struct drm_mode_object *obj,
+				      struct drm_property *property,
+				      uint64_t value)
+{
+	int ret = -EINVAL;
+	struct drm_plane *plane = obj_to_plane(obj);
+
+	if (plane->funcs->set_property)
+		ret = plane->funcs->set_property(plane, property, value);
+	if (!ret)
+		drm_object_property_set_value(obj, property, value);
+
+	return ret;
+}
+
+int drm_mode_obj_get_properties_ioctl(struct drm_device *dev, void *data,
+				      struct drm_file *file_priv)
+{
+	struct drm_mode_obj_get_properties *arg = data;
 	struct drm_mode_object *obj;
+	int ret = 0;
+	int i;
+	int copied = 0;
+	int props_count = 0;
+	uint32_t __user *props_ptr;
+	uint64_t __user *prop_values_ptr;
+
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -EINVAL;
+
+	sx_xlock(&dev->mode_config.mutex);
+
+	obj = drm_mode_object_find(dev, arg->obj_id, arg->obj_type);
+	if (!obj) {
+		ret = -EINVAL;
+		goto out;
+	}
+	if (!obj->properties) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+	props_count = obj->properties->count;
+
+	/* This ioctl is called twice, once to determine how much space is
+	 * needed, and the 2nd time to fill it. */
+	if ((arg->count_props >= props_count) && props_count) {
+		copied = 0;
+		props_ptr = (uint32_t __user *)(unsigned long)(arg->props_ptr);
+		prop_values_ptr = (uint64_t __user *)(unsigned long)
+				  (arg->prop_values_ptr);
+		for (i = 0; i < props_count; i++) {
+			if (copyout(props_ptr + copied,
+			    &obj->properties->ids[i], sizeof(uint32_t))) {
+				ret = -EFAULT;
+				goto out;
+			}
+			if (copyout(prop_values_ptr + copied,
+			    &obj->properties->values[i], sizeof(uint64_t))) {
+				ret = -EFAULT;
+				goto out;
+			}
+			copied++;
+		}
+	}
+	arg->count_props = props_count;
+out:
+	sx_xunlock(&dev->mode_config.mutex);
+	return ret;
+}
+
+int drm_mode_obj_set_property_ioctl(struct drm_device *dev, void *data,
+				    struct drm_file *file_priv)
+{
+	struct drm_mode_obj_set_property *arg = data;
+	struct drm_mode_object *arg_obj;
+	struct drm_mode_object *prop_obj;
 	struct drm_property *property;
-	struct drm_connector *connector;
 	int ret = -EINVAL;
 	int i;
 
@@ -3003,60 +3236,41 @@ int drm_mode_connector_property_set_ioctl(struct drm_device *dev,
 
 	sx_xlock(&dev->mode_config.mutex);
 
-	obj = drm_mode_object_find(dev, out_resp->connector_id, DRM_MODE_OBJECT_CONNECTOR);
-	if (!obj) {
+	arg_obj = drm_mode_object_find(dev, arg->obj_id, arg->obj_type);
+	if (!arg_obj)
+		goto out;
+	if (!arg_obj->properties)
 		goto out;
-	}
-	connector = obj_to_connector(obj);
 
-	for (i = 0; i < DRM_CONNECTOR_MAX_PROPERTY; i++) {
-		if (connector->property_ids[i] == out_resp->prop_id)
+	for (i = 0; i < arg_obj->properties->count; i++)
+		if (arg_obj->properties->ids[i] == arg->prop_id)
 			break;
-	}
 
-	if (i == DRM_CONNECTOR_MAX_PROPERTY) {
-		goto out;
-	}
-
-	obj = drm_mode_object_find(dev, out_resp->prop_id, DRM_MODE_OBJECT_PROPERTY);
-	if (!obj) {
-		goto out;
-	}
-	property = obj_to_property(obj);
-
-	if (property->flags & DRM_MODE_PROP_IMMUTABLE)
+	if (i == arg_obj->properties->count)
 		goto out;
 
-	if (property->flags & DRM_MODE_PROP_RANGE) {
-		if (out_resp->value < property->values[0])
-			goto out;
+	prop_obj = drm_mode_object_find(dev, arg->prop_id,
+					DRM_MODE_OBJECT_PROPERTY);
+	if (!prop_obj)
+		goto out;
+	property = obj_to_property(prop_obj);
 
-		if (out_resp->value > property->values[1])
-			goto out;
-	} else {
-		int found = 0;
-		for (i = 0; i < property->num_values; i++) {
-			if (property->values[i] == out_resp->value) {
-				found = 1;
-				break;
-			}
-		}
-		if (!found) {
-			goto out;
-		}
+	if (!drm_property_change_is_valid(property, arg->value))
+		goto out;
+
+	switch (arg_obj->type) {
+	case DRM_MODE_OBJECT_CONNECTOR:
+		ret = drm_mode_connector_set_obj_prop(arg_obj, property,
+						      arg->value);
+		break;
+	case DRM_MODE_OBJECT_CRTC:
+		ret = drm_mode_crtc_set_obj_prop(arg_obj, property, arg->value);
+		break;
+	case DRM_MODE_OBJECT_PLANE:
+		ret = drm_mode_plane_set_obj_prop(arg_obj, property, arg->value);
+		break;
 	}
 
-	/* Do DPMS ourselves */
-	if (property == connector->dev->mode_config.dpms_property) {
-		if (connector->funcs->dpms)
-			(*connector->funcs->dpms)(connector, (int) out_resp->value);
-		ret = 0;
-	} else if (connector->funcs->set_property)
-		ret = connector->funcs->set_property(connector, property, out_resp->value);
-
-	/* store the property value if successful */
-	if (!ret)
-		drm_connector_property_set_value(connector, property, out_resp->value);
 out:
 	sx_xunlock(&dev->mode_config.mutex);
 	return ret;
@@ -3176,6 +3390,11 @@ int drm_mode_gamma_get_ioctl(struct drm_device *dev,
 	}
 	crtc = obj_to_crtc(obj);
 
+	if (crtc->funcs->gamma_set == NULL) {
+		ret = -ENOSYS;
+		goto out;
+	}
+
 	/* memcpy into gamma store */
 	if (crtc_lut->gamma_size != crtc->gamma_size) {
 		ret = -EINVAL;
@@ -3417,3 +3636,136 @@ void drm_fb_get_bpp_depth(uint32_t format, unsigned int *depth,
 		break;
 	}
 }
+
+/**
+ * drm_format_num_planes - get the number of planes for format
+ * @format: pixel format (DRM_FORMAT_*)
+ *
+ * RETURNS:
+ * The number of planes used by the specified pixel format.
+ */
+int drm_format_num_planes(uint32_t format)
+{
+	switch (format) {
+	case DRM_FORMAT_YUV410:
+	case DRM_FORMAT_YVU410:
+	case DRM_FORMAT_YUV411:
+	case DRM_FORMAT_YVU411:
+	case DRM_FORMAT_YUV420:
+	case DRM_FORMAT_YVU420:
+	case DRM_FORMAT_YUV422:
+	case DRM_FORMAT_YVU422:
+	case DRM_FORMAT_YUV444:
+	case DRM_FORMAT_YVU444:
+		return 3;
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_NV21:
+	case DRM_FORMAT_NV16:
+	case DRM_FORMAT_NV61:
+		return 2;
+	default:
+		return 1;
+	}
+}
+
+/**
+ * drm_format_plane_cpp - determine the bytes per pixel value
+ * @format: pixel format (DRM_FORMAT_*)
+ * @plane: plane index
+ *
+ * RETURNS:
+ * The bytes per pixel value for the specified plane.
+ */
+int drm_format_plane_cpp(uint32_t format, int plane)
+{
+	unsigned int depth;
+	int bpp;
+
+	if (plane >= drm_format_num_planes(format))
+		return 0;
+
+	switch (format) {
+	case DRM_FORMAT_YUYV:
+	case DRM_FORMAT_YVYU:
+	case DRM_FORMAT_UYVY:
+	case DRM_FORMAT_VYUY:
+		return 2;
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_NV21:
+	case DRM_FORMAT_NV16:
+	case DRM_FORMAT_NV61:
+		return plane ? 2 : 1;
+	case DRM_FORMAT_YUV410:
+	case DRM_FORMAT_YVU410:
+	case DRM_FORMAT_YUV411:
+	case DRM_FORMAT_YVU411:
+	case DRM_FORMAT_YUV420:
+	case DRM_FORMAT_YVU420:
+	case DRM_FORMAT_YUV422:
+	case DRM_FORMAT_YVU422:
+	case DRM_FORMAT_YUV444:
+	case DRM_FORMAT_YVU444:
+		return 1;
+	default:
+		drm_fb_get_bpp_depth(format, &depth, &bpp);
+		return bpp >> 3;
+	}
+}
+
+/**
+ * drm_format_horz_chroma_subsampling - get the horizontal chroma subsampling factor
+ * @format: pixel format (DRM_FORMAT_*)
+ *
+ * RETURNS:
+ * The horizontal chroma subsampling factor for the
+ * specified pixel format.
+ */
+int drm_format_horz_chroma_subsampling(uint32_t format)
+{
+	switch (format) {
+	case DRM_FORMAT_YUV411:
+	case DRM_FORMAT_YVU411:
+	case DRM_FORMAT_YUV410:
+	case DRM_FORMAT_YVU410:
+		return 4;
+	case DRM_FORMAT_YUYV:
+	case DRM_FORMAT_YVYU:
+	case DRM_FORMAT_UYVY:
+	case DRM_FORMAT_VYUY:
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_NV21:
+	case DRM_FORMAT_NV16:
+	case DRM_FORMAT_NV61:
+	case DRM_FORMAT_YUV422:
+	case DRM_FORMAT_YVU422:
+	case DRM_FORMAT_YUV420:
+	case DRM_FORMAT_YVU420:
+		return 2;
+	default:
+		return 1;
+	}
+}
+
+/**
+ * drm_format_vert_chroma_subsampling - get the vertical chroma subsampling factor
+ * @format: pixel format (DRM_FORMAT_*)
+ *
+ * RETURNS:
+ * The vertical chroma subsampling factor for the
+ * specified pixel format.
+ */
+int drm_format_vert_chroma_subsampling(uint32_t format)
+{
+	switch (format) {
+	case DRM_FORMAT_YUV410:
+	case DRM_FORMAT_YVU410:
+		return 4;
+	case DRM_FORMAT_YUV420:
+	case DRM_FORMAT_YVU420:
+	case DRM_FORMAT_NV12:
+	case DRM_FORMAT_NV21:
+		return 2;
+	default:
+		return 1;
+	}
+}
diff --git a/sys/dev/drm2/drm_crtc.h b/sys/dev/drm2/drm_crtc.h
index 0ab2bf48ff8..36708c6de65 100644
--- a/sys/dev/drm2/drm_crtc.h
+++ b/sys/dev/drm2/drm_crtc.h
@@ -47,6 +47,14 @@ struct i2c_adapter;
 struct drm_mode_object {
 	uint32_t id;
 	uint32_t type;
+	struct drm_object_properties *properties;
+};
+
+#define DRM_OBJECT_MAX_PROPERTY 16
+struct drm_object_properties {
+	int count;
+	uint32_t ids[DRM_OBJECT_MAX_PROPERTY];
+	uint64_t values[DRM_OBJECT_MAX_PROPERTY];
 };
 
 /*
@@ -295,7 +303,8 @@ struct drm_plane;
  * @mode_fixup: fixup proposed mode
  * @mode_set: set the desired mode on the CRTC
  * @gamma_set: specify color ramp for CRTC
- * @destroy: deinit and free object.
+ * @destroy: deinit and free object
+ * @set_property: called when a property is changed
  *
  * The drm_crtc_funcs structure is the central CRTC management structure
  * in the DRM.  Each CRTC controls one or more connectors (note that the name
@@ -339,6 +348,8 @@ struct drm_crtc_funcs {
 	int (*page_flip)(struct drm_crtc *crtc,
 			 struct drm_framebuffer *fb,
 			 struct drm_pending_vblank_event *event);
+	int (*set_property)(struct drm_crtc *crtc,
+			    struct drm_property *property, uint64_t val);
 };
 
 /**
@@ -347,6 +358,7 @@ struct drm_crtc_funcs {
  * @x: x position on screen
  * @y: y position on screen
  * @funcs: CRTC control functions
+ * @properties: property tracking for this CRTC
  *
  * Each CRTC may have one or more connectors associated with it.  This structure
  * allows the CRTC to be controlled.
@@ -382,6 +394,8 @@ struct drm_crtc {
 
 	/* if you are using the helper */
 	void *helper_private;
+
+	struct drm_object_properties properties;
 };
 
 
@@ -431,7 +445,6 @@ struct drm_encoder_funcs {
 };
 
 #define DRM_CONNECTOR_MAX_UMODES 16
-#define DRM_CONNECTOR_MAX_PROPERTY 16
 #define DRM_CONNECTOR_LEN 32
 #define DRM_CONNECTOR_MAX_ENCODER 2
 
@@ -511,8 +524,7 @@ struct drm_connector {
 
 	struct list_head user_modes;
 	struct drm_property_blob *edid_blob_ptr;
-	u32 property_ids[DRM_CONNECTOR_MAX_PROPERTY];
-	uint64_t property_values[DRM_CONNECTOR_MAX_PROPERTY];
+	struct drm_object_properties properties;
 
 	uint8_t polled; /* DRM_CONNECTOR_POLL_* */
 
@@ -543,6 +555,7 @@ struct drm_connector {
  * @update_plane: update the plane configuration
  * @disable_plane: shut down the plane
  * @destroy: clean up plane resources
+ * @set_property: called when a property is changed
  */
 struct drm_plane_funcs {
 	int (*update_plane)(struct drm_plane *plane,
@@ -553,6 +566,8 @@ struct drm_plane_funcs {
 			    uint32_t src_w, uint32_t src_h);
 	int (*disable_plane)(struct drm_plane *plane);
 	void (*destroy)(struct drm_plane *plane);
+	int (*set_property)(struct drm_plane *plane,
+			    struct drm_property *property, uint64_t val);
 };
 
 /**
@@ -570,6 +585,7 @@ struct drm_plane_funcs {
  * @enabled: enabled flag
  * @funcs: helper functions
  * @helper_private: storage for drver layer
+ @properties: property tracking for this plane
  */
 struct drm_plane {
 	struct drm_device *dev;
@@ -592,6 +608,8 @@ struct drm_plane {
 
 	const struct drm_plane_funcs *funcs;
 	void *helper_private;
+
+	struct drm_object_properties properties;
 };
 
 /**
@@ -806,6 +824,15 @@ extern int drm_connector_property_set_value(struct drm_connector *connector,
 extern int drm_connector_property_get_value(struct drm_connector *connector,
 					 struct drm_property *property,
 					 uint64_t *value);
+void drm_object_attach_property(struct drm_mode_object *obj,
+				struct drm_property *property,
+				uint64_t init_val);
+extern int drm_object_property_set_value(struct drm_mode_object *obj,
+					 struct drm_property *property,
+					 uint64_t val);
+extern int drm_object_property_get_value(struct drm_mode_object *obj,
+					 struct drm_property *property,
+					 uint64_t *value);
 extern struct drm_display_mode *drm_crtc_mode_create(struct drm_device *dev);
 extern void drm_framebuffer_set_object(struct drm_device *dev,
 				       unsigned long handle);
@@ -818,7 +845,7 @@ extern int drmfb_remove(struct drm_device *dev, struct drm_framebuffer *fb);
 extern void drm_crtc_probe_connector_modes(struct drm_device *dev, int maxX, int maxY);
 extern bool drm_crtc_in_use(struct drm_crtc *crtc);
 
-extern int drm_connector_attach_property(struct drm_connector *connector,
+extern void drm_connector_attach_property(struct drm_connector *connector,
 				      struct drm_property *property, uint64_t init_val);
 extern struct drm_property *drm_property_create(struct drm_device *dev, int flags,
 						const char *name, int num_values);
@@ -826,6 +853,10 @@ extern struct drm_property *drm_property_create_enum(struct drm_device *dev, int
 					 const char *name,
 					 const struct drm_prop_enum_list *props,
 					 int num_values);
+struct drm_property *drm_property_create_bitmask(struct drm_device *dev,
+						 int flags, const char *name,
+						 const struct drm_prop_enum_list *props,
+						 int num_values);
 struct drm_property *drm_property_create_range(struct drm_device *dev, int flags,
 					 const char *name,
 					 uint64_t min, uint64_t max);
@@ -921,7 +952,8 @@ extern int drm_add_modes_noedid(struct drm_connector *connector,
 extern int drm_edid_header_is_valid(const u8 *raw_edid);
 extern bool drm_edid_is_valid(struct edid *edid);
 struct drm_display_mode *drm_mode_find_dmt(struct drm_device *dev,
-					   int hsize, int vsize, int fresh);
+					   int hsize, int vsize, int fresh,
+					   bool rb);
 
 extern int drm_mode_create_dumb_ioctl(struct drm_device *dev,
 				      void *data, struct drm_file *file_priv);
@@ -929,7 +961,16 @@ extern int drm_mode_mmap_dumb_ioctl(struct drm_device *dev,
 				    void *data, struct drm_file *file_priv);
 extern int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
 				      void *data, struct drm_file *file_priv);
+extern int drm_mode_obj_get_properties_ioctl(struct drm_device *dev, void *data,
+					     struct drm_file *file_priv);
+extern int drm_mode_obj_set_property_ioctl(struct drm_device *dev, void *data,
+					   struct drm_file *file_priv);
 
 extern void drm_fb_get_bpp_depth(uint32_t format, unsigned int *depth,
 				 int *bpp);
+extern int drm_format_num_planes(uint32_t format);
+extern int drm_format_plane_cpp(uint32_t format, int plane);
+extern int drm_format_horz_chroma_subsampling(uint32_t format);
+extern int drm_format_vert_chroma_subsampling(uint32_t format);
+
 #endif /* __DRM_CRTC_H__ */
diff --git a/sys/dev/drm2/drm_crtc_helper.c b/sys/dev/drm2/drm_crtc_helper.c
index 77e134640f9..9fb812006f0 100644
--- a/sys/dev/drm2/drm_crtc_helper.c
+++ b/sys/dev/drm2/drm_crtc_helper.c
@@ -549,7 +549,7 @@ int drm_crtc_helper_set_config(struct drm_mode_set *set)
 	int count = 0, ro, fail = 0;
 	struct drm_crtc_helper_funcs *crtc_funcs;
 	struct drm_mode_set save_set;
-	int ret = 0;
+	int ret;
 	int i;
 
 	DRM_DEBUG_KMS("\n");
diff --git a/sys/dev/drm2/drm_crtc_helper.h b/sys/dev/drm2/drm_crtc_helper.h
index 0110949b170..0f36b1fa3d4 100644
--- a/sys/dev/drm2/drm_crtc_helper.h
+++ b/sys/dev/drm2/drm_crtc_helper.h
@@ -78,7 +78,7 @@ struct drm_encoder_helper_funcs {
 	void (*restore)(struct drm_encoder *encoder);
 
 	bool (*mode_fixup)(struct drm_encoder *encoder,
-			   const struct drm_display_mode *mode,
+			   struct drm_display_mode *mode,
 			   struct drm_display_mode *adjusted_mode);
 	void (*prepare)(struct drm_encoder *encoder);
 	void (*commit)(struct drm_encoder *encoder);
diff --git a/sys/dev/drm2/drm_drv.c b/sys/dev/drm2/drm_drv.c
index d86cbbd27cc..b5150e1821c 100644
--- a/sys/dev/drm2/drm_drv.c
+++ b/sys/dev/drm2/drm_drv.c
@@ -188,6 +188,8 @@ static drm_ioctl_desc_t		  drm_ioctls[256] = {
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_CREATE_DUMB, drm_mode_create_dumb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_MAP_DUMB, drm_mode_mmap_dumb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 	DRM_IOCTL_DEF(DRM_IOCTL_MODE_DESTROY_DUMB, drm_mode_destroy_dumb_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_GETPROPERTIES, drm_mode_obj_get_properties_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
+	DRM_IOCTL_DEF(DRM_IOCTL_MODE_OBJ_SETPROPERTY, drm_mode_obj_set_property_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW|DRM_UNLOCKED),
 };
 
 static struct cdevsw drm_cdevsw = {
@@ -246,7 +248,8 @@ int drm_probe(device_t kdev, drm_pci_id_list_t *idlist)
 	device = pci_get_device(kdev);
 
 	if (pci_get_class(kdev) != PCIC_DISPLAY
-	    || pci_get_subclass(kdev) != PCIS_DISPLAY_VGA)
+	    || (pci_get_subclass(kdev) != PCIS_DISPLAY_VGA &&
+	    pci_get_subclass(kdev) != PCIS_DISPLAY_OTHER))
 		return ENXIO;
 
 	id_entry = drm_find_description(vendor, device, idlist);
@@ -1075,10 +1078,63 @@ SYSINIT(drm_register, SI_SUB_KLD, SI_ORDER_MIDDLE,
 SYSUNINIT(drm_unregister, SI_SUB_KLD, SI_ORDER_MIDDLE,
     drm_core_exit, NULL);
 
+static bool
+dmi_found(const struct dmi_system_id *dsi)
+{
+	char *hw_vendor, *hw_prod;
+	int i, slot;
+	bool res;
+
+	hw_vendor = kern_getenv("smbios.planar.maker");
+	hw_prod = kern_getenv("smbios.planar.product");
+	res = true;
+	for (i = 0; i < nitems(dsi->matches); i++) {
+		slot = dsi->matches[i].slot;
+		switch (slot) {
+		case DMI_NONE:
+			break;
+		case DMI_SYS_VENDOR:
+		case DMI_BOARD_VENDOR:
+			if (hw_vendor != NULL &&
+			    !strcmp(hw_vendor, dsi->matches[i].substr)) {
+				break;
+			} else {
+				res = false;
+				goto out;
+			}
+		case DMI_PRODUCT_NAME:
+		case DMI_BOARD_NAME:
+			if (hw_prod != NULL &&
+			    !strcmp(hw_prod, dsi->matches[i].substr)) {
+				break;
+			} else {
+				res = false;
+				goto out;
+			}
+		default:
+			res = false;
+			goto out;
+		}
+	}
+out:
+	freeenv(hw_vendor);
+	freeenv(hw_prod);
+
+	return (res);
+}
+
 bool
 dmi_check_system(const struct dmi_system_id *sysid)
 {
+	const struct dmi_system_id *dsi;
+	bool res;
 
-	/* XXXKIB */
-	return (false);
+	for (res = false, dsi = sysid; dsi->matches[0].slot != 0 ; dsi++) {
+		if (dmi_found(dsi)) {
+			res = true;
+			if (dsi->callback != NULL && dsi->callback(dsi))
+				break;
+		}
+	}
+	return (res);
 }
diff --git a/sys/dev/drm2/drm_edid.c b/sys/dev/drm2/drm_edid.c
index 8a317a08d7b..bf4cc9fd254 100644
--- a/sys/dev/drm2/drm_edid.c
+++ b/sys/dev/drm2/drm_edid.c
@@ -84,7 +84,7 @@ struct detailed_mode_closure {
 #define LEVEL_CVT	3
 
 static struct edid_quirk {
-	char *vendor;
+	char vendor[4];
 	int product_id;
 	u32 quirks;
 } edid_quirk_list[] = {
@@ -151,13 +151,13 @@ int drm_edid_header_is_valid(const u8 *raw_edid)
  * doesn't check out, or 1 if it's valid.
  */
 static bool
-drm_edid_block_valid(u8 *raw_edid)
+drm_edid_block_valid(u8 *raw_edid, int block)
 {
 	int i;
 	u8 csum = 0;
 	struct edid *edid = (struct edid *)raw_edid;
 
-	if (raw_edid[0] == 0x00) {
+	if (block == 0) {
 		int score = drm_edid_header_is_valid(raw_edid);
 		if (score == 8) ;
 		else if (score >= 6) {
@@ -230,7 +230,7 @@ bool drm_edid_is_valid(struct edid *edid)
 		return false;
 
 	for (i = 0; i <= edid->extensions; i++)
-		if (!drm_edid_block_valid(raw + i * EDID_LENGTH))
+		if (!drm_edid_block_valid(raw + i * EDID_LENGTH, i))
 			return false;
 
 	return true;
@@ -320,7 +320,7 @@ drm_do_get_edid(struct drm_connector *connector, device_t adapter)
 	for (i = 0; i < 4; i++) {
 		if (drm_do_probe_ddc_edid(adapter, block, 0, EDID_LENGTH))
 			goto out;
-		if (drm_edid_block_valid(block))
+		if (drm_edid_block_valid(block, 0))
 			break;
 		if (i == 0 && drm_edid_is_zero(block, EDID_LENGTH)) {
 			connector->null_edid_counter++;
@@ -344,7 +344,7 @@ drm_do_get_edid(struct drm_connector *connector, device_t adapter)
 				  block + (valid_extensions + 1) * EDID_LENGTH,
 				  j, EDID_LENGTH))
 				goto out;
-			if (drm_edid_block_valid(block + (valid_extensions + 1) * EDID_LENGTH)) {
+			if (drm_edid_block_valid(block + (valid_extensions + 1) * EDID_LENGTH, j)) {
 				valid_extensions++;
 				break;
 			}
@@ -504,23 +504,47 @@ static void edid_fixup_preferred(struct drm_connector *connector,
 	preferred_mode->type |= DRM_MODE_TYPE_PREFERRED;
 }
 
-struct drm_display_mode *drm_mode_find_dmt(struct drm_device *dev,
-					   int hsize, int vsize, int fresh)
+static bool
+mode_is_rb(const struct drm_display_mode *mode)
+{
+	return (mode->htotal - mode->hdisplay == 160) &&
+	       (mode->hsync_end - mode->hdisplay == 80) &&
+	       (mode->hsync_end - mode->hsync_start == 32) &&
+	       (mode->vsync_start - mode->vdisplay == 3);
+}
+
+/*
+ * drm_mode_find_dmt - Create a copy of a mode if present in DMT
+ * @dev: Device to duplicate against
+ * @hsize: Mode width
+ * @vsize: Mode height
+ * @fresh: Mode refresh rate
+ * @rb: Mode reduced-blanking-ness
+ *
+ * Walk the DMT mode list looking for a match for the given parameters.
+ * Return a newly allocated copy of the mode, or NULL if not found.
+ */
+struct drm_display_mode *drm_mode_find_dmt(struct drm_device *dev,
+					   int hsize, int vsize, int fresh,
+					   bool rb)
 {
-	struct drm_display_mode *mode = NULL;
 	int i;
 
 	for (i = 0; i < drm_num_dmt_modes; i++) {
 		struct drm_display_mode *ptr = &drm_dmt_modes[i];
-		if (hsize == ptr->hdisplay &&
-			vsize == ptr->vdisplay &&
-			fresh == drm_mode_vrefresh(ptr)) {
-			/* get the expected default mode */
-			mode = drm_mode_duplicate(dev, ptr);
-			break;
-		}
+		if (hsize != ptr->hdisplay)
+			continue;
+		if (vsize != ptr->vdisplay)
+			continue;
+		if (fresh != drm_mode_vrefresh(ptr))
+			continue;
+		if (rb != mode_is_rb(ptr))
+			continue;
+
+		return drm_mode_duplicate(dev, ptr);
 	}
-	return mode;
+
+	return NULL;
 }
 
 typedef void detailed_cb(struct detailed_timing *timing, void *closure);
@@ -763,10 +787,17 @@ drm_mode_std(struct drm_connector *connector, struct edid *edid,
 	}
 
 	/* check whether it can be found in default mode table */
-	mode = drm_mode_find_dmt(dev, hsize, vsize, vrefresh_rate);
+	if (drm_monitor_supports_rb(edid)) {
+		mode = drm_mode_find_dmt(dev, hsize, vsize, vrefresh_rate,
+					 true);
+		if (mode)
+			return mode;
+	}
+	mode = drm_mode_find_dmt(dev, hsize, vsize, vrefresh_rate, false);
 	if (mode)
 		return mode;
 
+	/* okay, generate it */
 	switch (timing_level) {
 	case LEVEL_DMT:
 		break;
@@ -780,6 +811,8 @@ drm_mode_std(struct drm_connector *connector, struct edid *edid,
 		 * secondary GTF curve.  Please don't do that.
 		 */
 		mode = drm_gtf_mode(dev, hsize, vsize, vrefresh_rate, 0, 0);
+		if (!mode)
+			return NULL;
 		if (drm_mode_hsync(mode) > drm_gtf2_hbreak(edid)) {
 			free(mode, DRM_MEM_KMS);
 			mode = drm_gtf_mode_complex(dev, hsize, vsize,
@@ -940,15 +973,6 @@ static struct drm_display_mode *drm_mode_detailed(struct drm_device *dev,
 	return mode;
 }
 
-static bool
-mode_is_rb(const struct drm_display_mode *mode)
-{
-	return (mode->htotal - mode->hdisplay == 160) &&
-	       (mode->hsync_end - mode->hdisplay == 80) &&
-	       (mode->hsync_end - mode->hsync_start == 32) &&
-	       (mode->vsync_start - mode->vdisplay == 3);
-}
-
 static bool
 mode_in_hsync_range(struct drm_display_mode *mode,
 		    struct edid *edid, u8 *t)
@@ -1026,12 +1050,8 @@ mode_in_range(struct drm_display_mode *mode, struct edid *edid,
 	return true;
 }
 
-/*
- * XXX If drm_dmt_modes ever regrows the CVT-R modes (and it will) this will
- * need to account for them.
- */
 static int
-drm_gtf_modes_for_range(struct drm_connector *connector, struct edid *edid,
+drm_dmt_modes_for_range(struct drm_connector *connector, struct edid *edid,
 			struct detailed_timing *timing)
 {
 	int i, modes = 0;
@@ -1051,17 +1071,110 @@ drm_gtf_modes_for_range(struct drm_connector *connector, struct edid *edid,
 	return modes;
 }
 
+/* fix up 1366x768 mode from 1368x768;
+ * GFT/CVT can't express 1366 width which isn't dividable by 8
+ */
+static void fixup_mode_1366x768(struct drm_display_mode *mode)
+{
+	if (mode->hdisplay == 1368 && mode->vdisplay == 768) {
+		mode->hdisplay = 1366;
+		mode->hsync_start--;
+		mode->hsync_end--;
+		drm_mode_set_name(mode);
+	}
+}
+
+static int
+drm_gtf_modes_for_range(struct drm_connector *connector, struct edid *edid,
+			struct detailed_timing *timing)
+{
+	int i, modes = 0;
+	struct drm_display_mode *newmode;
+	struct drm_device *dev = connector->dev;
+
+	for (i = 0; i < num_extra_modes; i++) {
+		const struct minimode *m = &extra_modes[i];
+		newmode = drm_gtf_mode(dev, m->w, m->h, m->r, 0, 0);
+		if (!newmode)
+			return modes;
+
+		fixup_mode_1366x768(newmode);
+		if (!mode_in_range(newmode, edid, timing)) {
+			drm_mode_destroy(dev, newmode);
+			continue;
+		}
+
+		drm_mode_probed_add(connector, newmode);
+		modes++;
+	}
+
+	return modes;
+}
+
+static int
+drm_cvt_modes_for_range(struct drm_connector *connector, struct edid *edid,
+			struct detailed_timing *timing)
+{
+	int i, modes = 0;
+	struct drm_display_mode *newmode;
+	struct drm_device *dev = connector->dev;
+	bool rb = drm_monitor_supports_rb(edid);
+
+	for (i = 0; i < num_extra_modes; i++) {
+		const struct minimode *m = &extra_modes[i];
+		newmode = drm_cvt_mode(dev, m->w, m->h, m->r, rb, 0, 0);
+		if (!newmode)
+			return modes;
+
+		fixup_mode_1366x768(newmode);
+		if (!mode_in_range(newmode, edid, timing)) {
+			drm_mode_destroy(dev, newmode);
+			continue;
+		}
+
+		drm_mode_probed_add(connector, newmode);
+		modes++;
+	}
+
+	return modes;
+}
+
 static void
 do_inferred_modes(struct detailed_timing *timing, void *c)
 {
 	struct detailed_mode_closure *closure = c;
 	struct detailed_non_pixel *data = &timing->data.other_data;
-	int gtf = (closure->edid->features & DRM_EDID_FEATURE_DEFAULT_GTF);
+	struct detailed_data_monitor_range *range = &data->data.range;
 
-	if (gtf && data->type == EDID_DETAIL_MONITOR_RANGE)
+	if (data->type != EDID_DETAIL_MONITOR_RANGE)
+		return;
+
+	closure->modes += drm_dmt_modes_for_range(closure->connector,
+						  closure->edid,
+						  timing);
+	
+	if (!version_greater(closure->edid, 1, 1))
+		return; /* GTF not defined yet */
+
+	switch (range->flags) {
+	case 0x02: /* secondary gtf, XXX could do more */
+	case 0x00: /* default gtf */
 		closure->modes += drm_gtf_modes_for_range(closure->connector,
 							  closure->edid,
 							  timing);
+		break;
+	case 0x04: /* cvt, only in 1.4+ */
+		if (!version_greater(closure->edid, 1, 3))
+			break;
+
+		closure->modes += drm_cvt_modes_for_range(closure->connector,
+							  closure->edid,
+							  timing);
+		break;
+	case 0x01: /* just the ranges, no formula */
+	default:
+		break;
+	}
 }
 
 static int
@@ -1094,8 +1207,8 @@ drm_est3_modes(struct drm_connector *connector, struct detailed_timing *timing)
 				mode = drm_mode_find_dmt(connector->dev,
 							 est3_modes[m].w,
 							 est3_modes[m].h,
-							 est3_modes[m].r
-							 /*, est3_modes[m].rb */);
+							 est3_modes[m].r,
+							 est3_modes[m].rb);
 				if (mode) {
 					drm_mode_probed_add(connector, mode);
 					modes++;
@@ -1343,6 +1456,8 @@ add_detailed_modes(struct drm_connector *connector, struct edid *edid,
 #define VENDOR_BLOCK    0x03
 #define SPEAKER_BLOCK	0x04
 #define EDID_BASIC_AUDIO	(1 << 6)
+#define EDID_CEA_YCRCB444	(1 << 5)
+#define EDID_CEA_YCRCB422	(1 << 4)
 
 /**
  * Search EDID for CEA extension block.
@@ -1647,13 +1762,29 @@ static void drm_add_display_info(struct edid *edid,
 	info->bpc = 0;
 	info->color_formats = 0;
 
-	/* Only defined for 1.4 with digital displays */
-	if (edid->revision < 4)
+	if (edid->revision < 3)
 		return;
 
 	if (!(edid->input & DRM_EDID_INPUT_DIGITAL))
 		return;
 
+	/* Get data from CEA blocks if present */
+	edid_ext = drm_find_cea_extension(edid);
+	if (edid_ext) {
+		info->cea_rev = edid_ext[1];
+
+		/* The existence of a CEA block should imply RGB support */
+		info->color_formats = DRM_COLOR_FORMAT_RGB444;
+		if (edid_ext[3] & EDID_CEA_YCRCB444)
+			info->color_formats |= DRM_COLOR_FORMAT_YCRCB444;
+		if (edid_ext[3] & EDID_CEA_YCRCB422)
+			info->color_formats |= DRM_COLOR_FORMAT_YCRCB422;
+	}
+
+	/* Only defined for 1.4 with digital displays */
+	if (edid->revision < 4)
+		return;
+
 	switch (edid->input & DRM_EDID_DIGITAL_DEPTH_MASK) {
 	case DRM_EDID_DIGITAL_DEPTH_6:
 		info->bpc = 6;
@@ -1679,18 +1810,11 @@ static void drm_add_display_info(struct edid *edid,
 		break;
 	}
 
-	info->color_formats = DRM_COLOR_FORMAT_RGB444;
-	if (info->color_formats & DRM_EDID_FEATURE_RGB_YCRCB444)
-		info->color_formats = DRM_COLOR_FORMAT_YCRCB444;
-	if (info->color_formats & DRM_EDID_FEATURE_RGB_YCRCB422)
-		info->color_formats = DRM_COLOR_FORMAT_YCRCB422;
-
-	/* Get data from CEA blocks if present */
-	edid_ext = drm_find_cea_extension(edid);
-	if (!edid_ext)
-		return;
-
-	info->cea_rev = edid_ext[1];
+	info->color_formats |= DRM_COLOR_FORMAT_RGB444;
+	if (edid->features & DRM_EDID_FEATURE_RGB_YCRCB444)
+		info->color_formats |= DRM_COLOR_FORMAT_YCRCB444;
+	if (edid->features & DRM_EDID_FEATURE_RGB_YCRCB422)
+		info->color_formats |= DRM_COLOR_FORMAT_YCRCB422;
 }
 
 /**
diff --git a/sys/dev/drm2/drm_edid.h b/sys/dev/drm2/drm_edid.h
index e0c14705438..3dfe6e3c133 100644
--- a/sys/dev/drm2/drm_edid.h
+++ b/sys/dev/drm2/drm_edid.h
@@ -93,12 +93,26 @@ struct detailed_data_monitor_range {
 	u8 min_hfreq_khz;
 	u8 max_hfreq_khz;
 	u8 pixel_clock_mhz; /* need to multiply by 10 */
-	u16 sec_gtf_toggle; /* A000=use above, 20=use below */
-	u8 hfreq_start_khz; /* need to multiply by 2 */
-	u8 c; /* need to divide by 2 */
-	u16 m;
-	u8 k;
-	u8 j; /* need to divide by 2 */
+	u8 flags;
+	union {
+		struct {
+			u8 reserved;
+			u8 hfreq_start_khz; /* need to multiply by 2 */
+			u8 c; /* need to divide by 2 */
+			u16 m;
+			u8 k;
+			u8 j; /* need to divide by 2 */
+		} __attribute__((packed)) gtf2;
+		struct {
+			u8 version;
+			u8 data1; /* high 6 bits: extra clock resolution */
+			u8 data2; /* plus low 2 of above: max hactive */
+			u8 supported_aspects;
+			u8 flags; /* preferred aspect and blanking support */
+			u8 supported_scalings;
+			u8 preferred_refresh;
+		} __attribute__((packed)) cvt;
+	} formula;
 } __attribute__((packed));
 
 struct detailed_data_wpindex {
diff --git a/sys/dev/drm2/drm_edid_modes.h b/sys/dev/drm2/drm_edid_modes.h
index beded269bf4..b6431d36c10 100644
--- a/sys/dev/drm2/drm_edid_modes.h
+++ b/sys/dev/drm2/drm_edid_modes.h
@@ -31,7 +31,6 @@
 /*
  * Autogenerated from the DMT spec.
  * This table is copied from xfree86/modes/xf86EdidModes.c.
- * But the mode with Reduced blank feature is deleted.
  */
 static struct drm_display_mode drm_dmt_modes[] = {
 	/* 640x350@85Hz */
@@ -82,6 +81,10 @@ static struct drm_display_mode drm_dmt_modes[] = {
 	{ DRM_MODE("800x600", DRM_MODE_TYPE_DRIVER, 56250, 800, 832,
 		   896, 1048, 0, 600, 601, 604, 631, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 800x600@120Hz RB */
+	{ DRM_MODE("800x600", DRM_MODE_TYPE_DRIVER, 73250, 800, 848,
+		   880, 960, 0, 600, 603, 607, 636, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 848x480@60Hz */
 	{ DRM_MODE("848x480", DRM_MODE_TYPE_DRIVER, 33750, 848, 864,
 		   976, 1088, 0, 480, 486, 494, 517, 0,
@@ -107,10 +110,18 @@ static struct drm_display_mode drm_dmt_modes[] = {
 	{ DRM_MODE("1024x768", DRM_MODE_TYPE_DRIVER, 94500, 1024, 1072,
 		   1168, 1376, 0, 768, 769, 772, 808, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 1024x768@120Hz RB */
+	{ DRM_MODE("1024x768", DRM_MODE_TYPE_DRIVER, 115500, 1024, 1072,
+		   1104, 1184, 0, 768, 771, 775, 813, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 1152x864@75Hz */
 	{ DRM_MODE("1152x864", DRM_MODE_TYPE_DRIVER, 108000, 1152, 1216,
 		   1344, 1600, 0, 864, 865, 868, 900, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 1280x768@60Hz RB */
+	{ DRM_MODE("1280x768", DRM_MODE_TYPE_DRIVER, 68250, 1280, 1328,
+		   1360, 1440, 0, 768, 771, 778, 790, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 1280x768@60Hz */
 	{ DRM_MODE("1280x768", DRM_MODE_TYPE_DRIVER, 79500, 1280, 1344,
 		   1472, 1664, 0, 768, 771, 778, 798, 0,
@@ -123,6 +134,14 @@ static struct drm_display_mode drm_dmt_modes[] = {
 	{ DRM_MODE("1280x768", DRM_MODE_TYPE_DRIVER, 117500, 1280, 1360,
 		   1496, 1712, 0, 768, 771, 778, 809, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 1280x768@120Hz RB */
+	{ DRM_MODE("1280x768", DRM_MODE_TYPE_DRIVER, 140250, 1280, 1328,
+		   1360, 1440, 0, 768, 771, 778, 813, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 1280x800@60Hz RB */
+	{ DRM_MODE("1280x800", DRM_MODE_TYPE_DRIVER, 71000, 1280, 1328,
+		   1360, 1440, 0, 800, 803, 809, 823, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 1280x800@60Hz */
 	{ DRM_MODE("1280x800", DRM_MODE_TYPE_DRIVER, 83500, 1280, 1352,
 		   1480, 1680, 0, 800, 803, 809, 831, 0,
@@ -135,6 +154,10 @@ static struct drm_display_mode drm_dmt_modes[] = {
 	{ DRM_MODE("1280x800", DRM_MODE_TYPE_DRIVER, 122500, 1280, 1360,
 		   1496, 1712, 0, 800, 803, 809, 843, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 1280x800@120Hz RB */
+	{ DRM_MODE("1280x800", DRM_MODE_TYPE_DRIVER, 146250, 1280, 1328,
+		   1360, 1440, 0, 800, 803, 809, 847, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 1280x960@60Hz */
 	{ DRM_MODE("1280x960", DRM_MODE_TYPE_DRIVER, 108000, 1280, 1376,
 		   1488, 1800, 0, 960, 961, 964, 1000, 0,
@@ -143,6 +166,10 @@ static struct drm_display_mode drm_dmt_modes[] = {
 	{ DRM_MODE("1280x960", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1344,
 		   1504, 1728, 0, 960, 961, 964, 1011, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 1280x960@120Hz RB */
+	{ DRM_MODE("1280x960", DRM_MODE_TYPE_DRIVER, 175500, 1280, 1328,
+		   1360, 1440, 0, 960, 963, 967, 1017, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 1280x1024@60Hz */
 	{ DRM_MODE("1280x1024", DRM_MODE_TYPE_DRIVER, 108000, 1280, 1328,
 		   1440, 1688, 0, 1024, 1025, 1028, 1066, 0,
@@ -155,22 +182,42 @@ static struct drm_display_mode drm_dmt_modes[] = {
 	{ DRM_MODE("1280x1024", DRM_MODE_TYPE_DRIVER, 157500, 1280, 1344,
 		   1504, 1728, 0, 1024, 1025, 1028, 1072, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 1280x1024@120Hz RB */
+	{ DRM_MODE("1280x1024", DRM_MODE_TYPE_DRIVER, 187250, 1280, 1328,
+		   1360, 1440, 0, 1024, 1027, 1034, 1084, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 1360x768@60Hz */
 	{ DRM_MODE("1360x768", DRM_MODE_TYPE_DRIVER, 85500, 1360, 1424,
 		   1536, 1792, 0, 768, 771, 777, 795, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
-	/* 1440x1050@60Hz */
+	/* 1360x768@120Hz RB */
+	{ DRM_MODE("1360x768", DRM_MODE_TYPE_DRIVER, 148250, 1360, 1408,
+		   1440, 1520, 0, 768, 771, 776, 813, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 1400x1050@60Hz RB */
+	{ DRM_MODE("1400x1050", DRM_MODE_TYPE_DRIVER, 101000, 1400, 1448,
+		   1480, 1560, 0, 1050, 1053, 1057, 1080, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 1400x1050@60Hz */
 	{ DRM_MODE("1400x1050", DRM_MODE_TYPE_DRIVER, 121750, 1400, 1488,
 		   1632, 1864, 0, 1050, 1053, 1057, 1089, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) },
-	/* 1440x1050@75Hz */
+	/* 1400x1050@75Hz */
 	{ DRM_MODE("1400x1050", DRM_MODE_TYPE_DRIVER, 156000, 1400, 1504,
 		   1648, 1896, 0, 1050, 1053, 1057, 1099, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) },
-	/* 1440x1050@85Hz */
+	/* 1400x1050@85Hz */
 	{ DRM_MODE("1400x1050", DRM_MODE_TYPE_DRIVER, 179500, 1400, 1504,
 		   1656, 1912, 0, 1050, 1053, 1057, 1105, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 1400x1050@120Hz RB */
+	{ DRM_MODE("1400x1050", DRM_MODE_TYPE_DRIVER, 208000, 1400, 1448,
+		   1480, 1560, 0, 1050, 1053, 1057, 1112, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 1440x900@60Hz RB */
+	{ DRM_MODE("1440x900", DRM_MODE_TYPE_DRIVER, 88750, 1440, 1488,
+		   1520, 1600, 0, 900, 903, 909, 926, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 1440x900@60Hz */
 	{ DRM_MODE("1440x900", DRM_MODE_TYPE_DRIVER, 106500, 1440, 1520,
 		   1672, 1904, 0, 900, 903, 909, 934, 0,
@@ -183,6 +230,10 @@ static struct drm_display_mode drm_dmt_modes[] = {
 	{ DRM_MODE("1440x900", DRM_MODE_TYPE_DRIVER, 157000, 1440, 1544,
 		   1696, 1952, 0, 900, 903, 909, 948, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 1440x900@120Hz RB */
+	{ DRM_MODE("1440x900", DRM_MODE_TYPE_DRIVER, 182750, 1440, 1488,
+		   1520, 1600, 0, 900, 903, 909, 953, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 1600x1200@60Hz */
 	{ DRM_MODE("1600x1200", DRM_MODE_TYPE_DRIVER, 162000, 1600, 1664,
 		   1856, 2160, 0, 1200, 1201, 1204, 1250, 0,
@@ -203,6 +254,14 @@ static struct drm_display_mode drm_dmt_modes[] = {
 	{ DRM_MODE("1600x1200", DRM_MODE_TYPE_DRIVER, 229500, 1600, 1664,
 		   1856, 2160, 0, 1200, 1201, 1204, 1250, 0,
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 1600x1200@120Hz RB */
+	{ DRM_MODE("1600x1200", DRM_MODE_TYPE_DRIVER, 268250, 1600, 1648,
+		   1680, 1760, 0, 1200, 1203, 1207, 1271, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 1680x1050@60Hz RB */
+	{ DRM_MODE("1680x1050", DRM_MODE_TYPE_DRIVER, 119000, 1680, 1728,
+		   1760, 1840, 0, 1050, 1053, 1059, 1080, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 1680x1050@60Hz */
 	{ DRM_MODE("1680x1050", DRM_MODE_TYPE_DRIVER, 146250, 1680, 1784,
 		   1960, 2240, 0, 1050, 1053, 1059, 1089, 0,
@@ -215,15 +274,23 @@ static struct drm_display_mode drm_dmt_modes[] = {
 	{ DRM_MODE("1680x1050", DRM_MODE_TYPE_DRIVER, 214750, 1680, 1808,
 		   1984, 2288, 0, 1050, 1053, 1059, 1105, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 1680x1050@120Hz RB */
+	{ DRM_MODE("1680x1050", DRM_MODE_TYPE_DRIVER, 245500, 1680, 1728,
+		   1760, 1840, 0, 1050, 1053, 1059, 1112, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 1792x1344@60Hz */
 	{ DRM_MODE("1792x1344", DRM_MODE_TYPE_DRIVER, 204750, 1792, 1920,
 		   2120, 2448, 0, 1344, 1345, 1348, 1394, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) },
-	/* 1729x1344@75Hz */
+	/* 1792x1344@75Hz */
 	{ DRM_MODE("1792x1344", DRM_MODE_TYPE_DRIVER, 261000, 1792, 1888,
 		   2104, 2456, 0, 1344, 1345, 1348, 1417, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) },
-	/* 1853x1392@60Hz */
+	/* 1792x1344@120Hz RB */
+	{ DRM_MODE("1792x1344", DRM_MODE_TYPE_DRIVER, 333250, 1792, 1840,
+		   1872, 1952, 0, 1344, 1347, 1351, 1423, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 1856x1392@60Hz */
 	{ DRM_MODE("1856x1392", DRM_MODE_TYPE_DRIVER, 218250, 1856, 1952,
 		   2176, 2528, 0, 1392, 1393, 1396, 1439, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) },
@@ -231,6 +298,14 @@ static struct drm_display_mode drm_dmt_modes[] = {
 	{ DRM_MODE("1856x1392", DRM_MODE_TYPE_DRIVER, 288000, 1856, 1984,
 		   2208, 2560, 0, 1392, 1395, 1399, 1500, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 1856x1392@120Hz RB */
+	{ DRM_MODE("1856x1392", DRM_MODE_TYPE_DRIVER, 356500, 1856, 1904,
+		   1936, 2016, 0, 1392, 1395, 1399, 1474, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 1920x1200@60Hz RB */
+	{ DRM_MODE("1920x1200", DRM_MODE_TYPE_DRIVER, 154000, 1920, 1968,
+		   2000, 2080, 0, 1200, 1203, 1209, 1235, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 1920x1200@60Hz */
 	{ DRM_MODE("1920x1200", DRM_MODE_TYPE_DRIVER, 193250, 1920, 2056,
 		   2256, 2592, 0, 1200, 1203, 1209, 1245, 0,
@@ -243,6 +318,10 @@ static struct drm_display_mode drm_dmt_modes[] = {
 	{ DRM_MODE("1920x1200", DRM_MODE_TYPE_DRIVER, 281250, 1920, 2064,
 		   2272, 2624, 0, 1200, 1203, 1209, 1262, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 1920x1200@120Hz RB */
+	{ DRM_MODE("1920x1200", DRM_MODE_TYPE_DRIVER, 317000, 1920, 1968,
+		   2000, 2080, 0, 1200, 1203, 1209, 1271, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 1920x1440@60Hz */
 	{ DRM_MODE("1920x1440", DRM_MODE_TYPE_DRIVER, 234000, 1920, 2048,
 		   2256, 2600, 0, 1440, 1441, 1444, 1500, 0,
@@ -251,6 +330,14 @@ static struct drm_display_mode drm_dmt_modes[] = {
 	{ DRM_MODE("1920x1440", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2064,
 		   2288, 2640, 0, 1440, 1441, 1444, 1500, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 1920x1440@120Hz RB */
+	{ DRM_MODE("1920x1440", DRM_MODE_TYPE_DRIVER, 380500, 1920, 1968,
+		   2000, 2080, 0, 1440, 1443, 1447, 1525, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 2560x1600@60Hz RB */
+	{ DRM_MODE("2560x1600", DRM_MODE_TYPE_DRIVER, 268500, 2560, 2608,
+		   2640, 2720, 0, 1600, 1603, 1609, 1646, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
 	/* 2560x1600@60Hz */
 	{ DRM_MODE("2560x1600", DRM_MODE_TYPE_DRIVER, 348500, 2560, 2752,
 		   3032, 3504, 0, 1600, 1603, 1609, 1658, 0,
@@ -263,6 +350,11 @@ static struct drm_display_mode drm_dmt_modes[] = {
 	{ DRM_MODE("2560x1600", DRM_MODE_TYPE_DRIVER, 505250, 2560, 2768,
 		   3048, 3536, 0, 1600, 1603, 1609, 1682, 0,
 		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 2560x1600@120Hz RB */
+	{ DRM_MODE("2560x1600", DRM_MODE_TYPE_DRIVER, 552750, 2560, 2608,
+		   2640, 2720, 0, 1600, 1603, 1609, 1694, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC) },
+
 };
 static const int drm_num_dmt_modes =
 	sizeof(drm_dmt_modes) / sizeof(struct drm_display_mode);
@@ -321,12 +413,14 @@ static struct drm_display_mode edid_est_modes[] = {
 		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) }, /* 1152x864@75Hz */
 };
 
-static const struct {
+struct minimode {
 	short w;
 	short h;
 	short r;
 	short rb;
-} est3_modes[] = {
+};
+
+static const struct minimode est3_modes[] = {
 	/* byte 6 */
 	{ 640, 350, 85, 0 },
 	{ 640, 400, 85, 0 },
@@ -378,4 +472,304 @@ static const struct {
 	{ 1920, 1440, 60, 0 },
 	{ 1920, 1440, 75, 0 },
 };
-static const int num_est3_modes = sizeof(est3_modes) / sizeof(est3_modes[0]);
+static const int num_est3_modes = DRM_ARRAY_SIZE(est3_modes);
+
+static const struct minimode extra_modes[] = {
+	{ 1024, 576,  60, 0 },
+	{ 1366, 768,  60, 0 },
+	{ 1600, 900,  60, 0 },
+	{ 1680, 945,  60, 0 },
+	{ 1920, 1080, 60, 0 },
+	{ 2048, 1152, 60, 0 },
+	{ 2048, 1536, 60, 0 },
+};
+static const int num_extra_modes = DRM_ARRAY_SIZE(extra_modes);
+
+/*
+ * Probably taken from CEA-861 spec.
+ * This table is converted from xorg's hw/xfree86/modes/xf86EdidModes.c.
+ */
+static const struct drm_display_mode edid_cea_modes[] = {
+	/* 1 - 640x480@60Hz */
+	{ DRM_MODE("640x480", DRM_MODE_TYPE_DRIVER, 25175, 640, 656,
+		   752, 800, 0, 480, 490, 492, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 2 - 720x480@60Hz */
+	{ DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 27000, 720, 736,
+		   798, 858, 0, 480, 489, 495, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 3 - 720x480@60Hz */
+	{ DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 27000, 720, 736,
+		   798, 858, 0, 480, 489, 495, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 4 - 1280x720@60Hz */
+	{ DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1390,
+		   1430, 1650, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 5 - 1920x1080i@60Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2008,
+		   2052, 2200, 0, 1080, 1084, 1094, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
+			DRM_MODE_FLAG_INTERLACE) },
+	/* 6 - 1440x480i@60Hz */
+	{ DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1478,
+		   1602, 1716, 0, 480, 488, 494, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
+	/* 7 - 1440x480i@60Hz */
+	{ DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1478,
+		   1602, 1716, 0, 480, 488, 494, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
+	/* 8 - 1440x240@60Hz */
+	{ DRM_MODE("1440x240", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1478,
+		   1602, 1716, 0, 240, 244, 247, 262, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_DBLCLK) },
+	/* 9 - 1440x240@60Hz */
+	{ DRM_MODE("1440x240", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1478,
+		   1602, 1716, 0, 240, 244, 247, 262, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_DBLCLK) },
+	/* 10 - 2880x480i@60Hz */
+	{ DRM_MODE("2880x480", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
+		   3204, 3432, 0, 480, 488, 494, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE) },
+	/* 11 - 2880x480i@60Hz */
+	{ DRM_MODE("2880x480", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
+		   3204, 3432, 0, 480, 488, 494, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE) },
+	/* 12 - 2880x240@60Hz */
+	{ DRM_MODE("2880x240", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
+		   3204, 3432, 0, 240, 244, 247, 262, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 13 - 2880x240@60Hz */
+	{ DRM_MODE("2880x240", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2956,
+		   3204, 3432, 0, 240, 244, 247, 262, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 14 - 1440x480@60Hz */
+	{ DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1472,
+		   1596, 1716, 0, 480, 489, 495, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 15 - 1440x480@60Hz */
+	{ DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1472,
+		   1596, 1716, 0, 480, 489, 495, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 16 - 1920x1080@60Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008,
+		   2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 17 - 720x576@50Hz */
+	{ DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 27000, 720, 732,
+		   796, 864, 0, 576, 581, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 18 - 720x576@50Hz */
+	{ DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 27000, 720, 732,
+		   796, 864, 0, 576, 581, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 19 - 1280x720@50Hz */
+	{ DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 1720,
+		   1760, 1980, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 20 - 1920x1080i@50Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2448,
+		   2492, 2640, 0, 1080, 1084, 1094, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
+			DRM_MODE_FLAG_INTERLACE) },
+	/* 21 - 1440x576i@50Hz */
+	{ DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1464,
+		   1590, 1728, 0, 576, 580, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
+	/* 22 - 1440x576i@50Hz */
+	{ DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1464,
+		   1590, 1728, 0, 576, 580, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
+	/* 23 - 1440x288@50Hz */
+	{ DRM_MODE("1440x288", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1464,
+		   1590, 1728, 0, 288, 290, 293, 312, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_DBLCLK) },
+	/* 24 - 1440x288@50Hz */
+	{ DRM_MODE("1440x288", DRM_MODE_TYPE_DRIVER, 27000, 1440, 1464,
+		   1590, 1728, 0, 288, 290, 293, 312, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_DBLCLK) },
+	/* 25 - 2880x576i@50Hz */
+	{ DRM_MODE("2880x576", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
+		   3180, 3456, 0, 576, 580, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE) },
+	/* 26 - 2880x576i@50Hz */
+	{ DRM_MODE("2880x576", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
+		   3180, 3456, 0, 576, 580, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE) },
+	/* 27 - 2880x288@50Hz */
+	{ DRM_MODE("2880x288", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
+		   3180, 3456, 0, 288, 290, 293, 312, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 28 - 2880x288@50Hz */
+	{ DRM_MODE("2880x288", DRM_MODE_TYPE_DRIVER, 54000, 2880, 2928,
+		   3180, 3456, 0, 288, 290, 293, 312, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 29 - 1440x576@50Hz */
+	{ DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1464,
+		   1592, 1728, 0, 576, 581, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 30 - 1440x576@50Hz */
+	{ DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1464,
+		   1592, 1728, 0, 576, 581, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 31 - 1920x1080@50Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448,
+		   2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 32 - 1920x1080@24Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2558,
+		   2602, 2750, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 33 - 1920x1080@25Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2448,
+		   2492, 2640, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 34 - 1920x1080@30Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 74250, 1920, 2008,
+		   2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 35 - 2880x480@60Hz */
+	{ DRM_MODE("2880x480", DRM_MODE_TYPE_DRIVER, 108000, 2880, 2944,
+		   3192, 3432, 0, 480, 489, 495, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 36 - 2880x480@60Hz */
+	{ DRM_MODE("2880x480", DRM_MODE_TYPE_DRIVER, 108000, 2880, 2944,
+		   3192, 3432, 0, 480, 489, 495, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 37 - 2880x576@50Hz */
+	{ DRM_MODE("2880x576", DRM_MODE_TYPE_DRIVER, 108000, 2880, 2928,
+		   3184, 3456, 0, 576, 581, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 38 - 2880x576@50Hz */
+	{ DRM_MODE("2880x576", DRM_MODE_TYPE_DRIVER, 108000, 2880, 2928,
+		   3184, 3456, 0, 576, 581, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 39 - 1920x1080i@50Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 72000, 1920, 1952,
+		   2120, 2304, 0, 1080, 1126, 1136, 1250, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE) },
+	/* 40 - 1920x1080i@100Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2448,
+		   2492, 2640, 0, 1080, 1084, 1094, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
+			DRM_MODE_FLAG_INTERLACE) },
+	/* 41 - 1280x720@100Hz */
+	{ DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1720,
+		   1760, 1980, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 42 - 720x576@100Hz */
+	{ DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 54000, 720, 732,
+		   796, 864, 0, 576, 581, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 43 - 720x576@100Hz */
+	{ DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 54000, 720, 732,
+		   796, 864, 0, 576, 581, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 44 - 1440x576i@100Hz */
+	{ DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1464,
+		   1590, 1728, 0, 576, 580, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_DBLCLK) },
+	/* 45 - 1440x576i@100Hz */
+	{ DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1464,
+		   1590, 1728, 0, 576, 580, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_DBLCLK) },
+	/* 46 - 1920x1080i@120Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 148500, 1920, 2008,
+		   2052, 2200, 0, 1080, 1084, 1094, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC |
+			DRM_MODE_FLAG_INTERLACE) },
+	/* 47 - 1280x720@120Hz */
+	{ DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 148500, 1280, 1390,
+		   1430, 1650, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 48 - 720x480@120Hz */
+	{ DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 54000, 720, 736,
+		   798, 858, 0, 480, 489, 495, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 49 - 720x480@120Hz */
+	{ DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 54000, 720, 736,
+		   798, 858, 0, 480, 489, 495, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 50 - 1440x480i@120Hz */
+	{ DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1478,
+		   1602, 1716, 0, 480, 488, 494, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
+	/* 51 - 1440x480i@120Hz */
+	{ DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 54000, 1440, 1478,
+		   1602, 1716, 0, 480, 488, 494, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
+	/* 52 - 720x576@200Hz */
+	{ DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 108000, 720, 732,
+		   796, 864, 0, 576, 581, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 53 - 720x576@200Hz */
+	{ DRM_MODE("720x576", DRM_MODE_TYPE_DRIVER, 108000, 720, 732,
+		   796, 864, 0, 576, 581, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 54 - 1440x576i@200Hz */
+	{ DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 108000, 1440, 1464,
+		   1590, 1728, 0, 576, 580, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
+	/* 55 - 1440x576i@200Hz */
+	{ DRM_MODE("1440x576", DRM_MODE_TYPE_DRIVER, 108000, 1440, 1464,
+		   1590, 1728, 0, 576, 580, 586, 625, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
+	/* 56 - 720x480@240Hz */
+	{ DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 108000, 720, 736,
+		   798, 858, 0, 480, 489, 495, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 57 - 720x480@240Hz */
+	{ DRM_MODE("720x480", DRM_MODE_TYPE_DRIVER, 108000, 720, 736,
+		   798, 858, 0, 480, 489, 495, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC) },
+	/* 58 - 1440x480i@240 */
+	{ DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 108000, 1440, 1478,
+		   1602, 1716, 0, 480, 488, 494, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
+	/* 59 - 1440x480i@240 */
+	{ DRM_MODE("1440x480", DRM_MODE_TYPE_DRIVER, 108000, 1440, 1478,
+		   1602, 1716, 0, 480, 488, 494, 525, 0,
+		   DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC |
+			DRM_MODE_FLAG_INTERLACE | DRM_MODE_FLAG_DBLCLK) },
+	/* 60 - 1280x720@24Hz */
+	{ DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 59400, 1280, 3040,
+		   3080, 3300, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 61 - 1280x720@25Hz */
+	{ DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 3700,
+		   3740, 3960, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 62 - 1280x720@30Hz */
+	{ DRM_MODE("1280x720", DRM_MODE_TYPE_DRIVER, 74250, 1280, 3040,
+		   3080, 3300, 0, 720, 725, 730, 750, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 63 - 1920x1080@120Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2008,
+		   2052, 2200, 0, 1080, 1084, 1089, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+	/* 64 - 1920x1080@100Hz */
+	{ DRM_MODE("1920x1080", DRM_MODE_TYPE_DRIVER, 297000, 1920, 2448,
+		   2492, 2640, 0, 1080, 1084, 1094, 1125, 0,
+		   DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC) },
+};
+static const int drm_num_cea_modes = DRM_ARRAY_SIZE(edid_cea_modes);
diff --git a/sys/dev/drm2/drm_fb_helper.c b/sys/dev/drm2/drm_fb_helper.c
index 2eb41a51dc7..562bf82f555 100644
--- a/sys/dev/drm2/drm_fb_helper.c
+++ b/sys/dev/drm2/drm_fb_helper.c
@@ -202,6 +202,9 @@ static void drm_fb_helper_restore_lut_atomic(struct drm_crtc *crtc)
 {
 	uint16_t *r_base, *g_base, *b_base;
 
+	if (crtc->funcs->gamma_set == NULL)
+		return;
+
 	r_base = crtc->gamma_store;
 	g_base = r_base + crtc->gamma_size;
 	b_base = g_base + crtc->gamma_size;
@@ -1225,7 +1228,7 @@ static bool drm_target_cloned(struct drm_fb_helper *fb_helper,
 
 	/* try and find a 1024x768 mode on each connector */
 	can_clone = true;
-	dmt_mode = drm_mode_find_dmt(fb_helper->dev, 1024, 768, 60);
+	dmt_mode = drm_mode_find_dmt(fb_helper->dev, 1024, 768, 60, false);
 
 	for (i = 0; i < fb_helper->connector_count; i++) {
 
diff --git a/sys/dev/drm2/drm_ioctl.c b/sys/dev/drm2/drm_ioctl.c
index c0af3761d29..e13eec787af 100644
--- a/sys/dev/drm2/drm_ioctl.c
+++ b/sys/dev/drm2/drm_ioctl.c
@@ -250,6 +250,10 @@ int drm_getcap(struct drm_device *dev, void *data, struct drm_file *file_priv)
 	case DRM_CAP_DUMB_PREFER_SHADOW:
 		req->value = dev->mode_config.prefer_shadow;
 		break;
+	case DRM_CAP_PRIME:
+		req->value |= false /* XXXKIB dev->driver->prime_fd_to_handle */ ? DRM_PRIME_CAP_IMPORT : 0;
+		req->value |= false /* XXXKIB dev->driver->prime_handle_to_fd */ ? DRM_PRIME_CAP_EXPORT : 0;
+		break;
 	case DRM_CAP_TIMESTAMP_MONOTONIC:
 		req->value = drm_timestamp_monotonic;
 		break;
diff --git a/sys/dev/drm2/drm_irq.c b/sys/dev/drm2/drm_irq.c
index 3a60fa971c0..4ab3de00fd0 100644
--- a/sys/dev/drm2/drm_irq.c
+++ b/sys/dev/drm2/drm_irq.c
@@ -639,7 +639,7 @@ drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev, int crtc,
 u32 drm_get_last_vbltimestamp(struct drm_device *dev, int crtc,
 			      struct timeval *tvblank, unsigned flags)
 {
-	int ret = 0;
+	int ret;
 
 	/* Define requested maximum error on timestamps (nanoseconds). */
 	int max_error = (int) drm_timestamp_precision * 1000;
@@ -930,18 +930,15 @@ int drm_modeset_ctl(struct drm_device *dev, void *data,
 		    struct drm_file *file_priv)
 {
 	struct drm_modeset_ctl *modeset = data;
-	int ret = 0;
 	unsigned int crtc;
 
 	/* If drm_vblank_init() hasn't been called yet, just no-op */
 	if (!dev->num_crtcs)
-		goto out;
+		return 0;
 
 	crtc = modeset->crtc;
-	if (crtc >= dev->num_crtcs) {
-		ret = -EINVAL;
-		goto out;
-	}
+	if (crtc >= dev->num_crtcs)
+		return -EINVAL;
 
 	switch (modeset->cmd) {
 	case _DRM_PRE_MODESET:
@@ -951,12 +948,11 @@ int drm_modeset_ctl(struct drm_device *dev, void *data,
 		drm_vblank_post_modeset(dev, crtc);
 		break;
 	default:
-		ret = -EINVAL;
+		return -EINVAL;
 		break;
 	}
 
-out:
-	return ret;
+	return 0;
 }
 
 static void
@@ -1054,7 +1050,7 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
 		    struct drm_file *file_priv)
 {
 	union drm_wait_vblank *vblwait = data;
-	int ret = 0;
+	int ret;
 	unsigned int flags, seq, crtc, high_crtc;
 
 	if (/*(!drm_dev_to_irq(dev)) || */(!dev->irq_enabled))
diff --git a/sys/dev/drm2/drm_memory.c b/sys/dev/drm2/drm_memory.c
index fa48197388c..af43a89bbfd 100644
--- a/sys/dev/drm2/drm_memory.c
+++ b/sys/dev/drm2/drm_memory.c
@@ -125,3 +125,11 @@ drm_clflush_pages(vm_page_t *pages, unsigned long num_pages)
 
 	pmap_invalidate_cache_pages(pages, num_pages);
 }
+
+void
+drm_clflush_virt_range(char *addr, unsigned long length)
+{
+
+	pmap_invalidate_cache_range((vm_offset_t)addr,
+	    (vm_offset_t)addr + length, TRUE);
+}
diff --git a/sys/dev/drm2/drm_mode.h b/sys/dev/drm2/drm_mode.h
index bc2824011d1..79cf933d4c8 100644
--- a/sys/dev/drm2/drm_mode.h
+++ b/sys/dev/drm2/drm_mode.h
@@ -228,6 +228,7 @@ struct drm_mode_get_connector {
 #define DRM_MODE_PROP_IMMUTABLE	(1<<2)
 #define DRM_MODE_PROP_ENUM	(1<<3) /* enumerated type with text strings */
 #define DRM_MODE_PROP_BLOB	(1<<4)
+#define DRM_MODE_PROP_BITMASK	(1<<5) /* bitmask of enumerated types */
 
 struct drm_mode_property_enum {
 	uint64_t value;
@@ -252,6 +253,21 @@ struct drm_mode_connector_set_property {
 	uint32_t connector_id;
 };
 
+struct drm_mode_obj_get_properties {
+	uint64_t props_ptr;
+	uint64_t prop_values_ptr;
+	uint32_t count_props;
+	uint32_t obj_id;
+	uint32_t obj_type;
+};
+
+struct drm_mode_obj_set_property {
+	uint64_t value;
+	uint32_t prop_id;
+	uint32_t obj_id;
+	uint32_t obj_type;
+};
+
 struct drm_mode_get_blob {
 	uint32_t blob_id;
 	uint32_t length;
diff --git a/sys/dev/drm2/drm_pciids.h b/sys/dev/drm2/drm_pciids.h
index 50fb932adc5..cad56a161ce 100644
--- a/sys/dev/drm2/drm_pciids.h
+++ b/sys/dev/drm2/drm_pciids.h
@@ -48,6 +48,13 @@
 	{0x8086, 0x0162, CHIP_I9XX|CHIP_I915, "Intel IvyBridge"}, \
 	{0x8086, 0x0166, CHIP_I9XX|CHIP_I915, "Intel IvyBridge (M)"}, \
 	{0x8086, 0x016A, CHIP_I9XX|CHIP_I915, "Intel IvyBridge (S)"}, \
+	{0x8086, 0x0402, CHIP_I9XX|CHIP_I915, "Intel Haswell"}, \
+	{0x8086, 0x0412, CHIP_I9XX|CHIP_I915, "Intel Haswell"}, \
+	{0x8086, 0x040a, CHIP_I9XX|CHIP_I915, "Intel Haswell (S)"}, \
+	{0x8086, 0x041a, CHIP_I9XX|CHIP_I915, "Intel Haswell (S)"}, \
+	{0x8086, 0x0406, CHIP_I9XX|CHIP_I915, "Intel Haswell (M)"}, \
+	{0x8086, 0x0416, CHIP_I9XX|CHIP_I915, "Intel Haswell (M)"}, \
+	{0x8086, 0x0c16, CHIP_I9XX|CHIP_I915, "Intel Haswell (SDV)"}, \
 	{0x8086, 0x2562, CHIP_I8XX, "Intel i845G GMCH"}, \
 	{0x8086, 0x2572, CHIP_I8XX, "Intel i865G GMCH"}, \
 	{0x8086, 0x2582, CHIP_I9XX|CHIP_I915, "Intel i915G"}, \
diff --git a/sys/dev/drm2/drm_stub.c b/sys/dev/drm2/drm_stub.c
index 2c87dec7915..6f6af51983d 100644
--- a/sys/dev/drm2/drm_stub.c
+++ b/sys/dev/drm2/drm_stub.c
@@ -45,7 +45,7 @@ drm_setmaster_ioctl(struct drm_device *dev, void *data,
 
 	if (file_priv->master != 0)
 		return (0);
-	return (-EPERM);
+	return (EPERM);
 }
 
 int
@@ -55,6 +55,6 @@ drm_dropmaster_ioctl(struct drm_device *dev, void *data,
 
 	DRM_DEBUG("dropmaster\n");
 	if (file_priv->master != 0)
-		return (-EINVAL);
+		return (EINVAL);
 	return (0);
 }
diff --git a/sys/dev/drm2/i915/i915_debug.c b/sys/dev/drm2/i915/i915_debug.c
index 384ed4ae843..5f44a994481 100644
--- a/sys/dev/drm2/i915/i915_debug.c
+++ b/sys/dev/drm2/i915/i915_debug.c
@@ -43,7 +43,6 @@ enum {
 	FLUSHING_LIST,
 	INACTIVE_LIST,
 	PINNED_LIST,
-	DEFERRED_FREE_LIST,
 };
 
 static const char *
@@ -135,6 +134,8 @@ describe_obj(struct sbuf *m, struct drm_i915_gem_object *obj)
 		   obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
 	if (obj->base.name)
 		sbuf_printf(m, " (name: %d)", obj->base.name);
+	if (obj->pin_display)
+		sbuf_printf(m, " (display)");
 	if (obj->fence_reg != I915_FENCE_REG_NONE)
 		sbuf_printf(m, " (fence: %d)", obj->fence_reg);
 	if (obj->gtt_space != NULL)
@@ -175,18 +176,10 @@ i915_gem_object_list_info(struct drm_device *dev, struct sbuf *m, void *data)
 		sbuf_printf(m, "Inactive:\n");
 		head = &dev_priv->mm.inactive_list;
 		break;
-	case PINNED_LIST:
-		sbuf_printf(m, "Pinned:\n");
-		head = &dev_priv->mm.pinned_list;
-		break;
 	case FLUSHING_LIST:
 		sbuf_printf(m, "Flushing:\n");
 		head = &dev_priv->mm.flushing_list;
 		break;
-	case DEFERRED_FREE_LIST:
-		sbuf_printf(m, "Deferred free:\n");
-		head = &dev_priv->mm.deferred_free_list;
-		break;
 	default:
 		DRM_UNLOCK(dev);
 		return (EINVAL);
@@ -244,21 +237,11 @@ i915_gem_object_info(struct drm_device *dev, struct sbuf *m, void *data)
 	sbuf_printf(m, "  %u [%u] active objects, %zu [%zu] bytes\n",
 		   count, mappable_count, size, mappable_size);
 
-	size = count = mappable_size = mappable_count = 0;
-	count_objects(&dev_priv->mm.pinned_list, mm_list);
-	sbuf_printf(m, "  %u [%u] pinned objects, %zu [%zu] bytes\n",
-		   count, mappable_count, size, mappable_size);
-
 	size = count = mappable_size = mappable_count = 0;
 	count_objects(&dev_priv->mm.inactive_list, mm_list);
 	sbuf_printf(m, "  %u [%u] inactive objects, %zu [%zu] bytes\n",
 		   count, mappable_count, size, mappable_size);
 
-	size = count = mappable_size = mappable_count = 0;
-	count_objects(&dev_priv->mm.deferred_free_list, mm_list);
-	sbuf_printf(m, "  %u [%u] freed objects, %zu [%zu] bytes\n",
-		   count, mappable_count, size, mappable_size);
-
 	size = count = mappable_size = mappable_count = 0;
 	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
 		if (obj->fault_mappable) {
@@ -283,9 +266,10 @@ i915_gem_object_info(struct drm_device *dev, struct sbuf *m, void *data)
 }
 
 static int
-i915_gem_gtt_info(struct drm_device *dev, struct sbuf *m, void* data)
+i915_gem_gtt_info(struct drm_device *dev, struct sbuf *m, void *data)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	uintptr_t list = (uintptr_t)data;
 	struct drm_i915_gem_object *obj;
 	size_t total_obj_size, total_gtt_size;
 	int count;
@@ -295,6 +279,9 @@ i915_gem_gtt_info(struct drm_device *dev, struct sbuf *m, void* data)
 
 	total_obj_size = total_gtt_size = count = 0;
 	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
+		if (list == PINNED_LIST && obj->pin_count == 0)
+			continue;
+
 		sbuf_printf(m, "   ");
 		describe_obj(m, obj);
 		sbuf_printf(m, "\n");
@@ -420,10 +407,6 @@ i915_ring_seqno_info(struct sbuf *m, struct intel_ring_buffer *ring)
 	if (ring->get_seqno) {
 		sbuf_printf(m, "Current sequence (%s): %d\n",
 			   ring->name, ring->get_seqno(ring));
-		sbuf_printf(m, "Waiter sequence (%s):  %d\n",
-			   ring->name, ring->waiting_seqno);
-		sbuf_printf(m, "IRQ sequence (%s):     %d\n",
-			   ring->name, ring->irq_seqno);
 	}
 }
 
@@ -451,7 +434,45 @@ i915_interrupt_info(struct drm_device *dev, struct sbuf *m, void *data)
 	if (sx_xlock_sig(&dev->dev_struct_lock))
 		return (EINTR);
 
-	if (!HAS_PCH_SPLIT(dev)) {
+	if (IS_VALLEYVIEW(dev)) {
+		sbuf_printf(m, "Display IER:\t%08x\n",
+			   I915_READ(VLV_IER));
+		sbuf_printf(m, "Display IIR:\t%08x\n",
+			   I915_READ(VLV_IIR));
+		sbuf_printf(m, "Display IIR_RW:\t%08x\n",
+			   I915_READ(VLV_IIR_RW));
+		sbuf_printf(m, "Display IMR:\t%08x\n",
+			   I915_READ(VLV_IMR));
+		for_each_pipe(pipe)
+			sbuf_printf(m, "Pipe %c stat:\t%08x\n",
+				   pipe_name(pipe),
+				   I915_READ(PIPESTAT(pipe)));
+
+		sbuf_printf(m, "Master IER:\t%08x\n",
+			   I915_READ(VLV_MASTER_IER));
+
+		sbuf_printf(m, "Render IER:\t%08x\n",
+			   I915_READ(GTIER));
+		sbuf_printf(m, "Render IIR:\t%08x\n",
+			   I915_READ(GTIIR));
+		sbuf_printf(m, "Render IMR:\t%08x\n",
+			   I915_READ(GTIMR));
+
+		sbuf_printf(m, "PM IER:\t\t%08x\n",
+			   I915_READ(GEN6_PMIER));
+		sbuf_printf(m, "PM IIR:\t\t%08x\n",
+			   I915_READ(GEN6_PMIIR));
+		sbuf_printf(m, "PM IMR:\t\t%08x\n",
+			   I915_READ(GEN6_PMIMR));
+
+		sbuf_printf(m, "Port hotplug:\t%08x\n",
+			   I915_READ(PORT_HOTPLUG_EN));
+		sbuf_printf(m, "DPFLIPSTAT:\t%08x\n",
+			   I915_READ(VLV_DPFLIPSTAT));
+		sbuf_printf(m, "DPINVGTT:\t%08x\n",
+			   I915_READ(DPINVGTT));
+
+	} else if (!HAS_PCH_SPLIT(dev)) {
 		sbuf_printf(m, "Interrupt enable:    %08x\n",
 			   I915_READ(IER));
 		sbuf_printf(m, "Interrupt identity:  %08x\n",
@@ -544,61 +565,6 @@ i915_hws_info(struct drm_device *dev, struct sbuf *m, void *data)
 	return (0);
 }
 
-static int
-i915_ringbuffer_data(struct drm_device *dev, struct sbuf *m, void *data)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	struct intel_ring_buffer *ring;
-
-	if (sx_xlock_sig(&dev->dev_struct_lock))
-		return (EINTR);
-	ring = &dev_priv->rings[(uintptr_t)data];
-	if (!ring->obj) {
-		sbuf_printf(m, "No ringbuffer setup\n");
-	} else {
-		u8 *virt = ring->virtual_start;
-		uint32_t off;
-
-		for (off = 0; off < ring->size; off += 4) {
-			uint32_t *ptr = (uint32_t *)(virt + off);
-			sbuf_printf(m, "%08x :  %08x\n", off, *ptr);
-		}
-	}
-	DRM_UNLOCK(dev);
-	return (0);
-}
-
-static int
-i915_ringbuffer_info(struct drm_device *dev, struct sbuf *m, void *data)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	struct intel_ring_buffer *ring;
-
-	ring = &dev_priv->rings[(uintptr_t)data];
-	if (ring->size == 0)
-		return (0);
-
-	if (sx_xlock_sig(&dev->dev_struct_lock))
-		return (EINTR);
-
-	sbuf_printf(m, "Ring %s:\n", ring->name);
-	sbuf_printf(m, "  Head :    %08x\n", I915_READ_HEAD(ring) & HEAD_ADDR);
-	sbuf_printf(m, "  Tail :    %08x\n", I915_READ_TAIL(ring) & TAIL_ADDR);
-	sbuf_printf(m, "  Size :    %08x\n", ring->size);
-	sbuf_printf(m, "  Active :  %08x\n", intel_ring_get_active_head(ring));
-	sbuf_printf(m, "  NOPID :   %08x\n", I915_READ_NOPID(ring));
-	if (IS_GEN6(dev) || IS_GEN7(dev)) {
-		sbuf_printf(m, "  Sync 0 :   %08x\n", I915_READ_SYNC_0(ring));
-		sbuf_printf(m, "  Sync 1 :   %08x\n", I915_READ_SYNC_1(ring));
-	}
-	sbuf_printf(m, "  Control : %08x\n", I915_READ_CTL(ring));
-	sbuf_printf(m, "  Start :   %08x\n", I915_READ_START(ring));
-
-	DRM_UNLOCK(dev);
-
-	return (0);
-}
-
 static const char *
 ring_str(int ring)
 {
@@ -677,6 +643,7 @@ i915_ring_error_state(struct sbuf *m, struct drm_device *dev,
     struct drm_i915_error_state *error, unsigned ring)
 {
 
+	MPASS((ring < I915_NUM_RINGS));	/* shut up confused gcc */
 	sbuf_printf(m, "%s command stream:\n", ring_str(ring));
 	sbuf_printf(m, "  HEAD: 0x%08x\n", error->head[ring]);
 	sbuf_printf(m, "  TAIL: 0x%08x\n", error->tail[ring]);
@@ -691,8 +658,8 @@ i915_ring_error_state(struct sbuf *m, struct drm_device *dev,
 	if (INTEL_INFO(dev)->gen >= 4)
 		sbuf_printf(m, "  INSTPS: 0x%08x\n", error->instps[ring]);
 	sbuf_printf(m, "  INSTPM: 0x%08x\n", error->instpm[ring]);
+	sbuf_printf(m, "  FADDR: 0x%08x\n", error->faddr[ring]);
 	if (INTEL_INFO(dev)->gen >= 6) {
-		sbuf_printf(m, "  FADDR: 0x%08x\n", error->faddr[ring]);
 		sbuf_printf(m, "  FAULT_REG: 0x%08x\n", error->fault_reg[ring]);
 		sbuf_printf(m, "  SYNC_0: 0x%08x\n",
 			   error->semaphore_mboxes[ring][0]);
@@ -700,21 +667,28 @@ i915_ring_error_state(struct sbuf *m, struct drm_device *dev,
 			   error->semaphore_mboxes[ring][1]);
 	}
 	sbuf_printf(m, "  seqno: 0x%08x\n", error->seqno[ring]);
+	sbuf_printf(m, "  waiting: %s\n", yesno(error->waiting[ring]));
 	sbuf_printf(m, "  ring->head: 0x%08x\n", error->cpu_ring_head[ring]);
 	sbuf_printf(m, "  ring->tail: 0x%08x\n", error->cpu_ring_tail[ring]);
 }
 
-static int i915_error_state(struct drm_device *dev, struct sbuf *m,
+static int
+i915_error_state(struct drm_device *dev, struct sbuf *m,
     void *unused)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_i915_error_state *error;
+	struct intel_ring_buffer *ring;
 	int i, j, page, offset, elt;
 
 	mtx_lock(&dev_priv->error_lock);
-	if (!dev_priv->first_error) {
+	error = dev_priv->first_error;
+	if (error != NULL)
+		refcount_acquire(&error->ref);
+	mtx_unlock(&dev_priv->error_lock);
+	if (error == NULL) {
 		sbuf_printf(m, "no error state collected\n");
-		goto out;
+		return (0);
 	}
 
 	error = dev_priv->first_error;
@@ -723,6 +697,7 @@ static int i915_error_state(struct drm_device *dev, struct sbuf *m,
 	    (intmax_t)error->time.tv_usec);
 	sbuf_printf(m, "PCI ID: 0x%04x\n", dev->pci_device);
 	sbuf_printf(m, "EIR: 0x%08x\n", error->eir);
+	sbuf_printf(m, "IER: 0x%08x\n", error->ier);
 	sbuf_printf(m, "PGTBL_ER: 0x%08x\n", error->pgtbl_er);
 
 	for (i = 0; i < dev_priv->num_fence_regs; i++)
@@ -734,11 +709,8 @@ static int i915_error_state(struct drm_device *dev, struct sbuf *m,
 		sbuf_printf(m, "DONE_REG: 0x%08x\n", error->done_reg);
 	}
 
-	i915_ring_error_state(m, dev, error, RCS);
-	if (HAS_BLT(dev))
-		i915_ring_error_state(m, dev, error, BCS);
-	if (HAS_BSD(dev))
-		i915_ring_error_state(m, dev, error, VCS);
+	for_each_ring(ring, dev_priv, i)
+		i915_ring_error_state(m, dev, error, i);
 
 	if (error->active_bo)
 		print_error_buffers(m, "Active",
@@ -801,12 +773,28 @@ static int i915_error_state(struct drm_device *dev, struct sbuf *m,
 	if (error->display)
 		intel_display_print_error_state(m, dev, error->display);
 
-out:
-	mtx_unlock(&dev_priv->error_lock);
+	if (refcount_release(&error->ref))
+		i915_error_state_free(error);
 
 	return (0);
 }
 
+static int
+i915_error_state_w(struct drm_device *dev, const char *str, void *unused)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct drm_i915_error_state *error;
+
+	DRM_DEBUG_DRIVER("Resetting error state\n");
+	mtx_lock(&dev_priv->error_lock);
+	error = dev_priv->first_error;
+	dev_priv->first_error = NULL;
+	mtx_unlock(&dev_priv->error_lock);
+	if (error != NULL && refcount_release(&error->ref))
+		i915_error_state_free(error);
+	return (0);
+}
+
 static int
 i915_rstdby_delays(struct drm_device *dev, struct sbuf *m, void *unused)
 {
@@ -1081,6 +1069,17 @@ gen6_drpc_info(struct drm_device *dev, struct sbuf *m)
 
 	sbuf_printf(m, "Core Power Down: %s\n",
 		   yesno(gt_core_status & GEN6_CORE_CPD_STATE_MASK));
+
+	/* Not exactly sure what this is */
+	sbuf_printf(m, "RC6 \"Locked to RPn\" residency since boot: %u\n",
+		   I915_READ(GEN6_GT_GFX_RC6_LOCKED));
+	sbuf_printf(m, "RC6 residency since boot: %u\n",
+		   I915_READ(GEN6_GT_GFX_RC6));
+	sbuf_printf(m, "RC6+ residency since boot: %u\n",
+		   I915_READ(GEN6_GT_GFX_RC6p));
+	sbuf_printf(m, "RC6++ residency since boot: %u\n",
+		   I915_READ(GEN6_GT_GFX_RC6pp));
+
 	return 0;
 }
 
@@ -1444,6 +1443,52 @@ i915_ppgtt_info(struct drm_device *dev, struct sbuf *m, void *data)
 	return (0);
 }
 
+static int i915_dpio_info(struct drm_device *dev, struct sbuf *m, void *data)
+{
+	struct drm_i915_private *dev_priv;
+	int ret;
+
+	if (!IS_VALLEYVIEW(dev)) {
+		sbuf_printf(m, "unsupported\n");
+		return 0;
+	}
+
+	dev_priv = dev->dev_private;
+
+	ret = sx_xlock_sig(&dev->mode_config.mutex);
+	if (ret != 0)
+		return (EINTR);
+
+	sbuf_printf(m, "DPIO_CTL: 0x%08x\n", I915_READ(DPIO_CTL));
+
+	sbuf_printf(m, "DPIO_DIV_A: 0x%08x\n",
+		   intel_dpio_read(dev_priv, _DPIO_DIV_A));
+	sbuf_printf(m, "DPIO_DIV_B: 0x%08x\n",
+		   intel_dpio_read(dev_priv, _DPIO_DIV_B));
+
+	sbuf_printf(m, "DPIO_REFSFR_A: 0x%08x\n",
+		   intel_dpio_read(dev_priv, _DPIO_REFSFR_A));
+	sbuf_printf(m, "DPIO_REFSFR_B: 0x%08x\n",
+		   intel_dpio_read(dev_priv, _DPIO_REFSFR_B));
+
+	sbuf_printf(m, "DPIO_CORE_CLK_A: 0x%08x\n",
+		   intel_dpio_read(dev_priv, _DPIO_CORE_CLK_A));
+	sbuf_printf(m, "DPIO_CORE_CLK_B: 0x%08x\n",
+		   intel_dpio_read(dev_priv, _DPIO_CORE_CLK_B));
+
+	sbuf_printf(m, "DPIO_LFP_COEFF_A: 0x%08x\n",
+		   intel_dpio_read(dev_priv, _DPIO_LFP_COEFF_A));
+	sbuf_printf(m, "DPIO_LFP_COEFF_B: 0x%08x\n",
+		   intel_dpio_read(dev_priv, _DPIO_LFP_COEFF_B));
+
+	sbuf_printf(m, "DPIO_FASTCLK_DISABLE: 0x%08x\n",
+		   intel_dpio_read(dev_priv, DPIO_FASTCLK_DISABLE));
+
+	sx_xunlock(&dev->mode_config.mutex);
+
+	return 0;
+}
+
 static int
 i915_debug_set_wedged(SYSCTL_HANDLER_ARGS)
 {
@@ -1520,57 +1565,77 @@ i915_cache_sharing(SYSCTL_HANDLER_ARGS)
 	return (0);
 }
 
+static int
+i915_stop_rings(SYSCTL_HANDLER_ARGS)
+{
+	struct drm_device *dev;
+	drm_i915_private_t *dev_priv;
+	int error, val;
+
+	dev = arg1;
+	dev_priv = dev->dev_private;
+	if (dev_priv == NULL)
+		return (EBUSY);
+	DRM_LOCK(dev);
+	val = dev_priv->stop_rings;
+	DRM_UNLOCK(dev);
+	error = sysctl_handle_int(oidp, &val, 0, req);
+	if (error || !req->newptr)
+		return (error);
+	DRM_DEBUG("Stopping rings 0x%08x\n", val);
+
+	DRM_LOCK(dev);
+	dev_priv->stop_rings = val;
+	DRM_UNLOCK(dev);
+	return (0);
+}
+
 static struct i915_info_sysctl_list {
 	const char *name;
 	int (*ptr)(struct drm_device *dev, struct sbuf *m, void *data);
+	int (*ptr_w)(struct drm_device *dev, const char *str, void *data);
 	int flags;
 	void *data;
 } i915_info_sysctl_list[] = {
-	{"i915_capabilities", i915_capabilities, 0},
-	{"i915_gem_objects", i915_gem_object_info, 0},
-	{"i915_gem_gtt", i915_gem_gtt_info, 0},
-	{"i915_gem_active", i915_gem_object_list_info, 0, (void *)ACTIVE_LIST},
-	{"i915_gem_flushing", i915_gem_object_list_info, 0,
+	{"i915_capabilities", i915_capabilities, NULL, 0},
+	{"i915_gem_objects", i915_gem_object_info, NULL, 0},
+	{"i915_gem_gtt", i915_gem_gtt_info, NULL, 0},
+	{"i915_gem_pinned", i915_gem_gtt_info, NULL, 0, (void *)PINNED_LIST},
+	{"i915_gem_active", i915_gem_object_list_info, NULL, 0,
+	    (void *)ACTIVE_LIST},
+	{"i915_gem_flushing", i915_gem_object_list_info, NULL, 0,
 	    (void *)FLUSHING_LIST},
-	{"i915_gem_inactive", i915_gem_object_list_info, 0,
+	{"i915_gem_inactive", i915_gem_object_list_info, NULL, 0,
 	    (void *)INACTIVE_LIST},
-	{"i915_gem_pinned", i915_gem_object_list_info, 0,
-	    (void *)PINNED_LIST},
-	{"i915_gem_deferred_free", i915_gem_object_list_info, 0,
-	    (void *)DEFERRED_FREE_LIST},
-	{"i915_gem_pageflip", i915_gem_pageflip_info, 0},
-	{"i915_gem_request", i915_gem_request_info, 0},
-	{"i915_gem_seqno", i915_gem_seqno_info, 0},
-	{"i915_gem_fence_regs", i915_gem_fence_regs_info, 0},
-	{"i915_gem_interrupt", i915_interrupt_info, 0},
-	{"i915_gem_hws", i915_hws_info, 0, (void *)RCS},
-	{"i915_gem_hws_blt", i915_hws_info, 0, (void *)BCS},
-	{"i915_gem_hws_bsd", i915_hws_info, 0, (void *)VCS},
-	{"i915_ringbuffer_data", i915_ringbuffer_data, 0, (void *)RCS},
-	{"i915_ringbuffer_info", i915_ringbuffer_info, 0, (void *)RCS},
-	{"i915_bsd_ringbuffer_data", i915_ringbuffer_data, 0, (void *)VCS},
-	{"i915_bsd_ringbuffer_info", i915_ringbuffer_info, 0, (void *)VCS},
-	{"i915_blt_ringbuffer_data", i915_ringbuffer_data, 0, (void *)BCS},
-	{"i915_blt_ringbuffer_info", i915_ringbuffer_info, 0, (void *)BCS},
-	{"i915_error_state", i915_error_state, 0},
-	{"i915_rstdby_delays", i915_rstdby_delays, 0},
-	{"i915_cur_delayinfo", i915_cur_delayinfo, 0},
-	{"i915_delayfreq_table", i915_delayfreq_table, 0},
-	{"i915_inttoext_table", i915_inttoext_table, 0},
-	{"i915_drpc_info", i915_drpc_info, 0},
-	{"i915_emon_status", i915_emon_status, 0},
-	{"i915_ring_freq_table", i915_ring_freq_table, 0},
-	{"i915_gfxec", i915_gfxec, 0},
-	{"i915_fbc_status", i915_fbc_status, 0},
-	{"i915_sr_status", i915_sr_status, 0},
+	{"i915_gem_pageflip", i915_gem_pageflip_info, NULL, 0},
+	{"i915_gem_request", i915_gem_request_info, NULL, 0},
+	{"i915_gem_seqno", i915_gem_seqno_info, NULL, 0},
+	{"i915_gem_fence_regs", i915_gem_fence_regs_info, NULL, 0},
+	{"i915_gem_interrupt", i915_interrupt_info, NULL, 0},
+	{"i915_gem_hws", i915_hws_info, NULL, 0, (void *)RCS},
+	{"i915_gem_hws_blt", i915_hws_info, NULL, 0, (void *)BCS},
+	{"i915_gem_hws_bsd", i915_hws_info, NULL, 0, (void *)VCS},
+	{"i915_error_state", i915_error_state, i915_error_state_w, 0},
+	{"i915_rstdby_delays", i915_rstdby_delays, NULL, 0},
+	{"i915_cur_delayinfo", i915_cur_delayinfo, NULL, 0},
+	{"i915_delayfreq_table", i915_delayfreq_table, NULL, 0},
+	{"i915_inttoext_table", i915_inttoext_table, NULL, 0},
+	{"i915_drpc_info", i915_drpc_info, NULL, 0},
+	{"i915_emon_status", i915_emon_status, NULL, 0},
+	{"i915_ring_freq_table", i915_ring_freq_table, NULL, 0},
+	{"i915_gfxec", i915_gfxec, NULL, 0},
+	{"i915_fbc_status", i915_fbc_status, NULL, 0},
+	{"i915_sr_status", i915_sr_status, NULL, 0},
 #if 0
-	{"i915_opregion", i915_opregion, 0},
+	{"i915_opregion", i915_opregion, NULL, 0},
 #endif
-	{"i915_gem_framebuffer", i915_gem_framebuffer_info, 0},
-	{"i915_context_status", i915_context_status, 0},
-	{"i915_gen6_forcewake_count_info", i915_gen6_forcewake_count_info, 0},
-	{"i915_swizzle_info", i915_swizzle_info, 0},
-	{"i915_ppgtt_info", i915_ppgtt_info, 0},
+	{"i915_gem_framebuffer", i915_gem_framebuffer_info, NULL, 0},
+	{"i915_context_status", i915_context_status, NULL, 0},
+	{"i915_gen6_forcewake_count_info", i915_gen6_forcewake_count_info,
+	    NULL, 0},
+	{"i915_swizzle_info", i915_swizzle_info, NULL, 0},
+	{"i915_ppgtt_info", i915_ppgtt_info, NULL, 0},
+	{"i915_dpio", i915_dpio_info, NULL, 0},
 };
 
 struct i915_info_sysctl_thunk {
@@ -1586,6 +1651,7 @@ i915_info_sysctl_handler(SYSCTL_HANDLER_ARGS)
 	struct i915_info_sysctl_thunk *thunk;
 	struct drm_device *dev;
 	drm_i915_private_t *dev_priv;
+	char *p;
 	int error;
 
 	thunk = arg1;
@@ -1602,6 +1668,19 @@ i915_info_sysctl_handler(SYSCTL_HANDLER_ARGS)
 	if (error == 0)
 		error = sbuf_finish(&m);
 	sbuf_delete(&m);
+	if (error != 0 || req->newptr == NULL)
+		return (error);
+	if (req->newlen > 2048)
+		return (E2BIG);
+	p = malloc(req->newlen + 1, M_TEMP, M_WAITOK);
+	error = SYSCTL_IN(req, p, req->newlen);
+	if (error != 0)
+		goto out;
+	p[req->newlen] = '\0';
+	error = i915_info_sysctl_list[thunk->idx].ptr_w(dev, p,
+	    thunk->arg);
+out:
+	free(p, M_TEMP);
 	return (error);
 }
 
@@ -1632,7 +1711,9 @@ i915_sysctl_init(struct drm_device *dev, struct sysctl_ctx_list *ctx,
 		return (ENOMEM);
 	for (i = 0; i < DRM_ARRAY_SIZE(i915_info_sysctl_list); i++) {
 		oid = SYSCTL_ADD_OID(ctx, SYSCTL_CHILDREN(info), OID_AUTO,
-		    i915_info_sysctl_list[i].name, CTLTYPE_STRING | CTLFLAG_RD,
+		    i915_info_sysctl_list[i].name, CTLTYPE_STRING |
+		    (i915_info_sysctl_list[i].ptr_w != NULL ? CTLFLAG_RW :
+		    CTLFLAG_RD),
 		    &thunks[i], 0, i915_info_sysctl_handler, "A", NULL);
 		if (oid == NULL)
 			return (ENOMEM);
@@ -1655,6 +1736,11 @@ i915_sysctl_init(struct drm_device *dev, struct sysctl_ctx_list *ctx,
 	    0, i915_cache_sharing, "I", NULL);
 	if (oid == NULL)
 		return (ENOMEM);
+	oid = SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(top), OID_AUTO,
+	    "stop_rings", CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE, dev,
+	    0, i915_stop_rings, "I", NULL);
+	if (oid == NULL)
+		return (ENOMEM);
 	oid = SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(top), OID_AUTO, "sync_exec",
 	    CTLFLAG_RW, &i915_gem_sync_exec_requests, 0, NULL);
 	if (oid == NULL)
diff --git a/sys/dev/drm2/i915/i915_dma.c b/sys/dev/drm2/i915/i915_dma.c
index 3fceaadb433..58dcae7b542 100644
--- a/sys/dev/drm2/i915/i915_dma.c
+++ b/sys/dev/drm2/i915/i915_dma.c
@@ -36,22 +36,56 @@ __FBSDID("$FreeBSD$");
 #include <dev/drm2/i915/intel_drv.h>
 #include <dev/drm2/i915/intel_ringbuffer.h>
 
-static struct drm_i915_private *i915_mch_dev;
-/*
- * Lock protecting IPS related data structures
- *   - i915_mch_dev
- *   - dev_priv->max_delay
- *   - dev_priv->min_delay
- *   - dev_priv->fmax
- *   - dev_priv->gpu_busy
- */
-static struct mtx mchdev_lock;
-MTX_SYSINIT(mchdev, &mchdev_lock, "mchdev", MTX_DEF);
+#define LP_RING(d) (&((struct drm_i915_private *)(d))->rings[RCS])
+
+#define BEGIN_LP_RING(n) \
+	intel_ring_begin(LP_RING(dev_priv), (n))
+
+#define OUT_RING(x) \
+	intel_ring_emit(LP_RING(dev_priv), x)
+
+#define ADVANCE_LP_RING() \
+	intel_ring_advance(LP_RING(dev_priv))
+
+#define RING_LOCK_TEST_WITH_RETURN(dev, file) do {			\
+	if (LP_RING(dev->dev_private)->obj == NULL)			\
+		LOCK_TEST_WITH_RETURN(dev, file);			\
+} while (0)
+
+static inline u32
+intel_read_legacy_status_page(struct drm_i915_private *dev_priv, int reg)
+{
+	if (I915_NEED_GFX_HWS(dev_priv->dev))
+		return ((volatile u32*)(dev_priv->dri1.gfx_hws_cpu_addr))[reg];
+	else
+		return intel_read_status_page(LP_RING(dev_priv), reg);
+}
+
+#define READ_HWSP(dev_priv, reg) intel_read_legacy_status_page(dev_priv, reg)
+#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX)
+#define I915_BREADCRUMB_INDEX		0x21
 
-static void i915_pineview_get_mem_freq(struct drm_device *dev);
-static void i915_ironlake_get_mem_freq(struct drm_device *dev);
 static int i915_driver_unload_int(struct drm_device *dev, bool locked);
 
+void i915_update_dri1_breadcrumb(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+#if 0
+	struct drm_i915_master_private *master_priv;
+
+	if (dev->primary->master) {
+		master_priv = dev->primary->master->driver_priv;
+		if (master_priv->sarea_priv)
+			master_priv->sarea_priv->last_dispatch =
+				READ_BREADCRUMB(dev_priv);
+	}
+#else
+	if (dev_priv->sarea_priv)
+		dev_priv->sarea_priv->last_dispatch =
+		    READ_BREADCRUMB(dev_priv);
+#endif
+}
+
 static void i915_write_hws_pga(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
@@ -115,7 +149,8 @@ static void i915_free_hws(struct drm_device *dev)
 	if (dev_priv->status_gfx_addr) {
 		dev_priv->status_gfx_addr = 0;
 		ring->status_page.gfx_addr = 0;
-		drm_core_ioremapfree(&dev_priv->hws_map, dev);
+		pmap_unmapdev((vm_offset_t)dev_priv->dri1.gfx_hws_cpu_addr,
+		    PAGE_SIZE);
 	}
 
 	/* Need to rewrite hardware status page */
@@ -214,7 +249,7 @@ static int i915_initialize(struct drm_device * dev, drm_i915_init_t * init)
 
 	/* Allow hardware batchbuffers unless told otherwise.
 	 */
-	dev_priv->allow_batchbuffer = 1;
+	dev_priv->dri1.allow_batchbuffer = 1;
 
 	return 0;
 }
@@ -226,7 +261,7 @@ static int i915_dma_resume(struct drm_device * dev)
 
 	DRM_DEBUG("\n");
 
-	if (ring->map.handle == NULL) {
+	if (ring->virtual_start == NULL) {
 		DRM_ERROR("can not ioremap virtual address for"
 			  " ring buffer\n");
 		return -ENOMEM;
@@ -254,6 +289,9 @@ static int i915_dma_init(struct drm_device *dev, void *data,
 	drm_i915_init_t *init = data;
 	int retcode = 0;
 
+	if (drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
+
 	switch (init->func) {
 	case I915_INIT_DMA:
 		retcode = i915_initialize(dev, init);
@@ -502,6 +540,9 @@ i915_dispatch_batchbuffer(struct drm_device * dev,
 	int nbox = batch->num_cliprects;
 	int i, count, ret;
 
+	if (drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
+
 	if ((batch->start | batch->used) & 0x7) {
 		DRM_ERROR("alignment\n");
 		return -EINVAL;
@@ -618,6 +659,9 @@ i915_flush_ioctl(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
 	int ret;
 
+	if (drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
+
 	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
 
 	DRM_LOCK(dev);
@@ -637,7 +681,7 @@ int i915_batchbuffer(struct drm_device *dev, void *data,
 	size_t cliplen;
 	int ret;
 
-	if (!dev_priv->allow_batchbuffer) {
+	if (!dev_priv->dri1.allow_batchbuffer) {
 		DRM_ERROR("Batchbuffer ioctl disabled\n");
 		return -EINVAL;
 	}
@@ -686,6 +730,9 @@ int i915_cmdbuffer(struct drm_device *dev, void *data,
 	void *batch_data;
 	int ret;
 
+	if (drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
+
 	DRM_DEBUG("i915 cmdbuffer, buf %p sz %d cliprects %d\n",
 		  cmdbuf->buf, cmdbuf->sz, cmdbuf->num_cliprects);
 
@@ -733,11 +780,199 @@ fail_batch_free:
 	return ret;
 }
 
+static int i915_emit_irq(struct drm_device * dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+#if 0
+	struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
+#endif
+
+	i915_kernel_lost_context(dev);
+
+	DRM_DEBUG("i915: emit_irq\n");
+
+	dev_priv->counter++;
+	if (dev_priv->counter > 0x7FFFFFFFUL)
+		dev_priv->counter = 1;
+#if 0
+	if (master_priv->sarea_priv)
+		master_priv->sarea_priv->last_enqueue = dev_priv->counter;
+#else
+	if (dev_priv->sarea_priv)
+		dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
+#endif
+
+	if (BEGIN_LP_RING(4) == 0) {
+		OUT_RING(MI_STORE_DWORD_INDEX);
+		OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+		OUT_RING(dev_priv->counter);
+		OUT_RING(MI_USER_INTERRUPT);
+		ADVANCE_LP_RING();
+	}
+
+	return dev_priv->counter;
+}
+
+static int i915_wait_irq(struct drm_device * dev, int irq_nr)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+#if 0
+	struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
+#endif
+	int ret;
+	struct intel_ring_buffer *ring = LP_RING(dev_priv);
+
+	DRM_DEBUG("irq_nr=%d breadcrumb=%d\n", irq_nr,
+		  READ_BREADCRUMB(dev_priv));
+
+#if 0
+	if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
+		if (master_priv->sarea_priv)
+			master_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
+		return 0;
+	}
+
+	if (master_priv->sarea_priv)
+		master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
+#else
+	if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
+		if (dev_priv->sarea_priv) {
+			dev_priv->sarea_priv->last_dispatch =
+				READ_BREADCRUMB(dev_priv);
+		}
+		return 0;
+	}
+
+	if (dev_priv->sarea_priv)
+		dev_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
+#endif
+
+	ret = 0;
+	mtx_lock(&dev_priv->irq_lock);
+	if (ring->irq_get(ring)) {
+		DRM_UNLOCK(dev);
+		while (ret == 0 && READ_BREADCRUMB(dev_priv) < irq_nr) {
+			ret = -msleep(ring, &dev_priv->irq_lock, PCATCH,
+			    "915wtq", 3 * hz);
+		}
+		ring->irq_put(ring);
+		mtx_unlock(&dev_priv->irq_lock);
+		DRM_LOCK(dev);
+	} else {
+		mtx_unlock(&dev_priv->irq_lock);
+		if (_intel_wait_for(dev, READ_BREADCRUMB(dev_priv) >= irq_nr,
+		     3000, 1, "915wir"))
+			ret = -EBUSY;
+	}
+
+	if (ret == -EBUSY) {
+		DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
+			  READ_BREADCRUMB(dev_priv), (int)dev_priv->counter);
+	}
+
+	return ret;
+}
+
+/* Needs the lock as it touches the ring.
+ */
+int i915_irq_emit(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	drm_i915_irq_emit_t *emit = data;
+	int result;
+
+	if (drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
+
+	if (!dev_priv || !LP_RING(dev_priv)->virtual_start) {
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
+	}
+
+	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
+
+	DRM_LOCK(dev);
+	result = i915_emit_irq(dev);
+	DRM_UNLOCK(dev);
+
+	if (DRM_COPY_TO_USER(emit->irq_seq, &result, sizeof(int))) {
+		DRM_ERROR("copy_to_user\n");
+		return -EFAULT;
+	}
+
+	return 0;
+}
+
+/* Doesn't need the hardware lock.
+ */
+static int i915_irq_wait(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	drm_i915_irq_wait_t *irqwait = data;
+
+	if (drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
+
+	if (!dev_priv) {
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
+	}
+
+	return i915_wait_irq(dev, irqwait->irq_seq);
+}
+
+static int i915_vblank_pipe_get(struct drm_device *dev, void *data,
+			 struct drm_file *file_priv)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	drm_i915_vblank_pipe_t *pipe = data;
+
+	if (drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
+
+	if (!dev_priv) {
+		DRM_ERROR("called with no initialization\n");
+		return -EINVAL;
+	}
+
+	pipe->pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
+
+	return 0;
+}
+
+/**
+ * Schedule buffer swap at given vertical blank.
+ */
+static int i915_vblank_swap(struct drm_device *dev, void *data,
+		     struct drm_file *file_priv)
+{
+	/* The delayed swap mechanism was fundamentally racy, and has been
+	 * removed.  The model was that the client requested a delayed flip/swap
+	 * from the kernel, then waited for vblank before continuing to perform
+	 * rendering.  The problem was that the kernel might wake the client
+	 * up before it dispatched the vblank swap (since the lock has to be
+	 * held while touching the ringbuffer), in which case the client would
+	 * clear and start the next frame before the swap occurred, and
+	 * flicker would occur in addition to likely missing the vblank.
+	 *
+	 * In the absence of this ioctl, userland falls back to a correct path
+	 * of waiting for a vblank, then dispatching the swap on its own.
+	 * Context switching to userland and back is plenty fast enough for
+	 * meeting the requirements of vblank swapping.
+	 */
+	return -EINVAL;
+}
+
 static int i915_flip_bufs(struct drm_device *dev, void *data,
 			  struct drm_file *file_priv)
 {
 	int ret;
 
+	if (drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
+
 	DRM_DEBUG("%s\n", __func__);
 
 	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
@@ -764,7 +999,7 @@ int i915_getparam(struct drm_device *dev, void *data,
 		value = dev->irq_enabled ? 1 : 0;
 		break;
 	case I915_PARAM_ALLOW_BATCHBUFFER:
-		value = dev_priv->allow_batchbuffer ? 1 : 0;
+		value = dev_priv->dri1.allow_batchbuffer ? 1 : 0;
 		break;
 	case I915_PARAM_LAST_DISPATCH:
 		value = READ_BREADCRUMB(dev_priv);
@@ -788,10 +1023,10 @@ int i915_getparam(struct drm_device *dev, void *data,
 		value = 1;
 		break;
 	case I915_PARAM_HAS_BSD:
-		value = HAS_BSD(dev);
+		value = intel_ring_initialized(&dev_priv->rings[VCS]);
 		break;
 	case I915_PARAM_HAS_BLT:
-		value = HAS_BLT(dev);
+		value = intel_ring_initialized(&dev_priv->rings[BCS]);
 		break;
 	case I915_PARAM_HAS_RELAXED_FENCING:
 		value = 1;
@@ -811,6 +1046,9 @@ int i915_getparam(struct drm_device *dev, void *data,
 	case I915_PARAM_HAS_LLC:
 		value = HAS_LLC(dev);
 		break;
+	case I915_PARAM_HAS_ALIASING_PPGTT:
+		value = dev_priv->mm.aliasing_ppgtt ? 1 : 0;
+		break;
 	default:
 		DRM_DEBUG_DRIVER("Unknown parameter %d\n",
 				 param->param);
@@ -840,10 +1078,9 @@ static int i915_setparam(struct drm_device *dev, void *data,
 	case I915_SETPARAM_USE_MI_BATCHBUFFER_START:
 		break;
 	case I915_SETPARAM_TEX_LRU_LOG_GRANULARITY:
-		dev_priv->tex_lru_log_granularity = param->value;
 		break;
 	case I915_SETPARAM_ALLOW_BATCHBUFFER:
-		dev_priv->allow_batchbuffer = param->value;
+		dev_priv->dri1.allow_batchbuffer = param->value ? 1 : 0;
 		break;
 	case I915_SETPARAM_NUM_USED_FENCES:
 		if (param->value > dev_priv->num_fence_regs ||
@@ -867,6 +1104,9 @@ static int i915_set_status_page(struct drm_device *dev, void *data,
 	drm_i915_hws_addr_t *hws = data;
 	struct intel_ring_buffer *ring = LP_RING(dev_priv);
 
+	if (drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
+
 	if (!I915_NEED_GFX_HWS(dev))
 		return -EINVAL;
 
@@ -884,24 +1124,18 @@ static int i915_set_status_page(struct drm_device *dev, void *data,
 	ring->status_page.gfx_addr = dev_priv->status_gfx_addr =
 	    hws->addr & (0x1ffff<<12);
 
-	dev_priv->hws_map.offset = dev->agp->base + hws->addr;
-	dev_priv->hws_map.size = 4*1024;
-	dev_priv->hws_map.type = 0;
-	dev_priv->hws_map.flags = 0;
-	dev_priv->hws_map.mtrr = 0;
-
-	drm_core_ioremap_wc(&dev_priv->hws_map, dev);
-	if (dev_priv->hws_map.virtual == NULL) {
+	dev_priv->dri1.gfx_hws_cpu_addr = pmap_mapdev_attr(
+	    dev->agp->base + hws->addr, PAGE_SIZE,
+	    VM_MEMATTR_WRITE_COMBINING);
+	if (dev_priv->dri1.gfx_hws_cpu_addr == NULL) {
 		i915_dma_cleanup(dev);
 		ring->status_page.gfx_addr = dev_priv->status_gfx_addr = 0;
 		DRM_ERROR("can not ioremap virtual address for"
 				" G33 hw status page\n");
 		return -ENOMEM;
 	}
-	ring->status_page.page_addr = dev_priv->hw_status_page =
-	    dev_priv->hws_map.virtual;
 
-	memset(dev_priv->hw_status_page, 0, PAGE_SIZE);
+	memset(dev_priv->dri1.gfx_hws_cpu_addr, 0, PAGE_SIZE);
 	I915_WRITE(HWS_PGA, dev_priv->status_gfx_addr);
 	DRM_DEBUG("load hws HWS_PGA with gfx mem 0x%x\n",
 			dev_priv->status_gfx_addr);
@@ -909,90 +1143,6 @@ static int i915_set_status_page(struct drm_device *dev, void *data,
 	return 0;
 }
 
-static bool
-intel_enable_ppgtt(struct drm_device *dev)
-{
-	if (i915_enable_ppgtt >= 0)
-		return i915_enable_ppgtt;
-
-	/* Disable ppgtt on SNB if VT-d is on. */
-	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_enabled)
-		return false;
-
-	return true;
-}
-
-static int
-i915_load_gem_init(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	unsigned long prealloc_size, gtt_size, mappable_size;
-	int ret;
-
-	prealloc_size = dev_priv->mm.gtt.stolen_size;
-	gtt_size = dev_priv->mm.gtt.gtt_total_entries << PAGE_SHIFT;
-	mappable_size = dev_priv->mm.gtt.gtt_mappable_entries << PAGE_SHIFT;
-
-	/* Basic memrange allocator for stolen space */
-	drm_mm_init(&dev_priv->mm.stolen, 0, prealloc_size);
-
-	DRM_LOCK(dev);
-	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
-		/* PPGTT pdes are stolen from global gtt ptes, so shrink the
-		 * aperture accordingly when using aliasing ppgtt. */
-		gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
-		/* For paranoia keep the guard page in between. */
-		gtt_size -= PAGE_SIZE;
-
-		i915_gem_do_init(dev, 0, mappable_size, gtt_size);
-
-		ret = i915_gem_init_aliasing_ppgtt(dev);
-		if (ret) {
-			DRM_UNLOCK(dev);
-			return ret;
-		}
-	} else {
-		/* Let GEM Manage all of the aperture.
-		 *
-		 * However, leave one page at the end still bound to the scratch
-		 * page.  There are a number of places where the hardware
-		 * apparently prefetches past the end of the object, and we've
-		 * seen multiple hangs with the GPU head pointer stuck in a
-		 * batchbuffer bound at the last page of the aperture.  One page
-		 * should be enough to keep any prefetching inside of the
-		 * aperture.
-		 */
-		i915_gem_do_init(dev, 0, mappable_size, gtt_size - PAGE_SIZE);
-	}
-
-	ret = i915_gem_init_hw(dev);
-	DRM_UNLOCK(dev);
-	if (ret != 0) {
-		i915_gem_cleanup_aliasing_ppgtt(dev);
-		return (ret);
-	}
-
-#if 0
-	/* Try to set up FBC with a reasonable compressed buffer size */
-	if (I915_HAS_FBC(dev) && i915_powersave) {
-		int cfb_size;
-
-		/* Leave 1M for line length buffer & misc. */
-
-		/* Try to get a 32M buffer... */
-		if (prealloc_size > (36*1024*1024))
-			cfb_size = 32*1024*1024;
-		else /* fall back to 7/8 of the stolen space */
-			cfb_size = prealloc_size * 7 / 8;
-		i915_setup_compression(dev, cfb_size);
-	}
-#endif
-
-	/* Allow hardware batchbuffers unless told otherwise. */
-	dev_priv->allow_batchbuffer = 1;
-	return 0;
-}
-
 static int
 i915_load_modeset_init(struct drm_device *dev)
 {
@@ -1007,15 +1157,18 @@ i915_load_modeset_init(struct drm_device *dev)
 	intel_register_dsm_handler();
 #endif
 
-	/* IIR "flip pending" bit means done if this bit is set */
-	if (IS_GEN3(dev) && (I915_READ(ECOSKPD) & ECO_FLIP_DONE))
-		dev_priv->flip_pending_is_done = true;
+	/* Initialise stolen first so that we may reserve preallocated
+	 * objects for the BIOS to KMS transition.
+	 */
+	ret = i915_gem_init_stolen(dev);
+	if (ret)
+		goto cleanup_vga_switcheroo;
 
 	intel_modeset_init(dev);
 
-	ret = i915_load_gem_init(dev);
+	ret = i915_gem_init(dev);
 	if (ret != 0)
-		goto cleanup_gem;
+		goto cleanup_gem_stolen;
 
 	intel_modeset_gem_init(dev);
 
@@ -1041,6 +1194,9 @@ cleanup_gem:
 	i915_gem_cleanup_ringbuffer(dev);
 	DRM_UNLOCK(dev);
 	i915_gem_cleanup_aliasing_ppgtt(dev);
+cleanup_gem_stolen:
+	i915_gem_cleanup_stolen(dev);
+cleanup_vga_switcheroo:
 	return (ret);
 }
 
@@ -1197,9 +1353,17 @@ int
 i915_driver_load(struct drm_device *dev, unsigned long flags)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	const struct intel_device_info *info;
 	unsigned long base, size;
 	int mmio_bar, ret;
 
+	info = i915_get_device_id(dev->pci_device);
+
+	/* Refuse to load on gen6+ without kms enabled. */
+	if (info->gen >= 6 && !drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
+
+
 	ret = 0;
 
 	/* i915 has 4 more counters */
@@ -1211,12 +1375,10 @@ i915_driver_load(struct drm_device *dev, unsigned long flags)
 
 	dev_priv = malloc(sizeof(drm_i915_private_t), DRM_MEM_DRIVER,
 	    M_ZERO | M_WAITOK);
-	if (dev_priv == NULL)
-		return -ENOMEM;
 
 	dev->dev_private = (void *)dev_priv;
 	dev_priv->dev = dev;
-	dev_priv->info = i915_get_device_id(dev->pci_device);
+	dev_priv->info = info;
 
 	if (i915_get_bridge_dev(dev)) {
 		free(dev_priv, DRM_MEM_DRIVER);
@@ -1239,8 +1401,8 @@ i915_driver_load(struct drm_device *dev, unsigned long flags)
 	mtx_init(&dev_priv->error_lock, "915err", NULL, MTX_DEF);
 	mtx_init(&dev_priv->error_completion_lock, "915cmp", NULL, MTX_DEF);
 	mtx_init(&dev_priv->rps_lock, "915rps", NULL, MTX_DEF);
+	mtx_init(&dev_priv->dpio_lock, "915dpi", NULL, MTX_DEF);
 
-	dev_priv->has_gem = 1;
 	intel_irq_init(dev);
 
 	intel_setup_mchbar(dev);
@@ -1262,14 +1424,9 @@ i915_driver_load(struct drm_device *dev, unsigned long flags)
 		}
 	}
 
-	if (IS_PINEVIEW(dev))
-		i915_pineview_get_mem_freq(dev);
-	else if (IS_GEN5(dev))
-		i915_ironlake_get_mem_freq(dev);
-
 	mtx_init(&dev_priv->irq_lock, "userirq", NULL, MTX_DEF);
 
-	if (IS_IVYBRIDGE(dev))
+	if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev))
 		dev_priv->num_pipe = 3;
 	else if (IS_MOBILE(dev) || !IS_GEN2(dev))
 		dev_priv->num_pipe = 2;
@@ -1301,12 +1458,8 @@ i915_driver_load(struct drm_device *dev, unsigned long flags)
 	callout_reset(&dev_priv->hangcheck_timer, DRM_I915_HANGCHECK_PERIOD,
 	    i915_hangcheck_elapsed, dev);
 
-	if (IS_GEN5(dev)) {
-		mtx_lock(&mchdev_lock);
-		i915_mch_dev = dev_priv;
-		dev_priv->mchdev_lock = &mchdev_lock;
-		mtx_unlock(&mchdev_lock);
-	}
+	if (IS_GEN5(dev))
+		intel_gpu_ips_init(dev_priv);
 
 	return (0);
 
@@ -1324,9 +1477,10 @@ i915_driver_unload_int(struct drm_device *dev, bool locked)
 
 	if (!locked)
 		DRM_LOCK(dev);
-	ret = i915_gpu_idle(dev, true);
+	ret = i915_gpu_idle(dev);
 	if (ret)
 		DRM_ERROR("failed to idle hardware: %d\n", ret);
+	i915_gem_retire_requests(dev);
 	if (!locked)
 		DRM_UNLOCK(dev);
 
@@ -1386,6 +1540,7 @@ i915_driver_unload_int(struct drm_device *dev, bool locked)
 	drm_rmmap(dev, dev_priv->mmio_map);
 	intel_teardown_gmbus(dev);
 
+	mtx_destroy(&dev_priv->dpio_lock);
 	mtx_destroy(&dev_priv->error_lock);
 	mtx_destroy(&dev_priv->error_completion_lock);
 	mtx_destroy(&dev_priv->rps_lock);
@@ -1466,7 +1621,7 @@ struct drm_ioctl_desc i915_ioctls[] = {
 	DRM_IOCTL_DEF(DRM_I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
 	DRM_IOCTL_DEF(DRM_I915_CMDBUFFER, i915_cmdbuffer, DRM_AUTH),
 	DRM_IOCTL_DEF(DRM_I915_DESTROY_HEAP,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY ),
-	DRM_IOCTL_DEF(DRM_I915_SET_VBLANK_PIPE,  i915_vblank_pipe_set, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY ),
+	DRM_IOCTL_DEF(DRM_I915_SET_VBLANK_PIPE,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY ),
 	DRM_IOCTL_DEF(DRM_I915_GET_VBLANK_PIPE,  i915_vblank_pipe_get, DRM_AUTH ),
 	DRM_IOCTL_DEF(DRM_I915_VBLANK_SWAP, i915_vblank_swap, DRM_AUTH),
 	DRM_IOCTL_DEF(DRM_I915_HWS_ADDR, i915_set_status_page, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
@@ -1541,550 +1696,13 @@ struct drm_driver_info i915_driver_info = {
 	.patchlevel	= DRIVER_PATCHLEVEL,
 };
 
-/**
- * Determine if the device really is AGP or not.
- *
- * All Intel graphics chipsets are treated as AGP, even if they are really
- * built-in.
- *
- * \param dev   The device to be tested.
- *
- * \returns
- * A value of 1 is always retured to indictate every i9x5 is AGP.
+/*
+ * This is really ugly: Because old userspace abused the linux agp interface to
+ * manage the gtt, we need to claim that all intel devices are agp.  For
+ * otherwise the drm core refuses to initialize the agp support code.
  */
 int i915_driver_device_is_agp(struct drm_device * dev)
 {
 	return 1;
 }
 
-static void i915_pineview_get_mem_freq(struct drm_device *dev)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	u32 tmp;
-
-	tmp = I915_READ(CLKCFG);
-
-	switch (tmp & CLKCFG_FSB_MASK) {
-	case CLKCFG_FSB_533:
-		dev_priv->fsb_freq = 533; /* 133*4 */
-		break;
-	case CLKCFG_FSB_800:
-		dev_priv->fsb_freq = 800; /* 200*4 */
-		break;
-	case CLKCFG_FSB_667:
-		dev_priv->fsb_freq =  667; /* 167*4 */
-		break;
-	case CLKCFG_FSB_400:
-		dev_priv->fsb_freq = 400; /* 100*4 */
-		break;
-	}
-
-	switch (tmp & CLKCFG_MEM_MASK) {
-	case CLKCFG_MEM_533:
-		dev_priv->mem_freq = 533;
-		break;
-	case CLKCFG_MEM_667:
-		dev_priv->mem_freq = 667;
-		break;
-	case CLKCFG_MEM_800:
-		dev_priv->mem_freq = 800;
-		break;
-	}
-
-	/* detect pineview DDR3 setting */
-	tmp = I915_READ(CSHRDDR3CTL);
-	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
-}
-
-static void i915_ironlake_get_mem_freq(struct drm_device *dev)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	u16 ddrpll, csipll;
-
-	ddrpll = I915_READ16(DDRMPLL1);
-	csipll = I915_READ16(CSIPLL0);
-
-	switch (ddrpll & 0xff) {
-	case 0xc:
-		dev_priv->mem_freq = 800;
-		break;
-	case 0x10:
-		dev_priv->mem_freq = 1066;
-		break;
-	case 0x14:
-		dev_priv->mem_freq = 1333;
-		break;
-	case 0x18:
-		dev_priv->mem_freq = 1600;
-		break;
-	default:
-		DRM_DEBUG("unknown memory frequency 0x%02x\n",
-				 ddrpll & 0xff);
-		dev_priv->mem_freq = 0;
-		break;
-	}
-
-	dev_priv->r_t = dev_priv->mem_freq;
-
-	switch (csipll & 0x3ff) {
-	case 0x00c:
-		dev_priv->fsb_freq = 3200;
-		break;
-	case 0x00e:
-		dev_priv->fsb_freq = 3733;
-		break;
-	case 0x010:
-		dev_priv->fsb_freq = 4266;
-		break;
-	case 0x012:
-		dev_priv->fsb_freq = 4800;
-		break;
-	case 0x014:
-		dev_priv->fsb_freq = 5333;
-		break;
-	case 0x016:
-		dev_priv->fsb_freq = 5866;
-		break;
-	case 0x018:
-		dev_priv->fsb_freq = 6400;
-		break;
-	default:
-		DRM_DEBUG("unknown fsb frequency 0x%04x\n",
-				 csipll & 0x3ff);
-		dev_priv->fsb_freq = 0;
-		break;
-	}
-
-	if (dev_priv->fsb_freq == 3200) {
-		dev_priv->c_m = 0;
-	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
-		dev_priv->c_m = 1;
-	} else {
-		dev_priv->c_m = 2;
-	}
-}
-
-static const struct cparams {
-	u16 i;
-	u16 t;
-	u16 m;
-	u16 c;
-} cparams[] = {
-	{ 1, 1333, 301, 28664 },
-	{ 1, 1066, 294, 24460 },
-	{ 1, 800, 294, 25192 },
-	{ 0, 1333, 276, 27605 },
-	{ 0, 1066, 276, 27605 },
-	{ 0, 800, 231, 23784 },
-};
-
-unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
-{
-	u64 total_count, diff, ret;
-	u32 count1, count2, count3, m = 0, c = 0;
-	unsigned long now = jiffies_to_msecs(jiffies), diff1;
-	int i;
-
-	diff1 = now - dev_priv->last_time1;
-	/*
-	 * sysctl(8) reads the value of sysctl twice in rapid
-	 * succession.  There is high chance that it happens in the
-	 * same timer tick.  Use the cached value to not divide by
-	 * zero and give the hw a chance to gather more samples.
-	 */
-	if (diff1 <= 10)
-		return (dev_priv->chipset_power);
-
-	count1 = I915_READ(DMIEC);
-	count2 = I915_READ(DDREC);
-	count3 = I915_READ(CSIEC);
-
-	total_count = count1 + count2 + count3;
-
-	/* FIXME: handle per-counter overflow */
-	if (total_count < dev_priv->last_count1) {
-		diff = ~0UL - dev_priv->last_count1;
-		diff += total_count;
-	} else {
-		diff = total_count - dev_priv->last_count1;
-	}
-
-	for (i = 0; i < DRM_ARRAY_SIZE(cparams); i++) {
-		if (cparams[i].i == dev_priv->c_m &&
-		    cparams[i].t == dev_priv->r_t) {
-			m = cparams[i].m;
-			c = cparams[i].c;
-			break;
-		}
-	}
-
-	diff = diff / diff1;
-	ret = ((m * diff) + c);
-	ret = ret / 10;
-
-	dev_priv->last_count1 = total_count;
-	dev_priv->last_time1 = now;
-
-	dev_priv->chipset_power = ret;
-	return (ret);
-}
-
-unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
-{
-	unsigned long m, x, b;
-	u32 tsfs;
-
-	tsfs = I915_READ(TSFS);
-
-	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
-	x = I915_READ8(I915_TR1);
-
-	b = tsfs & TSFS_INTR_MASK;
-
-	return ((m * x) / 127) - b;
-}
-
-static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
-{
-	static const struct v_table {
-		u16 vd; /* in .1 mil */
-		u16 vm; /* in .1 mil */
-	} v_table[] = {
-		{ 0, 0, },
-		{ 375, 0, },
-		{ 500, 0, },
-		{ 625, 0, },
-		{ 750, 0, },
-		{ 875, 0, },
-		{ 1000, 0, },
-		{ 1125, 0, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4125, 3000, },
-		{ 4250, 3125, },
-		{ 4375, 3250, },
-		{ 4500, 3375, },
-		{ 4625, 3500, },
-		{ 4750, 3625, },
-		{ 4875, 3750, },
-		{ 5000, 3875, },
-		{ 5125, 4000, },
-		{ 5250, 4125, },
-		{ 5375, 4250, },
-		{ 5500, 4375, },
-		{ 5625, 4500, },
-		{ 5750, 4625, },
-		{ 5875, 4750, },
-		{ 6000, 4875, },
-		{ 6125, 5000, },
-		{ 6250, 5125, },
-		{ 6375, 5250, },
-		{ 6500, 5375, },
-		{ 6625, 5500, },
-		{ 6750, 5625, },
-		{ 6875, 5750, },
-		{ 7000, 5875, },
-		{ 7125, 6000, },
-		{ 7250, 6125, },
-		{ 7375, 6250, },
-		{ 7500, 6375, },
-		{ 7625, 6500, },
-		{ 7750, 6625, },
-		{ 7875, 6750, },
-		{ 8000, 6875, },
-		{ 8125, 7000, },
-		{ 8250, 7125, },
-		{ 8375, 7250, },
-		{ 8500, 7375, },
-		{ 8625, 7500, },
-		{ 8750, 7625, },
-		{ 8875, 7750, },
-		{ 9000, 7875, },
-		{ 9125, 8000, },
-		{ 9250, 8125, },
-		{ 9375, 8250, },
-		{ 9500, 8375, },
-		{ 9625, 8500, },
-		{ 9750, 8625, },
-		{ 9875, 8750, },
-		{ 10000, 8875, },
-		{ 10125, 9000, },
-		{ 10250, 9125, },
-		{ 10375, 9250, },
-		{ 10500, 9375, },
-		{ 10625, 9500, },
-		{ 10750, 9625, },
-		{ 10875, 9750, },
-		{ 11000, 9875, },
-		{ 11125, 10000, },
-		{ 11250, 10125, },
-		{ 11375, 10250, },
-		{ 11500, 10375, },
-		{ 11625, 10500, },
-		{ 11750, 10625, },
-		{ 11875, 10750, },
-		{ 12000, 10875, },
-		{ 12125, 11000, },
-		{ 12250, 11125, },
-		{ 12375, 11250, },
-		{ 12500, 11375, },
-		{ 12625, 11500, },
-		{ 12750, 11625, },
-		{ 12875, 11750, },
-		{ 13000, 11875, },
-		{ 13125, 12000, },
-		{ 13250, 12125, },
-		{ 13375, 12250, },
-		{ 13500, 12375, },
-		{ 13625, 12500, },
-		{ 13750, 12625, },
-		{ 13875, 12750, },
-		{ 14000, 12875, },
-		{ 14125, 13000, },
-		{ 14250, 13125, },
-		{ 14375, 13250, },
-		{ 14500, 13375, },
-		{ 14625, 13500, },
-		{ 14750, 13625, },
-		{ 14875, 13750, },
-		{ 15000, 13875, },
-		{ 15125, 14000, },
-		{ 15250, 14125, },
-		{ 15375, 14250, },
-		{ 15500, 14375, },
-		{ 15625, 14500, },
-		{ 15750, 14625, },
-		{ 15875, 14750, },
-		{ 16000, 14875, },
-		{ 16125, 15000, },
-	};
-	if (dev_priv->info->is_mobile)
-		return v_table[pxvid].vm;
-	else
-		return v_table[pxvid].vd;
-}
-
-void i915_update_gfx_val(struct drm_i915_private *dev_priv)
-{
-	struct timespec now, diff1;
-	u64 diff;
-	unsigned long diffms;
-	u32 count;
-
-	if (dev_priv->info->gen != 5)
-		return;
-
-	nanotime(&now);
-	diff1 = now;
-	timespecsub(&diff1, &dev_priv->last_time2);
-
-	/* Don't divide by 0 */
-	diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
-	if (!diffms)
-		return;
-
-	count = I915_READ(GFXEC);
-
-	if (count < dev_priv->last_count2) {
-		diff = ~0UL - dev_priv->last_count2;
-		diff += count;
-	} else {
-		diff = count - dev_priv->last_count2;
-	}
-
-	dev_priv->last_count2 = count;
-	dev_priv->last_time2 = now;
-
-	/* More magic constants... */
-	diff = diff * 1181;
-	diff = diff / (diffms * 10);
-	dev_priv->gfx_power = diff;
-}
-
-unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
-{
-	unsigned long t, corr, state1, corr2, state2;
-	u32 pxvid, ext_v;
-
-	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->cur_delay * 4));
-	pxvid = (pxvid >> 24) & 0x7f;
-	ext_v = pvid_to_extvid(dev_priv, pxvid);
-
-	state1 = ext_v;
-
-	t = i915_mch_val(dev_priv);
-
-	/* Revel in the empirically derived constants */
-
-	/* Correction factor in 1/100000 units */
-	if (t > 80)
-		corr = ((t * 2349) + 135940);
-	else if (t >= 50)
-		corr = ((t * 964) + 29317);
-	else /* < 50 */
-		corr = ((t * 301) + 1004);
-
-	corr = corr * ((150142 * state1) / 10000 - 78642);
-	corr /= 100000;
-	corr2 = (corr * dev_priv->corr);
-
-	state2 = (corr2 * state1) / 10000;
-	state2 /= 100; /* convert to mW */
-
-	i915_update_gfx_val(dev_priv);
-
-	return dev_priv->gfx_power + state2;
-}
-
-/**
- * i915_read_mch_val - return value for IPS use
- *
- * Calculate and return a value for the IPS driver to use when deciding whether
- * we have thermal and power headroom to increase CPU or GPU power budget.
- */
-unsigned long i915_read_mch_val(void)
-{
-	struct drm_i915_private *dev_priv;
-	unsigned long chipset_val, graphics_val, ret = 0;
-
-	mtx_lock(&mchdev_lock);
-	if (!i915_mch_dev)
-		goto out_unlock;
-	dev_priv = i915_mch_dev;
-
-	chipset_val = i915_chipset_val(dev_priv);
-	graphics_val = i915_gfx_val(dev_priv);
-
-	ret = chipset_val + graphics_val;
-
-out_unlock:
-	mtx_unlock(&mchdev_lock);
-
-	return ret;
-}
-
-/**
- * i915_gpu_raise - raise GPU frequency limit
- *
- * Raise the limit; IPS indicates we have thermal headroom.
- */
-bool i915_gpu_raise(void)
-{
-	struct drm_i915_private *dev_priv;
-	bool ret = true;
-
-	mtx_lock(&mchdev_lock);
-	if (!i915_mch_dev) {
-		ret = false;
-		goto out_unlock;
-	}
-	dev_priv = i915_mch_dev;
-
-	if (dev_priv->max_delay > dev_priv->fmax)
-		dev_priv->max_delay--;
-
-out_unlock:
-	mtx_unlock(&mchdev_lock);
-
-	return ret;
-}
-
-/**
- * i915_gpu_lower - lower GPU frequency limit
- *
- * IPS indicates we're close to a thermal limit, so throttle back the GPU
- * frequency maximum.
- */
-bool i915_gpu_lower(void)
-{
-	struct drm_i915_private *dev_priv;
-	bool ret = true;
-
-	mtx_lock(&mchdev_lock);
-	if (!i915_mch_dev) {
-		ret = false;
-		goto out_unlock;
-	}
-	dev_priv = i915_mch_dev;
-
-	if (dev_priv->max_delay < dev_priv->min_delay)
-		dev_priv->max_delay++;
-
-out_unlock:
-	mtx_unlock(&mchdev_lock);
-
-	return ret;
-}
-
-/**
- * i915_gpu_busy - indicate GPU business to IPS
- *
- * Tell the IPS driver whether or not the GPU is busy.
- */
-bool i915_gpu_busy(void)
-{
-	struct drm_i915_private *dev_priv;
-	bool ret = false;
-
-	mtx_lock(&mchdev_lock);
-	if (!i915_mch_dev)
-		goto out_unlock;
-	dev_priv = i915_mch_dev;
-
-	ret = dev_priv->busy;
-
-out_unlock:
-	mtx_unlock(&mchdev_lock);
-
-	return ret;
-}
-
-/**
- * i915_gpu_turbo_disable - disable graphics turbo
- *
- * Disable graphics turbo by resetting the max frequency and setting the
- * current frequency to the default.
- */
-bool i915_gpu_turbo_disable(void)
-{
-	struct drm_i915_private *dev_priv;
-	bool ret = true;
-
-	mtx_lock(&mchdev_lock);
-	if (!i915_mch_dev) {
-		ret = false;
-		goto out_unlock;
-	}
-	dev_priv = i915_mch_dev;
-
-	dev_priv->max_delay = dev_priv->fstart;
-
-	if (!ironlake_set_drps(dev_priv->dev, dev_priv->fstart))
-		ret = false;
-
-out_unlock:
-	mtx_unlock(&mchdev_lock);
-
-	return ret;
-}
diff --git a/sys/dev/drm2/i915/i915_drm.h b/sys/dev/drm2/i915/i915_drm.h
index cf5227f20be..c6e5c7e6b8f 100644
--- a/sys/dev/drm2/i915/i915_drm.h
+++ b/sys/dev/drm2/i915/i915_drm.h
@@ -301,15 +301,15 @@ typedef struct drm_i915_irq_wait {
 
 /* Ioctl to query kernel params:
  */
-#define I915_PARAM_IRQ_ACTIVE            1
-#define I915_PARAM_ALLOW_BATCHBUFFER     2
-#define I915_PARAM_LAST_DISPATCH         3
-#define I915_PARAM_CHIPSET_ID            4
-#define I915_PARAM_HAS_GEM               5
-#define I915_PARAM_NUM_FENCES_AVAIL      6
-#define I915_PARAM_HAS_OVERLAY           7
+#define I915_PARAM_IRQ_ACTIVE		 1
+#define I915_PARAM_ALLOW_BATCHBUFFER	 2
+#define I915_PARAM_LAST_DISPATCH	 3
+#define I915_PARAM_CHIPSET_ID		 4
+#define I915_PARAM_HAS_GEM		 5
+#define I915_PARAM_NUM_FENCES_AVAIL	 6
+#define I915_PARAM_HAS_OVERLAY		 7
 #define I915_PARAM_HAS_PAGEFLIPPING	 8
-#define I915_PARAM_HAS_EXECBUF2          9
+#define I915_PARAM_HAS_EXECBUF2	 9
 #define I915_PARAM_HAS_BSD		 10
 #define I915_PARAM_HAS_BLT		 11
 #define I915_PARAM_HAS_RELAXED_FENCING	 12
@@ -317,7 +317,8 @@ typedef struct drm_i915_irq_wait {
 #define I915_PARAM_HAS_EXEC_CONSTANTS	 14
 #define I915_PARAM_HAS_RELAXED_DELTA	 15
 #define I915_PARAM_HAS_GEN7_SOL_RESET	 16
-#define I915_PARAM_HAS_LLC     	 17
+#define I915_PARAM_HAS_LLC		 17
+#define I915_PARAM_HAS_ALIASING_PPGTT	 18
 
 typedef struct drm_i915_getparam {
 	int param;
diff --git a/sys/dev/drm2/i915/i915_drv.c b/sys/dev/drm2/i915/i915_drv.c
index 2380d237f0f..b5a43a54df3 100644
--- a/sys/dev/drm2/i915/i915_drv.c
+++ b/sys/dev/drm2/i915/i915_drv.c
@@ -132,6 +132,7 @@ static const struct intel_device_info intel_ironlake_d_info = {
 	.gen = 5,
 	.need_gfx_hws = 1, .has_hotplug = 1,
 	.has_bsd_ring = 1,
+	.has_pch_split = 1,
 };
 
 static const struct intel_device_info intel_ironlake_m_info = {
@@ -139,6 +140,7 @@ static const struct intel_device_info intel_ironlake_m_info = {
 	.need_gfx_hws = 1, .has_hotplug = 1,
 	.has_fbc = 0, /* disabled due to buggy hardware */
 	.has_bsd_ring = 1,
+	.has_pch_split = 1,
 };
 
 static const struct intel_device_info intel_sandybridge_d_info = {
@@ -147,6 +149,7 @@ static const struct intel_device_info intel_sandybridge_d_info = {
 	.has_bsd_ring = 1,
 	.has_blt_ring = 1,
 	.has_llc = 1,
+	.has_pch_split = 1,
 };
 
 static const struct intel_device_info intel_sandybridge_m_info = {
@@ -156,6 +159,7 @@ static const struct intel_device_info intel_sandybridge_m_info = {
 	.has_bsd_ring = 1,
 	.has_blt_ring = 1,
 	.has_llc = 1,
+	.has_pch_split = 1,
 };
 
 static const struct intel_device_info intel_ivybridge_d_info = {
@@ -164,6 +168,7 @@ static const struct intel_device_info intel_ivybridge_d_info = {
 	.has_bsd_ring = 1,
 	.has_blt_ring = 1,
 	.has_llc = 1,
+	.has_pch_split = 1,
 };
 
 static const struct intel_device_info intel_ivybridge_m_info = {
@@ -173,6 +178,45 @@ static const struct intel_device_info intel_ivybridge_m_info = {
 	.has_bsd_ring = 1,
 	.has_blt_ring = 1,
 	.has_llc = 1,
+	.has_pch_split = 1,
+};
+
+#if 0
+static const struct intel_device_info intel_valleyview_m_info = {
+	.gen = 7, .is_mobile = 1,
+	.need_gfx_hws = 1, .has_hotplug = 1,
+	.has_fbc = 0,
+	.has_bsd_ring = 1,
+	.has_blt_ring = 1,
+	.is_valleyview = 1,
+};
+
+static const struct intel_device_info intel_valleyview_d_info = {
+	.gen = 7,
+	.need_gfx_hws = 1, .has_hotplug = 1,
+	.has_fbc = 0,
+	.has_bsd_ring = 1,
+	.has_blt_ring = 1,
+	.is_valleyview = 1,
+};
+#endif
+
+static const struct intel_device_info intel_haswell_d_info = {
+	.is_haswell = 1, .gen = 7,
+	.need_gfx_hws = 1, .has_hotplug = 1,
+	.has_bsd_ring = 1,
+	.has_blt_ring = 1,
+	.has_llc = 1,
+	.has_pch_split = 1,
+};
+
+static const struct intel_device_info intel_haswell_m_info = {
+	.is_haswell = 1, .gen = 7, .is_mobile = 1,
+	.need_gfx_hws = 1, .has_hotplug = 1,
+	.has_bsd_ring = 1,
+	.has_blt_ring = 1,
+	.has_llc = 1,
+	.has_pch_split = 1,
 };
 
 #define INTEL_VGA_DEVICE(id, info_) {		\
@@ -228,6 +272,13 @@ static const struct intel_gfx_device_id {
 	INTEL_VGA_DEVICE(0x0162, &intel_ivybridge_d_info), /* GT2 desktop */
 	INTEL_VGA_DEVICE(0x015a, &intel_ivybridge_d_info), /* GT1 server */
 	INTEL_VGA_DEVICE(0x016a, &intel_ivybridge_d_info), /* GT2 server */
+	INTEL_VGA_DEVICE(0x0402, &intel_haswell_d_info), /* GT1 desktop */
+	INTEL_VGA_DEVICE(0x0412, &intel_haswell_d_info), /* GT2 desktop */
+	INTEL_VGA_DEVICE(0x040a, &intel_haswell_d_info), /* GT1 server */
+	INTEL_VGA_DEVICE(0x041a, &intel_haswell_d_info), /* GT2 server */
+	INTEL_VGA_DEVICE(0x0406, &intel_haswell_m_info), /* GT1 mobile */
+	INTEL_VGA_DEVICE(0x0416, &intel_haswell_m_info), /* GT2 mobile */
+	INTEL_VGA_DEVICE(0x0c16, &intel_haswell_d_info), /* SDV */
 	{0, 0}
 };
 
@@ -304,14 +355,15 @@ static int i915_drm_thaw(struct drm_device *dev)
 
 	/* KMS EnterVT equivalent */
 	if (drm_core_check_feature(dev, DRIVER_MODESET)) {
-		dev_priv->mm.suspended = 0;
-
-		error = i915_gem_init_hw(dev);
-
 		if (HAS_PCH_SPLIT(dev))
 			ironlake_init_pch_refclk(dev);
 
+		dev_priv->mm.suspended = 0;
+
+		error = i915_gem_init_hw(dev);
 		DRM_UNLOCK(dev);
+
+		intel_modeset_init_hw(dev);
 		sx_xlock(&dev->mode_config.mutex);
 		drm_mode_config_reset(dev);
 		sx_xunlock(&dev->mode_config.mutex);
@@ -321,9 +373,6 @@ static int i915_drm_thaw(struct drm_device *dev)
 		/* Resume the modeset for every activated CRTC */
 		drm_helper_resume_force_mode(dev);
 		sx_xunlock(&dev->mode_config.mutex);
-
-		if (IS_IRONLAKE_M(dev))
-			ironlake_enable_rc6(dev);
 		DRM_LOCK(dev);
 	}
 
@@ -445,7 +494,11 @@ MODULE_DEPEND(i915kms, iicbb, 1, 1, 1);
 
 int intel_iommu_enabled = 0;
 TUNABLE_INT("drm.i915.intel_iommu_enabled", &intel_iommu_enabled);
+int intel_iommu_gfx_mapped = 0;
+TUNABLE_INT("drm.i915.intel_iommu_gfx_mapped", &intel_iommu_gfx_mapped);
 
+int i915_prefault_disable;
+TUNABLE_INT("drm.i915.prefault_disable", &i915_prefault_disable);
 int i915_semaphores = -1;
 TUNABLE_INT("drm.i915.semaphores", &i915_semaphores);
 static int i915_try_reset = 1;
@@ -460,10 +513,14 @@ int i915_enable_fbc = 0;
 TUNABLE_INT("drm.i915.enable_fbc", &i915_enable_fbc);
 int i915_enable_rc6 = 0;
 TUNABLE_INT("drm.i915.enable_rc6", &i915_enable_rc6);
+int i915_lvds_channel_mode;
+TUNABLE_INT("drm.i915.lvds_channel_mode", &i915_lvds_channel_mode);
 int i915_panel_use_ssc = -1;
 TUNABLE_INT("drm.i915.panel_use_ssc", &i915_panel_use_ssc);
 int i915_panel_ignore_lid = 0;
 TUNABLE_INT("drm.i915.panel_ignore_lid", &i915_panel_ignore_lid);
+int i915_panel_invert_brightness;
+TUNABLE_INT("drm.i915.panel_invert_brightness", &i915_panel_invert_brightness);
 int i915_modeset = 1;
 TUNABLE_INT("drm.i915.modeset", &i915_modeset);
 int i915_enable_ppgtt = -1;
@@ -476,9 +533,9 @@ TUNABLE_INT("drm.i915.enable_hangcheck", &i915_enable_hangcheck);
 #define INTEL_PCH_IBX_DEVICE_ID_TYPE	0x3b00
 #define INTEL_PCH_CPT_DEVICE_ID_TYPE	0x1c00
 #define INTEL_PCH_PPT_DEVICE_ID_TYPE	0x1e00
+#define INTEL_PCH_LPT_DEVICE_ID_TYPE	0x8c00
 
-void
-intel_detect_pch(struct drm_device *dev)
+void intel_detect_pch(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv;
 	device_t pch;
@@ -490,20 +547,44 @@ intel_detect_pch(struct drm_device *dev)
 		id = pci_get_device(pch) & INTEL_PCH_DEVICE_ID_MASK;
 		if (id == INTEL_PCH_IBX_DEVICE_ID_TYPE) {
 			dev_priv->pch_type = PCH_IBX;
+			dev_priv->num_pch_pll = 2;
 			DRM_DEBUG_KMS("Found Ibex Peak PCH\n");
 		} else if (id == INTEL_PCH_CPT_DEVICE_ID_TYPE) {
 			dev_priv->pch_type = PCH_CPT;
+			dev_priv->num_pch_pll = 2;
 			DRM_DEBUG_KMS("Found CougarPoint PCH\n");
 		} else if (id == INTEL_PCH_PPT_DEVICE_ID_TYPE) {
 			/* PantherPoint is CPT compatible */
 			dev_priv->pch_type = PCH_CPT;
+			dev_priv->num_pch_pll = 2;
 			DRM_DEBUG_KMS("Found PatherPoint PCH\n");
+		} else if (id == INTEL_PCH_LPT_DEVICE_ID_TYPE) {
+			dev_priv->pch_type = PCH_LPT;
+			dev_priv->num_pch_pll = 0;
+			DRM_DEBUG_KMS("Found LynxPoint PCH\n");
 		} else
 			DRM_DEBUG_KMS("No PCH detected\n");
+		KASSERT(dev_priv->num_pch_pll <= I915_NUM_PLLS,
+		    ("num_pch_pll %d\n", dev_priv->num_pch_pll));
 	} else
 		DRM_DEBUG_KMS("No Intel PCI-ISA bridge found\n");
 }
 
+bool i915_semaphore_is_enabled(struct drm_device *dev)
+{
+	if (INTEL_INFO(dev)->gen < 6)
+		return 0;
+
+	if (i915_semaphores >= 0)
+		return i915_semaphores;
+
+	/* Enable semaphores on SNB when IO remapping is off */
+	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
+		return false;
+
+	return 1;
+}
+
 void
 __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv)
 {
@@ -530,7 +611,7 @@ __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv)
 	while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_MT_ACK) & 1))
 		DELAY(10);
 
-	I915_WRITE_NOTRACE(FORCEWAKE_MT, (1<<16) | 1);
+	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_ENABLE(1));
 	POSTING_READ(FORCEWAKE_MT);
 
 	count = 0;
@@ -573,7 +654,7 @@ void
 __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv)
 {
 
-	I915_WRITE_NOTRACE(FORCEWAKE_MT, (1<<16) | 0);
+	I915_WRITE_NOTRACE(FORCEWAKE_MT, _MASKED_BIT_DISABLE(1));
 	/* The below doubles as a POSTING_READ */
 	gen6_gt_check_fifodbg(dev_priv);
 }
@@ -611,6 +692,31 @@ __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
 	return (ret);
 }
 
+void vlv_force_wake_get(struct drm_i915_private *dev_priv)
+{
+	int count;
+
+	count = 0;
+
+	/* Already awake? */
+	if ((I915_READ(0x130094) & 0xa1) == 0xa1)
+		return;
+
+	I915_WRITE_NOTRACE(FORCEWAKE_VLV, 0xffffffff);
+	POSTING_READ(FORCEWAKE_VLV);
+
+	count = 0;
+	while (count++ < 50 && (I915_READ_NOTRACE(FORCEWAKE_ACK_VLV) & 1) == 0)
+		DELAY(10);
+}
+
+void vlv_force_wake_put(struct drm_i915_private *dev_priv)
+{
+	I915_WRITE_NOTRACE(FORCEWAKE_VLV, 0xffff0000);
+	/* FIXME: confirm VLV behavior with Punit folks */
+	POSTING_READ(FORCEWAKE_VLV);
+}
+
 static int
 i8xx_do_reset(struct drm_device *dev)
 {
@@ -653,12 +759,13 @@ i965_reset_complete(struct drm_device *dev)
 	u8 gdrst;
 
 	gdrst = pci_read_config(dev->device, I965_GDRST, 1);
-	return (gdrst & 0x1);
+	return (gdrst & GRDOM_RESET_ENABLE) == 0;
 }
 
 static int
 i965_do_reset(struct drm_device *dev)
 {
+	int ret;
 	u8 gdrst;
 
 	/*
@@ -670,8 +777,16 @@ i965_do_reset(struct drm_device *dev)
 	pci_write_config(dev->device, I965_GDRST,
 	    gdrst | GRDOM_RENDER | GRDOM_RESET_ENABLE, 1);
 
-	return (_intel_wait_for(dev, i965_reset_complete(dev), 500, 1,
-	    "915rst"));
+	ret =  wait_for(i965_reset_complete(dev), 500);
+	if (ret)
+		return ret;
+
+	/* We can't reset render&media without also resetting display ... */
+	gdrst = pci_read_config(dev->device, I965_GDRST, 1);
+	pci_write_config(dev->device, I965_GDRST,
+			 gdrst | GRDOM_MEDIA | GRDOM_RESET_ENABLE, 1);
+ 
+ 	return wait_for(i965_reset_complete(dev), 500);
 }
 
 static int
@@ -679,14 +794,21 @@ ironlake_do_reset(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv;
 	u32 gdrst;
+	int ret;
 
 	dev_priv = dev->dev_private;
 	gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR);
 	I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
-	    gdrst | GRDOM_RENDER | GRDOM_RESET_ENABLE);
-	return (_intel_wait_for(dev,
-	    (I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1) != 0,
-	    500, 1, "915rst"));
+		   gdrst | GRDOM_RENDER | GRDOM_RESET_ENABLE);
+	ret = wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
+	if (ret)
+		return ret;
+
+	/* We can't reset render&media without also resetting display ... */
+	gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR);
+	I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR,
+		   gdrst | GRDOM_MEDIA | GRDOM_RESET_ENABLE);
+ 	return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500);
 }
 
 static int
@@ -713,7 +835,7 @@ gen6_do_reset(struct drm_device *dev)
 	/* Spin waiting for the device to ack the reset request */
 	ret = _intel_wait_for(dev,
 	    (I915_READ_NOTRACE(GEN6_GDRST) & GEN6_GRDOM_FULL) == 0,
-	    500, 1, "915rst");
+	    500, 0, "915rst");
 
 	/* If reset with a user forcewake, try to restore, otherwise turn it off */
  	if (dev_priv->forcewake_count)
@@ -728,7 +850,8 @@ gen6_do_reset(struct drm_device *dev)
 	return (ret);
 }
 
-int intel_gpu_reset(struct drm_device *dev)
+int
+intel_gpu_reset(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int ret = -ENODEV;
@@ -763,15 +886,9 @@ int intel_gpu_reset(struct drm_device *dev)
 	return ret;
 }
 
-int
-i915_reset(struct drm_device *dev)
+int i915_reset(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	/*
-	 * We really should only reset the display subsystem if we actually
-	 * need to
-	 */
-	bool need_display = true;
 	int ret;
 
 	if (!i915_try_reset)
@@ -780,12 +897,14 @@ i915_reset(struct drm_device *dev)
 	if (!sx_try_xlock(&dev->dev_struct_lock))
 		return (-EBUSY);
 
+	dev_priv->stop_rings = 0;
+
 	i915_gem_reset(dev);
 
 	ret = -ENODEV;
-	if (time_second - dev_priv->last_gpu_reset < 5) {
+	if (time_second - dev_priv->last_gpu_reset < 5)
 		DRM_ERROR("GPU hanging too fast, declaring wedged!\n");
-	} else
+	else
 		ret = intel_gpu_reset(dev);
 
 	dev_priv->last_gpu_reset = time_second;
@@ -797,36 +916,41 @@ i915_reset(struct drm_device *dev)
 
 	if (drm_core_check_feature(dev, DRIVER_MODESET) ||
 	    !dev_priv->mm.suspended) {
+		struct intel_ring_buffer *ring;
+		int i;
+
 		dev_priv->mm.suspended = 0;
 
 		i915_gem_init_swizzling(dev);
 
-		dev_priv->rings[RCS].init(&dev_priv->rings[RCS]);
-		if (HAS_BSD(dev))
-			dev_priv->rings[VCS].init(&dev_priv->rings[VCS]);
-		if (HAS_BLT(dev))
-			dev_priv->rings[BCS].init(&dev_priv->rings[BCS]);
+		for_each_ring(ring, dev_priv, i)
+			ring->init(ring);
 
 		i915_gem_context_init(dev);
 		i915_gem_init_ppgtt(dev);
 
+		DRM_UNLOCK(dev);
+
+		if (drm_core_check_feature(dev, DRIVER_MODESET))
+			intel_modeset_init_hw(dev);
+
+		DRM_LOCK(dev);
 		drm_irq_uninstall(dev);
-		drm_mode_config_reset(dev);
 		DRM_UNLOCK(dev);
 		drm_irq_install(dev);
-		DRM_LOCK(dev);
-	}
-	DRM_UNLOCK(dev);
-
-	if (need_display) {
-		sx_xlock(&dev->mode_config.mutex);
-		drm_helper_resume_force_mode(dev);
-		sx_xunlock(&dev->mode_config.mutex);
-	}
+	} else
+		DRM_UNLOCK(dev);
 
 	return (0);
 }
 
+/* We give fast paths for the really cool registers */
+#define NEEDS_FORCE_WAKE(dev_priv, reg) \
+       (((dev_priv)->info->gen >= 6) && \
+        ((reg) < 0x40000) &&            \
+        ((reg) != FORCEWAKE)) && \
+       (!IS_VALLEYVIEW((dev_priv)->dev))
+
 #define __i915_read(x, y) \
 u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg) { \
 	u##x val = 0; \
diff --git a/sys/dev/drm2/i915/i915_drv.h b/sys/dev/drm2/i915/i915_drv.h
index b84f4f39a74..c5f85d642ee 100644
--- a/sys/dev/drm2/i915/i915_drv.h
+++ b/sys/dev/drm2/i915/i915_drv.h
@@ -66,10 +66,31 @@ enum plane {
 };
 #define plane_name(p) ((p) + 'A')
 
-#define	I915_GEM_GPU_DOMAINS	(~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
+enum port {
+	PORT_A = 0,
+	PORT_B,
+	PORT_C,
+	PORT_D,
+	PORT_E,
+	I915_MAX_PORTS
+};
+#define port_name(p) ((p) + 'A')
+
+#define I915_GEM_GPU_DOMAINS	(~(I915_GEM_DOMAIN_CPU | I915_GEM_DOMAIN_GTT))
+
 
 #define for_each_pipe(p) for ((p) = 0; (p) < dev_priv->num_pipe; (p)++)
 
+struct intel_pch_pll {
+	int refcount; /* count of number of CRTCs sharing this PLL */
+	int active; /* count of number of active CRTCs (i.e. DPMS on) */
+	bool on; /* is the PLL actually active? Disabled during modeset */
+	int pll_reg;
+	int fp0_reg;
+	int fp1_reg;
+};
+#define I915_NUM_PLLS 2
+
 /* Interface history:
  *
  * 1.1: Original.
@@ -115,11 +136,15 @@ struct drm_i915_display_funcs {
 	void (*update_wm)(struct drm_device *dev);
 	void (*update_sprite_wm)(struct drm_device *dev, int pipe,
 				 uint32_t sprite_width, int pixel_size);
+	void (*sanitize_pm)(struct drm_device *dev);
+	void (*update_linetime_wm)(struct drm_device *dev, int pipe,
+				 struct drm_display_mode *mode);
 	int (*crtc_mode_set)(struct drm_crtc *crtc,
 			     struct drm_display_mode *mode,
 			     struct drm_display_mode *adjusted_mode,
 			     int x, int y,
 			     struct drm_framebuffer *old_fb);
+	void (*off)(struct drm_crtc *crtc);
 	void (*write_eld)(struct drm_connector *connector,
 			  struct drm_crtc *crtc);
 	void (*fdi_link_train)(struct drm_crtc *crtc);
@@ -152,6 +177,9 @@ struct intel_device_info {
 	u8 is_broadwater:1;
 	u8 is_crestline:1;
 	u8 is_ivybridge:1;
+	u8 is_valleyview:1;
+	u8 has_pch_split:1;
+	u8 is_haswell:1;
 	u8 has_fbc:1;
 	u8 has_pipe_cxsr:1;
 	u8 has_hotplug:1;
@@ -227,7 +255,6 @@ struct intel_opregion {
 struct drm_i915_fence_reg {
 	struct list_head lru_list;
 	struct drm_i915_gem_object *obj;
-	uint32_t setup_seqno;
 	int pin_count;
 };
 
@@ -243,10 +270,12 @@ struct sdvo_device_mapping {
 enum intel_pch {
 	PCH_IBX,	/* Ibexpeak PCH */
 	PCH_CPT,	/* Cougarpoint PCH */
+	PCH_LPT,	/* Lynxpoint PCH */
 };
 
 #define QUIRK_PIPEA_FORCE (1<<0)
 #define QUIRK_LVDS_SSC_DISABLE (1<<1)
+#define QUIRK_INVERT_BRIGHTNESS (1<<2)
 
 struct intel_fbdev;
 struct intel_fbc_work;
@@ -254,15 +283,15 @@ struct intel_fbc_work;
 typedef struct drm_i915_private {
 	struct drm_device *dev;
 
-	device_t *gmbus_bridge;
-	device_t *bbbus_bridge;
-	device_t *gmbus;
-	device_t *bbbus;
+	device_t gmbus_bridge[GMBUS_NUM_PORTS + 1];
+	device_t bbbus_bridge[GMBUS_NUM_PORTS + 1];
+	device_t gmbus[GMBUS_NUM_PORTS + 1];
+	device_t bbbus[GMBUS_NUM_PORTS + 1];
 	/** gmbus_sx protects against concurrent usage of the single hw gmbus
 	 * controller on different i2c buses. */
 	struct sx gmbus_sx;
+	uint32_t gpio_mmio_base;
 
-	int has_gem;
 	int relative_constants_mode;
 
 	drm_local_map_t *sarea;
@@ -286,7 +315,6 @@ typedef struct drm_i915_private {
 	dma_addr_t dma_status_page;
 	uint32_t counter;
 	unsigned int status_gfx_addr;
-	drm_local_map_t hws_map;
 	struct drm_gem_object *hws_obj;
 
 	struct drm_i915_gem_object *pwrctx;
@@ -308,23 +336,23 @@ typedef struct drm_i915_private {
 	u32 pch_irq_mask;
 	struct mtx irq_lock;
 
+	struct mtx dpio_lock;
+
 	u32 hotplug_supported_mask;
 
-	int tex_lru_log_granularity;
-	int allow_batchbuffer;
 	unsigned int sr01, adpa, ppcr, dvob, dvoc, lvds;
-	int vblank_pipe;
 	int num_pipe;
+	int num_pch_pll;
 
 	/* For hangcheck timer */
 #define DRM_I915_HANGCHECK_PERIOD ((1500 /* in ms */ * hz) / 1000)
 	int hangcheck_count;
-	uint32_t last_acthd;
-	uint32_t last_acthd_bsd;
-	uint32_t last_acthd_blt;
+	uint32_t last_acthd[I915_NUM_RINGS];
 	uint32_t last_instdone;
 	uint32_t last_instdone1;
 
+	unsigned int stop_rings;
+
 	struct intel_opregion opregion;
 
 
@@ -346,6 +374,8 @@ typedef struct drm_i915_private {
 	unsigned int lvds_use_ssc:1;
 	unsigned int display_clock_mode:1;
 	int lvds_ssc_freq;
+	unsigned int bios_lvds_val; /* initial [PCH_]LVDS reg val in VBIOS */
+	unsigned int lvds_val; /* used for checking LVDS channel mode */
 	struct {
 		int rate;
 		int lanes;
@@ -575,23 +605,9 @@ typedef struct drm_i915_private {
 		 */
 		struct list_head inactive_list;
 
-		/**
-		 * LRU list of objects which are not in the ringbuffer but
-		 * are still pinned in the GTT.
-		 */
-		struct list_head pinned_list;
-
 		/** LRU list of objects with fence regs on them. */
 		struct list_head fence_list;
 
-		/**
-		 * List of objects currently pending being freed.
-		 *
-		 * These objects are no longer in use, but due to a signal
-		 * we were prevented from freeing them at the appointed time.
-		 */
-		struct list_head deferred_free_list;
-
 		/**
 		 * We leave the user IRQ off as much as possible,
 		 * but this means that requests will finish and never
@@ -658,6 +674,15 @@ typedef struct drm_i915_private {
 
 	const struct intel_device_info *info;
 
+	/* Old dri1 support infrastructure, beware the dragons ya fools entering
+	 * here! */
+	struct {
+		unsigned allow_batchbuffer : 1;
+		u32 *gfx_hws_cpu_addr;
+	} dri1;
+
+	/* Kernel Modesetting */
+
 	struct sdvo_device_mapping sdvo_mappings[2];
 	/* indicate whether the LVDS_BORDER should be enabled or not */
 	unsigned int lvds_border_bits;
@@ -667,7 +692,8 @@ typedef struct drm_i915_private {
 	struct drm_crtc *plane_to_crtc_mapping[3];
 	struct drm_crtc *pipe_to_crtc_mapping[3];
 	/* wait_queue_head_t pending_flip_queue; XXXKIB */
-	bool flip_pending_is_done;
+
+	struct intel_pch_pll pch_plls[I915_NUM_PLLS];
 
 	/* Reclocking support */
 	bool render_reclock_avail;
@@ -711,7 +737,8 @@ typedef struct drm_i915_private {
 
 	enum no_fbc_reason no_fbc_reason;
 
-	unsigned int stop_rings;
+	struct drm_mm_node *compressed_fb;
+	struct drm_mm_node *compressed_llb;
 
 	unsigned long cfb_size;
 	unsigned int cfb_fb;
@@ -726,6 +753,7 @@ typedef struct drm_i915_private {
 	struct task hotplug_task;
 	int error_completion;
 	struct mtx error_completion_lock;
+	/* Protected by dev->error_lock. */
 	struct drm_i915_error_state *first_error;
 	struct mtx error_lock;
 	struct callout hangcheck_timer;
@@ -816,7 +844,14 @@ struct drm_i915_gem_object {
 	 * Current tiling mode for the object.
 	 */
 	unsigned int tiling_mode:2;
-	unsigned int tiling_changed:1;
+	/**
+	 * Whether the tiling parameters for the currently associated fence
+	 * register have changed. Note that for the purposes of tracking
+	 * tiling changes we also treat the unfenced register, the register
+	 * slot that the object occupies whilst it executes a fenced
+	 * command (such as BLT on gen2/3), as a "fence".
+	 */
+	unsigned int fence_dirty:1;
 
 	/** How many users have pinned this object in GTT space. The following
 	 * users can each hold at most one reference: pwrite/pread, pin_ioctl
@@ -843,6 +878,7 @@ struct drm_i915_gem_object {
 	 */
 	unsigned int fault_mappable:1;
 	unsigned int pin_mappable:1;
+	unsigned int pin_display:1;
 
 	/*
 	 * Is the GPU currently using a fence to access this buffer,
@@ -856,6 +892,7 @@ struct drm_i915_gem_object {
 	unsigned int has_global_gtt_mapping:1;
 
 	vm_page_t *pages;
+	int pages_pin_count;
 
 	/**
 	 * DMAR support
@@ -876,13 +913,12 @@ struct drm_i915_gem_object {
 	 */
 	uint32_t gtt_offset;
 
-	/** Breadcrumb of last rendering to the buffer. */
-	uint32_t last_rendering_seqno;
 	struct intel_ring_buffer *ring;
 
+	/** Breadcrumb of last rendering to the buffer. */
+	uint32_t last_rendering_seqno;
 	/** Breadcrumb of last fenced GPU access to the buffer. */
 	uint32_t last_fenced_seqno;
-	struct intel_ring_buffer *last_fenced_ring;
 
 	/** Current tiling stride for the object, if it's tiled. */
 	uint32_t stride;
@@ -890,12 +926,6 @@ struct drm_i915_gem_object {
 	/** Record of address bit 17 of each page at last unbind. */
 	unsigned long *bit_17;
 
-	/**
-	 * If present, while GEM_DOMAIN_CPU is in the read domain this array
-	 * flags which individual pages are valid.
-	 */
-	uint8_t *page_cpu_valid;
-
 	/** User space pin count and filp owning the pin */
 	uint32_t user_pin_count;
 	struct drm_file *pin_filp;
@@ -953,8 +983,11 @@ struct drm_i915_file_private {
 };
 
 struct drm_i915_error_state {
+	u_int ref;
 	u32 eir;
 	u32 pgtbl_er;
+	u32 ier;
+	bool waiting[I915_NUM_RINGS];
 	u32 pipestat[I915_MAX_PIPES];
 	u32 tail[I915_NUM_RINGS];
 	u32 head[I915_NUM_RINGS];
@@ -1037,9 +1070,12 @@ extern struct drm_driver_info i915_driver_info;
 extern struct cdev_pager_ops i915_gem_pager_ops;
 extern unsigned int i915_fbpercrtc;
 extern int i915_panel_ignore_lid;
+extern int i915_panel_invert_brightness;
 extern unsigned int i915_powersave;
+extern int i915_prefault_disable;
 extern int i915_semaphores;
 extern unsigned int i915_lvds_downclock;
+extern int i915_lvds_channel_mode;
 extern int i915_panel_use_ssc;
 extern int i915_vbt_sdvo_panel_type;
 extern int i915_enable_rc6;
@@ -1049,8 +1085,8 @@ extern int i915_enable_hangcheck;
 
 const struct intel_device_info *i915_get_device_id(int device);
 
-extern int intel_gpu_reset(struct drm_device *dev);
 int i915_reset(struct drm_device *dev);
+extern int intel_gpu_reset(struct drm_device *dev);
 
 /* i915_debug.c */
 int i915_sysctl_init(struct drm_device *dev, struct sysctl_ctx_list *ctx,
@@ -1064,6 +1100,7 @@ int i915_cmdbuffer(struct drm_device *dev, void *data,
     struct drm_file *file_priv);
 int i915_getparam(struct drm_device *dev, void *data,
     struct drm_file *file_priv);
+void i915_update_dri1_breadcrumb(struct drm_device *dev);
 extern void i915_kernel_lost_context(struct drm_device * dev);
 extern int i915_driver_load(struct drm_device *, unsigned long flags);
 extern int i915_driver_unload(struct drm_device *);
@@ -1095,20 +1132,12 @@ bool i915_gpu_turbo_disable(void);
 /* i915_irq.c */
 extern int i915_irq_emit(struct drm_device *dev, void *data,
 			 struct drm_file *file_priv);
-extern int i915_irq_wait(struct drm_device *dev, void *data,
-			 struct drm_file *file_priv);
-
 extern void intel_irq_init(struct drm_device *dev);
 
-extern int i915_vblank_pipe_set(struct drm_device *dev, void *data,
-				struct drm_file *file_priv);
-extern int i915_vblank_pipe_get(struct drm_device *dev, void *data,
-				struct drm_file *file_priv);
-extern int i915_vblank_swap(struct drm_device *dev, void *data,
-			    struct drm_file *file_priv);
 void intel_enable_asle(struct drm_device *dev);
 void i915_hangcheck_elapsed(void *context);
 void i915_handle_error(struct drm_device *dev, bool wedged);
+void i915_error_state_free(struct drm_i915_error_state *error);
 
 void i915_enable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask);
 void i915_disable_pipestat(drm_i915_private_t *dev_priv, int pipe, u32 mask);
@@ -1169,13 +1198,15 @@ int i915_gem_object_unbind(struct drm_i915_gem_object *obj);
 void i915_gem_lastclose(struct drm_device *dev);
 uint32_t i915_get_gem_seqno(struct drm_device *dev);
 
-static inline void
+static inline bool
 i915_gem_object_pin_fence(struct drm_i915_gem_object *obj)
 {
 	if (obj->fence_reg != I915_FENCE_REG_NONE) {
 		struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 		dev_priv->fence_regs[obj->fence_reg].pin_count++;
-	}
+		return true;
+	} else
+		return false;
 }
 
 static inline void
@@ -1192,36 +1223,38 @@ void i915_gem_retire_requests_ring(struct intel_ring_buffer *ring);
 void i915_gem_clflush_object(struct drm_i915_gem_object *obj);
 struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
     size_t size);
-int i915_gem_do_init(struct drm_device *dev, unsigned long start,
-    unsigned long mappable_end, unsigned long end);
 uint32_t i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
     uint32_t size, int tiling_mode);
 int i915_mutex_lock_interruptible(struct drm_device *dev);
 int i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj,
     bool write);
+int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
+    bool write);
 int i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
     u32 alignment, struct intel_ring_buffer *pipelined);
+void i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj);
 int i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj);
 int i915_gem_flush_ring(struct intel_ring_buffer *ring,
     uint32_t invalidate_domains, uint32_t flush_domains);
 void i915_gem_release_mmap(struct drm_i915_gem_object *obj);
 int i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj);
+int i915_gem_object_sync(struct drm_i915_gem_object *obj,
+    struct intel_ring_buffer *to);
 int i915_gem_object_put_fence(struct drm_i915_gem_object *obj);
 int i915_gem_idle(struct drm_device *dev);
+int i915_gem_init(struct drm_device *dev);
 int i915_gem_init_hw(struct drm_device *dev);
 void i915_gem_init_swizzling(struct drm_device *dev);
 void i915_gem_init_ppgtt(struct drm_device *dev);
 void i915_gem_cleanup_ringbuffer(struct drm_device *dev);
-int i915_gpu_idle(struct drm_device *dev, bool do_retire);
+int i915_gpu_idle(struct drm_device *dev);
 void i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
     struct intel_ring_buffer *ring, uint32_t seqno);
 int i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file,
     struct drm_i915_gem_request *request);
-int i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
-    struct intel_ring_buffer *pipelined);
+int i915_gem_object_get_fence(struct drm_i915_gem_object *obj);
 void i915_gem_reset(struct drm_device *dev);
-int i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno,
-    bool do_retire);
+int i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno);
 int i915_gem_mmap(struct drm_device *dev, uint64_t offset, int prot);
 int i915_gem_fault(struct drm_device *dev, uint64_t offset, int prot,
     uint64_t *phys);
@@ -1257,12 +1290,17 @@ int i915_gem_dumb_destroy(struct drm_file *file_priv, struct drm_device *dev,
 void i915_gem_detect_bit_6_swizzle(struct drm_device *dev);
 void i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj);
 void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj);
+void i915_gem_object_do_bit_17_swizzle_page(struct drm_i915_gem_object *obj,
+    struct vm_page *m);
 
 /* i915_gem_evict.c */
 int i915_gem_evict_something(struct drm_device *dev, int min_size,
     unsigned alignment, bool mappable);
 int i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only);
-int i915_gem_evict_inactive(struct drm_device *dev, bool purgeable_only);
+
+/* i915_gem_stolen.c */
+int i915_gem_init_stolen(struct drm_device *dev);
+void i915_gem_cleanup_stolen(struct drm_device *dev);
 
 /* i915_suspend.c */
 extern int i915_save_state(struct drm_device *dev);
@@ -1274,6 +1312,12 @@ extern void intel_teardown_gmbus(struct drm_device *dev);
 extern void intel_gmbus_set_speed(device_t idev, int speed);
 extern void intel_gmbus_force_bit(device_t idev, bool force_bit);
 extern void intel_iic_reset(struct drm_device *dev);
+static inline bool intel_gmbus_is_port_valid(unsigned port)
+{
+	return (port >= GMBUS_PORT_SSC && port <= GMBUS_PORT_DPD);
+}
+extern device_t intel_gmbus_get_adapter(struct drm_i915_private *dev_priv,
+    unsigned port);
 
 /* intel_opregion.c */
 int intel_opregion_setup(struct drm_device *dev);
@@ -1292,12 +1336,16 @@ void i915_ppgtt_unbind_object(struct i915_hw_ppgtt *ppgtt,
     struct drm_i915_gem_object *obj);
 
 void i915_gem_restore_gtt_mappings(struct drm_device *dev);
-int i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj);
+int i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj);
+void i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
+			      enum i915_cache_level cache_level);
 void i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj);
-void i915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj,
-    enum i915_cache_level cache_level);
+void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj);
+int i915_gem_init_global_gtt(struct drm_device *dev, unsigned long start,
+    unsigned long mappable_end, unsigned long end);
 
 /* modesetting */
+extern void intel_modeset_init_hw(struct drm_device *dev);
 extern void intel_modeset_init(struct drm_device *dev);
 extern void intel_modeset_gem_init(struct drm_device *dev);
 extern void intel_modeset_cleanup(struct drm_device *dev);
@@ -1310,12 +1358,19 @@ extern void ironlake_enable_rc6(struct drm_device *dev);
 extern void gen6_set_rps(struct drm_device *dev, u8 val);
 extern void intel_detect_pch(struct drm_device *dev);
 extern int intel_trans_dp_port_sel(struct drm_crtc *crtc);
+/* IPS */
+extern void intel_gpu_ips_init(struct drm_i915_private *dev_priv);
+extern void intel_gpu_ips_teardown(void);
 
+extern bool i915_semaphore_is_enabled(struct drm_device *dev);
 extern void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv);
 extern void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv);
 extern void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv);
 extern void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv);
 
+extern void vlv_force_wake_get(struct drm_i915_private *dev_priv);
+extern void vlv_force_wake_put(struct drm_i915_private *dev_priv);
+
 extern struct intel_overlay_error_state *intel_overlay_capture_error_state(
     struct drm_device *dev);
 extern void intel_overlay_print_error_state(struct sbuf *m,
@@ -1340,12 +1395,6 @@ void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv);
 void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv);
 int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv);
 
-/* We give fast paths for the really cool registers */
-#define NEEDS_FORCE_WAKE(dev_priv, reg) \
-	(((dev_priv)->info->gen >= 6) && \
-	 ((reg) < 0x40000) &&		 \
-	 ((reg) != FORCEWAKE))
-
 #define __i915_read(x, y) \
 	u##x i915_read##x(struct drm_i915_private *dev_priv, u32 reg);
 
@@ -1385,22 +1434,6 @@ __i915_write(64, 64)
 
 #define I915_VERBOSE 0
 
-#define LP_RING(d) (&((struct drm_i915_private *)(d))->rings[RCS])
-
-#define BEGIN_LP_RING(n) \
-	intel_ring_begin(LP_RING(dev_priv), (n))
-
-#define OUT_RING(x) \
-	intel_ring_emit(LP_RING(dev_priv), x)
-
-#define ADVANCE_LP_RING() \
-	intel_ring_advance(LP_RING(dev_priv))
-
-#define RING_LOCK_TEST_WITH_RETURN(dev, file) do {			\
-	if (LP_RING(dev->dev_private)->obj == NULL)			\
-		LOCK_TEST_WITH_RETURN(dev, file);			\
-} while (0)
-
 /**
  * Reads a dword out of the status page, which is written to from the command
  * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
@@ -1416,10 +1449,7 @@ __i915_write(64, 64)
  *
  * The area from dword 0x20 to 0x3ff is available for driver usage.
  */
-#define READ_HWSP(dev_priv, reg)  (((volatile u32*)(dev_priv->hw_status_page))[reg])
-#define READ_BREADCRUMB(dev_priv) READ_HWSP(dev_priv, I915_BREADCRUMB_INDEX)
 #define I915_GEM_HWS_INDEX		0x20
-#define I915_BREADCRUMB_INDEX		0x21
 
 #define INTEL_INFO(dev)	(((struct drm_i915_private *) (dev)->dev_private)->info)
 
@@ -1442,6 +1472,8 @@ __i915_write(64, 64)
 #define IS_IRONLAKE_D(dev)	((dev)->pci_device == 0x0042)
 #define IS_IRONLAKE_M(dev)	((dev)->pci_device == 0x0046)
 #define	IS_IVYBRIDGE(dev)	(INTEL_INFO(dev)->is_ivybridge)
+#define IS_VALLEYVIEW(dev)	(INTEL_INFO(dev)->is_valleyview)
+#define IS_HASWELL(dev)		(INTEL_INFO(dev)->is_haswell)
 #define IS_MOBILE(dev)		(INTEL_INFO(dev)->is_mobile)
 
 /* XXXKIB LEGACY */
@@ -1503,10 +1535,11 @@ __i915_write(64, 64)
 #define HAS_PIPE_CXSR(dev) (INTEL_INFO(dev)->has_pipe_cxsr)
 #define I915_HAS_FBC(dev) (INTEL_INFO(dev)->has_fbc)
 
-#define HAS_PCH_SPLIT(dev) (IS_GEN5(dev) || IS_GEN6(dev) || IS_IVYBRIDGE(dev))
+#define HAS_PCH_SPLIT(dev) (INTEL_INFO(dev)->has_pch_split)
 #define HAS_PIPE_CONTROL(dev) (INTEL_INFO(dev)->gen >= 5)
 
 #define INTEL_PCH_TYPE(dev) (((struct drm_i915_private *)(dev)->dev_private)->pch_type)
+#define HAS_PCH_LPT(dev) (INTEL_PCH_TYPE(dev) == PCH_LPT)
 #define HAS_PCH_CPT(dev) (INTEL_PCH_TYPE(dev) == PCH_CPT)
 #define HAS_PCH_IBX(dev) (INTEL_PCH_TYPE(dev) == PCH_IBX)
 
@@ -1519,6 +1552,23 @@ i915_seqno_passed(uint32_t seq1, uint32_t seq2)
 	return ((int32_t)(seq1 - seq2) >= 0);
 }
 
+static inline void i915_gem_chipset_flush(struct drm_device *dev)
+{
+	if (INTEL_INFO(dev)->gen < 6)
+		intel_gtt_chipset_flush();
+}
+
+static inline void i915_gem_object_pin_pages(struct drm_i915_gem_object *obj)
+{
+	/* KASSERT(obj->pages != NULL, ("pin and NULL pages")); */
+	obj->pages_pin_count++;
+}
+static inline void i915_gem_object_unpin_pages(struct drm_i915_gem_object *obj)
+{
+	KASSERT(obj->pages_pin_count != 0, ("zero pages_pin_count"));
+	obj->pages_pin_count--;
+}
+
 u32 i915_gem_next_request_seqno(struct intel_ring_buffer *ring);
 
 #endif
diff --git a/sys/dev/drm2/i915/i915_gem.c b/sys/dev/drm2/i915/i915_gem.c
index 45e770d30f3..cf346fc9a14 100644
--- a/sys/dev/drm2/i915/i915_gem.c
+++ b/sys/dev/drm2/i915/i915_gem.c
@@ -67,6 +67,8 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm.h>
 #include <vm/vm_pageout.h>
 
+#include <machine/md_var.h>
+
 static void i915_gem_object_flush_cpu_write_domain(
     struct drm_i915_gem_object *obj);
 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size,
@@ -78,32 +80,57 @@ static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
     int flags);
 static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj);
-static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
-    bool write);
-static void i915_gem_object_set_to_full_cpu_read_domain(
-    struct drm_i915_gem_object *obj);
-static int i915_gem_object_set_cpu_read_domain_range(
-    struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size);
+static void i915_gem_object_put_pages_range(struct drm_i915_gem_object *obj,
+    off_t start, off_t end);
+static int i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj,
+    off_t start, off_t end);
 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj);
 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj);
 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj);
 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj);
 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj);
-static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex);
+static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex,
+    bool *fresh);
 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
     uint32_t flush_domains);
-static void i915_gem_clear_fence_reg(struct drm_device *dev,
-    struct drm_i915_fence_reg *reg);
 static void i915_gem_reset_fences(struct drm_device *dev);
 static void i915_gem_retire_task_handler(void *arg, int pending);
-static int i915_gem_phys_pwrite(struct drm_device *dev,
-    struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset,
-    uint64_t size, struct drm_file *file_priv);
 static void i915_gem_lowmem(void *arg);
+static void i915_gem_write_fence(struct drm_device *dev, int reg,
+    struct drm_i915_gem_object *obj);
+static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
+    bool interruptible);
+static int i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno);
 
 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem");
 long i915_gem_wired_pages_cnt;
 
+static bool cpu_cache_is_coherent(struct drm_device *dev,
+				  enum i915_cache_level level)
+{
+	return HAS_LLC(dev) || level != I915_CACHE_NONE;
+}
+
+static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
+{
+	if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
+		return true;
+
+	return obj->pin_display;
+}
+
+static inline void i915_gem_object_fence_lost(struct drm_i915_gem_object *obj)
+{
+	if (obj->tiling_mode)
+		i915_gem_release_mmap(obj);
+
+	/* As we do not have an associated fence register, we will force
+	 * a tiling change if we ever need to acquire one.
+	 */
+	obj->fence_dirty = false;
+	obj->fence_reg = I915_FENCE_REG_NONE;
+}
+
 static void
 i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size)
 {
@@ -172,50 +199,42 @@ i915_mutex_lock_interruptible(struct drm_device *dev)
 }
 
 
-static void
-i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
+void
+i915_gem_free_object(struct drm_gem_object *gem_obj)
 {
+	struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
 	struct drm_device *dev;
 	drm_i915_private_t *dev_priv;
-	int ret;
 
 	dev = obj->base.dev;
 	dev_priv = dev->dev_private;
 
-	ret = i915_gem_object_unbind(obj);
-	if (ret == -ERESTART) {
-		list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list);
-		return;
+	CTR1(KTR_DRM, "object_destroy_tail %p", obj);
+
+	if (obj->phys_obj)
+		i915_gem_detach_phys_object(dev, obj);
+
+	obj->pin_count = 0;
+	if (i915_gem_object_unbind(obj) == -ERESTARTSYS) {
+		bool was_interruptible;
+
+		was_interruptible = dev_priv->mm.interruptible;
+		dev_priv->mm.interruptible = false;
+
+		if (i915_gem_object_unbind(obj))
+			printf("i915_gem_free_object: unbind\n");
+
+		dev_priv->mm.interruptible = was_interruptible;
 	}
 
-	CTR1(KTR_DRM, "object_destroy_tail %p", obj);
 	drm_gem_free_mmap_offset(&obj->base);
 	drm_gem_object_release(&obj->base);
 	i915_gem_info_remove_obj(dev_priv, obj->base.size);
 
-	free(obj->page_cpu_valid, DRM_I915_GEM);
 	free(obj->bit_17, DRM_I915_GEM);
 	free(obj, DRM_I915_GEM);
 }
 
-void
-i915_gem_free_object(struct drm_gem_object *gem_obj)
-{
-	struct drm_i915_gem_object *obj;
-	struct drm_device *dev;
-
-	obj = to_intel_bo(gem_obj);
-	dev = obj->base.dev;
-
-	while (obj->pin_count > 0)
-		i915_gem_object_unpin(obj);
-
-	if (obj->phys_obj != NULL)
-		i915_gem_detach_phys_object(dev, obj);
-
-	i915_gem_free_object_tail(obj);
-}
-
 static void
 init_ring_lists(struct intel_ring_buffer *ring)
 {
@@ -236,9 +255,7 @@ i915_gem_load(struct drm_device *dev)
 	INIT_LIST_HEAD(&dev_priv->mm.active_list);
 	INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
 	INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
-	INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
 	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
-	INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
 	INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
 	for (i = 0; i < I915_NUM_RINGS; i++)
 		init_ring_lists(&dev_priv->rings[i]);
@@ -250,16 +267,8 @@ i915_gem_load(struct drm_device *dev)
 
 	/* On GEN3 we really need to make sure the ARB C3 LP bit is set */
 	if (IS_GEN3(dev)) {
-		u32 tmp = I915_READ(MI_ARB_STATE);
-		if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
-			/*
-			 * arb state is a masked write, so set bit +
-			 * bit in mask.
-			 */
-			tmp = MI_ARB_C3_LP_WRITE_ENABLE |
-			    (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
-			I915_WRITE(MI_ARB_STATE, tmp);
-		}
+		I915_WRITE(MI_ARB_STATE,
+			   _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
 	}
 
 	dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
@@ -275,9 +284,8 @@ i915_gem_load(struct drm_device *dev)
 		dev_priv->num_fence_regs = 8;
 
 	/* Initialize fence registers to zero */
-	for (i = 0; i < dev_priv->num_fence_regs; i++) {
-		i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
-	}
+	i915_gem_reset_fences(dev);
+
 	i915_gem_detect_bit_6_swizzle(dev);
 	dev_priv->mm.interruptible = true;
 
@@ -285,35 +293,6 @@ i915_gem_load(struct drm_device *dev)
 	    i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY);
 }
 
-int
-i915_gem_do_init(struct drm_device *dev, unsigned long start,
-    unsigned long mappable_end, unsigned long end)
-{
-	drm_i915_private_t *dev_priv;
-	unsigned long mappable;
-	int error;
-
-	dev_priv = dev->dev_private;
-	mappable = min(end, mappable_end) - start;
-
-	drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
-
-	dev_priv->mm.gtt_start = start;
-	dev_priv->mm.gtt_mappable_end = mappable_end;
-	dev_priv->mm.gtt_end = end;
-	dev_priv->mm.gtt_total = end - start;
-	dev_priv->mm.mappable_gtt_total = mappable;
-
-	/* Take over this portion of the GTT */
-	intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
-	device_printf(dev->device,
-	    "taking over the fictitious range 0x%lx-0x%lx\n",
-	    dev->agp->base + start, dev->agp->base + start + mappable);
-	error = -vm_phys_fictitious_reg_range(dev->agp->base + start,
-	    dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
-	return (error);
-}
-
 int
 i915_gem_init_ioctl(struct drm_device *dev, void *data,
     struct drm_file *file)
@@ -321,6 +300,9 @@ i915_gem_init_ioctl(struct drm_device *dev, void *data,
 	struct drm_i915_gem_init *args;
 	drm_i915_private_t *dev_priv;
 
+	if (drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
+
 	dev_priv = dev->dev_private;
 	args = data;
 
@@ -330,11 +312,16 @@ i915_gem_init_ioctl(struct drm_device *dev, void *data,
 
 	if (mtx_initialized(&dev_priv->mm.gtt_space.unused_lock))
 		return (-EBUSY);
+
+	/* GEM with user mode setting was never supported on ilk and later. */
+	if (INTEL_INFO(dev)->gen >= 5)
+		return -ENODEV;
+
 	/*
 	 * XXXKIB. The second-time initialization should be guarded
 	 * against.
 	 */
-	return (i915_gem_do_init(dev, args->gtt_start, args->gtt_end,
+	return (i915_gem_init_global_gtt(dev, args->gtt_start, args->gtt_end,
 	    args->gtt_end));
 }
 
@@ -348,13 +335,14 @@ i915_gem_idle(struct drm_device *dev)
 	if (dev_priv->mm.suspended)
 		return (0);
 
-	ret = i915_gpu_idle(dev, true);
+	ret = i915_gpu_idle(dev);
 	if (ret != 0)
 		return (ret);
+	i915_gem_retire_requests(dev);
 
 	/* Under UMS, be paranoid and evict. */
 	if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
-		ret = i915_gem_evict_inactive(dev, false);
+		ret = i915_gem_evict_everything(dev, false);
 		if (ret != 0)
 			return ret;
 	}
@@ -393,15 +381,15 @@ i915_gem_init_swizzling(struct drm_device *dev)
 	if (IS_GEN5(dev))
 		return;
 
+
 	I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
 	if (IS_GEN6(dev))
-		I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_SNB));
+		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
 	else
-		I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_IVB));
+		I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
 }
 
-void
-i915_gem_init_ppgtt(struct drm_device *dev)
+void i915_gem_init_ppgtt(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv;
 	struct i915_hw_ppgtt *ppgtt;
@@ -429,21 +417,27 @@ i915_gem_init_ppgtt(struct drm_device *dev)
 	pd_offset <<= 16;
 
 	if (INTEL_INFO(dev)->gen == 6) {
-		uint32_t ecochk = I915_READ(GAM_ECOCHK);
+		uint32_t ecochk, gab_ctl, ecobits;
+
+		ecobits = I915_READ(GAC_ECO_BITS); 
+		I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_PPGTT_CACHE64B);
+
+		gab_ctl = I915_READ(GAB_CTL);
+		I915_WRITE(GAB_CTL, gab_ctl | GAB_CTL_CONT_AFTER_PAGEFAULT);
+
+		ecochk = I915_READ(GAM_ECOCHK);
 		I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT |
 				       ECOCHK_PPGTT_CACHE64B);
-		I915_WRITE(GFX_MODE, GFX_MODE_ENABLE(GFX_PPGTT_ENABLE));
+		I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 	} else if (INTEL_INFO(dev)->gen >= 7) {
 		I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B);
 		/* GFX_MODE is per-ring on gen7+ */
 	}
 
-	for (i = 0; i < I915_NUM_RINGS; i++) {
-		ring = &dev_priv->rings[i];
-
+	for_each_ring(ring, dev_priv, i) {
 		if (INTEL_INFO(dev)->gen >= 7)
 			I915_WRITE(RING_MODE_GEN7(ring),
-				   GFX_MODE_ENABLE(GFX_PPGTT_ENABLE));
+				   _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
 
 		I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
 		I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset);
@@ -488,6 +482,69 @@ cleanup_render_ring:
 	return (ret);
 }
 
+static bool
+intel_enable_ppgtt(struct drm_device *dev)
+{
+	if (i915_enable_ppgtt >= 0)
+		return i915_enable_ppgtt;
+
+	/* Disable ppgtt on SNB if VT-d is on. */
+	if (INTEL_INFO(dev)->gen == 6 && intel_iommu_enabled)
+		return false;
+
+	return true;
+}
+
+int i915_gem_init(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	unsigned long gtt_size, mappable_size;
+	int ret;
+
+	gtt_size = dev_priv->mm.gtt.gtt_total_entries << PAGE_SHIFT;
+	mappable_size = dev_priv->mm.gtt.gtt_mappable_entries << PAGE_SHIFT;
+
+	DRM_LOCK(dev);
+	if (intel_enable_ppgtt(dev) && HAS_ALIASING_PPGTT(dev)) {
+		/* PPGTT pdes are stolen from global gtt ptes, so shrink the
+		 * aperture accordingly when using aliasing ppgtt. */
+		gtt_size -= I915_PPGTT_PD_ENTRIES*PAGE_SIZE;
+
+		i915_gem_init_global_gtt(dev, 0, mappable_size, gtt_size);
+
+		ret = i915_gem_init_aliasing_ppgtt(dev);
+		if (ret) {
+			DRM_UNLOCK(dev);
+			return ret;
+		}
+	} else {
+		/* Let GEM Manage all of the aperture.
+		 *
+		 * However, leave one page at the end still bound to the scratch
+		 * page.  There are a number of places where the hardware
+		 * apparently prefetches past the end of the object, and we've
+		 * seen multiple hangs with the GPU head pointer stuck in a
+		 * batchbuffer bound at the last page of the aperture.  One page
+		 * should be enough to keep any prefetching inside of the
+		 * aperture.
+		 */
+		i915_gem_init_global_gtt(dev, 0, mappable_size,
+					 gtt_size);
+	}
+
+	ret = i915_gem_init_hw(dev);
+	DRM_UNLOCK(dev);
+	if (ret != 0) {
+		i915_gem_cleanup_aliasing_ppgtt(dev);
+		return (ret);
+	}
+
+	/* Allow hardware batchbuffers unless told otherwise, but not for KMS. */
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		dev_priv->dri1.allow_batchbuffer = 1;
+	return 0;
+}
+
 int
 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
     struct drm_file *file)
@@ -500,13 +557,11 @@ i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 	dev_priv = dev->dev_private;
 	args = data;
 
-	if (!(dev->driver->driver_features & DRIVER_GEM))
-		return (-ENODEV);
-
 	pinned = 0;
 	DRM_LOCK(dev);
-	list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
-		pinned += obj->gtt_space->size;
+	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list)
+		if (obj->pin_count)
+			pinned += obj->gtt_space->size;
 	DRM_UNLOCK(dev);
 
 	args->aper_size = dev_priv->mm.gtt_total;
@@ -519,15 +574,10 @@ int
 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment,
      bool map_and_fenceable)
 {
-	struct drm_device *dev;
-	struct drm_i915_private *dev_priv;
 	int ret;
 
-	dev = obj->base.dev;
-	dev_priv = dev->dev_private;
-
-	KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT,
-	    ("Max pin count"));
+	if (obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT)
+		return (-EBUSY);
 
 	if (obj->gtt_space != NULL) {
 		if ((alignment && obj->gtt_offset & (alignment - 1)) ||
@@ -551,47 +601,24 @@ i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment,
 			return (ret);
 	}
 
-	if (obj->pin_count++ == 0 && !obj->active)
-		list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
+	if (!obj->has_global_gtt_mapping && map_and_fenceable)
+		i915_gem_gtt_bind_object(obj, obj->cache_level);
+
+	obj->pin_count++;
 	obj->pin_mappable |= map_and_fenceable;
 
-#if 1
-	KIB_NOTYET();
-#else
-	WARN_ON(i915_verify_lists(dev));
-#endif
-	return (0);
+	return 0;
 }
 
 void
 i915_gem_object_unpin(struct drm_i915_gem_object *obj)
 {
-	struct drm_device *dev;
-	drm_i915_private_t *dev_priv;
-
-	dev = obj->base.dev;
-	dev_priv = dev->dev_private;
-
-#if 1
-	KIB_NOTYET();
-#else
-	WARN_ON(i915_verify_lists(dev));
-#endif
 	
 	KASSERT(obj->pin_count != 0, ("zero pin count"));
 	KASSERT(obj->gtt_space != NULL, ("No gtt mapping"));
 
-	if (--obj->pin_count == 0) {
-		if (!obj->active)
-			list_move_tail(&obj->mm_list,
-			    &dev_priv->mm.inactive_list);
+	if (--obj->pin_count == 0)
 		obj->pin_mappable = false;
-	}
-#if 1
-	KIB_NOTYET();
-#else
-	WARN_ON(i915_verify_lists(dev));
-#endif
 }
 
 int
@@ -693,7 +720,6 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
 {
 	struct drm_i915_gem_busy *args;
 	struct drm_i915_gem_object *obj;
-	struct drm_i915_gem_request *request;
 	int ret;
 
 	args = data;
@@ -713,13 +739,9 @@ i915_gem_busy_ioctl(struct drm_device *dev, void *data,
 		if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
 			ret = i915_gem_flush_ring(obj->ring,
 			    0, obj->base.write_domain);
-		} else if (obj->ring->outstanding_lazy_request ==
-		    obj->last_rendering_seqno) {
-			request = malloc(sizeof(*request), DRM_I915_GEM,
-			    M_WAITOK | M_ZERO);
-			ret = i915_add_request(obj->ring, NULL, request);
-			if (ret != 0)
-				free(request, DRM_I915_GEM);
+		} else {
+			ret = i915_gem_check_olr(obj->ring,
+						 obj->last_rendering_seqno);
 		}
 
 		i915_gem_retire_requests_ring(obj->ring);
@@ -763,26 +785,7 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
 	if (seqno == 0)
 		return (0);
 
-	ret = 0;
-	mtx_lock(&ring->irq_lock);
-	if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
-		if (ring->irq_get(ring)) {
-			while (ret == 0 &&
-			    !(i915_seqno_passed(ring->get_seqno(ring), seqno) ||
-			    atomic_load_acq_int(&dev_priv->mm.wedged)))
-				ret = -msleep(ring, &ring->irq_lock, PCATCH,
-				    "915thr", 0);
-			ring->irq_put(ring);
-			if (ret == 0 && atomic_load_acq_int(&dev_priv->mm.wedged))
-				ret = -EIO;
-		} else if (_intel_wait_for(dev,
-		    i915_seqno_passed(ring->get_seqno(ring), seqno) ||
-		    atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 0, "915rtr")) {
-			ret = -EBUSY;
-		}
-	}
-	mtx_unlock(&ring->irq_lock);
-
+	ret = __wait_seqno(ring, seqno, true);
 	if (ret == 0)
 		taskqueue_enqueue_timeout(dev_priv->tq,
 		    &dev_priv->mm.retire_task, 0);
@@ -847,11 +850,12 @@ void
 i915_gem_cleanup_ringbuffer(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv;
+	struct intel_ring_buffer *ring;
 	int i;
 
 	dev_priv = dev->dev_private;
-	for (i = 0; i < I915_NUM_RINGS; i++)
-		intel_cleanup_ring_buffer(&dev_priv->rings[i]);
+	for_each_ring(ring, dev_priv, i)
+		intel_cleanup_ring_buffer(ring);
 }
 
 int
@@ -859,7 +863,7 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
     struct drm_file *file_priv)
 {
 	drm_i915_private_t *dev_priv;
-	int ret, i;
+	int ret;
 
 	if (drm_core_check_feature(dev, DRIVER_MODESET))
 		return (0);
@@ -879,13 +883,6 @@ i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
 	KASSERT(list_empty(&dev_priv->mm.active_list), ("active list"));
 	KASSERT(list_empty(&dev_priv->mm.flushing_list), ("flushing list"));
 	KASSERT(list_empty(&dev_priv->mm.inactive_list), ("inactive list"));
-	for (i = 0; i < I915_NUM_RINGS; i++) {
-		KASSERT(list_empty(&dev_priv->rings[i].active_list),
-		    ("ring %d active list", i));
-		KASSERT(list_empty(&dev_priv->rings[i].request_list),
-		    ("ring %d request list", i));
-	}
-
 	DRM_UNLOCK(dev);
 	ret = drm_irq_install(dev);
 	DRM_LOCK(dev);
@@ -973,215 +970,810 @@ i915_gem_create_ioctl(struct drm_device *dev, void *data,
 	return (i915_gem_create(file, dev, args->size, &args->handle));
 }
 
-static int
-i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj,
-    uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw,
-    struct drm_file *file)
+#define __user
+#define __force
+#define __iomem
+#define	to_user_ptr(x) ((void *)(uintptr_t)(x))
+#define	offset_in_page(x) ((x) & PAGE_MASK)
+#define	page_to_phys(x) VM_PAGE_TO_PHYS(x)
+static inline long
+__copy_to_user(void __user *to, const void *from, unsigned long n)
+{
+	return (copyout(from, to, n) != 0 ? n : 0);
+}
+static inline int
+__copy_to_user_inatomic(void __user *to, const void *from, unsigned n)
+{
+	return (copyout_nofault(from, to, n) != 0 ? n : 0);
+}
+static inline unsigned long
+__copy_from_user(void *to, const void __user *from, unsigned long n)
+{
+	return ((copyin(__DECONST(void *, from), to, n) != 0 ? n : 0));
+}
+#define	copy_from_user(to, from, n) __copy_from_user((to), (from), (n))
+static inline unsigned long
+__copy_from_user_inatomic_nocache(void *to, const void __user *from,
+    unsigned long n)
 {
-	vm_object_t vm_obj;
-	vm_page_t m;
-	struct sf_buf *sf;
-	vm_offset_t mkva;
-	vm_pindex_t obj_pi;
-	int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po;
 
-	if (obj->gtt_offset != 0 && rw == UIO_READ)
-		do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
-	else
-		do_bit17_swizzling = 0;
+	/*
+	 * XXXKIB.  Equivalent Linux function is implemented using
+	 * MOVNTI for aligned moves.  For unaligned head and tail,
+	 * normal move is performed.  As such, it is not incorrect, if
+	 * only somewhat slower, to use normal copyin.  All uses
+	 * except shmem_pwrite_fast() have the destination mapped WC.
+	 */
+	return ((copyin_nofault(__DECONST(void *, from), to, n) != 0 ? n : 0));
+}
+static inline int
+fault_in_multipages_readable(const char __user *uaddr, int size)
+{
+	char c;
+	int ret = 0;
+	const char __user *end = uaddr + size - 1;
 
-	obj->dirty = 1;
-	vm_obj = obj->base.vm_obj;
-	ret = 0;
-
-	VM_OBJECT_WLOCK(vm_obj);
-	vm_object_pip_add(vm_obj, 1);
-	while (size > 0) {
-		obj_pi = OFF_TO_IDX(offset);
-		obj_po = offset & PAGE_MASK;
-
-		m = i915_gem_wire_page(vm_obj, obj_pi);
-		VM_OBJECT_WUNLOCK(vm_obj);
-
-		sched_pin();
-		sf = sf_buf_alloc(m, SFB_CPUPRIVATE);
-		mkva = sf_buf_kva(sf);
-		length = min(size, PAGE_SIZE - obj_po);
-		while (length > 0) {
-			if (do_bit17_swizzling &&
-			    (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) {
-				cnt = roundup2(obj_po + 1, 64);
-				cnt = min(cnt - obj_po, length);
-				swizzled_po = obj_po ^ 64;
-			} else {
-				cnt = length;
-				swizzled_po = obj_po;
-			}
-			if (rw == UIO_READ)
-				ret = -copyout_nofault(
-				    (char *)mkva + swizzled_po,
-				    (void *)(uintptr_t)data_ptr, cnt);
-			else
-				ret = -copyin_nofault(
-				    (void *)(uintptr_t)data_ptr,
-				    (char *)mkva + swizzled_po, cnt);
-			if (ret != 0)
-				break;
-			data_ptr += cnt;
-			size -= cnt;
-			length -= cnt;
-			offset += cnt;
-			obj_po += cnt;
-		}
-		sf_buf_free(sf);
-		sched_unpin();
-		VM_OBJECT_WLOCK(vm_obj);
-		if (rw == UIO_WRITE)
-			vm_page_dirty(m);
-		vm_page_reference(m);
-		vm_page_lock(m);
-		vm_page_unwire(m, PQ_ACTIVE);
-		vm_page_unlock(m);
-		atomic_add_long(&i915_gem_wired_pages_cnt, -1);
+	if (unlikely(size == 0))
+		return ret;
 
+	while (uaddr <= end) {
+		ret = copyin(uaddr, &c, 1);
 		if (ret != 0)
+			return -EFAULT;
+		uaddr += PAGE_SIZE;
+	}
+
+	/* Check whether the range spilled into the next page. */
+	if (((unsigned long)uaddr & ~PAGE_MASK) ==
+			((unsigned long)end & ~PAGE_MASK)) {
+		ret = copyin(end, &c, 1);
+	}
+
+	return -ret;
+}
+
+static inline int
+fault_in_multipages_writeable(char __user *uaddr, int size)
+{
+	int ret = 0;
+	char __user *end = uaddr + size - 1;
+
+	if (unlikely(size == 0))
+		return ret;
+
+	/*
+	 * Writing zeroes into userspace here is OK, because we know that if
+	 * the zero gets there, we'll be overwriting it.
+	 */
+	while (uaddr <= end) {
+		ret = subyte(uaddr, 0);
+		if (ret != 0)
+			return -EFAULT;
+		uaddr += PAGE_SIZE;
+	}
+
+	/* Check whether the range spilled into the next page. */
+	if (((unsigned long)uaddr & ~PAGE_MASK) ==
+			((unsigned long)end & ~PAGE_MASK))
+		ret = subyte(end, 0);
+
+	return ret;
+}
+
+static inline int
+__copy_to_user_swizzled(char __user *cpu_vaddr,
+			const char *gpu_vaddr, int gpu_offset,
+			int length)
+{
+	int ret, cpu_offset = 0;
+
+	while (length > 0) {
+		int cacheline_end = roundup2(gpu_offset + 1, 64);
+		int this_length = min(cacheline_end - gpu_offset, length);
+		int swizzled_gpu_offset = gpu_offset ^ 64;
+
+		ret = __copy_to_user(cpu_vaddr + cpu_offset,
+				     gpu_vaddr + swizzled_gpu_offset,
+				     this_length);
+		if (ret)
+			return ret + length;
+
+		cpu_offset += this_length;
+		gpu_offset += this_length;
+		length -= this_length;
+	}
+
+	return 0;
+}
+
+static inline int
+__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
+			  const char __user *cpu_vaddr,
+			  int length)
+{
+	int ret, cpu_offset = 0;
+
+	while (length > 0) {
+		int cacheline_end = roundup2(gpu_offset + 1, 64);
+		int this_length = min(cacheline_end - gpu_offset, length);
+		int swizzled_gpu_offset = gpu_offset ^ 64;
+
+		ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
+				       cpu_vaddr + cpu_offset,
+				       this_length);
+		if (ret)
+			return ret + length;
+
+		cpu_offset += this_length;
+		gpu_offset += this_length;
+		length -= this_length;
+	}
+
+	return 0;
+}
+
+static int
+i915_gem_phys_pwrite(struct drm_device *dev,
+		     struct drm_i915_gem_object *obj,
+		     struct drm_i915_gem_pwrite *args,
+		     struct drm_file *file_priv)
+{
+	void *vaddr = (char *)obj->phys_obj->handle->vaddr + args->offset;
+	char __user *user_data = to_user_ptr(args->data_ptr);
+
+	if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
+		unsigned long unwritten;
+
+		/* The physical object once assigned is fixed for the lifetime
+		 * of the obj, so we can safely drop the lock and continue
+		 * to access vaddr.
+		 */
+		DRM_UNLOCK(dev);
+		unwritten = copy_from_user(vaddr, user_data, args->size);
+		DRM_LOCK(dev);
+		if (unwritten)
+			return -EFAULT;
+	}
+
+	i915_gem_chipset_flush(dev);
+	return 0;
+}
+
+/* Per-page copy function for the shmem pread fastpath.
+ * Flushes invalid cachelines before reading the target if
+ * needs_clflush is set. */
+static int
+shmem_pread_fast(vm_page_t page, int shmem_page_offset, int page_length,
+		 char __user *user_data,
+		 bool page_do_bit17_swizzling, bool needs_clflush)
+{
+	char *vaddr;
+	struct sf_buf *sf;
+	int ret;
+
+	if (unlikely(page_do_bit17_swizzling))
+		return -EINVAL;
+
+	sched_pin();
+	sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE);
+	if (sf == NULL) {
+		sched_unpin();
+		return (-EFAULT);
+	}
+	vaddr = (char *)sf_buf_kva(sf);
+	if (needs_clflush)
+		drm_clflush_virt_range(vaddr + shmem_page_offset,
+				       page_length);
+	ret = __copy_to_user_inatomic(user_data,
+				      vaddr + shmem_page_offset,
+				      page_length);
+	sf_buf_free(sf);
+	sched_unpin();
+
+	return ret ? -EFAULT : 0;
+}
+
+static void
+shmem_clflush_swizzled_range(char *addr, unsigned long length,
+			     bool swizzled)
+{
+	if (unlikely(swizzled)) {
+		unsigned long start = (unsigned long) addr;
+		unsigned long end = (unsigned long) addr + length;
+
+		/* For swizzling simply ensure that we always flush both
+		 * channels. Lame, but simple and it works. Swizzled
+		 * pwrite/pread is far from a hotpath - current userspace
+		 * doesn't use it at all. */
+		start = rounddown2(start, 128);
+		end = roundup2(end, 128);
+
+		drm_clflush_virt_range((void *)start, end - start);
+	} else {
+		drm_clflush_virt_range(addr, length);
+	}
+
+}
+
+/* Only difference to the fast-path function is that this can handle bit17
+ * and uses non-atomic copy and kmap functions. */
+static int
+shmem_pread_slow(vm_page_t page, int shmem_page_offset, int page_length,
+		 char __user *user_data,
+		 bool page_do_bit17_swizzling, bool needs_clflush)
+{
+	char *vaddr;
+	struct sf_buf *sf;
+	int ret;
+
+	sf = sf_buf_alloc(page, 0);
+	vaddr = (char *)sf_buf_kva(sf);
+	if (needs_clflush)
+		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
+					     page_length,
+					     page_do_bit17_swizzling);
+
+	if (page_do_bit17_swizzling)
+		ret = __copy_to_user_swizzled(user_data,
+					      vaddr, shmem_page_offset,
+					      page_length);
+	else
+		ret = __copy_to_user(user_data,
+				     vaddr + shmem_page_offset,
+				     page_length);
+	sf_buf_free(sf);
+
+	return ret ? - EFAULT : 0;
+}
+
+static int
+i915_gem_shmem_pread(struct drm_device *dev,
+		     struct drm_i915_gem_object *obj,
+		     struct drm_i915_gem_pread *args,
+		     struct drm_file *file)
+{
+	char __user *user_data;
+	ssize_t remain, sremain;
+	off_t offset, soffset;
+	int shmem_page_offset, page_length, ret = 0;
+	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
+	int prefaulted = 0;
+	int needs_clflush = 0;
+
+	user_data = to_user_ptr(args->data_ptr);
+	sremain = remain = args->size;
+
+	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
+
+	if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
+		/* If we're not in the cpu read domain, set ourself into the gtt
+		 * read domain and manually flush cachelines (if required). This
+		 * optimizes for the case when the gpu will dirty the data
+		 * anyway again before the next pread happens. */
+		needs_clflush = !cpu_cache_is_coherent(dev, obj->cache_level);
+		ret = i915_gem_object_set_to_gtt_domain(obj, false);
+		if (ret)
+			return ret;
+	}
+
+	soffset = offset = args->offset;
+	ret = i915_gem_object_get_pages_range(obj, soffset, soffset + sremain);
+	if (ret)
+		return ret;
+
+	i915_gem_object_pin_pages(obj);
+
+	VM_OBJECT_WLOCK(obj->base.vm_obj);
+	for (vm_page_t page = vm_page_find_least(obj->base.vm_obj,
+	    OFF_TO_IDX(offset));; page = vm_page_next(page)) {
+		VM_OBJECT_WUNLOCK(obj->base.vm_obj);
+
+		if (remain <= 0)
 			break;
+
+		/* Operation in this page
+		 *
+		 * shmem_page_offset = offset within page in shmem file
+		 * page_length = bytes to copy for this page
+		 */
+		shmem_page_offset = offset_in_page(offset);
+		page_length = remain;
+		if ((shmem_page_offset + page_length) > PAGE_SIZE)
+			page_length = PAGE_SIZE - shmem_page_offset;
+
+		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
+			(page_to_phys(page) & (1 << 17)) != 0;
+
+		ret = shmem_pread_fast(page, shmem_page_offset, page_length,
+				       user_data, page_do_bit17_swizzling,
+				       needs_clflush);
+		if (ret == 0)
+			goto next_page;
+
+		DRM_UNLOCK(dev);
+
+		if (likely(!i915_prefault_disable) && !prefaulted) {
+			ret = fault_in_multipages_writeable(user_data, remain);
+			/* Userspace is tricking us, but we've already clobbered
+			 * its pages with the prefault and promised to write the
+			 * data up to the first fault. Hence ignore any errors
+			 * and just continue. */
+			(void)ret;
+			prefaulted = 1;
+		}
+
+		ret = shmem_pread_slow(page, shmem_page_offset, page_length,
+				       user_data, page_do_bit17_swizzling,
+				       needs_clflush);
+
+		DRM_LOCK(dev);
+
+next_page:
+		vm_page_reference(page);
+
+		if (ret)
+			goto out;
+
+		remain -= page_length;
+		user_data += page_length;
+		offset += page_length;
+		VM_OBJECT_WLOCK(obj->base.vm_obj);
 	}
-	vm_object_pip_wakeup(vm_obj);
-	VM_OBJECT_WUNLOCK(vm_obj);
 
-	return (ret);
+out:
+	i915_gem_object_unpin_pages(obj);
+	i915_gem_object_put_pages_range(obj, soffset, soffset + sremain);
+
+	return ret;
 }
 
-static int
-i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj,
-    uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file)
-{
-	vm_offset_t mkva;
-	vm_pindex_t obj_pi;
-	int obj_po, ret;
-
-	obj_pi = OFF_TO_IDX(offset);
-	obj_po = offset & PAGE_MASK;
-
-	mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset +
-	    IDX_TO_OFF(obj_pi), size, PAT_WRITE_COMBINING);
-	ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva +
-	    obj_po, size);
-	pmap_unmapdev(mkva, size);
-	return (ret);
-}
-
-static int
-i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr,
-    uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file)
+/**
+ * Reads data from the object referenced by handle.
+ *
+ * On error, the contents of *data are undefined.
+ */
+int
+i915_gem_pread_ioctl(struct drm_device *dev, void *data,
+		     struct drm_file *file)
 {
+	struct drm_i915_gem_pread *args = data;
 	struct drm_i915_gem_object *obj;
-	vm_page_t *ma;
-	vm_offset_t start, end;
-	int npages, ret;
+	int ret = 0;
 
-	if (size == 0)
-		return (0);
-	start = trunc_page(data_ptr);
-	end = round_page(data_ptr + size);
-	npages = howmany(end - start, PAGE_SIZE);
-	ma = malloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK |
-	    M_ZERO);
-	npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
-	    (vm_offset_t)data_ptr, size,
-	    (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages);
-	if (npages == -1) {
-		ret = -EFAULT;
-		goto free_ma;
-	}
+	if (args->size == 0)
+		return 0;
+
+	if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_WRITE))
+		return -EFAULT;
 
 	ret = i915_mutex_lock_interruptible(dev);
-	if (ret != 0)
-		goto unlocked;
+	if (ret)
+		return ret;
 
-	obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
+	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
 	if (&obj->base == NULL) {
 		ret = -ENOENT;
 		goto unlock;
 	}
-	if (offset > obj->base.size || size > obj->base.size - offset) {
+
+	/* Bounds check source.  */
+	if (args->offset > obj->base.size ||
+	    args->size > obj->base.size - args->offset) {
 		ret = -EINVAL;
 		goto out;
 	}
 
-	if (rw == UIO_READ) {
-		CTR3(KTR_DRM, "object_pread %p %jx %jx", obj, offset, size);
-		ret = i915_gem_object_set_cpu_read_domain_range(obj,
-		    offset, size);
-		if (ret != 0)
-			goto out;
-		ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
-		    UIO_READ, file);
-	} else {
-		if (obj->phys_obj) {
-			CTR3(KTR_DRM, "object_phys_write %p %jx %jx", obj,
-			    offset, size);
-			ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset,
-			    size, file);
-		} else if (obj->gtt_space &&
-		    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
-			CTR3(KTR_DRM, "object_gtt_write %p %jx %jx", obj,
-			    offset, size);
-			ret = i915_gem_object_pin(obj, 0, true);
-			if (ret != 0)
-				goto out;
-			ret = i915_gem_object_set_to_gtt_domain(obj, true);
-			if (ret != 0)
-				goto out_unpin;
-			ret = i915_gem_object_put_fence(obj);
-			if (ret != 0)
-				goto out_unpin;
-			ret = i915_gem_gtt_write(dev, obj, data_ptr, size,
-			    offset, file);
-out_unpin:
-			i915_gem_object_unpin(obj);
-		} else {
-			CTR3(KTR_DRM, "object_pwrite %p %jx %jx", obj,
-			    offset, size);
-			ret = i915_gem_object_set_to_cpu_domain(obj, true);
-			if (ret != 0)
-				goto out;
-			ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset,
-			    UIO_WRITE, file);
-		}
+#if 1
+	KIB_NOTYET();
+#else
+	/* prime objects have no backing filp to GEM pread/pwrite
+	 * pages from.
+	 */
+	if (!obj->base.filp) {
+		ret = -EINVAL;
+		goto out;
 	}
+#endif
+
+	CTR3(KTR_DRM, "pread %p %jx %jx", obj, args->offset, args->size);
+
+	ret = i915_gem_shmem_pread(dev, obj, args, file);
+
 out:
 	drm_gem_object_unreference(&obj->base);
 unlock:
 	DRM_UNLOCK(dev);
-unlocked:
-	vm_page_unhold_pages(ma, npages);
-free_ma:
-	free(ma, DRM_I915_GEM);
-	return (ret);
+	return ret;
 }
 
-int
-i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
+/* This is the fast write path which cannot handle
+ * page faults in the source data
+ */
+
+static inline int
+fast_user_write(struct drm_device *dev,
+		off_t page_base, int page_offset,
+		char __user *user_data,
+		int length)
 {
-	struct drm_i915_gem_pread *args;
+	void __iomem *vaddr_atomic;
+	void *vaddr;
+	unsigned long unwritten;
 
-	args = data;
-	return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
-	    args->offset, UIO_READ, file));
+	vaddr_atomic = pmap_mapdev_attr(dev->agp->base + page_base,
+	    length, PAT_WRITE_COMBINING);
+	/* We can use the cpu mem copy function because this is X86. */
+	vaddr = (char *)vaddr_atomic + page_offset;
+	unwritten = __copy_from_user_inatomic_nocache(vaddr,
+						      user_data, length);
+	pmap_unmapdev((vm_offset_t)vaddr_atomic, length);
+	return unwritten;
 }
 
-int
-i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
+/**
+ * This is the fast pwrite path, where we copy the data directly from the
+ * user into the GTT, uncached.
+ */
+static int
+i915_gem_gtt_pwrite_fast(struct drm_device *dev,
+			 struct drm_i915_gem_object *obj,
+			 struct drm_i915_gem_pwrite *args,
+			 struct drm_file *file)
 {
-	struct drm_i915_gem_pwrite *args;
+	ssize_t remain;
+	off_t offset, page_base;
+	char __user *user_data;
+	int page_offset, page_length, ret;
 
-	args = data;
-	return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size,
-	    args->offset, UIO_WRITE, file));
+	ret = i915_gem_object_pin(obj, 0, true);
+	/* XXXKIB ret = i915_gem_obj_ggtt_pin(obj, 0, true, true); */
+	if (ret != 0)
+		goto out;
+
+	ret = i915_gem_object_set_to_gtt_domain(obj, true);
+	if (ret)
+		goto out_unpin;
+
+	ret = i915_gem_object_put_fence(obj);
+	if (ret)
+		goto out_unpin;
+
+	user_data = to_user_ptr(args->data_ptr);
+	remain = args->size;
+
+	offset = obj->gtt_offset + args->offset;
+
+	while (remain > 0) {
+		/* Operation in this page
+		 *
+		 * page_base = page offset within aperture
+		 * page_offset = offset within page
+		 * page_length = bytes to copy for this page
+		 */
+		page_base = offset & ~PAGE_MASK;
+		page_offset = offset_in_page(offset);
+		page_length = remain;
+		if ((page_offset + remain) > PAGE_SIZE)
+			page_length = PAGE_SIZE - page_offset;
+
+		/* If we get a fault while copying data, then (presumably) our
+		 * source page isn't available.  Return the error and we'll
+		 * retry in the slow path.
+		 */
+		if (fast_user_write(dev, page_base,
+				    page_offset, user_data, page_length)) {
+			ret = -EFAULT;
+			goto out_unpin;
+		}
+
+		remain -= page_length;
+		user_data += page_length;
+		offset += page_length;
+	}
+
+out_unpin:
+	i915_gem_object_unpin(obj);
+out:
+	return ret;
 }
 
+/* Per-page copy function for the shmem pwrite fastpath.
+ * Flushes invalid cachelines before writing to the target if
+ * needs_clflush_before is set and flushes out any written cachelines after
+ * writing if needs_clflush is set. */
+static int
+shmem_pwrite_fast(vm_page_t page, int shmem_page_offset, int page_length,
+		  char __user *user_data,
+		  bool page_do_bit17_swizzling,
+		  bool needs_clflush_before,
+		  bool needs_clflush_after)
+{
+	char *vaddr;
+	struct sf_buf *sf;
+	int ret;
+
+	if (unlikely(page_do_bit17_swizzling))
+		return -EINVAL;
+
+	sched_pin();
+	sf = sf_buf_alloc(page, SFB_NOWAIT | SFB_CPUPRIVATE);
+	if (sf == NULL) {
+		sched_unpin();
+		return (-EFAULT);
+	}
+	vaddr = (char *)sf_buf_kva(sf);
+	if (needs_clflush_before)
+		drm_clflush_virt_range(vaddr + shmem_page_offset,
+				       page_length);
+	ret = __copy_from_user_inatomic_nocache(vaddr + shmem_page_offset,
+						user_data,
+						page_length);
+	if (needs_clflush_after)
+		drm_clflush_virt_range(vaddr + shmem_page_offset,
+				       page_length);
+	sf_buf_free(sf);
+	sched_unpin();
+
+	return ret ? -EFAULT : 0;
+}
+
+/* Only difference to the fast-path function is that this can handle bit17
+ * and uses non-atomic copy and kmap functions. */
+static int
+shmem_pwrite_slow(vm_page_t page, int shmem_page_offset, int page_length,
+		  char __user *user_data,
+		  bool page_do_bit17_swizzling,
+		  bool needs_clflush_before,
+		  bool needs_clflush_after)
+{
+	char *vaddr;
+	struct sf_buf *sf;
+	int ret;
+
+	sf = sf_buf_alloc(page, 0);
+	vaddr = (char *)sf_buf_kva(sf);
+	if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
+		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
+					     page_length,
+					     page_do_bit17_swizzling);
+	if (page_do_bit17_swizzling)
+		ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
+						user_data,
+						page_length);
+	else
+		ret = __copy_from_user(vaddr + shmem_page_offset,
+				       user_data,
+				       page_length);
+	if (needs_clflush_after)
+		shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
+					     page_length,
+					     page_do_bit17_swizzling);
+	sf_buf_free(sf);
+
+	return ret ? -EFAULT : 0;
+}
+
+static int
+i915_gem_shmem_pwrite(struct drm_device *dev,
+		      struct drm_i915_gem_object *obj,
+		      struct drm_i915_gem_pwrite *args,
+		      struct drm_file *file)
+{
+	ssize_t remain, sremain;
+	off_t offset, soffset;
+	char __user *user_data;
+	int shmem_page_offset, page_length, ret = 0;
+	int obj_do_bit17_swizzling, page_do_bit17_swizzling;
+	int hit_slowpath = 0;
+	int needs_clflush_after = 0;
+	int needs_clflush_before = 0;
+
+	user_data = to_user_ptr(args->data_ptr);
+	sremain = remain = args->size;
+
+	obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
+
+	if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
+		/* If we're not in the cpu write domain, set ourself into the gtt
+		 * write domain and manually flush cachelines (if required). This
+		 * optimizes for the case when the gpu will use the data
+		 * right away and we therefore have to clflush anyway. */
+		needs_clflush_after = cpu_write_needs_clflush(obj);
+		ret = i915_gem_object_set_to_gtt_domain(obj, true);
+		if (ret)
+			return ret;
+	}
+	/* Same trick applies to invalidate partially written cachelines read
+	 * before writing. */
+	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
+		needs_clflush_before =
+			!cpu_cache_is_coherent(dev, obj->cache_level);
+
+	soffset = offset = args->offset;
+	ret = i915_gem_object_get_pages_range(obj, soffset, soffset + sremain);
+	if (ret)
+		return ret;
+
+	i915_gem_object_pin_pages(obj);
+
+	obj->dirty = 1;
+
+	VM_OBJECT_WLOCK(obj->base.vm_obj);
+	for (vm_page_t page = vm_page_find_least(obj->base.vm_obj,
+	    OFF_TO_IDX(offset));; page = vm_page_next(page)) {
+		VM_OBJECT_WUNLOCK(obj->base.vm_obj);
+		int partial_cacheline_write;
+
+		if (remain <= 0)
+			break;
+
+		/* Operation in this page
+		 *
+		 * shmem_page_offset = offset within page in shmem file
+		 * page_length = bytes to copy for this page
+		 */
+		shmem_page_offset = offset_in_page(offset);
+
+		page_length = remain;
+		if ((shmem_page_offset + page_length) > PAGE_SIZE)
+			page_length = PAGE_SIZE - shmem_page_offset;
+
+		/* If we don't overwrite a cacheline completely we need to be
+		 * careful to have up-to-date data by first clflushing. Don't
+		 * overcomplicate things and flush the entire patch. */
+		partial_cacheline_write = needs_clflush_before &&
+			((shmem_page_offset | page_length)
+				& (cpu_clflush_line_size - 1));
+
+		page_do_bit17_swizzling = obj_do_bit17_swizzling &&
+			(page_to_phys(page) & (1 << 17)) != 0;
+
+		ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
+					user_data, page_do_bit17_swizzling,
+					partial_cacheline_write,
+					needs_clflush_after);
+		if (ret == 0)
+			goto next_page;
+
+		hit_slowpath = 1;
+		DRM_UNLOCK(dev);
+		ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
+					user_data, page_do_bit17_swizzling,
+					partial_cacheline_write,
+					needs_clflush_after);
+
+		DRM_LOCK(dev);
+
+next_page:
+		vm_page_dirty(page);
+		vm_page_reference(page);
+
+		if (ret)
+			goto out;
+
+		remain -= page_length;
+		user_data += page_length;
+		offset += page_length;
+		VM_OBJECT_WLOCK(obj->base.vm_obj);
+	}
+
+out:
+	i915_gem_object_unpin_pages(obj);
+	i915_gem_object_put_pages_range(obj, soffset, soffset + sremain);
+
+	if (hit_slowpath) {
+		/*
+		 * Fixup: Flush cpu caches in case we didn't flush the dirty
+		 * cachelines in-line while writing and the object moved
+		 * out of the cpu write domain while we've dropped the lock.
+		 */
+		if (!needs_clflush_after &&
+		    obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
+			i915_gem_clflush_object(obj);
+				i915_gem_chipset_flush(dev);
+		}
+	}
+
+	if (needs_clflush_after)
+		i915_gem_chipset_flush(dev);
+
+	return ret;
+}
+
+/**
+ * Writes data to the object referenced by handle.
+ *
+ * On error, the contents of the buffer that were to be modified are undefined.
+ */
+int
+i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
+		      struct drm_file *file)
+{
+	struct drm_i915_gem_pwrite *args = data;
+	struct drm_i915_gem_object *obj;
+	int ret;
+
+	if (args->size == 0)
+		return 0;
+
+	if (!useracc(to_user_ptr(args->data_ptr), args->size, VM_PROT_READ))
+		return -EFAULT;
+
+	if (likely(!i915_prefault_disable)) {
+		ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
+						   args->size);
+		if (ret)
+			return -EFAULT;
+	}
+
+	ret = i915_mutex_lock_interruptible(dev);
+	if (ret)
+		return ret;
+
+	obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
+	if (&obj->base == NULL) {
+		ret = -ENOENT;
+		goto unlock;
+	}
+
+	/* Bounds check destination. */
+	if (args->offset > obj->base.size ||
+	    args->size > obj->base.size - args->offset) {
+		ret = -EINVAL;
+		goto out;
+	}
+
+#if 1
+	KIB_NOTYET();
+#else
+	/* prime objects have no backing filp to GEM pread/pwrite
+	 * pages from.
+	 */
+	if (!obj->base.filp) {
+		ret = -EINVAL;
+		goto out;
+	}
+#endif
+
+	CTR3(KTR_DRM, "pwrite %p %jx %jx", obj, args->offset, args->size);
+
+	ret = -EFAULT;
+	/* We can only do the GTT pwrite on untiled buffers, as otherwise
+	 * it would end up going through the fenced access, and we'll get
+	 * different detiling behavior between reading and writing.
+	 * pread/pwrite currently are reading and writing from the CPU
+	 * perspective, requiring manual detiling by the client.
+	 */
+	if (obj->phys_obj) {
+		ret = i915_gem_phys_pwrite(dev, obj, args, file);
+		goto out;
+	}
+
+	if (obj->tiling_mode == I915_TILING_NONE &&
+	    obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
+	    cpu_write_needs_clflush(obj)) {
+		ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
+		/* Note that the gtt paths might fail with non-page-backed user
+		 * pointers (e.g. gtt mappings when moving data between
+		 * textures). Fallback to the shmem path in that case. */
+	}
+
+	if (ret == -EFAULT || ret == -ENOSPC)
+		ret = i915_gem_shmem_pwrite(dev, obj, args, file);
+
+out:
+	drm_gem_object_unreference(&obj->base);
+unlock:
+	DRM_UNLOCK(dev);
+	return ret;
+}
+#undef __user
+#undef __force
+#undef __iomem
+#undef to_user_ptr
+#undef offset_in_page
+#undef page_to_phys
+
 int
 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
     struct drm_file *file)
@@ -1192,9 +1784,6 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 	uint32_t write_domain;
 	int ret;
 
-	if ((dev->driver->driver_features & DRIVER_GEM) == 0)
-		return (-ENODEV);
-
 	args = data;
 	read_domains = args->read_domains;
 	write_domain = args->write_domain;
@@ -1236,9 +1825,7 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 	int ret;
 
 	args = data;
-	ret = 0;
-	if ((dev->driver->driver_features & DRIVER_GEM) == 0)
-		return (ENODEV);
+
 	ret = i915_mutex_lock_interruptible(dev);
 	if (ret != 0)
 		return (ret);
@@ -1269,9 +1856,6 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 
 	args = data;
 
-	if ((dev->driver->driver_features & DRIVER_GEM) == 0)
-		return (-ENODEV);
-
 	obj = drm_gem_object_lookup(dev, file, args->handle);
 	if (obj == NULL)
 		return (-ENOENT);
@@ -1415,10 +1999,10 @@ retry:
 		}
 	}
 
-	if (obj->tiling_mode == I915_TILING_NONE)
-		ret = i915_gem_object_put_fence(obj);
-	else
-		ret = i915_gem_object_get_fence(obj, NULL);
+	if (!obj->has_global_gtt_mapping)
+		i915_gem_gtt_bind_object(obj, obj->cache_level);
+
+	ret = i915_gem_object_get_fence(obj);
 	if (ret != 0) {
 		cause = 50;
 		goto unlock;
@@ -1517,9 +2101,6 @@ i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev,
 	struct drm_i915_gem_object *obj;
 	int ret;
 
-	if (!(dev->driver->driver_features & DRIVER_GEM))
-		return (-ENODEV);
-
 	dev_priv = dev->dev_private;
 
 	ret = i915_mutex_lock_interruptible(dev);
@@ -1679,6 +2260,7 @@ i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
 int
 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 {
+	drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
 	uint32_t old_write_domain, old_read_domains;
 	int ret;
 
@@ -1712,6 +2294,10 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 		obj->dirty = 1;
 	}
 
+	/* And bump the LRU for this access */
+	if (i915_gem_object_is_inactive(obj))
+		list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
+
 	CTR3(KTR_DRM, "object_change_domain set_to_gtt %p %x %x", obj,
 	    old_read_domains, old_write_domain);
 	return (0);
@@ -1752,7 +2338,8 @@ i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 				return (ret);
 		}
 
-		i915_gem_gtt_rebind_object(obj, cache_level);
+		if (obj->has_global_gtt_mapping)
+			i915_gem_gtt_bind_object(obj, cache_level);
 		if (obj->has_aliasing_ppgtt_mapping)
 			i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt,
 			    obj, cache_level);
@@ -1786,6 +2373,22 @@ i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 	return (0);
 }
 
+static bool is_pin_display(struct drm_i915_gem_object *obj)
+{
+	/* There are 3 sources that pin objects:
+	 *   1. The display engine (scanouts, sprites, cursors);
+	 *   2. Reservations for execbuffer;
+	 *   3. The user.
+	 *
+	 * We can ignore reservations as we hold the struct_mutex and
+	 * are only called outside of the reservation path.  The user
+	 * can only increment pin_count once, and so if after
+	 * subtracting the potential reference by the user, any pin_count
+	 * remains, it must be due to another use by the display engine.
+	 */
+	return obj->pin_count - !!obj->user_pin_count;
+}
+
 int
 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
     u32 alignment, struct intel_ring_buffer *pipelined)
@@ -1798,18 +2401,19 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 		return (ret);
 
 	if (pipelined != obj->ring) {
-		ret = i915_gem_object_wait_rendering(obj);
-		if (ret == -ERESTART || ret == -EINTR)
+		ret = i915_gem_object_sync(obj, pipelined);
+		if (ret)
 			return (ret);
 	}
 
+	obj->pin_display = true;
 	ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
 	if (ret != 0)
-		return (ret);
+		goto err_unpin_display;
 
 	ret = i915_gem_object_pin(obj, alignment, true);
 	if (ret != 0)
-		return (ret);
+		goto err_unpin_display;
 
 	i915_gem_object_flush_cpu_write_domain(obj);
 
@@ -1823,6 +2427,17 @@ i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 	CTR3(KTR_DRM, "object_change_domain pin_to_display_plan %p %x %x",
 	    obj, old_read_domains, obj->base.write_domain);
 	return (0);
+
+err_unpin_display:
+	obj->pin_display = is_pin_display(obj);
+	return ret;
+}
+
+void
+i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj)
+{
+	i915_gem_object_unpin(obj);
+	obj->pin_display = is_pin_display(obj);
 }
 
 int
@@ -1848,7 +2463,7 @@ i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
 	return (0);
 }
 
-static int
+int
 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 {
 	uint32_t old_write_domain, old_read_domains;
@@ -1861,12 +2476,13 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 	if (ret != 0)
 		return (ret);
 
-	ret = i915_gem_object_wait_rendering(obj);
-	if (ret != 0)
-		return (ret);
+	if (write || obj->pending_gpu_write) {
+		ret = i915_gem_object_wait_rendering(obj);
+		if (ret != 0)
+			return (ret);
+	}
 
 	i915_gem_object_flush_gtt_write_domain(obj);
-	i915_gem_object_set_to_full_cpu_read_domain(obj);
 
 	old_write_domain = obj->base.write_domain;
 	old_read_domains = obj->base.read_domains;
@@ -1889,74 +2505,6 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 	return (0);
 }
 
-static void
-i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
-{
-	int i;
-
-	if (obj->page_cpu_valid == NULL)
-		return;
-
-	if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) {
-		for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
-			if (obj->page_cpu_valid[i] != 0)
-				continue;
-			drm_clflush_pages(obj->pages + i, 1);
-		}
-	}
-
-	free(obj->page_cpu_valid, DRM_I915_GEM);
-	obj->page_cpu_valid = NULL;
-}
-
-static int
-i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
-    uint64_t offset, uint64_t size)
-{
-	uint32_t old_read_domains;
-	int i, ret;
-
-	if (offset == 0 && size == obj->base.size)
-		return (i915_gem_object_set_to_cpu_domain(obj, 0));
-
-	ret = i915_gem_object_flush_gpu_write_domain(obj);
-	if (ret != 0)
-		return (ret);
-	ret = i915_gem_object_wait_rendering(obj);
-	if (ret != 0)
-		return (ret);
-
-	i915_gem_object_flush_gtt_write_domain(obj);
-
-	if (obj->page_cpu_valid == NULL &&
-	    (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
-		return (0);
-
-	if (obj->page_cpu_valid == NULL) {
-		obj->page_cpu_valid = malloc(obj->base.size / PAGE_SIZE,
-		    DRM_I915_GEM, M_WAITOK | M_ZERO);
-	} else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
-		memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
-
-	for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
-	     i++) {
-		if (obj->page_cpu_valid[i])
-			continue;
-		drm_clflush_pages(obj->pages + i, 1);
-		obj->page_cpu_valid[i] = 1;
-	}
-
-	KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0,
-	    ("In gpu write domain"));
-
-	old_read_domains = obj->base.read_domains;
-	obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
-
-	CTR3(KTR_DRM, "object_change_domain set_cpu_read %p %x %x", obj,
-	    old_read_domains, obj->base.write_domain);
-	return (0);
-}
-
 static uint32_t
 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
 {
@@ -2107,7 +2655,7 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
 		return (ret);
 	}
 
-	ret = i915_gem_gtt_bind_object(obj);
+	ret = i915_gem_gtt_prepare_object(obj);
 	if (ret != 0) {
 		i915_gem_object_put_pages_gtt(obj);
 		drm_mm_put_block(obj->gtt_space);
@@ -2117,6 +2665,9 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
 		goto search_free;
 	}
 
+	if (!dev_priv->mm.aliasing_ppgtt)
+		i915_gem_gtt_bind_object(obj, obj->cache_level);
+
 	list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
 	list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 
@@ -2140,8 +2691,48 @@ i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
 	return (0);
 }
 
-static void
-i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
+int
+i915_gem_object_sync(struct drm_i915_gem_object *obj,
+		     struct intel_ring_buffer *to)
+{
+	struct intel_ring_buffer *from = obj->ring;
+	u32 seqno;
+	int ret, idx;
+
+	if (from == NULL || to == from)
+		return 0;
+
+	if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
+		return i915_gem_object_wait_rendering(obj);
+
+	idx = intel_ring_sync_index(from, to);
+
+	seqno = obj->last_rendering_seqno;
+	if (seqno <= from->sync_seqno[idx])
+		return 0;
+
+	if (seqno == from->outstanding_lazy_request) {
+		struct drm_i915_gem_request *request;
+
+		request = malloc(sizeof(*request), DRM_I915_GEM,
+		    M_WAITOK | M_ZERO);
+		ret = i915_add_request(from, NULL, request);
+		if (ret) {
+			free(request, DRM_I915_GEM);
+			return ret;
+		}
+		seqno = request->seqno;
+	}
+
+
+	ret = to->sync_to(to, from, seqno);
+	if (!ret)
+		from->sync_seqno[idx] = seqno;
+
+	return ret;
+}
+
+static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
 {
 	u32 old_write_domain, old_read_domains;
 
@@ -2196,14 +2787,17 @@ i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 	}
 
 	ret = i915_gem_object_put_fence(obj);
-	if (ret == -ERESTART)
+	if (ret)
 		return (ret);
 
-	i915_gem_gtt_unbind_object(obj);
+	if (obj->has_global_gtt_mapping)
+		i915_gem_gtt_unbind_object(obj);
 	if (obj->has_aliasing_ppgtt_mapping) {
 		i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj);
 		obj->has_aliasing_ppgtt_mapping = 0;
 	}
+	i915_gem_gtt_finish_object(obj);
+
 	i915_gem_object_put_pages_gtt(obj);
 
 	list_del_init(&obj->gtt_list);
@@ -2221,6 +2815,70 @@ i915_gem_object_unbind(struct drm_i915_gem_object *obj)
 	return (ret);
 }
 
+static void
+i915_gem_object_put_pages_range_locked(struct drm_i915_gem_object *obj,
+    vm_pindex_t si, vm_pindex_t ei)
+{
+	vm_object_t vm_obj;
+	vm_page_t m;
+	vm_pindex_t i;
+
+	vm_obj = obj->base.vm_obj;
+	VM_OBJECT_ASSERT_LOCKED(vm_obj);
+	for (i = si,  m = vm_page_lookup(vm_obj, i); i < ei;
+	    m = vm_page_next(m), i++) {
+		KASSERT(m->pindex == i, ("pindex %jx %jx",
+		    (uintmax_t)m->pindex, (uintmax_t)i));
+		vm_page_lock(m);
+		vm_page_unwire(m, PQ_INACTIVE);
+		if (m->wire_count == 0)
+			atomic_add_long(&i915_gem_wired_pages_cnt, -1);
+		vm_page_unlock(m);
+	}
+}
+
+static void
+i915_gem_object_put_pages_range(struct drm_i915_gem_object *obj,
+    off_t start, off_t end)
+{
+	vm_object_t vm_obj;
+
+	vm_obj = obj->base.vm_obj;
+	VM_OBJECT_WLOCK(vm_obj);
+	i915_gem_object_put_pages_range_locked(obj,
+	    OFF_TO_IDX(trunc_page(start)), OFF_TO_IDX(round_page(end)));
+	VM_OBJECT_WUNLOCK(vm_obj);
+}
+
+static int
+i915_gem_object_get_pages_range(struct drm_i915_gem_object *obj,
+    off_t start, off_t end)
+{
+	vm_object_t vm_obj;
+	vm_page_t m;
+	vm_pindex_t si, ei, i;
+	bool need_swizzle, fresh;
+
+	need_swizzle = i915_gem_object_needs_bit17_swizzle(obj) != 0;
+	vm_obj = obj->base.vm_obj;
+	si = OFF_TO_IDX(trunc_page(start));
+	ei = OFF_TO_IDX(round_page(end));
+	VM_OBJECT_WLOCK(vm_obj);
+	for (i = si; i < ei; i++) {
+		m = i915_gem_wire_page(vm_obj, i, &fresh);
+		if (m == NULL)
+			goto failed;
+		if (need_swizzle && fresh)
+			i915_gem_object_do_bit_17_swizzle_page(obj, m);
+	}
+	VM_OBJECT_WUNLOCK(vm_obj);
+	return (0);
+failed:
+	i915_gem_object_put_pages_range_locked(obj, si, i);
+	VM_OBJECT_WUNLOCK(vm_obj);
+	return (-EIO);
+}
+
 static int
 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
     int flags)
@@ -2228,36 +2886,30 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
 	struct drm_device *dev;
 	vm_object_t vm_obj;
 	vm_page_t m;
-	int page_count, i, j;
+	vm_pindex_t i, page_count;
+	int res;
 
 	dev = obj->base.dev;
 	KASSERT(obj->pages == NULL, ("Obj already has pages"));
-	page_count = obj->base.size / PAGE_SIZE;
+	page_count = OFF_TO_IDX(obj->base.size);
 	obj->pages = malloc(page_count * sizeof(vm_page_t), DRM_I915_GEM,
 	    M_WAITOK);
+	res = i915_gem_object_get_pages_range(obj, 0, obj->base.size);
+	if (res != 0) {
+		free(obj->pages, DRM_I915_GEM);
+		obj->pages = NULL;
+		return (res);
+	}
 	vm_obj = obj->base.vm_obj;
 	VM_OBJECT_WLOCK(vm_obj);
-	for (i = 0; i < page_count; i++) {
-		if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL)
-			goto failed;
+	for (i = 0, m = vm_page_lookup(vm_obj, 0); i < page_count;
+	    i++, m = vm_page_next(m)) {
+		KASSERT(m->pindex == i, ("pindex %jx %jx",
+		    (uintmax_t)m->pindex, (uintmax_t)i));
+		obj->pages[i] = m;
 	}
 	VM_OBJECT_WUNLOCK(vm_obj);
-	if (i915_gem_object_needs_bit17_swizzle(obj))
-		i915_gem_object_do_bit_17_swizzle(obj);
 	return (0);
-
-failed:
-	for (j = 0; j < i; j++) {
-		m = obj->pages[j];
-		vm_page_lock(m);
-		vm_page_unwire(m, PQ_INACTIVE);
-		vm_page_unlock(m);
-		atomic_add_long(&i915_gem_wired_pages_cnt, -1);
-	}
-	VM_OBJECT_WUNLOCK(vm_obj);
-	free(obj->pages, DRM_I915_GEM);
-	obj->pages = NULL;
-	return (-EIO);
 }
 
 #define	GEM_PARANOID_CHECK_GTT 0
@@ -2366,10 +3018,10 @@ i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
 	    obj->ring != NULL ? obj->ring->name : "none", obj->gtt_offset,
 	    obj->active, obj->last_rendering_seqno);
 	if (obj->active) {
-		ret = i915_wait_request(obj->ring, obj->last_rendering_seqno,
-		    true);
+		ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
 		if (ret != 0)
 			return (ret);
+		i915_gem_retire_requests_ring(obj->ring);
 	}
 	return (0);
 }
@@ -2398,7 +3050,6 @@ i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
 	obj->last_rendering_seqno = seqno;
 	if (obj->fenced_gpu_access) {
 		obj->last_fenced_seqno = seqno;
-		obj->last_fenced_ring = ring;
 
 		/* Bump MRU to take account of the delayed flush */
 		if (obj->fence_reg != I915_FENCE_REG_NONE) {
@@ -2435,15 +3086,11 @@ i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
 	struct drm_device *dev = obj->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
-	if (obj->pin_count != 0)
-		list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
-	else
-		list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
+	list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
 
 	KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list"));
 	KASSERT(obj->active, ("Object not active"));
 	obj->ring = NULL;
-	obj->last_fenced_ring = NULL;
 
 	i915_gem_object_move_off_active(obj);
 	obj->fenced_gpu_access = false;
@@ -2468,6 +3115,7 @@ i915_gem_object_truncate(struct drm_i915_gem_object *obj)
 	VM_OBJECT_WLOCK(vm_obj);
 	vm_object_page_remove(vm_obj, 0, 0, false);
 	VM_OBJECT_WUNLOCK(vm_obj);
+	drm_gem_free_mmap_offset(&obj->base);
 	obj->madv = I915_MADV_PURGED_INTERNAL;
 }
 
@@ -2511,7 +3159,7 @@ i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 }
 
 static vm_page_t
-i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex)
+i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex, bool *fresh)
 {
 	vm_page_t m;
 	int rv;
@@ -2530,11 +3178,17 @@ i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex)
 				vm_page_unlock(m);
 				return (NULL);
 			}
+			if (fresh != NULL)
+				*fresh = true;
 		} else {
 			pmap_zero_page(m);
 			m->valid = VM_PAGE_BITS_ALL;
 			m->dirty = 0;
+			if (fresh != NULL)
+				*fresh = false;
 		}
+	} else if (fresh != NULL) {
+		*fresh = false;
 	}
 	vm_page_lock(m);
 	vm_page_wire(m);
@@ -2565,7 +3219,7 @@ i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains,
 }
 
 static int
-i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire)
+i915_ring_idle(struct intel_ring_buffer *ring)
 {
 	int ret;
 
@@ -2579,12 +3233,11 @@ i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire)
 			return ret;
 	}
 
-	return (i915_wait_request(ring, i915_gem_next_request_seqno(ring),
-	    do_retire));
+	return (i915_wait_request(ring, i915_gem_next_request_seqno(ring)));
 }
 
 int
-i915_gpu_idle(struct drm_device *dev, bool do_retire)
+i915_gpu_idle(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct intel_ring_buffer *ring;
@@ -2596,29 +3249,25 @@ i915_gpu_idle(struct drm_device *dev, bool do_retire)
 		if (ret)
 			return ret;
 
-		ret = i915_ring_idle(ring, do_retire);
+		ret = i915_ring_idle(ring);
 		if (ret)
 			return ret;
+
+		/* Is the device fubar? */
+		if (!list_empty(&ring->gpu_write_list))
+			return -EBUSY;
 	}
 
 	return 0;
 }
 
-int
-i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire)
+static int
+i915_gem_check_wedge(struct drm_i915_private *dev_priv)
 {
-	drm_i915_private_t *dev_priv;
-	struct drm_i915_gem_request *request;
-	uint32_t ier;
-	int flags, ret;
-	bool recovery_complete;
-
-	KASSERT(seqno != 0, ("Zero seqno"));
-
-	dev_priv = ring->dev->dev_private;
-	ret = 0;
+	DRM_LOCK_ASSERT(dev_priv->dev);
 
 	if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) {
+		bool recovery_complete;
 		/* Give the error handler a chance to run. */
 		mtx_lock(&dev_priv->error_completion_lock);
 		recovery_complete = (&dev_priv->error_completion) > 0;
@@ -2626,11 +3275,25 @@ i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire
 		return (recovery_complete ? -EIO : -EAGAIN);
 	}
 
+	return 0;
+}
+
+/*
+ * Compare seqno against outstanding lazy request. Emit a request if they are
+ * equal.
+ */
+static int
+i915_gem_check_olr(struct intel_ring_buffer *ring, u32 seqno)
+{
+	int ret = 0;
+
+	DRM_LOCK_ASSERT(ring->dev);
+
 	if (seqno == ring->outstanding_lazy_request) {
+		struct drm_i915_gem_request *request;
+
 		request = malloc(sizeof(*request), DRM_I915_GEM,
 		    M_WAITOK | M_ZERO);
-		if (request == NULL)
-			return (-ENOMEM);
 
 		ret = i915_add_request(ring, NULL, request);
 		if (ret != 0) {
@@ -2638,59 +3301,64 @@ i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire
 			return (ret);
 		}
 
-		seqno = request->seqno;
+		MPASS(seqno == request->seqno);
+	}
+	return ret;
+}
+
+static int __wait_seqno(struct intel_ring_buffer *ring, u32 seqno,
+			bool interruptible)
+{
+	drm_i915_private_t *dev_priv = ring->dev->dev_private;
+	int ret = 0, flags;
+
+	if (i915_seqno_passed(ring->get_seqno(ring), seqno))
+		return 0;
+
+	CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno);
+
+	mtx_lock(&dev_priv->irq_lock);
+	if (!ring->irq_get(ring)) {
+		mtx_unlock(&dev_priv->irq_lock);
+		return (-ENODEV);
 	}
 
-	if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
-		if (HAS_PCH_SPLIT(ring->dev))
-			ier = I915_READ(DEIER) | I915_READ(GTIER);
-		else
-			ier = I915_READ(IER);
-		if (!ier) {
-			DRM_ERROR("something (likely vbetool) disabled "
-				  "interrupts, re-enabling\n");
-			ring->dev->driver->irq_preinstall(ring->dev);
-			ring->dev->driver->irq_postinstall(ring->dev);
-		}
+	flags = interruptible ? PCATCH : 0;
+	while (!i915_seqno_passed(ring->get_seqno(ring), seqno)
+	    && !atomic_load_acq_int(&dev_priv->mm.wedged) &&
+	    ret == 0)
+		ret = -msleep(ring, &dev_priv->irq_lock, flags, "915gwr", 0);
+	ring->irq_put(ring);
+	mtx_unlock(&dev_priv->irq_lock);
 
-		CTR2(KTR_DRM, "request_wait_begin %s %d", ring->name, seqno);
+	CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno, ret);
 
-		ring->waiting_seqno = seqno;
-		mtx_lock(&ring->irq_lock);
-		if (ring->irq_get(ring)) {
-			flags = dev_priv->mm.interruptible ? PCATCH : 0;
-			while (!i915_seqno_passed(ring->get_seqno(ring), seqno)
-			    && !atomic_load_acq_int(&dev_priv->mm.wedged) &&
-			    ret == 0) {
-				ret = -msleep(ring, &ring->irq_lock, flags,
-				    "915gwr", 0);
-			}
-			ring->irq_put(ring);
-			mtx_unlock(&ring->irq_lock);
-		} else {
-			mtx_unlock(&ring->irq_lock);
-			if (_intel_wait_for(ring->dev,
-			    i915_seqno_passed(ring->get_seqno(ring), seqno) ||
-			    atomic_load_acq_int(&dev_priv->mm.wedged), 3000,
-			    0, "i915wrq") != 0)
-				ret = -EBUSY;
-		}
-		ring->waiting_seqno = 0;
+	return ret;
+}
 
-		CTR3(KTR_DRM, "request_wait_end %s %d %d", ring->name, seqno,
-		    ret);
-	}
+int
+i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno)
+{
+	drm_i915_private_t *dev_priv;
+	int ret;
+
+	KASSERT(seqno != 0, ("Zero seqno"));
+
+	dev_priv = ring->dev->dev_private;
+	ret = 0;
+
+	ret = i915_gem_check_wedge(dev_priv);
+	if (ret)
+		return ret;
+
+	ret = i915_gem_check_olr(ring, seqno);
+	if (ret)
+		return ret;
+
+	ret = __wait_seqno(ring, seqno, dev_priv->mm.interruptible);
 	if (atomic_load_acq_int(&dev_priv->mm.wedged))
 		ret = -EAGAIN;
 
-	/* Directly dispatch request retiring.  While we have the work queue
-	 * to handle this, the waiter on a request often wants an associated
-	 * buffer to have made it to the inactive list, and we would need
-	 * a separate wait queue to handle that.
-	 */
-	if (ret == 0 && do_retire)
-		i915_gem_retire_requests_ring(ring);
-
 	return (ret);
 }
 
@@ -2851,20 +3519,18 @@ i915_gem_reset_fences(struct drm_device *dev)
 
 	for (i = 0; i < dev_priv->num_fence_regs; i++) {
 		struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
-		struct drm_i915_gem_object *obj = reg->obj;
 
-		if (!obj)
-			continue;
+		i915_gem_write_fence(dev, i, NULL);
 
-		if (obj->tiling_mode)
-			i915_gem_release_mmap(obj);
+		if (reg->obj)
+			i915_gem_object_fence_lost(reg->obj);
 
-		reg->obj->fence_reg = I915_FENCE_REG_NONE;
-		reg->obj->fenced_gpu_access = false;
-		reg->obj->last_fenced_seqno = 0;
-		reg->obj->last_fenced_ring = NULL;
-		i915_gem_clear_fence_reg(dev, reg);
+		reg->pin_count = 0;
+		reg->obj = NULL;
+		INIT_LIST_HEAD(&reg->lru_list);
 	}
+
+	INIT_LIST_HEAD(&dev_priv->mm.fence_list);
 }
 
 void
@@ -2872,10 +3538,11 @@ i915_gem_reset(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_i915_gem_object *obj;
+	struct intel_ring_buffer *ring;
 	int i;
 
-	for (i = 0; i < I915_NUM_RINGS; i++)
-		i915_gem_reset_ring_lists(dev_priv, &dev_priv->rings[i]);
+	for_each_ring(ring, dev_priv, i)
+		i915_gem_reset_ring_lists(dev_priv, ring);
 
 	/* Remove anything from the flushing lists. The GPU cache is likely
 	 * to be lost on reset along with the data, so simply move the
@@ -2961,9 +3628,10 @@ i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
 
 	if (ring->trace_irq_seqno &&
 	    i915_seqno_passed(seqno, ring->trace_irq_seqno)) {
-		mtx_lock(&ring->irq_lock);
+		struct drm_i915_private *dev_priv = ring->dev->dev_private;
+		mtx_lock(&dev_priv->irq_lock);
 		ring->irq_put(ring);
-		mtx_unlock(&ring->irq_lock);
+		mtx_unlock(&dev_priv->irq_lock);
 		ring->trace_irq_seqno = 0;
 	}
 }
@@ -2972,209 +3640,188 @@ void
 i915_gem_retire_requests(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	struct drm_i915_gem_object *obj, *next;
+	struct intel_ring_buffer *ring;
 	int i;
 
-	if (!list_empty(&dev_priv->mm.deferred_free_list)) {
-		list_for_each_entry_safe(obj, next,
-		    &dev_priv->mm.deferred_free_list, mm_list)
-			i915_gem_free_object_tail(obj);
-	}
-
-	for (i = 0; i < I915_NUM_RINGS; i++)
-		i915_gem_retire_requests_ring(&dev_priv->rings[i]);
+	for_each_ring(ring, dev_priv, i)
+		i915_gem_retire_requests_ring(ring);
 }
 
-static int
-sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
-    struct intel_ring_buffer *pipelined)
+static void sandybridge_write_fence_reg(struct drm_device *dev, int reg,
+					struct drm_i915_gem_object *obj)
 {
-	struct drm_device *dev = obj->base.dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	u32 size = obj->gtt_space->size;
-	int regnum = obj->fence_reg;
 	uint64_t val;
 
-	val = (uint64_t)((obj->gtt_offset + size - 4096) &
-			 0xfffff000) << 32;
-	val |= obj->gtt_offset & 0xfffff000;
-	val |= (uint64_t)((obj->stride / 128) - 1) <<
-		SANDYBRIDGE_FENCE_PITCH_SHIFT;
+	if (obj) {
+		u32 size = obj->gtt_space->size;
 
-	if (obj->tiling_mode == I915_TILING_Y)
-		val |= 1 << I965_FENCE_TILING_Y_SHIFT;
-	val |= I965_FENCE_REG_VALID;
+		val = (uint64_t)((obj->gtt_offset + size - 4096) &
+				 0xfffff000) << 32;
+		val |= obj->gtt_offset & 0xfffff000;
+		val |= (uint64_t)((obj->stride / 128) - 1) <<
+			SANDYBRIDGE_FENCE_PITCH_SHIFT;
 
-	if (pipelined) {
-		int ret = intel_ring_begin(pipelined, 6);
-		if (ret)
-			return ret;
-
-		intel_ring_emit(pipelined, MI_NOOP);
-		intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
-		intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
-		intel_ring_emit(pipelined, (u32)val);
-		intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
-		intel_ring_emit(pipelined, (u32)(val >> 32));
-		intel_ring_advance(pipelined);
+		if (obj->tiling_mode == I915_TILING_Y)
+			val |= 1 << I965_FENCE_TILING_Y_SHIFT;
+		val |= I965_FENCE_REG_VALID;
 	} else
-		I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
+		val = 0;
 
-	return 0;
+	I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + reg * 8, val);
+	POSTING_READ(FENCE_REG_SANDYBRIDGE_0 + reg * 8);
 }
 
-static int
-i965_write_fence_reg(struct drm_i915_gem_object *obj,
-    struct intel_ring_buffer *pipelined)
+static void i965_write_fence_reg(struct drm_device *dev, int reg,
+				 struct drm_i915_gem_object *obj)
 {
-	struct drm_device *dev = obj->base.dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	u32 size = obj->gtt_space->size;
-	int regnum = obj->fence_reg;
 	uint64_t val;
 
-	val = (uint64_t)((obj->gtt_offset + size - 4096) &
-		    0xfffff000) << 32;
-	val |= obj->gtt_offset & 0xfffff000;
-	val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
-	if (obj->tiling_mode == I915_TILING_Y)
-		val |= 1 << I965_FENCE_TILING_Y_SHIFT;
-	val |= I965_FENCE_REG_VALID;
+	if (obj) {
+		u32 size = obj->gtt_space->size;
 
-	if (pipelined) {
-		int ret = intel_ring_begin(pipelined, 6);
-		if (ret)
-			return ret;
-
-		intel_ring_emit(pipelined, MI_NOOP);
-		intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
-		intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
-		intel_ring_emit(pipelined, (u32)val);
-		intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
-		intel_ring_emit(pipelined, (u32)(val >> 32));
-		intel_ring_advance(pipelined);
+		val = (uint64_t)((obj->gtt_offset + size - 4096) &
+				 0xfffff000) << 32;
+		val |= obj->gtt_offset & 0xfffff000;
+		val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
+		if (obj->tiling_mode == I915_TILING_Y)
+			val |= 1 << I965_FENCE_TILING_Y_SHIFT;
+		val |= I965_FENCE_REG_VALID;
 	} else
-		I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
+		val = 0;
 
-	return 0;
+	I915_WRITE64(FENCE_REG_965_0 + reg * 8, val);
+	POSTING_READ(FENCE_REG_965_0 + reg * 8);
 }
 
-static int
-i915_write_fence_reg(struct drm_i915_gem_object *obj,
-    struct intel_ring_buffer *pipelined)
+static void i915_write_fence_reg(struct drm_device *dev, int reg,
+				 struct drm_i915_gem_object *obj)
 {
-	struct drm_device *dev = obj->base.dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	u32 size = obj->gtt_space->size;
-	u32 fence_reg, val, pitch_val;
-	int tile_width;
+	u32 val;
 
-	if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
-	    (size & -size) != size || (obj->gtt_offset & (size - 1))) {
-		printf(
-"object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
-		 obj->gtt_offset, obj->map_and_fenceable, size);
-		return -EINVAL;
-	}
+	if (obj) {
+		u32 size = obj->gtt_space->size;
+		int pitch_val;
+		int tile_width;
 
-	if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
-		tile_width = 128;
-	else
-		tile_width = 512;
+		if ((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
+		     (size & -size) != size ||
+		     (obj->gtt_offset & (size - 1)))
+			printf(
+		     "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
+		     obj->gtt_offset, obj->map_and_fenceable, size);
 
-	/* Note: pitch better be a power of two tile widths */
-	pitch_val = obj->stride / tile_width;
-	pitch_val = ffs(pitch_val) - 1;
+		if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
+			tile_width = 128;
+		else
+			tile_width = 512;
 
-	val = obj->gtt_offset;
-	if (obj->tiling_mode == I915_TILING_Y)
-		val |= 1 << I830_FENCE_TILING_Y_SHIFT;
-	val |= I915_FENCE_SIZE_BITS(size);
-	val |= pitch_val << I830_FENCE_PITCH_SHIFT;
-	val |= I830_FENCE_REG_VALID;
+		/* Note: pitch better be a power of two tile widths */
+		pitch_val = obj->stride / tile_width;
+		pitch_val = ffs(pitch_val) - 1;
 
-	fence_reg = obj->fence_reg;
-	if (fence_reg < 8)
-		fence_reg = FENCE_REG_830_0 + fence_reg * 4;
-	else
-		fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
-
-	if (pipelined) {
-		int ret = intel_ring_begin(pipelined, 4);
-		if (ret)
-			return ret;
-
-		intel_ring_emit(pipelined, MI_NOOP);
-		intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
-		intel_ring_emit(pipelined, fence_reg);
-		intel_ring_emit(pipelined, val);
-		intel_ring_advance(pipelined);
+		val = obj->gtt_offset;
+		if (obj->tiling_mode == I915_TILING_Y)
+			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
+		val |= I915_FENCE_SIZE_BITS(size);
+		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
+		val |= I830_FENCE_REG_VALID;
 	} else
-		I915_WRITE(fence_reg, val);
+		val = 0;
 
-	return 0;
+	if (reg < 8)
+		reg = FENCE_REG_830_0 + reg * 4;
+	else
+		reg = FENCE_REG_945_8 + (reg - 8) * 4;
+
+	I915_WRITE(reg, val);
+	POSTING_READ(reg);
 }
 
-static int
-i830_write_fence_reg(struct drm_i915_gem_object *obj,
-    struct intel_ring_buffer *pipelined)
+static void i830_write_fence_reg(struct drm_device *dev, int reg,
+				struct drm_i915_gem_object *obj)
 {
-	struct drm_device *dev = obj->base.dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	u32 size = obj->gtt_space->size;
-	int regnum = obj->fence_reg;
 	uint32_t val;
-	uint32_t pitch_val;
 
-	if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
-	    (size & -size) != size || (obj->gtt_offset & (size - 1))) {
-		printf(
-"object 0x%08x not 512K or pot-size 0x%08x aligned\n",
-		    obj->gtt_offset, size);
-		return -EINVAL;
-	}
+	if (obj) {
+		u32 size = obj->gtt_space->size;
+		uint32_t pitch_val;
 
-	pitch_val = obj->stride / 128;
-	pitch_val = ffs(pitch_val) - 1;
+		if ((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
+		     (size & -size) != size ||
+		     (obj->gtt_offset & (size - 1)))
+		    printf(
+		     "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
+		     obj->gtt_offset, size);
 
-	val = obj->gtt_offset;
-	if (obj->tiling_mode == I915_TILING_Y)
-		val |= 1 << I830_FENCE_TILING_Y_SHIFT;
-	val |= I830_FENCE_SIZE_BITS(size);
-	val |= pitch_val << I830_FENCE_PITCH_SHIFT;
-	val |= I830_FENCE_REG_VALID;
+		pitch_val = obj->stride / 128;
+		pitch_val = ffs(pitch_val) - 1;
 
-	if (pipelined) {
-		int ret = intel_ring_begin(pipelined, 4);
-		if (ret)
-			return ret;
-
-		intel_ring_emit(pipelined, MI_NOOP);
-		intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
-		intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
-		intel_ring_emit(pipelined, val);
-		intel_ring_advance(pipelined);
+		val = obj->gtt_offset;
+		if (obj->tiling_mode == I915_TILING_Y)
+			val |= 1 << I830_FENCE_TILING_Y_SHIFT;
+		val |= I830_FENCE_SIZE_BITS(size);
+		val |= pitch_val << I830_FENCE_PITCH_SHIFT;
+		val |= I830_FENCE_REG_VALID;
 	} else
-		I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
+		val = 0;
 
-	return 0;
+	I915_WRITE(FENCE_REG_830_0 + reg * 4, val);
+	POSTING_READ(FENCE_REG_830_0 + reg * 4);
 }
 
-static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
+static void i915_gem_write_fence(struct drm_device *dev, int reg,
+				 struct drm_i915_gem_object *obj)
 {
-	return i915_seqno_passed(ring->get_seqno(ring), seqno);
+	switch (INTEL_INFO(dev)->gen) {
+	case 7:
+	case 6: sandybridge_write_fence_reg(dev, reg, obj); break;
+	case 5:
+	case 4: i965_write_fence_reg(dev, reg, obj); break;
+	case 3: i915_write_fence_reg(dev, reg, obj); break;
+	case 2: i830_write_fence_reg(dev, reg, obj); break;
+	default: break;
+	}
+}
+
+static inline int fence_number(struct drm_i915_private *dev_priv,
+			       struct drm_i915_fence_reg *fence)
+{
+	return fence - dev_priv->fence_regs;
+}
+
+static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
+					 struct drm_i915_fence_reg *fence,
+					 bool enable)
+{
+	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+	int reg = fence_number(dev_priv, fence);
+
+	i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
+
+	if (enable) {
+		obj->fence_reg = reg;
+		fence->obj = obj;
+		list_move_tail(&fence->lru_list, &dev_priv->mm.fence_list);
+	} else {
+		obj->fence_reg = I915_FENCE_REG_NONE;
+		fence->obj = NULL;
+		list_del_init(&fence->lru_list);
+	}
 }
 
 static int
-i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
-    struct intel_ring_buffer *pipelined)
+i915_gem_object_flush_fence(struct drm_i915_gem_object *obj)
 {
 	int ret;
 
 	if (obj->fenced_gpu_access) {
 		if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
-			ret = i915_gem_flush_ring(obj->last_fenced_ring, 0,
-			    obj->base.write_domain);
+			ret = i915_gem_flush_ring(obj->ring,
+						  0, obj->base.write_domain);
 			if (ret)
 				return ret;
 		}
@@ -3182,18 +3829,13 @@ i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
 		obj->fenced_gpu_access = false;
 	}
 
-	if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
-		if (!ring_passed_seqno(obj->last_fenced_ring,
-				       obj->last_fenced_seqno)) {
-			ret = i915_wait_request(obj->last_fenced_ring,
-						obj->last_fenced_seqno,
-						true);
-			if (ret)
-				return ret;
-		}
+	if (obj->last_fenced_seqno) {
+		ret = i915_wait_request(obj->ring,
+					obj->last_fenced_seqno);
+		if (ret)
+			return ret;
 
 		obj->last_fenced_seqno = 0;
-		obj->last_fenced_ring = NULL;
 	}
 
 	/* Ensure that all CPU reads are completed before installing a fence
@@ -3208,35 +3850,29 @@ i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
 int
 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
 {
+	struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
 	int ret;
 
-	if (obj->tiling_mode)
-		i915_gem_release_mmap(obj);
-
-	ret = i915_gem_object_flush_fence(obj, NULL);
+	ret = i915_gem_object_flush_fence(obj);
 	if (ret)
 		return ret;
 
-	if (obj->fence_reg != I915_FENCE_REG_NONE) {
-		struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+	if (obj->fence_reg == I915_FENCE_REG_NONE)
+		return 0;
 
-		if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0)
-			printf("%s: pin_count %d\n", __func__,
-			    dev_priv->fence_regs[obj->fence_reg].pin_count);
-		i915_gem_clear_fence_reg(obj->base.dev,
-					 &dev_priv->fence_regs[obj->fence_reg]);
-
-		obj->fence_reg = I915_FENCE_REG_NONE;
-	}
+	i915_gem_object_update_fence(obj,
+				     &dev_priv->fence_regs[obj->fence_reg],
+				     false);
+	i915_gem_object_fence_lost(obj);
 
 	return 0;
 }
 
 static struct drm_i915_fence_reg *
-i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined)
+i915_find_fence_reg(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_i915_fence_reg *reg, *first, *avail;
+	struct drm_i915_fence_reg *reg, *avail;
 	int i;
 
 	/* First try to find a free reg */
@@ -3254,195 +3890,64 @@ i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined)
 		return NULL;
 
 	/* None available, try to steal one or wait for a user to finish */
-	avail = first = NULL;
 	list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
 		if (reg->pin_count)
 			continue;
 
-		if (first == NULL)
-			first = reg;
-
-		if (!pipelined ||
-		    !reg->obj->last_fenced_ring ||
-		    reg->obj->last_fenced_ring == pipelined) {
-			avail = reg;
-			break;
-		}
+		return reg;
 	}
 
-	if (avail == NULL)
-		avail = first;
-
-	return avail;
+	return NULL;
 }
 
 int
-i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
-    struct intel_ring_buffer *pipelined)
+i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
 {
 	struct drm_device *dev = obj->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	bool enable = obj->tiling_mode != I915_TILING_NONE;
 	struct drm_i915_fence_reg *reg;
 	int ret;
 
-	pipelined = NULL;
+	/* Have we updated the tiling parameters upon the object and so
+	 * will need to serialise the write to the associated fence register?
+	 */
+	if (obj->fence_dirty) {
+		ret = i915_gem_object_flush_fence(obj);
+		if (ret)
+			return ret;
+	}
+
 	ret = 0;
 
 	if (obj->fence_reg != I915_FENCE_REG_NONE) {
 		reg = &dev_priv->fence_regs[obj->fence_reg];
-		list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
+		if (!obj->fence_dirty) {
+			list_move_tail(&reg->lru_list,
+				       &dev_priv->mm.fence_list);
+			return 0;
+		}
+	} else if (enable) {
+		reg = i915_find_fence_reg(dev);
+		if (reg == NULL)
+			return -EDEADLK;
 
-		if (obj->tiling_changed) {
-			ret = i915_gem_object_flush_fence(obj, pipelined);
+		if (reg->obj) {
+			struct drm_i915_gem_object *old = reg->obj;
+
+			ret = i915_gem_object_flush_fence(old);
 			if (ret)
 				return ret;
 
-			if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
-				pipelined = NULL;
-
-			if (pipelined) {
-				reg->setup_seqno =
-					i915_gem_next_request_seqno(pipelined);
-				obj->last_fenced_seqno = reg->setup_seqno;
-				obj->last_fenced_ring = pipelined;
-			}
-
-			goto update;
+			i915_gem_object_fence_lost(old);
 		}
-
-		if (!pipelined) {
-			if (reg->setup_seqno) {
-				if (!ring_passed_seqno(obj->last_fenced_ring,
-				    reg->setup_seqno)) {
-					ret = i915_wait_request(
-					    obj->last_fenced_ring,
-					    reg->setup_seqno,
-					    true);
-					if (ret)
-						return ret;
-				}
-
-				reg->setup_seqno = 0;
-			}
-		} else if (obj->last_fenced_ring &&
-			   obj->last_fenced_ring != pipelined) {
-			ret = i915_gem_object_flush_fence(obj, pipelined);
-			if (ret)
-				return ret;
-		}
-
-		if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
-			pipelined = NULL;
-		KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined"));
-
-		if (obj->tiling_changed) {
-			if (pipelined) {
-				reg->setup_seqno =
-					i915_gem_next_request_seqno(pipelined);
-				obj->last_fenced_seqno = reg->setup_seqno;
-				obj->last_fenced_ring = pipelined;
-			}
-			goto update;
-		}
-
+	} else
 		return 0;
-	}
 
-	reg = i915_find_fence_reg(dev, pipelined);
-	if (reg == NULL)
-		return -EDEADLK;
+	i915_gem_object_update_fence(obj, reg, enable);
+	obj->fence_dirty = false;
 
-	ret = i915_gem_object_flush_fence(obj, pipelined);
-	if (ret)
-		return ret;
-
-	if (reg->obj) {
-		struct drm_i915_gem_object *old = reg->obj;
-
-		drm_gem_object_reference(&old->base);
-
-		if (old->tiling_mode)
-			i915_gem_release_mmap(old);
-
-		ret = i915_gem_object_flush_fence(old, pipelined);
-		if (ret) {
-			drm_gem_object_unreference(&old->base);
-			return ret;
-		}
-
-		if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
-			pipelined = NULL;
-
-		old->fence_reg = I915_FENCE_REG_NONE;
-		old->last_fenced_ring = pipelined;
-		old->last_fenced_seqno =
-			pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
-
-		drm_gem_object_unreference(&old->base);
-	} else if (obj->last_fenced_seqno == 0)
-		pipelined = NULL;
-
-	reg->obj = obj;
-	list_move_tail(&reg->lru_list, &dev_priv->mm.fence_list);
-	obj->fence_reg = reg - dev_priv->fence_regs;
-	obj->last_fenced_ring = pipelined;
-
-	reg->setup_seqno =
-		pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
-	obj->last_fenced_seqno = reg->setup_seqno;
-
-update:
-	obj->tiling_changed = false;
-	switch (INTEL_INFO(dev)->gen) {
-	case 7:
-	case 6:
-		ret = sandybridge_write_fence_reg(obj, pipelined);
-		break;
-	case 5:
-	case 4:
-		ret = i965_write_fence_reg(obj, pipelined);
-		break;
-	case 3:
-		ret = i915_write_fence_reg(obj, pipelined);
-		break;
-	case 2:
-		ret = i830_write_fence_reg(obj, pipelined);
-		break;
-	}
-
-	return ret;
-}
-
-static void
-i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	uint32_t fence_reg = reg - dev_priv->fence_regs;
-
-	switch (INTEL_INFO(dev)->gen) {
-	case 7:
-	case 6:
-		I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
-		break;
-	case 5:
-	case 4:
-		I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
-		break;
-	case 3:
-		if (fence_reg >= 8)
-			fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
-		else
-	case 2:
-			fence_reg = FENCE_REG_830_0 + fence_reg * 4;
-
-		I915_WRITE(fence_reg, 0);
-		break;
-	}
-
-	list_del_init(&reg->lru_list);
-	reg->obj = NULL;
-	reg->setup_seqno = 0;
-	reg->pin_count = 0;
+	return 0;
 }
 
 int
@@ -3457,7 +3962,7 @@ static bool
 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
 {
 
-	return (obj->gtt_space && !obj->active && obj->pin_count == 0);
+	return !obj->active;
 }
 
 static void
@@ -3465,6 +3970,7 @@ i915_gem_retire_task_handler(void *arg, int pending)
 {
 	drm_i915_private_t *dev_priv;
 	struct drm_device *dev;
+	struct intel_ring_buffer *ring;
 	bool idle;
 	int i;
 
@@ -3486,7 +3992,7 @@ i915_gem_retire_task_handler(void *arg, int pending)
 	 * objects indefinitely.
 	 */
 	idle = true;
-	for (i = 0; i < I915_NUM_RINGS; i++) {
+	for_each_ring(ring, dev_priv, i) {
 		struct intel_ring_buffer *ring = &dev_priv->rings[i];
 
 		if (!list_empty(&ring->gpu_write_list)) {
@@ -3602,7 +4108,7 @@ i915_gem_detach_phys_object(struct drm_device *dev,
 	page_count = obj->base.size / PAGE_SIZE;
 	VM_OBJECT_WLOCK(obj->base.vm_obj);
 	for (i = 0; i < page_count; i++) {
-		m = i915_gem_wire_page(obj->base.vm_obj, i);
+		m = i915_gem_wire_page(obj->base.vm_obj, i, NULL);
 		if (m == NULL)
 			continue; /* XXX */
 
@@ -3668,7 +4174,7 @@ i915_gem_attach_phys_object(struct drm_device *dev,
 	VM_OBJECT_WLOCK(obj->base.vm_obj);
 	ret = 0;
 	for (i = 0; i < page_count; i++) {
-		m = i915_gem_wire_page(obj->base.vm_obj, i);
+		m = i915_gem_wire_page(obj->base.vm_obj, i, NULL);
 		if (m == NULL) {
 			ret = -EIO;
 			break;
@@ -3693,33 +4199,6 @@ i915_gem_attach_phys_object(struct drm_device *dev,
 	return (0);
 }
 
-static int
-i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj,
-    uint64_t data_ptr, uint64_t offset, uint64_t size,
-    struct drm_file *file_priv)
-{
-	char *user_data, *vaddr;
-	int ret;
-
-	vaddr = (char *)obj->phys_obj->handle->vaddr + offset;
-	user_data = (char *)(uintptr_t)data_ptr;
-
-	if (copyin_nofault(user_data, vaddr, size) != 0) {
-		/* The physical object once assigned is fixed for the lifetime
-		 * of the obj, so we can safely drop the lock and continue
-		 * to access vaddr.
-		 */
-		DRM_UNLOCK(dev);
-		ret = -copyin(user_data, vaddr, size);
-		DRM_LOCK(dev);
-		if (ret != 0)
-			return (ret);
-	}
-
-	intel_gtt_chipset_flush();
-	return (0);
-}
-
 static int
 i915_gpu_is_active(struct drm_device *dev)
 {
@@ -3777,7 +4256,7 @@ rescan:
 		 * This has a dramatic impact to reduce the number of
 		 * OOM-killer events whilst running the GPU aggressively.
 		 */
-		if (i915_gpu_idle(dev, true) == 0)
+		if (i915_gpu_idle(dev) == 0)
 			goto rescan;
 	}
 	DRM_UNLOCK(dev);
diff --git a/sys/dev/drm2/i915/i915_gem_context.c b/sys/dev/drm2/i915/i915_gem_context.c
index 07d7a66d09e..dc4204320b6 100644
--- a/sys/dev/drm2/i915/i915_gem_context.c
+++ b/sys/dev/drm2/i915/i915_gem_context.c
@@ -405,7 +405,7 @@ static int do_switch(struct i915_hw_context *to)
 	}
 
 	if (!to->obj->has_global_gtt_mapping)
-		i915_gem_gtt_bind_object(to->obj);
+		i915_gem_gtt_bind_object(to->obj, to->obj->cache_level);
 
 	if (!to->is_initialized || is_default_context(to))
 		hw_flags |= MI_RESTORE_INHIBIT;
diff --git a/sys/dev/drm2/i915/i915_gem_evict.c b/sys/dev/drm2/i915/i915_gem_evict.c
index 0d8ac80d4d7..cadd9ff1a26 100644
--- a/sys/dev/drm2/i915/i915_gem_evict.c
+++ b/sys/dev/drm2/i915/i915_gem_evict.c
@@ -37,6 +37,9 @@ __FBSDID("$FreeBSD$");
 static bool
 mark_free(struct drm_i915_gem_object *obj, struct list_head *unwind)
 {
+	if (obj->pin_count)
+		return false;
+
 	list_add(&obj->exec_list, unwind);
 	return drm_mm_scan_add_block(obj->gtt_space);
 }
@@ -93,7 +96,7 @@ i915_gem_evict_something(struct drm_device *dev, int min_size,
 	/* Now merge in the soon-to-be-expired objects... */
 	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
 		/* Does the object require an outstanding flush? */
-		if (obj->base.write_domain || obj->pin_count)
+		if (obj->base.write_domain)
 			continue;
 
 		if (mark_free(obj, &unwind_list))
@@ -102,14 +105,11 @@ i915_gem_evict_something(struct drm_device *dev, int min_size,
 
 	/* Finally add anything with a pending flush (in order of retirement) */
 	list_for_each_entry(obj, &dev_priv->mm.flushing_list, mm_list) {
-		if (obj->pin_count)
-			continue;
-
 		if (mark_free(obj, &unwind_list))
 			goto found;
 	}
 	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list) {
-		if (!obj->base.write_domain || obj->pin_count)
+		if (!obj->base.write_domain)
 			continue;
 
 		if (mark_free(obj, &unwind_list))
@@ -169,8 +169,9 @@ int
 i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	int ret;
+	struct drm_i915_gem_object *obj, *next;
 	bool lists_empty;
+	int ret;
 
 	lists_empty = (list_empty(&dev_priv->mm.inactive_list) &&
 		       list_empty(&dev_priv->mm.flushing_list) &&
@@ -180,32 +181,25 @@ i915_gem_evict_everything(struct drm_device *dev, bool purgeable_only)
 
 	CTR2(KTR_DRM, "evict_everything %p %d", dev, purgeable_only);
 
-	/* Flush everything (on to the inactive lists) and evict */
-	ret = i915_gpu_idle(dev, true);
+	/* The gpu_idle will flush everything in the write domain to the
+	 * active list. Then we must move everything off the active list
+	 * with retire requests.
+	 */
+	ret = i915_gpu_idle(dev);
 	if (ret)
 		return ret;
 
+	i915_gem_retire_requests(dev);
+
 	KASSERT(list_empty(&dev_priv->mm.flushing_list),
 	    ("flush list not empty"));
 
-	return i915_gem_evict_inactive(dev, purgeable_only);
-}
-
-/** Unbinds all inactive objects. */
-int
-i915_gem_evict_inactive(struct drm_device *dev, bool purgeable_only)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	struct drm_i915_gem_object *obj, *next;
-
-	CTR2(KTR_DRM, "evict_inactive %p %d", dev, purgeable_only);
-
+	/* Having flushed everything, unbind() should never raise an error */
 	list_for_each_entry_safe(obj, next,
 				 &dev_priv->mm.inactive_list, mm_list) {
 		if (!purgeable_only || obj->madv != I915_MADV_WILLNEED) {
-			int ret = i915_gem_object_unbind(obj);
-			if (ret)
-				return ret;
+			if (obj->pin_count == 0)
+				i915_gem_object_unbind(obj);
 		}
 	}
 
diff --git a/sys/dev/drm2/i915/i915_gem_execbuffer.c b/sys/dev/drm2/i915/i915_gem_execbuffer.c
index e47141a29aa..b84dab55fdc 100644
--- a/sys/dev/drm2/i915/i915_gem_execbuffer.c
+++ b/sys/dev/drm2/i915/i915_gem_execbuffer.c
@@ -263,6 +263,12 @@ eb_destroy(struct eb_objects *eb)
 	free(eb, DRM_I915_GEM);
 }
 
+static inline int use_cpu_reloc(struct drm_i915_gem_object *obj)
+{
+	return (obj->base.write_domain == I915_GEM_DOMAIN_CPU ||
+		obj->cache_level != I915_CACHE_NONE);
+}
+
 static int
 i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 				   struct eb_objects *eb,
@@ -270,6 +276,7 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 {
 	struct drm_device *dev = obj->base.dev;
 	struct drm_gem_object *target_obj;
+	struct drm_i915_gem_object *target_i915_obj;
 	uint32_t target_offset;
 	int ret = -EINVAL;
 
@@ -278,7 +285,8 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 	if (unlikely(target_obj == NULL))
 		return -ENOENT;
 
-	target_offset = to_intel_bo(target_obj)->gtt_offset;
+	target_i915_obj = to_intel_bo(target_obj);
+	target_offset = target_i915_obj->gtt_offset;
 
 #if WATCH_RELOC
 	DRM_INFO("%s: obj %p offset %08x target %d "
@@ -364,12 +372,20 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 		return ret;
 	}
 
+	/* We can't wait for rendering with pagefaults disabled */
+	if (obj->active && (curthread->td_pflags & TDP_NOFAULTING) != 0)
+		return (-EFAULT);
+
 	reloc->delta += target_offset;
-	if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) {
+	if (use_cpu_reloc(obj)) {
 		uint32_t page_offset = reloc->offset & PAGE_MASK;
 		char *vaddr;
 		struct sf_buf *sf;
 
+		ret = i915_gem_object_set_to_cpu_domain(obj, 1);
+		if (ret)
+			return ret;
+
 		sf = sf_buf_alloc(obj->pages[OFF_TO_IDX(reloc->offset)],
 		    SFB_NOWAIT);
 		if (sf == NULL)
@@ -381,10 +397,11 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 		uint32_t *reloc_entry;
 		char *reloc_page;
 
-		/* We can't wait for rendering with pagefaults disabled */
-		if (obj->active && (curthread->td_pflags & TDP_NOFAULTING) != 0)
-			return (-EFAULT);
-		ret = i915_gem_object_set_to_gtt_domain(obj, 1);
+		ret = i915_gem_object_set_to_gtt_domain(obj, true);
+		if (ret)
+			return ret;
+
+		ret = i915_gem_object_put_fence(obj);
 		if (ret)
 			return ret;
 
@@ -401,6 +418,16 @@ i915_gem_execbuffer_relocate_entry(struct drm_i915_gem_object *obj,
 		pmap_unmapdev((vm_offset_t)reloc_page, PAGE_SIZE);
 	}
 
+	/* Sandybridge PPGTT errata: We need a global gtt mapping for MI and
+	 * pipe_control writes because the gpu doesn't properly redirect them
+	 * through the ppgtt for non_secure batchbuffers. */
+	if (unlikely(IS_GEN6(dev) &&
+	    reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION &&
+	    !target_i915_obj->has_global_gtt_mapping)) {
+		i915_gem_gtt_bind_object(target_i915_obj,
+					 target_i915_obj->cache_level);
+	}
+
 	/* and update the user's relocation entry */
 	reloc->presumed_offset = target_offset;
 
@@ -411,28 +438,44 @@ static int
 i915_gem_execbuffer_relocate_object(struct drm_i915_gem_object *obj,
     struct eb_objects *eb)
 {
+#define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry))
+	struct drm_i915_gem_relocation_entry stack_reloc[N_RELOC(512)];
 	struct drm_i915_gem_relocation_entry *user_relocs;
 	struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
-	struct drm_i915_gem_relocation_entry reloc;
-	int i, ret;
+	int remain, ret;
 
 	user_relocs = (void *)(uintptr_t)entry->relocs_ptr;
-	for (i = 0; i < entry->relocation_count; i++) {
-		ret = -copyin_nofault(user_relocs + i, &reloc, sizeof(reloc));
+	remain = entry->relocation_count;
+	while (remain) {
+		struct drm_i915_gem_relocation_entry *r = stack_reloc;
+		int count = remain;
+		if (count > DRM_ARRAY_SIZE(stack_reloc))
+			count = DRM_ARRAY_SIZE(stack_reloc);
+		remain -= count;
+
+		ret = -copyin_nofault(user_relocs, r, count*sizeof(r[0]));
 		if (ret != 0)
 			return (ret);
 
-		ret = i915_gem_execbuffer_relocate_entry(obj, eb, &reloc);
-		if (ret != 0)
-			return (ret);
+		do {
+			u64 offset = r->presumed_offset;
+ 
+			ret = i915_gem_execbuffer_relocate_entry(obj, eb, r);
+			if (ret)
+				return ret;
 
-		ret = -copyout_nofault(&reloc.presumed_offset,
-		    &user_relocs[i].presumed_offset,
-		    sizeof(reloc.presumed_offset));
-		if (ret != 0)
-			return (ret);
+			if (r->presumed_offset != offset &&
+			    copyout_nofault(&r->presumed_offset,
+					    &user_relocs->presumed_offset,
+					    sizeof(r->presumed_offset))) {
+				return -EFAULT;
+			}
+
+			user_relocs++;
+			r++;
+		} while (--count);
 	}
-
+#undef N_RELOC
 	return (0);
 }
 
@@ -486,6 +529,13 @@ i915_gem_execbuffer_relocate(struct drm_device *dev,
 
 #define  __EXEC_OBJECT_HAS_FENCE (1<<31)
 
+static int
+need_reloc_mappable(struct drm_i915_gem_object *obj)
+{
+	struct drm_i915_gem_exec_object2 *entry = obj->exec_entry;
+	return entry->relocation_count && !use_cpu_reloc(obj);
+}
+
 static int
 pin_and_fence_object(struct drm_i915_gem_object *obj,
 		     struct intel_ring_buffer *ring)
@@ -499,8 +549,7 @@ pin_and_fence_object(struct drm_i915_gem_object *obj,
 		has_fenced_gpu_access &&
 		entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 		obj->tiling_mode != I915_TILING_NONE;
-	need_mappable =
-		entry->relocation_count ? true : need_fence;
+	need_mappable = need_fence || need_reloc_mappable(obj);
 
 	ret = i915_gem_object_pin(obj, entry->alignment, need_mappable);
 	if (ret)
@@ -508,18 +557,13 @@ pin_and_fence_object(struct drm_i915_gem_object *obj,
 
 	if (has_fenced_gpu_access) {
 		if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
-			if (obj->tiling_mode) {
-				ret = i915_gem_object_get_fence(obj, ring);
-				if (ret)
-					goto err_unpin;
+			ret = i915_gem_object_get_fence(obj);
+			if (ret)
+				goto err_unpin;
 
+			if (i915_gem_object_pin_fence(obj))
 				entry->flags |= __EXEC_OBJECT_HAS_FENCE;
-				i915_gem_object_pin_fence(obj);
-			} else {
-				ret = i915_gem_object_put_fence(obj);
-				if (ret)
-					goto err_unpin;
-			}
+
 			obj->pending_fenced_gpu_access = true;
 		}
 	}
@@ -558,8 +602,7 @@ i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
 			has_fenced_gpu_access &&
 			entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 			obj->tiling_mode != I915_TILING_NONE;
-		need_mappable =
-			entry->relocation_count ? true : need_fence;
+		need_mappable = need_fence || need_reloc_mappable(obj);
 
 		if (need_mappable)
 			list_move(&obj->exec_list, &ordered_objects);
@@ -600,8 +643,7 @@ i915_gem_execbuffer_reserve(struct intel_ring_buffer *ring,
 				has_fenced_gpu_access &&
 				entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
 				obj->tiling_mode != I915_TILING_NONE;
-			need_mappable =
-				entry->relocation_count ? true : need_fence;
+			need_mappable = need_fence || need_reloc_mappable(obj);
 
 			if ((entry->alignment && obj->gtt_offset & (entry->alignment - 1)) ||
 			    (need_mappable && !obj->map_and_fenceable))
@@ -815,62 +857,6 @@ i915_gem_execbuffer_flush(struct drm_device *dev,
 	return 0;
 }
 
-static bool
-intel_enable_semaphores(struct drm_device *dev)
-{
-	if (INTEL_INFO(dev)->gen < 6)
-		return 0;
-
-	if (i915_semaphores >= 0)
-		return i915_semaphores;
-
-	/* Enable semaphores on SNB when IO remapping is off */
-	if (INTEL_INFO(dev)->gen == 6)
-		return !intel_iommu_enabled;
-
-	return 1;
-}
-
-static int
-i915_gem_execbuffer_sync_rings(struct drm_i915_gem_object *obj,
-			       struct intel_ring_buffer *to)
-{
-	struct intel_ring_buffer *from = obj->ring;
-	u32 seqno;
-	int ret, idx;
-
-	if (from == NULL || to == from)
-		return 0;
-
-	/* XXX gpu semaphores are implicated in various hard hangs on SNB */
-	if (!intel_enable_semaphores(obj->base.dev))
-		return i915_gem_object_wait_rendering(obj);
-
-	idx = intel_ring_sync_index(from, to);
-
-	seqno = obj->last_rendering_seqno;
-	if (seqno <= from->sync_seqno[idx])
-		return 0;
-
-	if (seqno == from->outstanding_lazy_request) {
-		struct drm_i915_gem_request *request;
-
-		request = malloc(sizeof(*request), DRM_I915_GEM,
-		    M_WAITOK | M_ZERO);
-		ret = i915_add_request(from, NULL, request);
-		if (ret) {
-			free(request, DRM_I915_GEM);
-			return ret;
-		}
-
-		seqno = request->seqno;
-	}
-
-	from->sync_seqno[idx] = seqno;
-
-	return to->sync_to(to, from, seqno - 1);
-}
-
 static int
 i915_gem_execbuffer_wait_for_flips(struct intel_ring_buffer *ring, u32 flips)
 {
@@ -937,7 +923,7 @@ i915_gem_execbuffer_move_to_gpu(struct intel_ring_buffer *ring,
 	}
 
 	list_for_each_entry(obj, objects, exec_list) {
-		ret = i915_gem_execbuffer_sync_rings(obj, ring);
+		ret = i915_gem_object_sync(obj, ring);
 		if (ret)
 			return ret;
 	}
@@ -1015,11 +1001,14 @@ i915_gem_execbuffer_move_to_active(struct list_head *objects,
 			obj->pending_gpu_write = true;
 			list_move_tail(&obj->gpu_write_list,
 				       &ring->gpu_write_list);
-			intel_mark_busy(ring->dev, obj);
+			if (obj->pin_count) /* check for potential scanout */
+				intel_mark_busy(ring->dev, obj);
 		}
 		CTR3(KTR_DRM, "object_change_domain move_to_active %p %x %x",
 		    obj, old_read, old_write);
 	}
+
+	intel_mark_busy(ring->dev, NULL);
 }
 
 int i915_gem_sync_exec_requests;
@@ -1051,8 +1040,10 @@ i915_gem_execbuffer_retire_commands(struct drm_device *dev,
 	if (request == NULL || i915_add_request(ring, file, request)) {
 		i915_gem_next_request_seqno(ring);
 		free(request, DRM_I915_GEM);
-	} else if (i915_gem_sync_exec_requests)
-		i915_wait_request(ring, request->seqno, true);
+	} else if (i915_gem_sync_exec_requests) {
+		i915_wait_request(ring, request->seqno);
+		i915_gem_retire_requests(dev);
+	}
 }
 
 static void
@@ -1154,10 +1145,6 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 		ring = &dev_priv->rings[RCS];
 		break;
 	case I915_EXEC_BSD:
-		if (!HAS_BSD(dev)) {
-			DRM_DEBUG("execbuf with invalid ring (BSD)\n");
-			return -EINVAL;
-		}
 		ring = &dev_priv->rings[VCS];
 		if (ctx_id != 0) {
 			DRM_DEBUG("Ring %s doesn't support contexts\n",
@@ -1166,10 +1153,6 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 		}
 		break;
 	case I915_EXEC_BLT:
-		if (!HAS_BLT(dev)) {
-			DRM_DEBUG("execbuf with invalid ring (BLT)\n");
-			return -EINVAL;
-		}
 		ring = &dev_priv->rings[BCS];
 		if (ctx_id != 0) {
 			DRM_DEBUG("Ring %s doesn't support contexts\n",
@@ -1183,6 +1166,11 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 		ret = -EINVAL;
 		goto pre_struct_lock_err;
 	}
+	if (!intel_ring_initialized(ring)) {
+		DRM_DEBUG("execbuf with invalid ring: %d\n",
+			  (int)(args->flags & I915_EXEC_RING_MASK));
+		return -EINVAL;
+	}
 
 	mode = args->flags & I915_EXEC_CONSTANTS_MASK;
 	mask = I915_EXEC_CONSTANTS_MASK;
@@ -1227,6 +1215,12 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 			goto pre_struct_lock_err;
 		}
 
+		if (INTEL_INFO(dev)->gen >= 5) {
+			DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
+			ret = -EINVAL;
+			goto pre_struct_lock_err;
+		}
+
 		if (args->num_cliprects > UINT_MAX / sizeof(*cliprects)) {
 			DRM_DEBUG("execbuf with %u cliprects\n",
 				  args->num_cliprects);
@@ -1328,9 +1322,10 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 			 * so every billion or so execbuffers, we need to stall
 			 * the GPU in order to reset the counters.
 			 */
-			ret = i915_gpu_idle(dev, true);
+			ret = i915_gpu_idle(dev);
 			if (ret)
 				goto err;
+			i915_gem_retire_requests(dev);
 
 			KASSERT(ring->sync_seqno[i] == 0, ("Non-zero sync_seqno"));
 		}
diff --git a/sys/dev/drm2/i915/i915_gem_gtt.c b/sys/dev/drm2/i915/i915_gem_gtt.c
index 6204c06349c..ce6c53b494a 100644
--- a/sys/dev/drm2/i915/i915_gem_gtt.c
+++ b/sys/dev/drm2/i915/i915_gem_gtt.c
@@ -245,7 +245,7 @@ do_idling(struct drm_i915_private *dev_priv)
 
 	if (dev_priv->mm.gtt.do_idle_maps) {
 		dev_priv->mm.interruptible = false;
-		if (i915_gpu_idle(dev_priv->dev, false)) {
+		if (i915_gpu_idle(dev_priv->dev)) {
 			DRM_ERROR("Couldn't idle GPU\n");
 			/* Wait a bit, in hopes it avoids the hang */
 			DELAY(10);
@@ -277,28 +277,21 @@ i915_gem_restore_gtt_mappings(struct drm_device *dev)
 
 	list_for_each_entry(obj, &dev_priv->mm.gtt_list, gtt_list) {
 		i915_gem_clflush_object(obj);
-		i915_gem_gtt_rebind_object(obj, obj->cache_level);
+		i915_gem_gtt_bind_object(obj, obj->cache_level);
 	}
 
 	intel_gtt_chipset_flush();
 }
 
 int
-i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj)
+i915_gem_gtt_prepare_object(struct drm_i915_gem_object *obj)
 {
-	unsigned int agp_type;
-
-	agp_type = cache_level_to_agp_type(obj->base.dev, obj->cache_level);
-	intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
-	    obj->base.size >> PAGE_SHIFT, obj->pages, agp_type);
-
-	obj->has_global_gtt_mapping = 1;
 
 	return (0);
 }
 
 void
-i915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj,
+i915_gem_gtt_bind_object(struct drm_i915_gem_object *obj,
     enum i915_cache_level cache_level)
 {
 	struct drm_device *dev;
@@ -312,11 +305,21 @@ i915_gem_gtt_rebind_object(struct drm_i915_gem_object *obj,
 	intel_gtt_insert_pages(obj->gtt_space->start >> PAGE_SHIFT,
 	    obj->base.size >> PAGE_SHIFT, obj->pages, agp_type);
 
-	obj->has_global_gtt_mapping = 0;
+	obj->has_global_gtt_mapping = 1;
 }
 
 void
 i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
+{
+
+	intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
+	    obj->base.size >> PAGE_SHIFT);
+
+	obj->has_global_gtt_mapping = 0;
+}
+
+void
+i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
 {
 	struct drm_device *dev = obj->base.dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
@@ -327,8 +330,35 @@ i915_gem_gtt_unbind_object(struct drm_i915_gem_object *obj)
 
 	interruptible = do_idling(dev_priv);
 
-	intel_gtt_clear_range(obj->gtt_space->start >> PAGE_SHIFT,
-	    obj->base.size >> PAGE_SHIFT);
-
 	undo_idling(dev_priv, interruptible);
 }
+
+int
+i915_gem_init_global_gtt(struct drm_device *dev, unsigned long start,
+    unsigned long mappable_end, unsigned long end)
+{
+	drm_i915_private_t *dev_priv;
+	unsigned long mappable;
+	int error;
+
+	dev_priv = dev->dev_private;
+	mappable = min(end, mappable_end) - start;
+
+	/* Substract the guard page ... */
+	drm_mm_init(&dev_priv->mm.gtt_space, start, end - start - PAGE_SIZE);
+
+	dev_priv->mm.gtt_start = start;
+	dev_priv->mm.gtt_mappable_end = mappable_end;
+	dev_priv->mm.gtt_end = end;
+	dev_priv->mm.gtt_total = end - start;
+	dev_priv->mm.mappable_gtt_total = mappable;
+
+	/* ... but ensure that we clear the entire range. */
+	intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
+	device_printf(dev->device,
+	    "taking over the fictitious range 0x%lx-0x%lx\n",
+	    dev->agp->base + start, dev->agp->base + start + mappable);
+	error = -vm_phys_fictitious_reg_range(dev->agp->base + start,
+	    dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING);
+	return (error);
+}
diff --git a/sys/dev/drm2/i915/i915_gem_stolen.c b/sys/dev/drm2/i915/i915_gem_stolen.c
new file mode 100644
index 00000000000..1955a112f4a
--- /dev/null
+++ b/sys/dev/drm2/i915/i915_gem_stolen.c
@@ -0,0 +1,205 @@
+/*
+ * Copyright © 2008-2012 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Eric Anholt <eric@anholt.net>
+ *    Chris Wilson <chris@chris-wilson.co.uk>
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <dev/drm2/drmP.h>
+#include <dev/drm2/drm.h>
+#include <dev/drm2/i915/i915_drm.h>
+#include <dev/drm2/i915/i915_drv.h>
+
+/*
+ * The BIOS typically reserves some of the system's memory for the exclusive
+ * use of the integrated graphics. This memory is no longer available for
+ * use by the OS and so the user finds that his system has less memory
+ * available than he put in. We refer to this memory as stolen.
+ *
+ * The BIOS will allocate its framebuffer from the stolen memory. Our
+ * goal is try to reuse that object for our own fbcon which must always
+ * be available for panics. Anything else we can reuse the stolen memory
+ * for is a boon.
+ */
+
+#define PTE_ADDRESS_MASK		0xfffff000
+#define PTE_ADDRESS_MASK_HIGH		0x000000f0 /* i915+ */
+#define PTE_MAPPING_TYPE_UNCACHED	(0 << 1)
+#define PTE_MAPPING_TYPE_DCACHE		(1 << 1) /* i830 only */
+#define PTE_MAPPING_TYPE_CACHED		(3 << 1)
+#define PTE_MAPPING_TYPE_MASK		(3 << 1)
+#define PTE_VALID			(1 << 0)
+
+/**
+ * i915_stolen_to_phys - take an offset into stolen memory and turn it into
+ *                       a physical one
+ * @dev: drm device
+ * @offset: address to translate
+ *
+ * Some chip functions require allocations from stolen space and need the
+ * physical address of the memory in question.
+ */
+static unsigned long i915_stolen_to_phys(struct drm_device *dev, u32 offset)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	device_t pdev = dev_priv->bridge_dev;
+	u32 base;
+
+#if 0
+	/* On the machines I have tested the Graphics Base of Stolen Memory
+	 * is unreliable, so compute the base by subtracting the stolen memory
+	 * from the Top of Low Usable DRAM which is where the BIOS places
+	 * the graphics stolen memory.
+	 */
+	if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) {
+		/* top 32bits are reserved = 0 */
+		pci_read_config_dword(pdev, 0xA4, &base);
+	} else {
+		/* XXX presume 8xx is the same as i915 */
+		pci_bus_read_config_dword(pdev->bus, 2, 0x5C, &base);
+	}
+#else
+	if (INTEL_INFO(dev)->gen > 3 || IS_G33(dev)) {
+		u16 val;
+		val = pci_read_config(pdev, 0xb0, 2);
+		base = val >> 4 << 20;
+	} else {
+		u8 val;
+		val = pci_read_config(pdev, 0x9c, 1);
+		base = val >> 3 << 27;
+	}
+	base -= dev_priv->mm.gtt.stolen_size;
+#endif
+
+	return base + offset;
+}
+
+static void i915_warn_stolen(struct drm_device *dev)
+{
+	DRM_INFO("not enough stolen space for compressed buffer, disabling\n");
+	DRM_INFO("hint: you may be able to increase stolen memory size in the BIOS to avoid this\n");
+}
+
+static void i915_setup_compression(struct drm_device *dev, int size)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_mm_node *compressed_fb, *compressed_llb;
+	unsigned long cfb_base;
+	unsigned long ll_base = 0;
+
+	/* Just in case the BIOS is doing something questionable. */
+	intel_disable_fbc(dev);
+
+	compressed_fb = drm_mm_search_free(&dev_priv->mm.stolen, size, 4096, 0);
+	if (compressed_fb)
+		compressed_fb = drm_mm_get_block(compressed_fb, size, 4096);
+	if (!compressed_fb)
+		goto err;
+
+	cfb_base = i915_stolen_to_phys(dev, compressed_fb->start);
+	if (!cfb_base)
+		goto err_fb;
+
+	if (!(IS_GM45(dev) || HAS_PCH_SPLIT(dev))) {
+		compressed_llb = drm_mm_search_free(&dev_priv->mm.stolen,
+						    4096, 4096, 0);
+		if (compressed_llb)
+			compressed_llb = drm_mm_get_block(compressed_llb,
+							  4096, 4096);
+		if (!compressed_llb)
+			goto err_fb;
+
+		ll_base = i915_stolen_to_phys(dev, compressed_llb->start);
+		if (!ll_base)
+			goto err_llb;
+	}
+
+	dev_priv->cfb_size = size;
+
+	dev_priv->compressed_fb = compressed_fb;
+	if (HAS_PCH_SPLIT(dev))
+		I915_WRITE(ILK_DPFC_CB_BASE, compressed_fb->start);
+	else if (IS_GM45(dev)) {
+		I915_WRITE(DPFC_CB_BASE, compressed_fb->start);
+	} else {
+		I915_WRITE(FBC_CFB_BASE, cfb_base);
+		I915_WRITE(FBC_LL_BASE, ll_base);
+		dev_priv->compressed_llb = compressed_llb;
+	}
+
+	DRM_DEBUG_KMS("FBC base 0x%08lx, ll base 0x%08lx, size %dM\n",
+		      cfb_base, ll_base, size >> 20);
+	return;
+
+err_llb:
+	drm_mm_put_block(compressed_llb);
+err_fb:
+	drm_mm_put_block(compressed_fb);
+err:
+	dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
+	i915_warn_stolen(dev);
+}
+
+static void i915_cleanup_compression(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	drm_mm_put_block(dev_priv->compressed_fb);
+	if (dev_priv->compressed_llb)
+		drm_mm_put_block(dev_priv->compressed_llb);
+}
+
+void i915_gem_cleanup_stolen(struct drm_device *dev)
+{
+	if (I915_HAS_FBC(dev) && i915_powersave)
+		i915_cleanup_compression(dev);
+}
+
+int i915_gem_init_stolen(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	unsigned long prealloc_size = dev_priv->mm.gtt.stolen_size;
+
+	/* Basic memrange allocator for stolen space */
+	drm_mm_init(&dev_priv->mm.stolen, 0, prealloc_size);
+
+	/* Try to set up FBC with a reasonable compressed buffer size */
+	if (I915_HAS_FBC(dev) && i915_powersave) {
+		int cfb_size;
+
+		/* Leave 1M for line length buffer & misc. */
+
+		/* Try to get a 32M buffer... */
+		if (prealloc_size > (36*1024*1024))
+			cfb_size = 32*1024*1024;
+		else /* fall back to 7/8 of the stolen space */
+			cfb_size = prealloc_size * 7 / 8;
+		i915_setup_compression(dev, cfb_size);
+	}
+
+	return 0;
+}
diff --git a/sys/dev/drm2/i915/i915_gem_tiling.c b/sys/dev/drm2/i915/i915_gem_tiling.c
index b3d98c8768d..6ca39627f05 100644
--- a/sys/dev/drm2/i915/i915_gem_tiling.c
+++ b/sys/dev/drm2/i915/i915_gem_tiling.c
@@ -357,11 +357,18 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 		/* We need to rebind the object if its current allocation
 		 * no longer meets the alignment restrictions for its new
 		 * tiling mode. Otherwise we can just leave it alone, but
-		 * need to ensure that any fence register is cleared.
+		 * need to ensure that any fence register is updated before
+		 * the next fenced (either through the GTT or by the BLT unit
+		 * on older GPUs) access.
+		 *
+		 * After updating the tiling parameters, we then flag whether
+		 * we need to update an associated fence register. Note this
+		 * has to also include the unfenced register the GPU uses
+		 * whilst executing a fenced command for an untiled object.
 		 */
-		i915_gem_release_mmap(obj);
 
-		obj->map_and_fenceable = obj->gtt_space == NULL ||
+		obj->map_and_fenceable =
+			obj->gtt_space == NULL ||
 		    (obj->gtt_offset + obj->base.size <=
 		    dev_priv->mm.gtt_mappable_end &&
 		    i915_gem_object_fence_ok(obj, args->tiling_mode));
@@ -374,12 +381,20 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 			if (obj->gtt_offset & (unfenced_alignment - 1))
 				ret = i915_gem_object_unbind(obj);
 		}
+
 		if (ret == 0) {
-			obj->tiling_changed = true;
+			obj->fence_dirty =
+				obj->fenced_gpu_access ||
+				obj->fence_reg != I915_FENCE_REG_NONE;
+
+
 			obj->tiling_mode = args->tiling_mode;
 			obj->stride = args->stride;
+
+			/* Force the fence to be reacquired for GTT access */
+			i915_gem_release_mmap(obj);
 		}
- 	}
+	}
 	/* we have to maintain this existing ABI... */
 	args->stride = obj->stride;
 	args->tiling_mode = obj->tiling_mode;
@@ -455,6 +470,22 @@ i915_gem_swizzle_page(vm_page_t m)
 	sf_buf_free(sf);
 }
 
+void
+i915_gem_object_do_bit_17_swizzle_page(struct drm_i915_gem_object *obj,
+    vm_page_t m)
+{
+	char new_bit_17;
+
+	if (obj->bit_17 == NULL)
+		return;
+
+	new_bit_17 = VM_PAGE_TO_PHYS(m) >> 17;
+	if ((new_bit_17 & 0x1) != (test_bit(m->pindex, obj->bit_17) != 0)) {
+		i915_gem_swizzle_page(m);
+		vm_page_dirty(m);
+	}
+}
+
 void
 i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
 {
diff --git a/sys/dev/drm2/i915/i915_irq.c b/sys/dev/drm2/i915/i915_irq.c
index 15d0a613d10..346d9c07467 100644
--- a/sys/dev/drm2/i915/i915_irq.c
+++ b/sys/dev/drm2/i915/i915_irq.c
@@ -36,37 +36,11 @@ __FBSDID("$FreeBSD$");
 #include <dev/drm2/i915/intel_drv.h>
 #include <sys/sched.h>
 #include <sys/sf_buf.h>
+#include <sys/sleepqueue.h>
 
 static void i915_capture_error_state(struct drm_device *dev);
 static u32 ring_last_seqno(struct intel_ring_buffer *ring);
 
-/**
- * Interrupts that are always left unmasked.
- *
- * Since pipe events are edge-triggered from the PIPESTAT register to IIR,
- * we leave them always unmasked in IMR and then control enabling them through
- * PIPESTAT alone.
- */
-#define I915_INTERRUPT_ENABLE_FIX			\
-	(I915_ASLE_INTERRUPT |				\
-	 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |		\
-	 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |		\
-	 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |	\
-	 I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |	\
-	 I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
-
-/** Interrupts that we mask and unmask at runtime. */
-#define I915_INTERRUPT_ENABLE_VAR (I915_USER_INTERRUPT | I915_BSD_USER_INTERRUPT)
-
-#define I915_PIPE_VBLANK_STATUS	(PIPE_START_VBLANK_INTERRUPT_STATUS |\
-				 PIPE_VBLANK_INTERRUPT_STATUS)
-
-#define I915_PIPE_VBLANK_ENABLE	(PIPE_START_VBLANK_INTERRUPT_ENABLE |\
-				 PIPE_VBLANK_INTERRUPT_ENABLE)
-
-#define DRM_I915_VBLANK_PIPE_ALL	(DRM_I915_VBLANK_PIPE_A | \
-					 DRM_I915_VBLANK_PIPE_B)
-
 /* For display hotplug interrupt */
 static void
 ironlake_enable_display_irq(drm_i915_private_t *dev_priv, u32 mask)
@@ -120,6 +94,10 @@ void intel_enable_asle(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 
+	/* FIXME: opregion/asle for VLV */
+	if (IS_VALLEYVIEW(dev))
+		return;
+
 	mtx_lock(&dev_priv->irq_lock);
 
 	if (HAS_PCH_SPLIT(dev))
@@ -368,18 +346,16 @@ static void notify_ring(struct drm_device *dev,
 			struct intel_ring_buffer *ring)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 seqno;
 
 	if (ring->obj == NULL)
 		return;
 
-	seqno = ring->get_seqno(ring);
-	CTR2(KTR_DRM, "request_complete %s %d", ring->name, seqno);
+	CTR2(KTR_DRM, "request_complete %s %d", ring->name,
+	    ring->get_seqno(ring));
 
-	mtx_lock(&ring->irq_lock);
-	ring->irq_seqno = seqno;
+	mtx_lock(&dev_priv->irq_lock);
 	wakeup(ring);
-	mtx_unlock(&ring->irq_lock);
+	mtx_unlock(&dev_priv->irq_lock);
 
 	if (i915_enable_hangcheck) {
 		dev_priv->hangcheck_count = 0;
@@ -445,14 +421,141 @@ gen6_pm_rps_work_func(void *arg, int pending)
 	DRM_UNLOCK(dev);
 }
 
-static void pch_irq_handler(struct drm_device *dev)
+static void snb_gt_irq_handler(struct drm_device *dev,
+			       struct drm_i915_private *dev_priv,
+			       u32 gt_iir)
+{
+
+	if (gt_iir & (GEN6_RENDER_USER_INTERRUPT |
+		      GEN6_RENDER_PIPE_CONTROL_NOTIFY_INTERRUPT))
+		notify_ring(dev, &dev_priv->rings[RCS]);
+	if (gt_iir & GEN6_BSD_USER_INTERRUPT)
+		notify_ring(dev, &dev_priv->rings[VCS]);
+	if (gt_iir & GEN6_BLITTER_USER_INTERRUPT)
+		notify_ring(dev, &dev_priv->rings[BCS]);
+
+	if (gt_iir & (GT_GEN6_BLT_CS_ERROR_INTERRUPT |
+		      GT_GEN6_BSD_CS_ERROR_INTERRUPT |
+		      GT_RENDER_CS_ERROR_INTERRUPT)) {
+		DRM_ERROR("GT error interrupt 0x%08x\n", gt_iir);
+		i915_handle_error(dev, false);
+	}
+}
+
+static void gen6_queue_rps_work(struct drm_i915_private *dev_priv,
+				u32 pm_iir)
+{
+
+	/*
+	 * IIR bits should never already be set because IMR should
+	 * prevent an interrupt from being shown in IIR. The warning
+	 * displays a case where we've unsafely cleared
+	 * dev_priv->pm_iir. Although missing an interrupt of the same
+	 * type is not a problem, it displays a problem in the logic.
+	 *
+	 * The mask bit in IMR is cleared by rps_work.
+	 */
+
+	mtx_lock(&dev_priv->rps_lock);
+	if (dev_priv->pm_iir & pm_iir)
+		printf("Missed a PM interrupt\n");
+	dev_priv->pm_iir |= pm_iir;
+	I915_WRITE(GEN6_PMIMR, dev_priv->pm_iir);
+	POSTING_READ(GEN6_PMIMR);
+	mtx_unlock(&dev_priv->rps_lock);
+
+	taskqueue_enqueue(dev_priv->tq, &dev_priv->rps_task);
+}
+
+static void valleyview_irq_handler(void *arg)
+{
+	struct drm_device *dev = (struct drm_device *) arg;
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	u32 iir, gt_iir, pm_iir;
+	int pipe;
+	u32 pipe_stats[I915_MAX_PIPES];
+	u32 vblank_status;
+	int vblank = 0;
+	bool blc_event;
+
+	atomic_inc(&dev_priv->irq_received);
+
+	vblank_status = PIPE_START_VBLANK_INTERRUPT_STATUS |
+		PIPE_VBLANK_INTERRUPT_STATUS;
+
+	while (true) {
+		iir = I915_READ(VLV_IIR);
+		gt_iir = I915_READ(GTIIR);
+		pm_iir = I915_READ(GEN6_PMIIR);
+
+		if (gt_iir == 0 && pm_iir == 0 && iir == 0)
+			goto out;
+
+		snb_gt_irq_handler(dev, dev_priv, gt_iir);
+
+		mtx_lock(&dev_priv->irq_lock);
+		for_each_pipe(pipe) {
+			int reg = PIPESTAT(pipe);
+			pipe_stats[pipe] = I915_READ(reg);
+
+			/*
+			 * Clear the PIPE*STAT regs before the IIR
+			 */
+			if (pipe_stats[pipe] & 0x8000ffff) {
+				if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
+					DRM_DEBUG_DRIVER("pipe %c underrun\n",
+							 pipe_name(pipe));
+				I915_WRITE(reg, pipe_stats[pipe]);
+			}
+		}
+		mtx_unlock(&dev_priv->irq_lock);
+
+		/* Consume port.  Then clear IIR or we'll miss events */
+		if (iir & I915_DISPLAY_PORT_INTERRUPT) {
+			u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
+
+			DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
+					 hotplug_status);
+			if (hotplug_status & dev_priv->hotplug_supported_mask)
+				taskqueue_enqueue(dev_priv->tq,
+				    &dev_priv->hotplug_task);
+
+			I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
+			I915_READ(PORT_HOTPLUG_STAT);
+		}
+
+
+		if (iir & I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT) {
+			drm_handle_vblank(dev, 0);
+			vblank++;
+			intel_finish_page_flip(dev, 0);
+		}
+
+		if (iir & I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT) {
+			drm_handle_vblank(dev, 1);
+			vblank++;
+			intel_finish_page_flip(dev, 0);
+		}
+
+		if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
+			blc_event = true;
+
+		if (pm_iir & GEN6_PM_DEFERRED_EVENTS)
+			gen6_queue_rps_work(dev_priv, pm_iir);
+
+		I915_WRITE(GTIIR, gt_iir);
+		I915_WRITE(GEN6_PMIIR, pm_iir);
+		I915_WRITE(VLV_IIR, iir);
+	}
+
+out:;
+}
+
+static void pch_irq_handler(struct drm_device *dev, u32 pch_iir)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	u32 pch_iir;
 	int pipe;
 
-	pch_iir = I915_READ(SDEIIR);
-
 	if (pch_iir & SDE_AUDIO_POWER_MASK)
 		DRM_DEBUG("i915: PCH audio power change on port %d\n",
 				 (pch_iir & SDE_AUDIO_POWER_MASK) >>
@@ -493,10 +596,8 @@ ivybridge_irq_handler(void *arg)
 {
 	struct drm_device *dev = (struct drm_device *) arg;
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir;
-#if 0
-	struct drm_i915_master_private *master_priv;
-#endif
+	u32 de_iir, gt_iir, de_ier, pm_iir;
+	int i;
 
 	atomic_inc(&dev_priv->irq_received);
 
@@ -505,84 +606,64 @@ ivybridge_irq_handler(void *arg)
 	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
 	POSTING_READ(DEIER);
 
+ 	gt_iir = I915_READ(GTIIR);
+	if (gt_iir) {
+		snb_gt_irq_handler(dev, dev_priv, gt_iir);
+		I915_WRITE(GTIIR, gt_iir);
+	}
+
 	de_iir = I915_READ(DEIIR);
-	gt_iir = I915_READ(GTIIR);
-	pch_iir = I915_READ(SDEIIR);
+	if (de_iir) {
+		if (de_iir & DE_GSE_IVB)
+			intel_opregion_gse_intr(dev);
+
+		for (i = 0; i < 3; i++) {
+			if (de_iir & (DE_PLANEA_FLIP_DONE_IVB << (5 * i))) {
+				intel_prepare_page_flip(dev, i);
+				intel_finish_page_flip_plane(dev, i);
+			}
+			if (de_iir & (DE_PIPEA_VBLANK_IVB << (5 * i)))
+				drm_handle_vblank(dev, i);
+		}
+
+		/* check event from PCH */
+		if (de_iir & DE_PCH_EVENT_IVB) {
+			u32 pch_iir = I915_READ(SDEIIR);
+
+			if (pch_iir & SDE_HOTPLUG_MASK_CPT)
+				taskqueue_enqueue(dev_priv->tq,
+				    &dev_priv->hotplug_task);
+			pch_irq_handler(dev, pch_iir);
+
+			/* clear PCH hotplug event before clear CPU irq */
+			I915_WRITE(SDEIIR, pch_iir);
+		}
+
+		I915_WRITE(DEIIR, de_iir);
+	}
+
 	pm_iir = I915_READ(GEN6_PMIIR);
-
-	CTR4(KTR_DRM, "ivybridge_irq de %x gt %x pch %x pm %x", de_iir,
-	    gt_iir, pch_iir, pm_iir);
-
-	if (de_iir == 0 && gt_iir == 0 && pch_iir == 0 && pm_iir == 0)
-		goto done;
-
-#if 0
-	if (dev->primary->master) {
-		master_priv = dev->primary->master->driver_priv;
-		if (master_priv->sarea_priv)
-			master_priv->sarea_priv->last_dispatch =
-				READ_BREADCRUMB(dev_priv);
-	}
-#else
-	if (dev_priv->sarea_priv)
-		dev_priv->sarea_priv->last_dispatch =
-		    READ_BREADCRUMB(dev_priv);
-#endif
-
-	if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY))
-		notify_ring(dev, &dev_priv->rings[RCS]);
-	if (gt_iir & GT_GEN6_BSD_USER_INTERRUPT)
-		notify_ring(dev, &dev_priv->rings[VCS]);
-	if (gt_iir & GT_BLT_USER_INTERRUPT)
-		notify_ring(dev, &dev_priv->rings[BCS]);
-
-	if (de_iir & DE_GSE_IVB) {
-		intel_opregion_gse_intr(dev);
+	if (pm_iir) {
+		if (pm_iir & GEN6_PM_DEFERRED_EVENTS)
+			gen6_queue_rps_work(dev_priv, pm_iir);
+		I915_WRITE(GEN6_PMIIR, pm_iir);
 	}
 
-	if (de_iir & DE_PLANEA_FLIP_DONE_IVB) {
-		intel_prepare_page_flip(dev, 0);
-		intel_finish_page_flip_plane(dev, 0);
-	}
-
-	if (de_iir & DE_PLANEB_FLIP_DONE_IVB) {
-		intel_prepare_page_flip(dev, 1);
-		intel_finish_page_flip_plane(dev, 1);
-	}
-
-	if (de_iir & DE_PIPEA_VBLANK_IVB)
-		drm_handle_vblank(dev, 0);
-
-	if (de_iir & DE_PIPEB_VBLANK_IVB)
-		drm_handle_vblank(dev, 1);
-
-	/* check event from PCH */
-	if (de_iir & DE_PCH_EVENT_IVB) {
-		if (pch_iir & SDE_HOTPLUG_MASK_CPT)
-			taskqueue_enqueue(dev_priv->tq, &dev_priv->hotplug_task);
-		pch_irq_handler(dev);
-	}
-
-	if (pm_iir & GEN6_PM_DEFERRED_EVENTS) {
-		mtx_lock(&dev_priv->rps_lock);
-		if ((dev_priv->pm_iir & pm_iir) != 0)
-			printf("Missed a PM interrupt\n");
-		dev_priv->pm_iir |= pm_iir;
-		I915_WRITE(GEN6_PMIMR, dev_priv->pm_iir);
-		POSTING_READ(GEN6_PMIMR);
-		mtx_unlock(&dev_priv->rps_lock);
-		taskqueue_enqueue(dev_priv->tq, &dev_priv->rps_task);
-	}
-
-	/* should clear PCH hotplug event before clear CPU irq */
-	I915_WRITE(SDEIIR, pch_iir);
-	I915_WRITE(GTIIR, gt_iir);
-	I915_WRITE(DEIIR, de_iir);
-	I915_WRITE(GEN6_PMIIR, pm_iir);
-
-done:
 	I915_WRITE(DEIER, de_ier);
 	POSTING_READ(DEIER);
+
+	CTR3(KTR_DRM, "ivybridge_irq de %x gt %x pm %x", de_iir,
+	    gt_iir, pm_iir);
+}
+
+static void ilk_gt_irq_handler(struct drm_device *dev,
+			       struct drm_i915_private *dev_priv,
+			       u32 gt_iir)
+{
+	if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY))
+		notify_ring(dev, &dev_priv->rings[RCS]);
+	if (gt_iir & GT_BSD_USER_INTERRUPT)
+		notify_ring(dev, &dev_priv->rings[VCS]);
 }
 
 static void
@@ -592,16 +673,9 @@ ironlake_irq_handler(void *arg)
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	u32 de_iir, gt_iir, de_ier, pch_iir, pm_iir;
 	u32 hotplug_mask;
-#if 0
-	struct drm_i915_master_private *master_priv;
-#endif
-	u32 bsd_usr_interrupt = GT_BSD_USER_INTERRUPT;
 
 	atomic_inc(&dev_priv->irq_received);
 
-	if (IS_GEN6(dev))
-		bsd_usr_interrupt = GT_GEN6_BSD_USER_INTERRUPT;
-
 	/* disable master interrupt before clearing iir  */
 	de_ier = I915_READ(DEIER);
 	I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
@@ -624,25 +698,10 @@ ironlake_irq_handler(void *arg)
 	else
 		hotplug_mask = SDE_HOTPLUG_MASK;
 
-#if 0
-	if (dev->primary->master) {
-		master_priv = dev->primary->master->driver_priv;
-		if (master_priv->sarea_priv)
-			master_priv->sarea_priv->last_dispatch =
-				READ_BREADCRUMB(dev_priv);
-	}
-#else
-		if (dev_priv->sarea_priv)
-			dev_priv->sarea_priv->last_dispatch =
-			    READ_BREADCRUMB(dev_priv);
-#endif
-
-	if (gt_iir & (GT_USER_INTERRUPT | GT_PIPE_NOTIFY))
-		notify_ring(dev, &dev_priv->rings[RCS]);
-	if (gt_iir & bsd_usr_interrupt)
-		notify_ring(dev, &dev_priv->rings[VCS]);
-	if (gt_iir & GT_BLT_USER_INTERRUPT)
-		notify_ring(dev, &dev_priv->rings[BCS]);
+	if (IS_GEN5(dev))
+		ilk_gt_irq_handler(dev, dev_priv, gt_iir);
+	else
+		snb_gt_irq_handler(dev, dev_priv, gt_iir);
 
 	if (de_iir & DE_GSE) {
 		intel_opregion_gse_intr(dev);
@@ -669,7 +728,7 @@ ironlake_irq_handler(void *arg)
 		if (pch_iir & hotplug_mask)
 			taskqueue_enqueue(dev_priv->tq,
 			    &dev_priv->hotplug_task);
-		pch_irq_handler(dev);
+		pch_irq_handler(dev, pch_iir);
 	}
 
 	if (de_iir & DE_PCU_EVENT) {
@@ -677,16 +736,8 @@ ironlake_irq_handler(void *arg)
 		i915_handle_rps_change(dev);
 	}
 
-	if (pm_iir & GEN6_PM_DEFERRED_EVENTS) {
-		mtx_lock(&dev_priv->rps_lock);
-		if ((dev_priv->pm_iir & pm_iir) != 0)
-			printf("Missed a PM interrupt\n");
-		dev_priv->pm_iir |= pm_iir;
-		I915_WRITE(GEN6_PMIMR, dev_priv->pm_iir);
-		POSTING_READ(GEN6_PMIMR);
-		mtx_unlock(&dev_priv->rps_lock);
-		taskqueue_enqueue(dev_priv->tq, &dev_priv->rps_task);
-	}
+	if (IS_GEN6(dev) && pm_iir & GEN6_PM_DEFERRED_EVENTS)
+		gen6_queue_rps_work(dev_priv, pm_iir);
 
 	/* should clear PCH hotplug event before clear CPU irq */
 	I915_WRITE(SDEIIR, pch_iir);
@@ -728,6 +779,8 @@ i915_error_work_func(void *context, int pending)
 	}
 }
 
+#define pr_err(...) printf(__VA_ARGS__)
+
 static void i915_report_and_clear_eir(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
@@ -743,26 +796,20 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
 		if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
 			u32 ipeir = I915_READ(IPEIR_I965);
 
-			printf("  IPEIR: 0x%08x\n",
-			       I915_READ(IPEIR_I965));
-			printf("  IPEHR: 0x%08x\n",
-			       I915_READ(IPEHR_I965));
-			printf("  INSTDONE: 0x%08x\n",
+			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
+			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
+			pr_err("  INSTDONE: 0x%08x\n",
 			       I915_READ(INSTDONE_I965));
-			printf("  INSTPS: 0x%08x\n",
-			       I915_READ(INSTPS));
-			printf("  INSTDONE1: 0x%08x\n",
-			       I915_READ(INSTDONE1));
-			printf("  ACTHD: 0x%08x\n",
-			       I915_READ(ACTHD_I965));
+			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
+			pr_err("  INSTDONE1: 0x%08x\n", I915_READ(INSTDONE1));
+			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
 			I915_WRITE(IPEIR_I965, ipeir);
 			POSTING_READ(IPEIR_I965);
 		}
 		if (eir & GM45_ERROR_PAGE_TABLE) {
 			u32 pgtbl_err = I915_READ(PGTBL_ER);
-			printf("page table error\n");
-			printf("  PGTBL_ER: 0x%08x\n",
-			       pgtbl_err);
+			pr_err("page table error\n");
+			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
 			I915_WRITE(PGTBL_ER, pgtbl_err);
 			POSTING_READ(PGTBL_ER);
 		}
@@ -771,53 +818,42 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
 	if (!IS_GEN2(dev)) {
 		if (eir & I915_ERROR_PAGE_TABLE) {
 			u32 pgtbl_err = I915_READ(PGTBL_ER);
-			printf("page table error\n");
-			printf("  PGTBL_ER: 0x%08x\n",
-			       pgtbl_err);
+			pr_err("page table error\n");
+			pr_err("  PGTBL_ER: 0x%08x\n", pgtbl_err);
 			I915_WRITE(PGTBL_ER, pgtbl_err);
 			POSTING_READ(PGTBL_ER);
 		}
 	}
 
 	if (eir & I915_ERROR_MEMORY_REFRESH) {
-		printf("memory refresh error:\n");
+		pr_err("memory refresh error:\n");
 		for_each_pipe(pipe)
-			printf("pipe %c stat: 0x%08x\n",
+			pr_err("pipe %c stat: 0x%08x\n",
 			       pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
 		/* pipestat has already been acked */
 	}
 	if (eir & I915_ERROR_INSTRUCTION) {
-		printf("instruction error\n");
-		printf("  INSTPM: 0x%08x\n",
-		       I915_READ(INSTPM));
+		pr_err("instruction error\n");
+		pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
 		if (INTEL_INFO(dev)->gen < 4) {
 			u32 ipeir = I915_READ(IPEIR);
 
-			printf("  IPEIR: 0x%08x\n",
-			       I915_READ(IPEIR));
-			printf("  IPEHR: 0x%08x\n",
-			       I915_READ(IPEHR));
-			printf("  INSTDONE: 0x%08x\n",
-			       I915_READ(INSTDONE));
-			printf("  ACTHD: 0x%08x\n",
-			       I915_READ(ACTHD));
+			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
+			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR));
+			pr_err("  INSTDONE: 0x%08x\n", I915_READ(INSTDONE));
+			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD));
 			I915_WRITE(IPEIR, ipeir);
 			POSTING_READ(IPEIR);
 		} else {
 			u32 ipeir = I915_READ(IPEIR_I965);
 
-			printf("  IPEIR: 0x%08x\n",
-			       I915_READ(IPEIR_I965));
-			printf("  IPEHR: 0x%08x\n",
-			       I915_READ(IPEHR_I965));
-			printf("  INSTDONE: 0x%08x\n",
+			pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR_I965));
+			pr_err("  IPEHR: 0x%08x\n", I915_READ(IPEHR_I965));
+			pr_err("  INSTDONE: 0x%08x\n",
 			       I915_READ(INSTDONE_I965));
-			printf("  INSTPS: 0x%08x\n",
-			       I915_READ(INSTPS));
-			printf("  INSTDONE1: 0x%08x\n",
-			       I915_READ(INSTDONE1));
-			printf("  ACTHD: 0x%08x\n",
-			       I915_READ(ACTHD_I965));
+			pr_err("  INSTPS: 0x%08x\n", I915_READ(INSTPS));
+			pr_err("  INSTDONE1: 0x%08x\n", I915_READ(INSTDONE1));
+			pr_err("  ACTHD: 0x%08x\n", I915_READ(ACTHD_I965));
 			I915_WRITE(IPEIR_I965, ipeir);
 			POSTING_READ(IPEIR_I965);
 		}
@@ -850,6 +886,8 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
 void i915_handle_error(struct drm_device *dev, bool wedged)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_ring_buffer *ring;
+	int i;
 
 	i915_capture_error_state(dev);
 	i915_report_and_clear_eir(dev);
@@ -864,18 +902,10 @@ void i915_handle_error(struct drm_device *dev, bool wedged)
 		/*
 		 * Wakeup waiting processes so they don't hang
 		 */
-		mtx_lock(&dev_priv->rings[RCS].irq_lock);
-		wakeup(&dev_priv->rings[RCS]);
-		mtx_unlock(&dev_priv->rings[RCS].irq_lock);
-		if (HAS_BSD(dev)) {
-			mtx_lock(&dev_priv->rings[VCS].irq_lock);
-			wakeup(&dev_priv->rings[VCS]);
-			mtx_unlock(&dev_priv->rings[VCS].irq_lock);
-		}
-		if (HAS_BLT(dev)) {
-			mtx_lock(&dev_priv->rings[BCS].irq_lock);
-			wakeup(&dev_priv->rings[BCS]);
-			mtx_unlock(&dev_priv->rings[BCS].irq_lock);
+		for_each_ring(ring, dev_priv, i) {
+			mtx_lock(&dev_priv->irq_lock);
+			wakeup(ring);
+			mtx_unlock(&dev_priv->irq_lock);
 		}
 	}
 
@@ -908,7 +938,8 @@ static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
 	obj = work->pending_flip_obj;
 	if (INTEL_INFO(dev)->gen >= 4) {
 		int dspsurf = DSPSURF(intel_crtc->plane);
-		stall_detected = I915_READ(dspsurf) == obj->gtt_offset;
+		stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) ==
+					obj->gtt_offset;
 	} else {
 		int dspaddr = DSPADDR(intel_crtc->plane);
 		stall_detected = I915_READ(dspaddr) == (obj->gtt_offset +
@@ -924,288 +955,6 @@ static void i915_pageflip_stall_check(struct drm_device *dev, int pipe)
 	}
 }
 
-static void
-i915_driver_irq_handler(void *arg)
-{
-	struct drm_device *dev = (struct drm_device *)arg;
-	drm_i915_private_t *dev_priv = (drm_i915_private_t *)dev->dev_private;
-#if 0
-	struct drm_i915_master_private *master_priv;
-#endif
-	u32 iir, new_iir;
-	u32 pipe_stats[I915_MAX_PIPES];
-	u32 vblank_status;
-	int vblank = 0;
-	int irq_received;
-	int pipe;
-	bool blc_event = false;
-
-	atomic_inc(&dev_priv->irq_received);
-
-	iir = I915_READ(IIR);
-
-	CTR1(KTR_DRM, "driver_irq_handler %x", iir);
-
-	if (INTEL_INFO(dev)->gen >= 4)
-		vblank_status = PIPE_START_VBLANK_INTERRUPT_STATUS;
-	else
-		vblank_status = PIPE_VBLANK_INTERRUPT_STATUS;
-
-	for (;;) {
-		irq_received = iir != 0;
-
-		/* Can't rely on pipestat interrupt bit in iir as it might
-		 * have been cleared after the pipestat interrupt was received.
-		 * It doesn't set the bit in iir again, but it still produces
-		 * interrupts (for non-MSI).
-		 */
-		mtx_lock(&dev_priv->irq_lock);
-		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
-			i915_handle_error(dev, false);
-
-		for_each_pipe(pipe) {
-			int reg = PIPESTAT(pipe);
-			pipe_stats[pipe] = I915_READ(reg);
-
-			/*
-			 * Clear the PIPE*STAT regs before the IIR
-			 */
-			if (pipe_stats[pipe] & 0x8000ffff) {
-				if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
-					DRM_DEBUG("pipe %c underrun\n",
-							 pipe_name(pipe));
-				I915_WRITE(reg, pipe_stats[pipe]);
-				irq_received = 1;
-			}
-		}
-		mtx_unlock(&dev_priv->irq_lock);
-
-		if (!irq_received)
-			break;
-
-		/* Consume port.  Then clear IIR or we'll miss events */
-		if ((I915_HAS_HOTPLUG(dev)) &&
-		    (iir & I915_DISPLAY_PORT_INTERRUPT)) {
-			u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
-
-			DRM_DEBUG("i915: hotplug event received, stat 0x%08x\n",
-				  hotplug_status);
-			if (hotplug_status & dev_priv->hotplug_supported_mask)
-				taskqueue_enqueue(dev_priv->tq,
-				    &dev_priv->hotplug_task);
-
-			I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
-			I915_READ(PORT_HOTPLUG_STAT);
-		}
-
-		I915_WRITE(IIR, iir);
-		new_iir = I915_READ(IIR); /* Flush posted writes */
-
-#if 0
-		if (dev->primary->master) {
-			master_priv = dev->primary->master->driver_priv;
-			if (master_priv->sarea_priv)
-				master_priv->sarea_priv->last_dispatch =
-					READ_BREADCRUMB(dev_priv);
-		}
-#else
-		if (dev_priv->sarea_priv)
-			dev_priv->sarea_priv->last_dispatch =
-			    READ_BREADCRUMB(dev_priv);
-#endif
-
-		if (iir & I915_USER_INTERRUPT)
-			notify_ring(dev, &dev_priv->rings[RCS]);
-		if (iir & I915_BSD_USER_INTERRUPT)
-			notify_ring(dev, &dev_priv->rings[VCS]);
-
-		if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) {
-			intel_prepare_page_flip(dev, 0);
-			if (dev_priv->flip_pending_is_done)
-				intel_finish_page_flip_plane(dev, 0);
-		}
-
-		if (iir & I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT) {
-			intel_prepare_page_flip(dev, 1);
-			if (dev_priv->flip_pending_is_done)
-				intel_finish_page_flip_plane(dev, 1);
-		}
-
-		for_each_pipe(pipe) {
-			if (pipe_stats[pipe] & vblank_status &&
-			    drm_handle_vblank(dev, pipe)) {
-				vblank++;
-				if (!dev_priv->flip_pending_is_done) {
-					i915_pageflip_stall_check(dev, pipe);
-					intel_finish_page_flip(dev, pipe);
-				}
-			}
-
-			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
-				blc_event = true;
-		}
-
-
-		if (blc_event || (iir & I915_ASLE_INTERRUPT)) {
-			intel_opregion_asle_intr(dev);
-		}
-
-		/* With MSI, interrupts are only generated when iir
-		 * transitions from zero to nonzero.  If another bit got
-		 * set while we were handling the existing iir bits, then
-		 * we would never get another interrupt.
-		 *
-		 * This is fine on non-MSI as well, as if we hit this path
-		 * we avoid exiting the interrupt handler only to generate
-		 * another one.
-		 *
-		 * Note that for MSI this could cause a stray interrupt report
-		 * if an interrupt landed in the time between writing IIR and
-		 * the posting read.  This should be rare enough to never
-		 * trigger the 99% of 100,000 interrupts test for disabling
-		 * stray interrupts.
-		 */
-		iir = new_iir;
-	}
-}
-
-static int i915_emit_irq(struct drm_device * dev)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-#if 0
-	struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
-#endif
-
-	i915_kernel_lost_context(dev);
-
-	DRM_DEBUG("i915: emit_irq\n");
-
-	dev_priv->counter++;
-	if (dev_priv->counter > 0x7FFFFFFFUL)
-		dev_priv->counter = 1;
-#if 0
-	if (master_priv->sarea_priv)
-		master_priv->sarea_priv->last_enqueue = dev_priv->counter;
-#else
-	if (dev_priv->sarea_priv)
-		dev_priv->sarea_priv->last_enqueue = dev_priv->counter;
-#endif
-
-	if (BEGIN_LP_RING(4) == 0) {
-		OUT_RING(MI_STORE_DWORD_INDEX);
-		OUT_RING(I915_BREADCRUMB_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-		OUT_RING(dev_priv->counter);
-		OUT_RING(MI_USER_INTERRUPT);
-		ADVANCE_LP_RING();
-	}
-
-	return dev_priv->counter;
-}
-
-static int i915_wait_irq(struct drm_device * dev, int irq_nr)
-{
-	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-#if 0
-	struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
-#endif
-	int ret;
-	struct intel_ring_buffer *ring = LP_RING(dev_priv);
-
-	DRM_DEBUG("irq_nr=%d breadcrumb=%d\n", irq_nr,
-		  READ_BREADCRUMB(dev_priv));
-
-#if 0
-	if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
-		if (master_priv->sarea_priv)
-			master_priv->sarea_priv->last_dispatch = READ_BREADCRUMB(dev_priv);
-		return 0;
-	}
-
-	if (master_priv->sarea_priv)
-		master_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
-#else
-	if (READ_BREADCRUMB(dev_priv) >= irq_nr) {
-		if (dev_priv->sarea_priv) {
-			dev_priv->sarea_priv->last_dispatch =
-				READ_BREADCRUMB(dev_priv);
-		}
-		return 0;
-	}
-
-	if (dev_priv->sarea_priv)
-		dev_priv->sarea_priv->perf_boxes |= I915_BOX_WAIT;
-#endif
-
-	ret = 0;
-	mtx_lock(&ring->irq_lock);
-	if (ring->irq_get(ring)) {
-		DRM_UNLOCK(dev);
-		while (ret == 0 && READ_BREADCRUMB(dev_priv) < irq_nr) {
-			ret = -msleep(ring, &ring->irq_lock, PCATCH,
-			    "915wtq", 3 * hz);
-		}
-		ring->irq_put(ring);
-		mtx_unlock(&ring->irq_lock);
-		DRM_LOCK(dev);
-	} else {
-		mtx_unlock(&ring->irq_lock);
-		if (_intel_wait_for(dev, READ_BREADCRUMB(dev_priv) >= irq_nr,
-		     3000, 1, "915wir"))
-			ret = -EBUSY;
-	}
-
-	if (ret == -EBUSY) {
-		DRM_ERROR("EBUSY -- rec: %d emitted: %d\n",
-			  READ_BREADCRUMB(dev_priv), (int)dev_priv->counter);
-	}
-
-	return ret;
-}
-
-/* Needs the lock as it touches the ring.
- */
-int i915_irq_emit(struct drm_device *dev, void *data,
-			 struct drm_file *file_priv)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_i915_irq_emit_t *emit = data;
-	int result;
-
-	if (!dev_priv || !LP_RING(dev_priv)->virtual_start) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
-	}
-
-	RING_LOCK_TEST_WITH_RETURN(dev, file_priv);
-
-	DRM_LOCK(dev);
-	result = i915_emit_irq(dev);
-	DRM_UNLOCK(dev);
-
-	if (DRM_COPY_TO_USER(emit->irq_seq, &result, sizeof(int))) {
-		DRM_ERROR("copy_to_user\n");
-		return -EFAULT;
-	}
-
-	return 0;
-}
-
-/* Doesn't need the hardware lock.
- */
-int i915_irq_wait(struct drm_device *dev, void *data,
-			 struct drm_file *file_priv)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_i915_irq_wait_t *irqwait = data;
-
-	if (!dev_priv) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
-	}
-
-	return i915_wait_irq(dev, irqwait->irq_seq);
-}
-
 /* Called from drm generic code, passed 'crtc' which
  * we use as a pipe index
  */
@@ -1227,7 +976,7 @@ i915_enable_vblank(struct drm_device *dev, int pipe)
 
 	/* maintain vblank delivery even in deep C-states */
 	if (dev_priv->info->gen == 3)
-		I915_WRITE(INSTPM, INSTPM_AGPBUSY_DIS << 16);
+		I915_WRITE(INSTPM, _MASKED_BIT_DISABLE(INSTPM_AGPBUSY_DIS));
 	mtx_unlock(&dev_priv->irq_lock);
 	CTR1(KTR_DRM, "i915_enable_vblank %d", pipe);
 
@@ -1260,14 +1009,38 @@ ivybridge_enable_vblank(struct drm_device *dev, int pipe)
 		return -EINVAL;
 
 	mtx_lock(&dev_priv->irq_lock);
-	ironlake_enable_display_irq(dev_priv, (pipe == 0) ?
-				    DE_PIPEA_VBLANK_IVB : DE_PIPEB_VBLANK_IVB);
+	ironlake_enable_display_irq(dev_priv,
+				    DE_PIPEA_VBLANK_IVB << (5 * pipe));
 	mtx_unlock(&dev_priv->irq_lock);
 	CTR1(KTR_DRM, "ivybridge_enable_vblank %d", pipe);
 
 	return 0;
 }
 
+static int valleyview_enable_vblank(struct drm_device *dev, int pipe)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	u32 dpfl, imr;
+
+	if (!i915_pipe_enabled(dev, pipe))
+		return -EINVAL;
+
+	mtx_lock(&dev_priv->irq_lock);
+	dpfl = I915_READ(VLV_DPFLIPSTAT);
+	imr = I915_READ(VLV_IMR);
+	if (pipe == 0) {
+		dpfl |= PIPEA_VBLANK_INT_EN;
+		imr &= ~I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
+	} else {
+		dpfl |= PIPEA_VBLANK_INT_EN;
+		imr &= ~I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
+	}
+	I915_WRITE(VLV_DPFLIPSTAT, dpfl);
+	I915_WRITE(VLV_IMR, imr);
+	mtx_unlock(&dev_priv->irq_lock);
+
+	return 0;
+}
 
 /* Called from drm generic code, passed 'crtc' which
  * we use as a pipe index
@@ -1279,8 +1052,7 @@ i915_disable_vblank(struct drm_device *dev, int pipe)
 
 	mtx_lock(&dev_priv->irq_lock);
 	if (dev_priv->info->gen == 3)
-		I915_WRITE(INSTPM,
-			   INSTPM_AGPBUSY_DIS << 16 | INSTPM_AGPBUSY_DIS);
+		I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_AGPBUSY_DIS));
 
 	i915_disable_pipestat(dev_priv, pipe,
 	    PIPE_VBLANK_INTERRUPT_ENABLE |
@@ -1307,64 +1079,30 @@ ivybridge_disable_vblank(struct drm_device *dev, int pipe)
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 
 	mtx_lock(&dev_priv->irq_lock);
-	ironlake_disable_display_irq(dev_priv, (pipe == 0) ?
-				     DE_PIPEA_VBLANK_IVB : DE_PIPEB_VBLANK_IVB);
+	ironlake_disable_display_irq(dev_priv,
+				     DE_PIPEA_VBLANK_IVB << (pipe * 5));
 	mtx_unlock(&dev_priv->irq_lock);
 	CTR1(KTR_DRM, "ivybridge_disable_vblank %d", pipe);
 }
 
-/* Set the vblank monitor pipe
- */
-int i915_vblank_pipe_set(struct drm_device *dev, void *data,
-			 struct drm_file *file_priv)
+static void valleyview_disable_vblank(struct drm_device *dev, int pipe)
 {
-	drm_i915_private_t *dev_priv = dev->dev_private;
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	u32 dpfl, imr;
 
-	if (!dev_priv) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
+	mtx_lock(&dev_priv->irq_lock);
+	dpfl = I915_READ(VLV_DPFLIPSTAT);
+	imr = I915_READ(VLV_IMR);
+	if (pipe == 0) {
+		dpfl &= ~PIPEA_VBLANK_INT_EN;
+		imr |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT;
+	} else {
+		dpfl &= ~PIPEB_VBLANK_INT_EN;
+		imr |= I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
 	}
-
-	return 0;
-}
-
-int i915_vblank_pipe_get(struct drm_device *dev, void *data,
-			 struct drm_file *file_priv)
-{
-	drm_i915_private_t *dev_priv = dev->dev_private;
-	drm_i915_vblank_pipe_t *pipe = data;
-
-	if (!dev_priv) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
-	}
-
-	pipe->pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
-
-	return 0;
-}
-
-/**
- * Schedule buffer swap at given vertical blank.
- */
-int i915_vblank_swap(struct drm_device *dev, void *data,
-		     struct drm_file *file_priv)
-{
-	/* The delayed swap mechanism was fundamentally racy, and has been
-	 * removed.  The model was that the client requested a delayed flip/swap
-	 * from the kernel, then waited for vblank before continuing to perform
-	 * rendering.  The problem was that the kernel might wake the client
-	 * up before it dispatched the vblank swap (since the lock has to be
-	 * held while touching the ringbuffer), in which case the client would
-	 * clear and start the next frame before the swap occurred, and
-	 * flicker would occur in addition to likely missing the vblank.
-	 *
-	 * In the absence of this ioctl, userland falls back to a correct path
-	 * of waiting for a vblank, then dispatching the swap on its own.
-	 * Context switching to userland and back is plenty fast enough for
-	 * meeting the requirements of vblank swapping.
-	 */
-	return -EINVAL;
+	I915_WRITE(VLV_IMR, imr);
+	I915_WRITE(VLV_DPFLIPSTAT, dpfl);
+	mtx_unlock(&dev_priv->irq_lock);
 }
 
 static u32
@@ -1383,15 +1121,15 @@ static bool i915_hangcheck_ring_idle(struct intel_ring_buffer *ring, bool *err)
 	if (list_empty(&ring->request_list) ||
 	    i915_seqno_passed(ring->get_seqno(ring), ring_last_seqno(ring))) {
 		/* Issue a wake-up to catch stuck h/w. */
-		if (ring->waiting_seqno) {
-			DRM_ERROR(
-"Hangcheck timer elapsed... %s idle [waiting on %d, at %d], missed IRQ?\n",
-				  ring->name,
-				  ring->waiting_seqno,
-				  ring->get_seqno(ring));
+		sleepq_lock(ring);
+		if (sleepq_sleepcnt(ring, 0) != 0) {
+			sleepq_release(ring);
+			DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
+				  ring->name);
 			wakeup(ring);
 			*err = true;
-		}
+		} else
+			sleepq_release(ring);
 		return true;
 	}
 	return false;
@@ -1411,6 +1149,35 @@ static bool kick_ring(struct intel_ring_buffer *ring)
 	return false;
 }
 
+static bool i915_hangcheck_hung(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+
+	if (dev_priv->hangcheck_count++ > 1) {
+		bool hung = true;
+
+		DRM_ERROR("Hangcheck timer elapsed... GPU hung\n");
+		i915_handle_error(dev, true);
+
+		if (!IS_GEN2(dev)) {
+			struct intel_ring_buffer *ring;
+			int i;
+
+			/* Is the chip hanging on a WAIT_FOR_EVENT?
+			 * If so we can simply poke the RB_WAIT bit
+			 * and break the hang. This should work on
+			 * all but the second generation chipsets.
+			 */
+			for_each_ring(ring, dev_priv, i)
+				hung &= !kick_ring(ring);
+		}
+
+		return hung;
+	}
+
+	return false;
+}
+
 /**
  * This is called when the chip hasn't reported back with completed
  * batchbuffers in a long time. The first time this is called we simply record
@@ -1422,19 +1189,31 @@ i915_hangcheck_elapsed(void *context)
 {
 	struct drm_device *dev = (struct drm_device *)context;
 	drm_i915_private_t *dev_priv = dev->dev_private;
-	uint32_t acthd, instdone, instdone1, acthd_bsd, acthd_blt;
-	bool err = false;
+	uint32_t acthd[I915_NUM_RINGS], instdone, instdone1;
+	struct intel_ring_buffer *ring;
+	bool err = false, idle;
+	int i;
 
 	if (!i915_enable_hangcheck)
 		return;
 
+	memset(acthd, 0, sizeof(acthd));
+	idle = true;
+	for_each_ring(ring, dev_priv, i) {
+	    idle &= i915_hangcheck_ring_idle(ring, &err);
+	    acthd[i] = intel_ring_get_active_head(ring);
+	}
+
 	/* If all work is done then ACTHD clearly hasn't advanced. */
-	if (i915_hangcheck_ring_idle(&dev_priv->rings[RCS], &err) &&
-	    i915_hangcheck_ring_idle(&dev_priv->rings[VCS], &err) &&
-	    i915_hangcheck_ring_idle(&dev_priv->rings[BCS], &err)) {
-		dev_priv->hangcheck_count = 0;
-		if (err)
+	if (idle) {
+		if (err) {
+			if (i915_hangcheck_hung(dev))
+				return;
+
 			goto repeat;
+		}
+
+		dev_priv->hangcheck_count = 0;
 		return;
 	}
 
@@ -1445,47 +1224,15 @@ i915_hangcheck_elapsed(void *context)
 		instdone = I915_READ(INSTDONE_I965);
 		instdone1 = I915_READ(INSTDONE1);
 	}
-	acthd = intel_ring_get_active_head(&dev_priv->rings[RCS]);
-	acthd_bsd = HAS_BSD(dev) ?
-		intel_ring_get_active_head(&dev_priv->rings[VCS]) : 0;
-	acthd_blt = HAS_BLT(dev) ?
-		intel_ring_get_active_head(&dev_priv->rings[BCS]) : 0;
-
-	if (dev_priv->last_acthd == acthd &&
-	    dev_priv->last_acthd_bsd == acthd_bsd &&
-	    dev_priv->last_acthd_blt == acthd_blt &&
+	if (memcmp(dev_priv->last_acthd, acthd, sizeof(acthd)) == 0 &&
 	    dev_priv->last_instdone == instdone &&
 	    dev_priv->last_instdone1 == instdone1) {
-		if (dev_priv->hangcheck_count++ > 1) {
-			DRM_ERROR("Hangcheck timer elapsed... GPU hung\n");
-			i915_handle_error(dev, true);
-
-			if (!IS_GEN2(dev)) {
-				/* Is the chip hanging on a WAIT_FOR_EVENT?
-				 * If so we can simply poke the RB_WAIT bit
-				 * and break the hang. This should work on
-				 * all but the second generation chipsets.
-				 */
-				if (kick_ring(&dev_priv->rings[RCS]))
-					goto repeat;
-
-				if (HAS_BSD(dev) &&
-				    kick_ring(&dev_priv->rings[VCS]))
-					goto repeat;
-
-				if (HAS_BLT(dev) &&
-				    kick_ring(&dev_priv->rings[BCS]))
-					goto repeat;
-			}
-
+		if (i915_hangcheck_hung(dev))
 			return;
-		}
 	} else {
 		dev_priv->hangcheck_count = 0;
 
-		dev_priv->last_acthd = acthd;
-		dev_priv->last_acthd_bsd = acthd_bsd;
-		dev_priv->last_acthd_blt = acthd_blt;
+		memcpy(dev_priv->last_acthd, acthd, sizeof(acthd));
 		dev_priv->last_instdone = instdone;
 		dev_priv->last_instdone1 = instdone1;
 	}
@@ -1504,13 +1251,6 @@ ironlake_irq_preinstall(struct drm_device *dev)
 
 	atomic_set(&dev_priv->irq_received, 0);
 
-	TASK_INIT(&dev_priv->hotplug_task, 0, i915_hotplug_work_func,
-	    dev->dev_private);
-	TASK_INIT(&dev_priv->error_task, 0, i915_error_work_func,
-	    dev->dev_private);
-	TASK_INIT(&dev_priv->rps_task, 0, gen6_pm_rps_work_func,
-	    dev->dev_private);
-
 	I915_WRITE(HWSTAM, 0xeffe);
 
 	/* XXX hotplug from PCH */
@@ -1530,6 +1270,38 @@ ironlake_irq_preinstall(struct drm_device *dev)
 	POSTING_READ(SDEIER);
 }
 
+static void valleyview_irq_preinstall(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	int pipe;
+
+	atomic_set(&dev_priv->irq_received, 0);
+
+	/* VLV magic */
+	I915_WRITE(VLV_IMR, 0);
+	I915_WRITE(RING_IMR(RENDER_RING_BASE), 0);
+	I915_WRITE(RING_IMR(GEN6_BSD_RING_BASE), 0);
+	I915_WRITE(RING_IMR(BLT_RING_BASE), 0);
+
+	/* and GT */
+	I915_WRITE(GTIIR, I915_READ(GTIIR));
+	I915_WRITE(GTIIR, I915_READ(GTIIR));
+	I915_WRITE(GTIMR, 0xffffffff);
+	I915_WRITE(GTIER, 0x0);
+	POSTING_READ(GTIER);
+
+	I915_WRITE(DPINVGTT, 0xff);
+
+	I915_WRITE(PORT_HOTPLUG_EN, 0);
+	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
+	for_each_pipe(pipe)
+		I915_WRITE(PIPESTAT(pipe), 0xffff);
+	I915_WRITE(VLV_IIR, 0xffffffff);
+	I915_WRITE(VLV_IMR, 0xffffffff);
+	I915_WRITE(VLV_IER, 0x0);
+	POSTING_READ(VLV_IER);
+}
+
 /*
  * Enable digital hotplug on the PCH, and configure the DP short pulse
  * duration to 2ms (which is the minimum in the Display Port spec)
@@ -1559,7 +1331,6 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
 	u32 render_irqs;
 	u32 hotplug_mask;
 
-	dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
 	dev_priv->irq_mask = ~display_mask;
 
 	/* should always can generate irq */
@@ -1576,8 +1347,8 @@ static int ironlake_irq_postinstall(struct drm_device *dev)
 	if (IS_GEN6(dev))
 		render_irqs =
 			GT_USER_INTERRUPT |
-			GT_GEN6_BSD_USER_INTERRUPT |
-			GT_BLT_USER_INTERRUPT;
+			GEN6_BSD_USER_INTERRUPT |
+			GEN6_BLITTER_USER_INTERRUPT;
 	else
 		render_irqs =
 			GT_USER_INTERRUPT |
@@ -1623,20 +1394,24 @@ ivybridge_irq_postinstall(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	/* enable kind of interrupts always enabled */
-	u32 display_mask = DE_MASTER_IRQ_CONTROL | DE_GSE_IVB |
-		DE_PCH_EVENT_IVB | DE_PLANEA_FLIP_DONE_IVB |
-		DE_PLANEB_FLIP_DONE_IVB;
+	u32 display_mask =
+		DE_MASTER_IRQ_CONTROL | DE_GSE_IVB | DE_PCH_EVENT_IVB |
+		DE_PLANEC_FLIP_DONE_IVB |
+		DE_PLANEB_FLIP_DONE_IVB |
+		DE_PLANEA_FLIP_DONE_IVB;
 	u32 render_irqs;
 	u32 hotplug_mask;
 
-	dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
 	dev_priv->irq_mask = ~display_mask;
 
 	/* should always can generate irq */
 	I915_WRITE(DEIIR, I915_READ(DEIIR));
 	I915_WRITE(DEIMR, dev_priv->irq_mask);
-	I915_WRITE(DEIER, display_mask | DE_PIPEA_VBLANK_IVB |
-		   DE_PIPEB_VBLANK_IVB);
+	I915_WRITE(DEIER,
+		   display_mask |
+		   DE_PIPEC_VBLANK_IVB |
+		   DE_PIPEB_VBLANK_IVB |
+		   DE_PIPEA_VBLANK_IVB);
 	POSTING_READ(DEIER);
 
 	dev_priv->gt_irq_mask = ~0;
@@ -1644,8 +1419,8 @@ ivybridge_irq_postinstall(struct drm_device *dev)
 	I915_WRITE(GTIIR, I915_READ(GTIIR));
 	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
 
-	render_irqs = GT_USER_INTERRUPT | GT_GEN6_BSD_USER_INTERRUPT |
-		GT_BLT_USER_INTERRUPT;
+	render_irqs = GT_USER_INTERRUPT | GEN6_BSD_USER_INTERRUPT |
+		GEN6_BLITTER_USER_INTERRUPT;
 	I915_WRITE(GTIER, render_irqs);
 	POSTING_READ(GTIER);
 
@@ -1665,20 +1440,494 @@ ivybridge_irq_postinstall(struct drm_device *dev)
 	return 0;
 }
 
+static int valleyview_irq_postinstall(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	u32 render_irqs;
+	u32 enable_mask;
+	u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
+	u16 msid;
+
+	enable_mask = I915_DISPLAY_PORT_INTERRUPT;
+	enable_mask |= I915_DISPLAY_PIPE_A_VBLANK_INTERRUPT |
+		I915_DISPLAY_PIPE_B_VBLANK_INTERRUPT;
+
+	dev_priv->irq_mask = ~enable_mask;
+
+	dev_priv->pipestat[0] = 0;
+	dev_priv->pipestat[1] = 0;
+
+	/* Hack for broken MSIs on VLV */
+	pci_write_config(dev->device, 0x94, 0xfee00000, 4);
+	msid = pci_read_config(dev->device, 0x98, 2);
+	msid &= 0xff; /* mask out delivery bits */
+	msid |= (1<<14);
+	pci_write_config(dev->device, 0x98, msid, 2);
+
+	I915_WRITE(VLV_IMR, dev_priv->irq_mask);
+	I915_WRITE(VLV_IER, enable_mask);
+	I915_WRITE(VLV_IIR, 0xffffffff);
+	I915_WRITE(PIPESTAT(0), 0xffff);
+	I915_WRITE(PIPESTAT(1), 0xffff);
+	POSTING_READ(VLV_IER);
+
+	I915_WRITE(VLV_IIR, 0xffffffff);
+	I915_WRITE(VLV_IIR, 0xffffffff);
+
+	render_irqs = GT_GEN6_BLT_FLUSHDW_NOTIFY_INTERRUPT |
+		GT_GEN6_BLT_CS_ERROR_INTERRUPT |
+		GT_GEN6_BLT_USER_INTERRUPT |
+		GT_GEN6_BSD_USER_INTERRUPT |
+		GT_GEN6_BSD_CS_ERROR_INTERRUPT |
+		GT_GEN7_L3_PARITY_ERROR_INTERRUPT |
+		GT_PIPE_NOTIFY |
+		GT_RENDER_CS_ERROR_INTERRUPT |
+		GT_SYNC_STATUS |
+		GT_USER_INTERRUPT;
+
+	dev_priv->gt_irq_mask = ~render_irqs;
+
+	I915_WRITE(GTIIR, I915_READ(GTIIR));
+	I915_WRITE(GTIIR, I915_READ(GTIIR));
+	I915_WRITE(GTIMR, 0);
+	I915_WRITE(GTIER, render_irqs);
+	POSTING_READ(GTIER);
+
+	/* ack & enable invalid PTE error interrupts */
+#if 0 /* FIXME: add support to irq handler for checking these bits */
+	I915_WRITE(DPINVGTT, DPINVGTT_STATUS_MASK);
+	I915_WRITE(DPINVGTT, DPINVGTT_EN_MASK);
+#endif
+
+	I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
+#if 0 /* FIXME: check register definitions; some have moved */
+	/* Note HDMI and DP share bits */
+	if (dev_priv->hotplug_supported_mask & HDMIB_HOTPLUG_INT_STATUS)
+		hotplug_en |= HDMIB_HOTPLUG_INT_EN;
+	if (dev_priv->hotplug_supported_mask & HDMIC_HOTPLUG_INT_STATUS)
+		hotplug_en |= HDMIC_HOTPLUG_INT_EN;
+	if (dev_priv->hotplug_supported_mask & HDMID_HOTPLUG_INT_STATUS)
+		hotplug_en |= HDMID_HOTPLUG_INT_EN;
+	if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS)
+		hotplug_en |= SDVOC_HOTPLUG_INT_EN;
+	if (dev_priv->hotplug_supported_mask & SDVOB_HOTPLUG_INT_STATUS)
+		hotplug_en |= SDVOB_HOTPLUG_INT_EN;
+	if (dev_priv->hotplug_supported_mask & CRT_HOTPLUG_INT_STATUS) {
+		hotplug_en |= CRT_HOTPLUG_INT_EN;
+		hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
+	}
+#endif
+
+	I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
+
+	return 0;
+}
+
+static void valleyview_irq_uninstall(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	int pipe;
+
+	if (!dev_priv)
+		return;
+
+	for_each_pipe(pipe)
+		I915_WRITE(PIPESTAT(pipe), 0xffff);
+
+	I915_WRITE(HWSTAM, 0xffffffff);
+	I915_WRITE(PORT_HOTPLUG_EN, 0);
+	I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
+	for_each_pipe(pipe)
+		I915_WRITE(PIPESTAT(pipe), 0xffff);
+	I915_WRITE(VLV_IIR, 0xffffffff);
+	I915_WRITE(VLV_IMR, 0xffffffff);
+	I915_WRITE(VLV_IER, 0x0);
+	POSTING_READ(VLV_IER);
+}
+
 static void
-i915_driver_irq_preinstall(struct drm_device * dev)
+ironlake_irq_uninstall(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+
+	if (dev_priv == NULL)
+		return;
+
+	I915_WRITE(HWSTAM, 0xffffffff);
+
+	I915_WRITE(DEIMR, 0xffffffff);
+	I915_WRITE(DEIER, 0x0);
+	I915_WRITE(DEIIR, I915_READ(DEIIR));
+
+	I915_WRITE(GTIMR, 0xffffffff);
+	I915_WRITE(GTIER, 0x0);
+	I915_WRITE(GTIIR, I915_READ(GTIIR));
+
+	I915_WRITE(SDEIMR, 0xffffffff);
+	I915_WRITE(SDEIER, 0x0);
+	I915_WRITE(SDEIIR, I915_READ(SDEIIR));
+}
+
+static void i8xx_irq_preinstall(struct drm_device * dev)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	int pipe;
+ 
+	atomic_set(&dev_priv->irq_received, 0);
+ 
+ 	for_each_pipe(pipe)
+ 		I915_WRITE(PIPESTAT(pipe), 0);
+	I915_WRITE16(IMR, 0xffff);
+	I915_WRITE16(IER, 0x0);
+	POSTING_READ16(IER);
+}
+
+static int i8xx_irq_postinstall(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+
+	dev_priv->pipestat[0] = 0;
+	dev_priv->pipestat[1] = 0;
+
+	I915_WRITE16(EMR,
+		     ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
+
+	/* Unmask the interrupts that we always want on. */
+	dev_priv->irq_mask =
+		~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
+		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
+		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
+		  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
+	I915_WRITE16(IMR, dev_priv->irq_mask);
+
+	I915_WRITE16(IER,
+		     I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
+		     I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+		     I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
+		     I915_USER_INTERRUPT);
+	POSTING_READ16(IER);
+
+	return 0;
+}
+
+static void i8xx_irq_handler(void *arg)
+{
+	struct drm_device *dev = (struct drm_device *) arg;
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	u16 iir, new_iir;
+	u32 pipe_stats[2];
+	int irq_received;
+	int pipe;
+	u16 flip_mask =
+		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
+		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
+
+	atomic_inc(&dev_priv->irq_received);
+
+	iir = I915_READ16(IIR);
+	if (iir == 0)
+		return;
+
+	while (iir & ~flip_mask) {
+		/* Can't rely on pipestat interrupt bit in iir as it might
+		 * have been cleared after the pipestat interrupt was received.
+		 * It doesn't set the bit in iir again, but it still produces
+		 * interrupts (for non-MSI).
+		 */
+		mtx_lock(&dev_priv->irq_lock);
+		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
+			i915_handle_error(dev, false);
+
+		for_each_pipe(pipe) {
+			int reg = PIPESTAT(pipe);
+			pipe_stats[pipe] = I915_READ(reg);
+
+			/*
+			 * Clear the PIPE*STAT regs before the IIR
+			 */
+			if (pipe_stats[pipe] & 0x8000ffff) {
+				if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
+					DRM_DEBUG_DRIVER("pipe %c underrun\n",
+							 pipe_name(pipe));
+				I915_WRITE(reg, pipe_stats[pipe]);
+				irq_received = 1;
+			}
+		}
+		mtx_unlock(&dev_priv->irq_lock);
+
+		I915_WRITE16(IIR, iir & ~flip_mask);
+		new_iir = I915_READ16(IIR); /* Flush posted writes */
+
+		i915_update_dri1_breadcrumb(dev);
+
+		if (iir & I915_USER_INTERRUPT)
+			notify_ring(dev, &dev_priv->rings[RCS]);
+
+		if (pipe_stats[0] & PIPE_VBLANK_INTERRUPT_STATUS &&
+		    drm_handle_vblank(dev, 0)) {
+			if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT) {
+				intel_prepare_page_flip(dev, 0);
+				intel_finish_page_flip(dev, 0);
+				flip_mask &= ~I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT;
+			}
+		}
+
+		if (pipe_stats[1] & PIPE_VBLANK_INTERRUPT_STATUS &&
+		    drm_handle_vblank(dev, 1)) {
+			if (iir & I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT) {
+				intel_prepare_page_flip(dev, 1);
+				intel_finish_page_flip(dev, 1);
+				flip_mask &= ~I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
+			}
+		}
+
+		iir = new_iir;
+	}
+}
+
+static void i8xx_irq_uninstall(struct drm_device * dev)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	int pipe;
+
+	for_each_pipe(pipe) {
+		/* Clear enable bits; then clear status bits */
+		I915_WRITE(PIPESTAT(pipe), 0);
+		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
+	}
+	I915_WRITE16(IMR, 0xffff);
+	I915_WRITE16(IER, 0x0);
+	I915_WRITE16(IIR, I915_READ16(IIR));
+}
+
+static void i915_irq_preinstall(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	int pipe;
 
 	atomic_set(&dev_priv->irq_received, 0);
 
-	TASK_INIT(&dev_priv->hotplug_task, 0, i915_hotplug_work_func,
-	    dev->dev_private);
-	TASK_INIT(&dev_priv->error_task, 0, i915_error_work_func,
-	    dev->dev_private);
-	TASK_INIT(&dev_priv->rps_task, 0, gen6_pm_rps_work_func,
-	    dev->dev_private);
+	if (I915_HAS_HOTPLUG(dev)) {
+		I915_WRITE(PORT_HOTPLUG_EN, 0);
+		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
+	}
+
+	I915_WRITE16(HWSTAM, 0xeffe);
+	for_each_pipe(pipe)
+		I915_WRITE(PIPESTAT(pipe), 0);
+	I915_WRITE(IMR, 0xffffffff);
+	I915_WRITE(IER, 0x0);
+	POSTING_READ(IER);
+}
+
+static int i915_irq_postinstall(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	u32 enable_mask;
+
+	dev_priv->pipestat[0] = 0;
+	dev_priv->pipestat[1] = 0;
+
+	I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
+
+	/* Unmask the interrupts that we always want on. */
+	dev_priv->irq_mask =
+		~(I915_ASLE_INTERRUPT |
+		  I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
+		  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+		  I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
+		  I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
+		  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
+
+	enable_mask =
+		I915_ASLE_INTERRUPT |
+		I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
+		I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+		I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
+		I915_USER_INTERRUPT;
+
+	if (I915_HAS_HOTPLUG(dev)) {
+		/* Enable in IER... */
+		enable_mask |= I915_DISPLAY_PORT_INTERRUPT;
+		/* and unmask in IMR */
+		dev_priv->irq_mask &= ~I915_DISPLAY_PORT_INTERRUPT;
+	}
+
+	I915_WRITE(IMR, dev_priv->irq_mask);
+	I915_WRITE(IER, enable_mask);
+	POSTING_READ(IER);
+
+	if (I915_HAS_HOTPLUG(dev)) {
+		u32 hotplug_en = I915_READ(PORT_HOTPLUG_EN);
+
+		if (dev_priv->hotplug_supported_mask & HDMIB_HOTPLUG_INT_STATUS)
+			hotplug_en |= HDMIB_HOTPLUG_INT_EN;
+		if (dev_priv->hotplug_supported_mask & HDMIC_HOTPLUG_INT_STATUS)
+			hotplug_en |= HDMIC_HOTPLUG_INT_EN;
+		if (dev_priv->hotplug_supported_mask & HDMID_HOTPLUG_INT_STATUS)
+			hotplug_en |= HDMID_HOTPLUG_INT_EN;
+		if (dev_priv->hotplug_supported_mask & SDVOC_HOTPLUG_INT_STATUS)
+			hotplug_en |= SDVOC_HOTPLUG_INT_EN;
+		if (dev_priv->hotplug_supported_mask & SDVOB_HOTPLUG_INT_STATUS)
+			hotplug_en |= SDVOB_HOTPLUG_INT_EN;
+		if (dev_priv->hotplug_supported_mask & CRT_HOTPLUG_INT_STATUS) {
+			hotplug_en |= CRT_HOTPLUG_INT_EN;
+			hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
+		}
+
+		/* Ignore TV since it's buggy */
+
+		I915_WRITE(PORT_HOTPLUG_EN, hotplug_en);
+	}
+
+	intel_opregion_enable_asle(dev);
+
+	return 0;
+}
+
+static void i915_irq_handler(void *arg)
+{
+	struct drm_device *dev = (struct drm_device *) arg;
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
+	u32 flip_mask =
+		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
+		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT;
+	u32 flip[2] = {
+		I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT,
+		I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT
+	};
+	int pipe;
+
+	atomic_inc(&dev_priv->irq_received);
+
+	iir = I915_READ(IIR);
+	do {
+		bool irq_received = (iir & ~flip_mask) != 0;
+		bool blc_event = false;
+
+		/* Can't rely on pipestat interrupt bit in iir as it might
+		 * have been cleared after the pipestat interrupt was received.
+		 * It doesn't set the bit in iir again, but it still produces
+		 * interrupts (for non-MSI).
+		 */
+		mtx_lock(&dev_priv->irq_lock);
+		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
+			i915_handle_error(dev, false);
+
+		for_each_pipe(pipe) {
+			int reg = PIPESTAT(pipe);
+			pipe_stats[pipe] = I915_READ(reg);
+
+			/* Clear the PIPE*STAT regs before the IIR */
+			if (pipe_stats[pipe] & 0x8000ffff) {
+				if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
+					DRM_DEBUG_DRIVER("pipe %c underrun\n",
+							 pipe_name(pipe));
+				I915_WRITE(reg, pipe_stats[pipe]);
+				irq_received = true;
+			}
+		}
+		mtx_unlock(&dev_priv->irq_lock);
+
+		if (!irq_received)
+			break;
+
+		/* Consume port.  Then clear IIR or we'll miss events */
+		if ((I915_HAS_HOTPLUG(dev)) &&
+		    (iir & I915_DISPLAY_PORT_INTERRUPT)) {
+			u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
+
+			DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
+				  hotplug_status);
+			if (hotplug_status & dev_priv->hotplug_supported_mask)
+				taskqueue_enqueue(dev_priv->tq,
+				    &dev_priv->hotplug_task);
+
+			I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
+			POSTING_READ(PORT_HOTPLUG_STAT);
+		}
+
+		I915_WRITE(IIR, iir & ~flip_mask);
+		new_iir = I915_READ(IIR); /* Flush posted writes */
+
+		if (iir & I915_USER_INTERRUPT)
+			notify_ring(dev, &dev_priv->rings[RCS]);
+
+		for_each_pipe(pipe) {
+			int plane = pipe;
+			if (IS_MOBILE(dev))
+				plane = !plane;
+			if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
+			    drm_handle_vblank(dev, pipe)) {
+				if (iir & flip[plane]) {
+					intel_prepare_page_flip(dev, plane);
+					intel_finish_page_flip(dev, pipe);
+					flip_mask &= ~flip[plane];
+				}
+			}
+
+			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
+				blc_event = true;
+		}
+
+		if (blc_event || (iir & I915_ASLE_INTERRUPT))
+			intel_opregion_asle_intr(dev);
+
+
+		/* With MSI, interrupts are only generated when iir
+		 * transitions from zero to nonzero.  If another bit got
+		 * set while we were handling the existing iir bits, then
+		 * we would never get another interrupt.
+		 *
+		 * This is fine on non-MSI as well, as if we hit this path
+		 * we avoid exiting the interrupt handler only to generate
+		 * another one.
+		 *
+		 * Note that for MSI this could cause a stray interrupt report
+		 * if an interrupt landed in the time between writing IIR and
+		 * the posting read.  This should be rare enough to never
+		 * trigger the 99% of 100,000 interrupts test for disabling
+		 * stray interrupts.
+		 */
+		iir = new_iir;
+	} while (iir & ~flip_mask);
+
+	i915_update_dri1_breadcrumb(dev);
+}
+
+static void i915_irq_uninstall(struct drm_device * dev)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	int pipe;
+
+	if (!dev_priv)
+		return;
+
+	if (I915_HAS_HOTPLUG(dev)) {
+		I915_WRITE(PORT_HOTPLUG_EN, 0);
+		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
+	}
+
+	I915_WRITE16(HWSTAM, 0xffff);
+	for_each_pipe(pipe) {
+		/* Clear enable bits; then clear status bits */
+		I915_WRITE(PIPESTAT(pipe), 0);
+		I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
+	}
+	I915_WRITE(IMR, 0xffffffff);
+	I915_WRITE(IER, 0x0);
+
+	I915_WRITE(IIR, I915_READ(IIR));
+}
+
+static void i965_irq_preinstall(struct drm_device * dev)
+{
+	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	int pipe;
+
+	atomic_set(&dev_priv->irq_received, 0);
 
 	if (I915_HAS_HOTPLUG(dev)) {
 		I915_WRITE(PORT_HOTPLUG_EN, 0);
@@ -1693,21 +1942,25 @@ i915_driver_irq_preinstall(struct drm_device * dev)
 	POSTING_READ(IER);
 }
 
-/*
- * Must be called after intel_modeset_init or hotplug interrupts won't be
- * enabled correctly.
- */
-static int
-i915_driver_irq_postinstall(struct drm_device *dev)
+static int i965_irq_postinstall(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
-	u32 enable_mask = I915_INTERRUPT_ENABLE_FIX | I915_INTERRUPT_ENABLE_VAR;
+	u32 enable_mask;
 	u32 error_mask;
 
-	dev_priv->vblank_pipe = DRM_I915_VBLANK_PIPE_A | DRM_I915_VBLANK_PIPE_B;
-
 	/* Unmask the interrupts that we always want on. */
-	dev_priv->irq_mask = ~I915_INTERRUPT_ENABLE_FIX;
+	dev_priv->irq_mask = ~(I915_ASLE_INTERRUPT |
+			       I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
+			       I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+			       I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
+			       I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT |
+			       I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
+
+	enable_mask = ~dev_priv->irq_mask;
+	enable_mask |= I915_USER_INTERRUPT;
+
+	if (IS_G4X(dev))
+		enable_mask |= I915_BSD_USER_INTERRUPT;
 
 	dev_priv->pipestat[0] = 0;
 	dev_priv->pipestat[1] = 0;
@@ -1774,45 +2027,123 @@ i915_driver_irq_postinstall(struct drm_device *dev)
 	return 0;
 }
 
-static void
-ironlake_irq_uninstall(struct drm_device *dev)
+static void i965_irq_handler(void *arg)
 {
+	struct drm_device *dev = (struct drm_device *) arg;
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
+	u32 iir, new_iir;
+	u32 pipe_stats[I915_MAX_PIPES];
+	int irq_received;
+	int pipe;
 
-	if (dev_priv == NULL)
-		return;
+	atomic_inc(&dev_priv->irq_received);
 
-	dev_priv->vblank_pipe = 0;
+	iir = I915_READ(IIR);
 
-	I915_WRITE(HWSTAM, 0xffffffff);
+	for (;;) {
+		bool blc_event = false;
 
-	I915_WRITE(DEIMR, 0xffffffff);
-	I915_WRITE(DEIER, 0x0);
-	I915_WRITE(DEIIR, I915_READ(DEIIR));
+		irq_received = iir != 0;
 
-	I915_WRITE(GTIMR, 0xffffffff);
-	I915_WRITE(GTIER, 0x0);
-	I915_WRITE(GTIIR, I915_READ(GTIIR));
+		/* Can't rely on pipestat interrupt bit in iir as it might
+		 * have been cleared after the pipestat interrupt was received.
+		 * It doesn't set the bit in iir again, but it still produces
+		 * interrupts (for non-MSI).
+		 */
+		mtx_lock(&dev_priv->irq_lock);
+		if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
+			i915_handle_error(dev, false);
 
-	I915_WRITE(SDEIMR, 0xffffffff);
-	I915_WRITE(SDEIER, 0x0);
-	I915_WRITE(SDEIIR, I915_READ(SDEIIR));
+		for_each_pipe(pipe) {
+			int reg = PIPESTAT(pipe);
+			pipe_stats[pipe] = I915_READ(reg);
 
-	taskqueue_drain(dev_priv->tq, &dev_priv->hotplug_task);
-	taskqueue_drain(dev_priv->tq, &dev_priv->error_task);
-	taskqueue_drain(dev_priv->tq, &dev_priv->rps_task);
+			/*
+			 * Clear the PIPE*STAT regs before the IIR
+			 */
+			if (pipe_stats[pipe] & 0x8000ffff) {
+				if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
+					DRM_DEBUG_DRIVER("pipe %c underrun\n",
+							 pipe_name(pipe));
+				I915_WRITE(reg, pipe_stats[pipe]);
+				irq_received = 1;
+			}
+		}
+		mtx_unlock(&dev_priv->irq_lock);
+
+		if (!irq_received)
+			break;
+
+		/* Consume port.  Then clear IIR or we'll miss events */
+		if ((I915_HAS_HOTPLUG(dev)) &&
+		    (iir & I915_DISPLAY_PORT_INTERRUPT)) {
+			u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
+
+			DRM_DEBUG_DRIVER("hotplug event received, stat 0x%08x\n",
+				  hotplug_status);
+			if (hotplug_status & dev_priv->hotplug_supported_mask)
+				taskqueue_enqueue(dev_priv->tq,
+				    &dev_priv->hotplug_task);
+
+			I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
+			I915_READ(PORT_HOTPLUG_STAT);
+		}
+
+		I915_WRITE(IIR, iir);
+		new_iir = I915_READ(IIR); /* Flush posted writes */
+
+		if (iir & I915_USER_INTERRUPT)
+			notify_ring(dev, &dev_priv->rings[RCS]);
+		if (iir & I915_BSD_USER_INTERRUPT)
+			notify_ring(dev, &dev_priv->rings[VCS]);
+
+		if (iir & I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT)
+			intel_prepare_page_flip(dev, 0);
+
+		if (iir & I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT)
+			intel_prepare_page_flip(dev, 1);
+
+		for_each_pipe(pipe) {
+			if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
+			    drm_handle_vblank(dev, pipe)) {
+				i915_pageflip_stall_check(dev, pipe);
+				intel_finish_page_flip(dev, pipe);
+			}
+
+			if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
+				blc_event = true;
+		}
+
+
+		if (blc_event || (iir & I915_ASLE_INTERRUPT))
+			intel_opregion_asle_intr(dev);
+
+		/* With MSI, interrupts are only generated when iir
+		 * transitions from zero to nonzero.  If another bit got
+		 * set while we were handling the existing iir bits, then
+		 * we would never get another interrupt.
+		 *
+		 * This is fine on non-MSI as well, as if we hit this path
+		 * we avoid exiting the interrupt handler only to generate
+		 * another one.
+		 *
+		 * Note that for MSI this could cause a stray interrupt report
+		 * if an interrupt landed in the time between writing IIR and
+		 * the posting read.  This should be rare enough to never
+		 * trigger the 99% of 100,000 interrupts test for disabling
+		 * stray interrupts.
+		 */
+		iir = new_iir;
+	}
+
+	i915_update_dri1_breadcrumb(dev);
 }
 
-static void i915_driver_irq_uninstall(struct drm_device * dev)
+static void i965_irq_uninstall(struct drm_device * dev)
 {
 	drm_i915_private_t *dev_priv = (drm_i915_private_t *) dev->dev_private;
 	int pipe;
 
-	if (!dev_priv)
-		return;
-
-	dev_priv->vblank_pipe = 0;
-
 	if (I915_HAS_HOTPLUG(dev)) {
 		I915_WRITE(PORT_HOTPLUG_EN, 0);
 		I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
@@ -1828,30 +2159,39 @@ static void i915_driver_irq_uninstall(struct drm_device * dev)
 		I915_WRITE(PIPESTAT(pipe),
 			   I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
 	I915_WRITE(IIR, I915_READ(IIR));
-
-	taskqueue_drain(dev_priv->tq, &dev_priv->hotplug_task);
-	taskqueue_drain(dev_priv->tq, &dev_priv->error_task);
-	taskqueue_drain(dev_priv->tq, &dev_priv->rps_task);
 }
 
-void
-intel_irq_init(struct drm_device *dev)
+void intel_irq_init(struct drm_device *dev)
 {
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	TASK_INIT(&dev_priv->hotplug_task, 0, i915_hotplug_work_func,
+	    dev->dev_private);
+	TASK_INIT(&dev_priv->error_task, 0, i915_error_work_func,
+	    dev->dev_private);
+	TASK_INIT(&dev_priv->rps_task, 0, gen6_pm_rps_work_func,
+	    dev->dev_private);
 
 	dev->driver->get_vblank_counter = i915_get_vblank_counter;
 	dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
-	if (IS_G4X(dev) || IS_GEN5(dev) || IS_GEN6(dev) || IS_IVYBRIDGE(dev)) {
+	if (IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
 		dev->max_vblank_count = 0xffffffff; /* full 32 bit counter */
 		dev->driver->get_vblank_counter = gm45_get_vblank_counter;
 	}
-
 	if (drm_core_check_feature(dev, DRIVER_MODESET))
 		dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
 	else
 		dev->driver->get_vblank_timestamp = NULL;
 	dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
 
-	if (IS_IVYBRIDGE(dev)) {
+	if (IS_VALLEYVIEW(dev)) {
+		dev->driver->irq_handler = valleyview_irq_handler;
+		dev->driver->irq_preinstall = valleyview_irq_preinstall;
+		dev->driver->irq_postinstall = valleyview_irq_postinstall;
+		dev->driver->irq_uninstall = valleyview_irq_uninstall;
+		dev->driver->enable_vblank = valleyview_enable_vblank;
+		dev->driver->disable_vblank = valleyview_disable_vblank;
+	} else if (IS_IVYBRIDGE(dev)) {
 		/* Share pre & uninstall handlers with ILK/SNB */
 		dev->driver->irq_handler = ivybridge_irq_handler;
 		dev->driver->irq_preinstall = ironlake_irq_preinstall;
@@ -1859,6 +2199,14 @@ intel_irq_init(struct drm_device *dev)
 		dev->driver->irq_uninstall = ironlake_irq_uninstall;
 		dev->driver->enable_vblank = ivybridge_enable_vblank;
 		dev->driver->disable_vblank = ivybridge_disable_vblank;
+	} else if (IS_HASWELL(dev)) {
+		/* Share interrupts handling with IVB */
+		dev->driver->irq_handler = ivybridge_irq_handler;
+		dev->driver->irq_preinstall = ironlake_irq_preinstall;
+		dev->driver->irq_postinstall = ivybridge_irq_postinstall;
+		dev->driver->irq_uninstall = ironlake_irq_uninstall;
+		dev->driver->enable_vblank = ivybridge_enable_vblank;
+		dev->driver->disable_vblank = ivybridge_disable_vblank;
 	} else if (HAS_PCH_SPLIT(dev)) {
 		dev->driver->irq_handler = ironlake_irq_handler;
 		dev->driver->irq_preinstall = ironlake_irq_preinstall;
@@ -1867,10 +2215,25 @@ intel_irq_init(struct drm_device *dev)
 		dev->driver->enable_vblank = ironlake_enable_vblank;
 		dev->driver->disable_vblank = ironlake_disable_vblank;
 	} else {
-		dev->driver->irq_preinstall = i915_driver_irq_preinstall;
-		dev->driver->irq_postinstall = i915_driver_irq_postinstall;
-		dev->driver->irq_uninstall = i915_driver_irq_uninstall;
-		dev->driver->irq_handler = i915_driver_irq_handler;
+		if (INTEL_INFO(dev)->gen == 2) {
+			dev->driver->irq_preinstall = i8xx_irq_preinstall;
+			dev->driver->irq_postinstall = i8xx_irq_postinstall;
+			dev->driver->irq_handler = i8xx_irq_handler;
+			dev->driver->irq_uninstall = i8xx_irq_uninstall;
+		} else if (INTEL_INFO(dev)->gen == 3) {
+			/* IIR "flip pending" means done if this bit is set */
+			I915_WRITE(ECOSKPD, _MASKED_BIT_DISABLE(ECO_FLIP_DONE));
+
+			dev->driver->irq_preinstall = i915_irq_preinstall;
+			dev->driver->irq_postinstall = i915_irq_postinstall;
+			dev->driver->irq_uninstall = i915_irq_uninstall;
+			dev->driver->irq_handler = i915_irq_handler;
+		} else {
+			dev->driver->irq_preinstall = i965_irq_preinstall;
+			dev->driver->irq_postinstall = i965_irq_postinstall;
+			dev->driver->irq_uninstall = i965_irq_uninstall;
+			dev->driver->irq_handler = i965_irq_handler;
+		}
 		dev->driver->enable_vblank = i915_enable_vblank;
 		dev->driver->disable_vblank = i915_disable_vblank;
 	}
@@ -1902,7 +2265,8 @@ i915_error_object_create(struct drm_i915_private *dev_priv,
 		if (d == NULL)
 			goto unwind;
 
-		if (reloc_offset < dev_priv->mm.gtt_mappable_end) {
+		if (reloc_offset < dev_priv->mm.gtt_mappable_end &&
+		    src->has_global_gtt_mapping) {
 			/* Simply ignore tiling or any overlapping fence.
 			 * It's part of the error state, and this hopefully
 			 * captures what the GPU read.
@@ -1960,9 +2324,8 @@ i915_error_object_free(struct drm_i915_error_object *obj)
 	free(obj, DRM_I915_GEM);
 }
 
-static void
-i915_error_state_free(struct drm_device *dev,
-		      struct drm_i915_error_state *error)
+void
+i915_error_state_free(struct drm_i915_error_state *error)
 {
 	int i;
 
@@ -1977,39 +2340,59 @@ i915_error_state_free(struct drm_device *dev,
 	free(error, DRM_I915_GEM);
 }
 
-static u32
-capture_bo_list(struct drm_i915_error_buffer *err, int count,
-    struct list_head *head)
+static void capture_bo(struct drm_i915_error_buffer *err,
+		       struct drm_i915_gem_object *obj)
+{
+	err->size = obj->base.size;
+	err->name = obj->base.name;
+	err->seqno = obj->last_rendering_seqno;
+	err->gtt_offset = obj->gtt_offset;
+	err->read_domains = obj->base.read_domains;
+	err->write_domain = obj->base.write_domain;
+	err->fence_reg = obj->fence_reg;
+	err->pinned = 0;
+	if (obj->pin_count > 0)
+		err->pinned = 1;
+	if (obj->user_pin_count > 0)
+		err->pinned = -1;
+	err->tiling = obj->tiling_mode;
+	err->dirty = obj->dirty;
+	err->purgeable = obj->madv != I915_MADV_WILLNEED;
+	err->ring = obj->ring ? obj->ring->id : -1;
+	err->cache_level = obj->cache_level;
+}
+
+static u32 capture_active_bo(struct drm_i915_error_buffer *err,
+			     int count, struct list_head *head)
 {
 	struct drm_i915_gem_object *obj;
 	int i = 0;
 
 	list_for_each_entry(obj, head, mm_list) {
-		err->size = obj->base.size;
-		err->name = obj->base.name;
-		err->seqno = obj->last_rendering_seqno;
-		err->gtt_offset = obj->gtt_offset;
-		err->read_domains = obj->base.read_domains;
-		err->write_domain = obj->base.write_domain;
-		err->fence_reg = obj->fence_reg;
-		err->pinned = 0;
-		if (obj->pin_count > 0)
-			err->pinned = 1;
-		if (obj->user_pin_count > 0)
-			err->pinned = -1;
-		err->tiling = obj->tiling_mode;
-		err->dirty = obj->dirty;
-		err->purgeable = obj->madv != I915_MADV_WILLNEED;
-		err->ring = obj->ring ? obj->ring->id : -1;
-		err->cache_level = obj->cache_level;
-
+		capture_bo(err++, obj);
 		if (++i == count)
 			break;
-
-		err++;
 	}
 
-	return (i);
+	return i;
+}
+
+static u32 capture_pinned_bo(struct drm_i915_error_buffer *err,
+			     int count, struct list_head *head)
+{
+	struct drm_i915_gem_object *obj;
+	int i = 0;
+
+	list_for_each_entry(obj, head, gtt_list) {
+		if (obj->pin_count == 0)
+			continue;
+
+		capture_bo(err++, obj);
+		if (++i == count)
+			break;
+	}
+
+	return i;
 }
 
 static void
@@ -2083,7 +2466,6 @@ i915_record_ring_state(struct drm_device *dev,
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
 	if (INTEL_INFO(dev)->gen >= 6) {
-		error->faddr[ring->id] = I915_READ(RING_DMA_FADD(ring->mmio_base));
 		error->fault_reg[ring->id] = I915_READ(RING_FAULT_REG(ring));
 		error->semaphore_mboxes[ring->id][0]
 			= I915_READ(RING_SYNC_0(ring->mmio_base));
@@ -2092,6 +2474,7 @@ i915_record_ring_state(struct drm_device *dev,
 	}
 
 	if (INTEL_INFO(dev)->gen >= 4) {
+		error->faddr[ring->id] = I915_READ(RING_DMA_FADD(ring->mmio_base));
 		error->ipeir[ring->id] = I915_READ(RING_IPEIR(ring->mmio_base));
 		error->ipehr[ring->id] = I915_READ(RING_IPEHR(ring->mmio_base));
 		error->instdone[ring->id] = I915_READ(RING_INSTDONE(ring->mmio_base));
@@ -2101,11 +2484,15 @@ i915_record_ring_state(struct drm_device *dev,
 			error->bbaddr = I915_READ64(BB_ADDR);
 		}
 	} else {
+		error->faddr[ring->id] = I915_READ(DMA_FADD_I8XX);
 		error->ipeir[ring->id] = I915_READ(IPEIR);
 		error->ipehr[ring->id] = I915_READ(IPEHR);
 		error->instdone[ring->id] = I915_READ(INSTDONE);
 	}
 
+	sleepq_lock(ring);
+	error->waiting[ring->id] = sleepq_sleepcnt(ring, 0) != 0;
+	sleepq_release(ring);
 	error->instpm[ring->id] = I915_READ(RING_INSTPM(ring->mmio_base));
 	error->seqno[ring->id] = ring->get_seqno(ring);
 	error->acthd[ring->id] = intel_ring_get_active_head(ring);
@@ -2121,15 +2508,11 @@ i915_gem_record_rings(struct drm_device *dev,
     struct drm_i915_error_state *error)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_ring_buffer *ring;
 	struct drm_i915_gem_request *request;
 	int i, count;
 
-	for (i = 0; i < I915_NUM_RINGS; i++) {
-		struct intel_ring_buffer *ring = &dev_priv->rings[i];
-
-		if (ring->obj == NULL)
-			continue;
-
+	for_each_ring(ring, dev_priv, i) {
 		i915_record_ring_state(dev, error, ring);
 
 		error->ring[i].batchbuffer =
@@ -2187,8 +2570,19 @@ i915_capture_error_state(struct drm_device *dev)
 	DRM_INFO("capturing error event; look for more information in "
 	    "sysctl hw.dri.%d.info.i915_error_state\n", dev->sysctl_node_idx);
 
+	refcount_init(&error->ref, 1);
 	error->eir = I915_READ(EIR);
 	error->pgtbl_er = I915_READ(PGTBL_ER);
+
+	if (HAS_PCH_SPLIT(dev))
+		error->ier = I915_READ(DEIER) | I915_READ(GTIER);
+	else if (IS_VALLEYVIEW(dev))
+		error->ier = I915_READ(GTIER) | I915_READ(VLV_IER);
+	else if (IS_GEN2(dev))
+		error->ier = I915_READ16(IER);
+	else
+		error->ier = I915_READ(IER);
+
 	for_each_pipe(pipe)
 		error->pipestat[pipe] = I915_READ(PIPESTAT(pipe));
 
@@ -2208,8 +2602,9 @@ i915_capture_error_state(struct drm_device *dev)
 	list_for_each_entry(obj, &dev_priv->mm.active_list, mm_list)
 		i++;
 	error->active_bo_count = i;
-	list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
-		i++;
+	list_for_each_entry(obj, &dev_priv->mm.gtt_list, mm_list)
+		if (obj->pin_count)
+			i++;
 	error->pinned_bo_count = i - error->active_bo_count;
 
 	error->active_bo = NULL;
@@ -2223,12 +2618,16 @@ i915_capture_error_state(struct drm_device *dev)
 	}
 
 	if (error->active_bo)
-		error->active_bo_count = capture_bo_list(error->active_bo,
-		    error->active_bo_count, &dev_priv->mm.active_list);
+		error->active_bo_count =
+			capture_active_bo(error->active_bo,
+					  error->active_bo_count,
+					  &dev_priv->mm.active_list);
 
 	if (error->pinned_bo)
-		error->pinned_bo_count = capture_bo_list(error->pinned_bo,
-		    error->pinned_bo_count, &dev_priv->mm.pinned_list);
+		error->pinned_bo_count =
+			capture_pinned_bo(error->pinned_bo,
+					  error->pinned_bo_count,
+					  &dev_priv->mm.gtt_list);
 
 	microtime(&error->time);
 
@@ -2243,7 +2642,7 @@ i915_capture_error_state(struct drm_device *dev)
 	mtx_unlock(&dev_priv->error_lock);
 
 	if (error != NULL)
-		i915_error_state_free(dev, error);
+		i915_error_state_free(error);
 }
 
 void
@@ -2257,6 +2656,6 @@ i915_destroy_error_state(struct drm_device *dev)
 	dev_priv->first_error = NULL;
 	mtx_unlock(&dev_priv->error_lock);
 
-	if (error != NULL)
-		i915_error_state_free(dev, error);
+	if (error != NULL && refcount_release(&error->ref))
+		i915_error_state_free(error);
 }
diff --git a/sys/dev/drm2/i915/i915_reg.h b/sys/dev/drm2/i915/i915_reg.h
index 492fac48042..0eed2767477 100644
--- a/sys/dev/drm2/i915/i915_reg.h
+++ b/sys/dev/drm2/i915/i915_reg.h
@@ -30,6 +30,11 @@ __FBSDID("$FreeBSD$");
 
 #define _PIPE(pipe, a, b) ((a) + (pipe)*((b)-(a)))
 
+#define _PORT(port, a, b) ((a) + (port)*((b)-(a)))
+
+#define _MASKED_BIT_ENABLE(a) (((a) << 16) | (a))
+#define _MASKED_BIT_DISABLE(a) ((a) << 16)
+
 /*
  * The Bridge device's PCI config space has information about the
  * fb aperture size and the amount of pre-reserved memory.
@@ -129,6 +134,13 @@ __FBSDID("$FreeBSD$");
 #define   ECOCHK_PPGTT_CACHE64B		(0x3<<3)
 #define   ECOCHK_PPGTT_CACHE4B		(0x0<<3)
 
+#define GAC_ECO_BITS			0x14090
+#define   ECOBITS_PPGTT_CACHE64B	(3<<8)
+#define   ECOBITS_PPGTT_CACHE4B		(0<<8)
+
+#define GAB_CTL				0x24000
+#define   GAB_CTL_CONT_AFTER_PAGEFAULT	(1<<8)
+
 /* VGA stuff */
 
 #define VGA_ST01_MDA 0x3ba
@@ -230,6 +242,7 @@ __FBSDID("$FreeBSD$");
 #define   MI_BATCH_NON_SECURE	(1)
 #define   MI_BATCH_NON_SECURE_I965 (1<<8)
 #define MI_BATCH_BUFFER_START	MI_INSTR(0x31, 0)
+#define   MI_BATCH_GTT		    (2<<6) /* aliased with (1<<7) on gen4 */
 #define MI_SEMAPHORE_MBOX	MI_INSTR(0x16, 1) /* gen6+ */
 #define  MI_SEMAPHORE_GLOBAL_GTT    (1<<22)
 #define  MI_SEMAPHORE_UPDATE	    (1<<21)
@@ -309,6 +322,61 @@ __FBSDID("$FreeBSD$");
 #define  DEBUG_RESET_RENDER		(1<<8)
 #define  DEBUG_RESET_DISPLAY		(1<<9)
 
+/*
+ * DPIO - a special bus for various display related registers to hide behind:
+ *  0x800c: m1, m2, n, p1, p2, k dividers
+ *  0x8014: REF and SFR select
+ *  0x8014: N divider, VCO select
+ *  0x801c/3c: core clock bits
+ *  0x8048/68: low pass filter coefficients
+ *  0x8100: fast clock controls
+ */
+#define DPIO_PKT			0x2100
+#define  DPIO_RID			(0<<24)
+#define  DPIO_OP_WRITE			(1<<16)
+#define  DPIO_OP_READ			(0<<16)
+#define  DPIO_PORTID			(0x12<<8)
+#define  DPIO_BYTE			(0xf<<4)
+#define  DPIO_BUSY			(1<<0) /* status only */
+#define DPIO_DATA			0x2104
+#define DPIO_REG			0x2108
+#define DPIO_CTL			0x2110
+#define  DPIO_MODSEL1			(1<<3) /* if ref clk b == 27 */
+#define  DPIO_MODSEL0			(1<<2) /* if ref clk a == 27 */
+#define  DPIO_SFR_BYPASS		(1<<1)
+#define  DPIO_RESET			(1<<0)
+
+#define _DPIO_DIV_A			0x800c
+#define   DPIO_POST_DIV_SHIFT		(28) /* 3 bits */
+#define   DPIO_K_SHIFT			(24) /* 4 bits */
+#define   DPIO_P1_SHIFT			(21) /* 3 bits */
+#define   DPIO_P2_SHIFT			(16) /* 5 bits */
+#define   DPIO_N_SHIFT			(12) /* 4 bits */
+#define   DPIO_ENABLE_CALIBRATION	(1<<11)
+#define   DPIO_M1DIV_SHIFT		(8) /* 3 bits */
+#define   DPIO_M2DIV_MASK		0xff
+#define _DPIO_DIV_B			0x802c
+#define DPIO_DIV(pipe) _PIPE(pipe, _DPIO_DIV_A, _DPIO_DIV_B)
+
+#define _DPIO_REFSFR_A			0x8014
+#define   DPIO_REFSEL_OVERRIDE		27
+#define   DPIO_PLL_MODESEL_SHIFT	24 /* 3 bits */
+#define   DPIO_BIAS_CURRENT_CTL_SHIFT	21 /* 3 bits, always 0x7 */
+#define   DPIO_PLL_REFCLK_SEL_SHIFT	16 /* 2 bits */
+#define   DPIO_DRIVER_CTL_SHIFT		12 /* always set to 0x8 */
+#define   DPIO_CLK_BIAS_CTL_SHIFT	8 /* always set to 0x5 */
+#define _DPIO_REFSFR_B			0x8034
+#define DPIO_REFSFR(pipe) _PIPE(pipe, _DPIO_REFSFR_A, _DPIO_REFSFR_B)
+
+#define _DPIO_CORE_CLK_A		0x801c
+#define _DPIO_CORE_CLK_B		0x803c
+#define DPIO_CORE_CLK(pipe) _PIPE(pipe, _DPIO_CORE_CLK_A, _DPIO_CORE_CLK_B)
+
+#define _DPIO_LFP_COEFF_A		0x8048
+#define _DPIO_LFP_COEFF_B		0x8068
+#define DPIO_LFP_COEFF(pipe) _PIPE(pipe, _DPIO_LFP_COEFF_A, _DPIO_LFP_COEFF_B)
+
+#define DPIO_FASTCLK_DISABLE		0x8100
 
 /*
  * Fence registers
@@ -368,8 +436,6 @@ __FBSDID("$FreeBSD$");
 #define ARB_MODE		0x04030
 #define   ARB_MODE_SWIZZLE_SNB	(1<<4)
 #define   ARB_MODE_SWIZZLE_IVB	(1<<5)
-#define   ARB_MODE_ENABLE(x)	GFX_MODE_ENABLE(x)
-#define   ARB_MODE_DISABLE(x)	GFX_MODE_DISABLE(x)
 #define RENDER_HWS_PGA_GEN7	(0x04080)
 #define RING_FAULT_REG(ring)	(0x4094 + 0x100*(ring)->id)
 #define DONE_REG		0x40b0
@@ -425,6 +491,7 @@ __FBSDID("$FreeBSD$");
 #define INSTDONE	0x02090
 #define NOPID		0x02094
 #define HWSTAM		0x02098
+#define DMA_FADD_I8XX	0x020d0
 
 #define ERROR_GEN6	0x040a0
 
@@ -440,6 +507,7 @@ __FBSDID("$FreeBSD$");
  */
 # define _3D_CHICKEN2_WM_READ_PIPELINED			(1 << 14)
 #define _3D_CHICKEN3	0x02090
+#define  _3D_CHICKEN_SF_DISABLE_FASTCLIP_CULL		(1 << 5)
 
 #define MI_MODE		0x0209c
 # define VS_TIMER_DISPATCH				(1 << 6)
@@ -455,14 +523,16 @@ __FBSDID("$FreeBSD$");
 #define   GFX_PSMI_GRANULARITY		(1<<10)
 #define   GFX_PPGTT_ENABLE		(1<<9)
 
-#define GFX_MODE_ENABLE(bit) (((bit) << 16) | (bit))
-#define GFX_MODE_DISABLE(bit) (((bit) << 16) | (0))
-
 #define SCPD0		0x0209c /* 915+ only */
 #define IER		0x020a0
 #define IIR		0x020a4
 #define IMR		0x020a8
 #define ISR		0x020ac
+#define VLV_IIR_RW	0x182084
+#define VLV_IER		0x1820a0
+#define VLV_IIR		0x1820a4
+#define VLV_IMR		0x1820a8
+#define VLV_ISR		0x1820ac
 #define   I915_PIPE_CONTROL_NOTIFY_INTERRUPT		(1<<18)
 #define   I915_DISPLAY_PORT_INTERRUPT			(1<<17)
 #define   I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT	(1<<15)
@@ -508,7 +578,6 @@ __FBSDID("$FreeBSD$");
 #define LM_BURST_LENGTH     0x00000700
 #define LM_FIFO_WATERMARK   0x0000001F
 #define MI_ARB_STATE	0x020e4 /* 915+ only */
-#define   MI_ARB_MASK_SHIFT	  16	/* shift for enable bits */
 
 /* Make render/texture TLB fetches lower priorty than associated data
  *   fetches. This is not turned on by default
@@ -573,7 +642,6 @@ __FBSDID("$FreeBSD$");
 #define   MI_ARB_DISPLAY_PRIORITY_B_A		(1 << 0)	/* display B > display A */
 
 #define CACHE_MODE_0	0x02120 /* 915+ only */
-#define   CM0_MASK_SHIFT          16
 #define   CM0_IZ_OPT_DISABLE      (1<<6)
 #define   CM0_ZR_OPT_DISABLE      (1<<5)
 #define	  CM0_STC_EVICT_DISABLE_LRA_SNB	(1<<5)
@@ -587,7 +655,12 @@ __FBSDID("$FreeBSD$");
 #define   ECO_GATING_CX_ONLY	(1<<3)
 #define   ECO_FLIP_DONE		(1<<0)
 
-/* GEN6 interrupt control */
+#define CACHE_MODE_1		0x7004 /* IVB+ */
+#define   PIXEL_SUBSPAN_COLLECT_OPT_DISABLE (1<<6)
+
+/* GEN6 interrupt control
+ * Note that the per-ring interrupt bits do alias with the global interrupt bits
+ * in GTIMR. */
 #define GEN6_RENDER_HWSTAM	0x2098
 #define GEN6_RENDER_IMR		0x20a8
 #define   GEN6_RENDER_CONTEXT_SWITCH_INTERRUPT		(1 << 8)
@@ -623,6 +696,21 @@ __FBSDID("$FreeBSD$");
 
 #define GEN6_BSD_RNCID			0x12198
 
+#define GEN7_FF_THREAD_MODE		0x20a0
+#define   GEN7_FF_SCHED_MASK		0x0077070
+#define   GEN7_FF_TS_SCHED_HS1		(0x5<<16)
+#define   GEN7_FF_TS_SCHED_HS0		(0x3<<16)
+#define   GEN7_FF_TS_SCHED_LOAD_BALANCE	(0x1<<16)
+#define   GEN7_FF_TS_SCHED_HW		(0x0<<16) /* Default */
+#define   GEN7_FF_VS_SCHED_HS1		(0x5<<12)
+#define   GEN7_FF_VS_SCHED_HS0		(0x3<<12)
+#define   GEN7_FF_VS_SCHED_LOAD_BALANCE	(0x1<<12) /* Default */
+#define   GEN7_FF_VS_SCHED_HW		(0x0<<12)
+#define   GEN7_FF_DS_SCHED_HS1		(0x5<<4)
+#define   GEN7_FF_DS_SCHED_HS0		(0x3<<4)
+#define   GEN7_FF_DS_SCHED_LOAD_BALANCE	(0x1<<4)  /* Default */
+#define   GEN7_FF_DS_SCHED_HW		(0x0<<4)
+
 /*
  * Framebuffer compression (915+ only)
  */
@@ -751,9 +839,9 @@ __FBSDID("$FreeBSD$");
 #define   GMBUS_PORT_PANEL	3
 #define   GMBUS_PORT_DPC	4 /* HDMIC */
 #define   GMBUS_PORT_DPB	5 /* SDVO, HDMIB */
-				  /* 6 reserved */
-#define   GMBUS_PORT_DPD	7 /* HDMID */
-#define   GMBUS_NUM_PORTS       8
+#define   GMBUS_PORT_DPD	6 /* HDMID */
+#define   GMBUS_PORT_RESERVED	7 /* 7 reserved */
+#define   GMBUS_NUM_PORTS	(GMBUS_PORT_DPD - GMBUS_PORT_SSC + 1)
 #define GMBUS1			0x5104 /* command/status */
 #define   GMBUS_SW_CLR_INT	(1<<31)
 #define   GMBUS_SW_RDY		(1<<30)
@@ -805,7 +893,9 @@ __FBSDID("$FreeBSD$");
 #define DPLL(pipe) _PIPE(pipe, _DPLL_A, _DPLL_B)
 #define   DPLL_VCO_ENABLE		(1 << 31)
 #define   DPLL_DVO_HIGH_SPEED		(1 << 30)
+#define   DPLL_EXT_BUFFER_ENABLE_VLV	(1 << 30)
 #define   DPLL_SYNCLOCK_ENABLE		(1 << 29)
+#define   DPLL_REFA_CLK_ENABLE_VLV	(1 << 29)
 #define   DPLL_VGA_MODE_DIS		(1 << 28)
 #define   DPLLB_MODE_DAC_SERIAL		(1 << 26) /* i915 */
 #define   DPLLB_MODE_LVDS		(2 << 26) /* i915 */
@@ -817,6 +907,7 @@ __FBSDID("$FreeBSD$");
 #define   DPLL_P2_CLOCK_DIV_MASK	0x03000000 /* i915 */
 #define   DPLL_FPA01_P1_POST_DIV_MASK	0x00ff0000 /* i915 */
 #define   DPLL_FPA01_P1_POST_DIV_MASK_PINEVIEW	0x00ff8000 /* Pineview */
+#define   DPLL_INTEGRATED_CLOCK_VLV	(1<<13)
 
 #define SRX_INDEX		0x3c4
 #define SRX_DATA		0x3c5
@@ -912,6 +1003,7 @@ __FBSDID("$FreeBSD$");
 #define   DPLL_MD_VGA_UDI_MULTIPLIER_SHIFT	0
 #define _DPLL_B_MD 0x06020 /* 965+ only */
 #define DPLL_MD(pipe) _PIPE(pipe, _DPLL_A_MD, _DPLL_B_MD)
+
 #define _FPA0	0x06040
 #define _FPA1	0x06044
 #define _FPB0	0x06048
@@ -1052,6 +1144,9 @@ __FBSDID("$FreeBSD$");
 #define RAMCLK_GATE_D		0x6210		/* CRL only */
 #define DEUC			0x6214          /* CRL only */
 
+#define FW_BLC_SELF_VLV		0x6500
+#define  FW_CSPWRDWNEN		(1<<15)
+
 /*
  * Palette regs
  */
@@ -1634,9 +1729,12 @@ __FBSDID("$FreeBSD$");
 /* Video Data Island Packet control */
 #define VIDEO_DIP_DATA		0x61178
 #define VIDEO_DIP_CTL		0x61170
+/* Pre HSW: */
 #define   VIDEO_DIP_ENABLE		(1 << 31)
 #define   VIDEO_DIP_PORT_B		(1 << 29)
 #define   VIDEO_DIP_PORT_C		(2 << 29)
+#define   VIDEO_DIP_PORT_D		(3 << 29)
+#define   VIDEO_DIP_PORT_MASK		(3 << 29)
 #define   VIDEO_DIP_ENABLE_AVI		(1 << 21)
 #define   VIDEO_DIP_ENABLE_VENDOR	(2 << 21)
 #define   VIDEO_DIP_ENABLE_SPD		(8 << 21)
@@ -1647,6 +1745,10 @@ __FBSDID("$FreeBSD$");
 #define   VIDEO_DIP_FREQ_ONCE		(0 << 16)
 #define   VIDEO_DIP_FREQ_VSYNC		(1 << 16)
 #define   VIDEO_DIP_FREQ_2VSYNC		(2 << 16)
+#define   VIDEO_DIP_FREQ_MASK		(3 << 16)
+/* HSW and later: */
+#define   VIDEO_DIP_ENABLE_AVI_HSW	(1 << 12)
+#define   VIDEO_DIP_ENABLE_SPD_HSW	(1 << 0)
 
 /* Panel power sequencing */
 #define PP_STATUS	0x61200
@@ -2413,7 +2515,8 @@ __FBSDID("$FreeBSD$");
 
 /* Pipe A */
 #define _PIPEADSL		0x70000
-#define   DSL_LINEMASK		0x00000fff
+#define   DSL_LINEMASK_GEN2	0x00000fff
+#define   DSL_LINEMASK_GEN3	0x00001fff
 #define _PIPEACONF		0x70008
 #define   PIPECONF_ENABLE	(1<<31)
 #define   PIPECONF_DISABLE	0
@@ -2455,23 +2558,30 @@ __FBSDID("$FreeBSD$");
 #define   PIPECONF_DITHER_TYPE_TEMP (3<<2)
 #define _PIPEASTAT		0x70024
 #define   PIPE_FIFO_UNDERRUN_STATUS		(1UL<<31)
+#define   SPRITE1_FLIPDONE_INT_EN_VLV		(1UL<<30)
 #define   PIPE_CRC_ERROR_ENABLE			(1UL<<29)
 #define   PIPE_CRC_DONE_ENABLE			(1UL<<28)
 #define   PIPE_GMBUS_EVENT_ENABLE		(1UL<<27)
+#define   PLANE_FLIP_DONE_INT_EN_VLV		(1UL<<26)
 #define   PIPE_HOTPLUG_INTERRUPT_ENABLE		(1UL<<26)
 #define   PIPE_VSYNC_INTERRUPT_ENABLE		(1UL<<25)
 #define   PIPE_DISPLAY_LINE_COMPARE_ENABLE	(1UL<<24)
 #define   PIPE_DPST_EVENT_ENABLE		(1UL<<23)
+#define   SPRITE0_FLIP_DONE_INT_EN_VLV		(1UL<<26)
 #define   PIPE_LEGACY_BLC_EVENT_ENABLE		(1UL<<22)
 #define   PIPE_ODD_FIELD_INTERRUPT_ENABLE	(1UL<<21)
 #define   PIPE_EVEN_FIELD_INTERRUPT_ENABLE	(1UL<<20)
 #define   PIPE_HOTPLUG_TV_INTERRUPT_ENABLE	(1UL<<18) /* pre-965 */
 #define   PIPE_START_VBLANK_INTERRUPT_ENABLE	(1UL<<18) /* 965 or later */
 #define   PIPE_VBLANK_INTERRUPT_ENABLE		(1UL<<17)
+#define   PIPEA_HBLANK_INT_EN_VLV		(1UL<<16)
 #define   PIPE_OVERLAY_UPDATED_ENABLE		(1UL<<16)
+#define   SPRITE1_FLIPDONE_INT_STATUS_VLV	(1UL<<15)
+#define   SPRITE0_FLIPDONE_INT_STATUS_VLV	(1UL<<15)
 #define   PIPE_CRC_ERROR_INTERRUPT_STATUS	(1UL<<13)
 #define   PIPE_CRC_DONE_INTERRUPT_STATUS	(1UL<<12)
 #define   PIPE_GMBUS_INTERRUPT_STATUS		(1UL<<11)
+#define   PLANE_FLIPDONE_INT_STATUS_VLV		(1UL<<10)
 #define   PIPE_HOTPLUG_INTERRUPT_STATUS		(1UL<<10)
 #define   PIPE_VSYNC_INTERRUPT_STATUS		(1UL<<9)
 #define   PIPE_DISPLAY_LINE_COMPARE_STATUS	(1UL<<8)
@@ -2496,6 +2606,40 @@ __FBSDID("$FreeBSD$");
 #define PIPEFRAMEPIXEL(pipe)  _PIPE(pipe, _PIPEAFRAMEPIXEL, _PIPEBFRAMEPIXEL)
 #define PIPESTAT(pipe) _PIPE(pipe, _PIPEASTAT, _PIPEBSTAT)
 
+#define VLV_DPFLIPSTAT				0x70028
+#define   PIPEB_LINE_COMPARE_STATUS		(1<<29)
+#define   PIPEB_HLINE_INT_EN			(1<<28)
+#define   PIPEB_VBLANK_INT_EN			(1<<27)
+#define   SPRITED_FLIPDONE_INT_EN		(1<<26)
+#define   SPRITEC_FLIPDONE_INT_EN		(1<<25)
+#define   PLANEB_FLIPDONE_INT_EN		(1<<24)
+#define   PIPEA_LINE_COMPARE_STATUS		(1<<21)
+#define   PIPEA_HLINE_INT_EN			(1<<20)
+#define   PIPEA_VBLANK_INT_EN			(1<<19)
+#define   SPRITEB_FLIPDONE_INT_EN		(1<<18)
+#define   SPRITEA_FLIPDONE_INT_EN		(1<<17)
+#define   PLANEA_FLIPDONE_INT_EN		(1<<16)
+
+#define DPINVGTT				0x7002c /* VLV only */
+#define   CURSORB_INVALID_GTT_INT_EN		(1<<23)
+#define   CURSORA_INVALID_GTT_INT_EN		(1<<22)
+#define   SPRITED_INVALID_GTT_INT_EN		(1<<21)
+#define   SPRITEC_INVALID_GTT_INT_EN		(1<<20)
+#define   PLANEB_INVALID_GTT_INT_EN		(1<<19)
+#define   SPRITEB_INVALID_GTT_INT_EN		(1<<18)
+#define   SPRITEA_INVALID_GTT_INT_EN		(1<<17)
+#define   PLANEA_INVALID_GTT_INT_EN		(1<<16)
+#define   DPINVGTT_EN_MASK			0xff0000
+#define   CURSORB_INVALID_GTT_STATUS		(1<<7)
+#define   CURSORA_INVALID_GTT_STATUS		(1<<6)
+#define   SPRITED_INVALID_GTT_STATUS		(1<<5)
+#define   SPRITEC_INVALID_GTT_STATUS		(1<<4)
+#define   PLANEB_INVALID_GTT_STATUS		(1<<3)
+#define   SPRITEB_INVALID_GTT_STATUS		(1<<2)
+#define   SPRITEA_INVALID_GTT_STATUS		(1<<1)
+#define   PLANEA_INVALID_GTT_STATUS		(1<<0)
+#define   DPINVGTT_STATUS_MASK			0xff
+
 #define DSPARB			0x70030
 #define   DSPARB_CSTART_MASK	(0x7f << 7)
 #define   DSPARB_CSTART_SHIFT	7
@@ -2525,11 +2669,28 @@ __FBSDID("$FreeBSD$");
 #define   DSPFW_HPLL_CURSOR_MASK	(0x3f<<16)
 #define   DSPFW_HPLL_SR_MASK		(0x1ff)
 
+/* drain latency register values*/
+#define DRAIN_LATENCY_PRECISION_32	32
+#define DRAIN_LATENCY_PRECISION_16	16
+#define VLV_DDL1			0x70050
+#define DDL_CURSORA_PRECISION_32	(1<<31)
+#define DDL_CURSORA_PRECISION_16	(0<<31)
+#define DDL_CURSORA_SHIFT		24
+#define DDL_PLANEA_PRECISION_32		(1<<7)
+#define DDL_PLANEA_PRECISION_16		(0<<7)
+#define VLV_DDL2			0x70054
+#define DDL_CURSORB_PRECISION_32	(1<<31)
+#define DDL_CURSORB_PRECISION_16	(0<<31)
+#define DDL_CURSORB_SHIFT		24
+#define DDL_PLANEB_PRECISION_32		(1<<7)
+#define DDL_PLANEB_PRECISION_16		(0<<7)
+
 /* FIFO watermark sizes etc */
 #define G4X_FIFO_LINE_SIZE	64
 #define I915_FIFO_LINE_SIZE	64
 #define I830_FIFO_LINE_SIZE	32
 
+#define VALLEYVIEW_FIFO_SIZE	255
 #define G4X_FIFO_SIZE		127
 #define I965_FIFO_SIZE		512
 #define I945_FIFO_SIZE		127
@@ -2537,6 +2698,7 @@ __FBSDID("$FreeBSD$");
 #define I855GM_FIFO_SIZE	127 /* In cachelines */
 #define I830_FIFO_SIZE		95
 
+#define VALLEYVIEW_MAX_WM	0xff
 #define G4X_MAX_WM		0x3f
 #define I915_MAX_WM		0x3f
 
@@ -2551,6 +2713,7 @@ __FBSDID("$FreeBSD$");
 #define PINEVIEW_CURSOR_DFT_WM	0
 #define PINEVIEW_CURSOR_GUARD_WM	5
 
+#define VALLEYVIEW_CURSOR_MAX_WM 64
 #define I965_CURSOR_FIFO	64
 #define I965_CURSOR_MAX_WM	32
 #define I965_CURSOR_DFT_WM	8
@@ -2759,6 +2922,13 @@ __FBSDID("$FreeBSD$");
 #define DSPSURF(plane) _PIPE(plane, _DSPASURF, _DSPBSURF)
 #define DSPTILEOFF(plane) _PIPE(plane, _DSPATILEOFF, _DSPBTILEOFF)
 
+/* Display/Sprite base address macros */
+#define DISP_BASEADDR_MASK	(0xfffff000)
+#define I915_LO_DISPBASE(val)	(val & ~DISP_BASEADDR_MASK)
+#define I915_HI_DISPBASE(val)	(val & DISP_BASEADDR_MASK)
+#define I915_MODIFY_DISPBASE(reg, gfx_addr) \
+		(I915_WRITE(reg, gfx_addr | I915_LO_DISPBASE(I915_READ(reg))))
+
 /* VBIOS flags */
 #define SWF00			0x71410
 #define SWF01			0x71414
@@ -3091,26 +3261,38 @@ __FBSDID("$FreeBSD$");
 #define DE_PCH_EVENT_IVB		(1<<28)
 #define DE_DP_A_HOTPLUG_IVB		(1<<27)
 #define DE_AUX_CHANNEL_A_IVB		(1<<26)
+#define DE_SPRITEC_FLIP_DONE_IVB	(1<<14)
+#define DE_PLANEC_FLIP_DONE_IVB		(1<<13)
+#define DE_PIPEC_VBLANK_IVB		(1<<10)
 #define DE_SPRITEB_FLIP_DONE_IVB	(1<<9)
-#define DE_SPRITEA_FLIP_DONE_IVB	(1<<4)
 #define DE_PLANEB_FLIP_DONE_IVB		(1<<8)
-#define DE_PLANEA_FLIP_DONE_IVB		(1<<3)
 #define DE_PIPEB_VBLANK_IVB		(1<<5)
+#define DE_SPRITEA_FLIP_DONE_IVB	(1<<4)
+#define DE_PLANEA_FLIP_DONE_IVB		(1<<3)
 #define DE_PIPEA_VBLANK_IVB		(1<<0)
 
+#define VLV_MASTER_IER			0x4400c /* Gunit master IER */
+#define   MASTER_INTERRUPT_ENABLE	(1<<31)
+
 #define DEISR   0x44000
 #define DEIMR   0x44004
 #define DEIIR   0x44008
 #define DEIER   0x4400c
 
-/* GT interrupt */
-#define GT_PIPE_NOTIFY		(1 << 4)
-#define GT_RENDER_CS_ERROR	(1 << 3)
-#define GT_SYNC_STATUS          (1 << 2)
-#define GT_USER_INTERRUPT       (1 << 0)
-#define GT_BSD_USER_INTERRUPT   (1 << 5)
-#define GT_GEN6_BSD_USER_INTERRUPT	(1 << 12)
-#define GT_BLT_USER_INTERRUPT	(1 << 22)
+/* GT interrupt.
+ * Note that for gen6+ the ring-specific interrupt bits do alias with the
+ * corresponding bits in the per-ring interrupt control registers. */
+#define GT_GEN6_BLT_FLUSHDW_NOTIFY_INTERRUPT	(1 << 26)
+#define GT_GEN6_BLT_CS_ERROR_INTERRUPT		(1 << 25)
+#define GT_GEN6_BLT_USER_INTERRUPT		(1 << 22)
+#define GT_GEN6_BSD_CS_ERROR_INTERRUPT		(1 << 15)
+#define GT_GEN6_BSD_USER_INTERRUPT		(1 << 12)
+#define GT_BSD_USER_INTERRUPT			(1 << 5) /* ilk only */
+#define GT_GEN7_L3_PARITY_ERROR_INTERRUPT	(1 << 5)
+#define GT_PIPE_NOTIFY				(1 << 4)
+#define GT_RENDER_CS_ERROR_INTERRUPT		(1 << 3)
+#define GT_SYNC_STATUS				(1 << 2)
+#define GT_USER_INTERRUPT			(1 << 0)
 
 #define GTISR   0x44010
 #define GTIMR   0x44014
@@ -3260,15 +3442,15 @@ __FBSDID("$FreeBSD$");
 
 #define _PCH_DPLL_A              0xc6014
 #define _PCH_DPLL_B              0xc6018
-#define PCH_DPLL(pipe) (pipe == 0 ?  _PCH_DPLL_A : _PCH_DPLL_B)
+#define _PCH_DPLL(pll) (pll == 0 ? _PCH_DPLL_A : _PCH_DPLL_B)
 
 #define _PCH_FPA0                0xc6040
 #define  FP_CB_TUNE		(0x3<<22)
 #define _PCH_FPA1                0xc6044
 #define _PCH_FPB0                0xc6048
 #define _PCH_FPB1                0xc604c
-#define PCH_FP0(pipe) (pipe == 0 ? _PCH_FPA0 : _PCH_FPB0)
-#define PCH_FP1(pipe) (pipe == 0 ? _PCH_FPA1 : _PCH_FPB1)
+#define _PCH_FP0(pll) (pll == 0 ? _PCH_FPA0 : _PCH_FPB0)
+#define _PCH_FP1(pll) (pll == 0 ? _PCH_FPA1 : _PCH_FPB1)
 
 #define PCH_DPLL_TEST           0xc606c
 
@@ -3363,6 +3545,57 @@ __FBSDID("$FreeBSD$");
 #define TVIDEO_DIP_DATA(pipe) _PIPE(pipe, _VIDEO_DIP_DATA_A, _VIDEO_DIP_DATA_B)
 #define TVIDEO_DIP_GCP(pipe) _PIPE(pipe, _VIDEO_DIP_GCP_A, _VIDEO_DIP_GCP_B)
 
+#define VLV_VIDEO_DIP_CTL_A		0x60220
+#define VLV_VIDEO_DIP_DATA_A		0x60208
+#define VLV_VIDEO_DIP_GDCP_PAYLOAD_A	0x60210
+
+#define VLV_VIDEO_DIP_CTL_B		0x61170
+#define VLV_VIDEO_DIP_DATA_B		0x61174
+#define VLV_VIDEO_DIP_GDCP_PAYLOAD_B	0x61178
+
+#define VLV_TVIDEO_DIP_CTL(pipe) \
+	 _PIPE(pipe, VLV_VIDEO_DIP_CTL_A, VLV_VIDEO_DIP_CTL_B)
+#define VLV_TVIDEO_DIP_DATA(pipe) \
+	 _PIPE(pipe, VLV_VIDEO_DIP_DATA_A, VLV_VIDEO_DIP_DATA_B)
+#define VLV_TVIDEO_DIP_GCP(pipe) \
+	_PIPE(pipe, VLV_VIDEO_DIP_GDCP_PAYLOAD_A, VLV_VIDEO_DIP_GDCP_PAYLOAD_B)
+
+/* Haswell DIP controls */
+#define HSW_VIDEO_DIP_CTL_A		0x60200
+#define HSW_VIDEO_DIP_AVI_DATA_A	0x60220
+#define HSW_VIDEO_DIP_VS_DATA_A		0x60260
+#define HSW_VIDEO_DIP_SPD_DATA_A	0x602A0
+#define HSW_VIDEO_DIP_GMP_DATA_A	0x602E0
+#define HSW_VIDEO_DIP_VSC_DATA_A	0x60320
+#define HSW_VIDEO_DIP_AVI_ECC_A		0x60240
+#define HSW_VIDEO_DIP_VS_ECC_A		0x60280
+#define HSW_VIDEO_DIP_SPD_ECC_A		0x602C0
+#define HSW_VIDEO_DIP_GMP_ECC_A		0x60300
+#define HSW_VIDEO_DIP_VSC_ECC_A		0x60344
+#define HSW_VIDEO_DIP_GCP_A		0x60210
+
+#define HSW_VIDEO_DIP_CTL_B		0x61200
+#define HSW_VIDEO_DIP_AVI_DATA_B	0x61220
+#define HSW_VIDEO_DIP_VS_DATA_B		0x61260
+#define HSW_VIDEO_DIP_SPD_DATA_B	0x612A0
+#define HSW_VIDEO_DIP_GMP_DATA_B	0x612E0
+#define HSW_VIDEO_DIP_VSC_DATA_B	0x61320
+#define HSW_VIDEO_DIP_BVI_ECC_B		0x61240
+#define HSW_VIDEO_DIP_VS_ECC_B		0x61280
+#define HSW_VIDEO_DIP_SPD_ECC_B		0x612C0
+#define HSW_VIDEO_DIP_GMP_ECC_B		0x61300
+#define HSW_VIDEO_DIP_VSC_ECC_B		0x61344
+#define HSW_VIDEO_DIP_GCP_B		0x61210
+
+#define HSW_TVIDEO_DIP_CTL(pipe) \
+	 _PIPE(pipe, HSW_VIDEO_DIP_CTL_A, HSW_VIDEO_DIP_CTL_B)
+#define HSW_TVIDEO_DIP_AVI_DATA(pipe) \
+	 _PIPE(pipe, HSW_VIDEO_DIP_AVI_DATA_A, HSW_VIDEO_DIP_AVI_DATA_B)
+#define HSW_TVIDEO_DIP_SPD_DATA(pipe) \
+	 _PIPE(pipe, HSW_VIDEO_DIP_SPD_DATA_A, HSW_VIDEO_DIP_SPD_DATA_B)
+#define HSW_TVIDEO_DIP_GCP(pipe) \
+	_PIPE(pipe, HSW_VIDEO_DIP_GCP_A, HSW_VIDEO_DIP_GCP_B)
+
 #define _TRANS_HTOTAL_B          0xe1000
 #define _TRANS_HBLANK_B          0xe1004
 #define _TRANS_HSYNC_B           0xe1008
@@ -3522,6 +3755,9 @@ __FBSDID("$FreeBSD$");
 #define  FDI_LINK_TRAIN_PATTERN_IDLE_CPT	(2<<8)
 #define  FDI_LINK_TRAIN_NORMAL_CPT		(3<<8)
 #define  FDI_LINK_TRAIN_PATTERN_MASK_CPT	(3<<8)
+/* LPT */
+#define  FDI_PORT_WIDTH_2X_LPT			(1<<19)
+#define  FDI_PORT_WIDTH_1X_LPT			(0<<19)
 
 #define _FDI_RXA_MISC            0xf0010
 #define _FDI_RXB_MISC            0xf1010
@@ -3582,6 +3818,7 @@ __FBSDID("$FreeBSD$");
 #define  ADPA_CRT_HOTPLUG_FORCE_TRIGGER (1<<16)
 
 /* or SDVOB */
+#define VLV_HDMIB 0x61140
 #define HDMIB   0xe1140
 #define  PORT_ENABLE    (1 << 31)
 #define  TRANSCODER(pipe)       ((pipe) << 30)
@@ -3747,6 +3984,8 @@ __FBSDID("$FreeBSD$");
 #define  EDP_LINK_TRAIN_VOL_EMP_MASK_IVB	(0x3f<<22)
 
 #define  FORCEWAKE				0xA18C
+#define  FORCEWAKE_VLV				0x1300b0
+#define  FORCEWAKE_ACK_VLV			0x1300b4
 #define  FORCEWAKE_ACK				0x130090
 #define  FORCEWAKE_MT				0xa188 /* multi-threaded */
 #define  FORCEWAKE_MT_ACK			0x130040
@@ -3764,6 +4003,7 @@ __FBSDID("$FreeBSD$");
 
 #define GEN6_UCGCTL1				0x9400
 # define GEN6_BLBUNIT_CLOCK_GATE_DISABLE		(1 << 5)
+# define GEN6_CSUNIT_CLOCK_GATE_DISABLE			(1 << 7)
 
 #define GEN6_UCGCTL2				0x9404
 # define GEN6_RCZUNIT_CLOCK_GATE_DISABLE		(1 << 13)
@@ -3841,8 +4081,13 @@ __FBSDID("$FreeBSD$");
 #define  GEN6_PM_RP_UP_EI_EXPIRED		(1<<2)
 #define  GEN6_PM_RP_DOWN_EI_EXPIRED		(1<<1)
 #define  GEN6_PM_DEFERRED_EVENTS     (GEN6_PM_RP_UP_THRESHOLD | \
-                  GEN6_PM_RP_DOWN_THRESHOLD | \
-                  GEN6_PM_RP_DOWN_TIMEOUT)
+						 GEN6_PM_RP_DOWN_THRESHOLD | \
+						 GEN6_PM_RP_DOWN_TIMEOUT)
+
+#define GEN6_GT_GFX_RC6_LOCKED			0x138104
+#define GEN6_GT_GFX_RC6				0x138108
+#define GEN6_GT_GFX_RC6p			0x13810C
+#define GEN6_GT_GFX_RC6pp			0x138110
 
 #define GEN6_PCODE_MAILBOX			0x138124
 #define   GEN6_PCODE_READY			(1<<31)
@@ -3903,4 +4148,197 @@ __FBSDID("$FreeBSD$");
 #define   AUD_CONFIG_PIXEL_CLOCK_HDMI		(0xf << 16)
 #define   AUD_CONFIG_DISABLE_NCTS		(1 << 3)
 
+/* HSW Power Wells */
+#define HSW_PWR_WELL_CTL1		0x45400		/* BIOS */
+#define HSW_PWR_WELL_CTL2		0x45404		/* Driver */
+#define HSW_PWR_WELL_CTL3		0x45408		/* KVMR */
+#define HSW_PWR_WELL_CTL4		0x4540C		/* Debug */
+#define   HSW_PWR_WELL_ENABLE				(1<<31)
+#define   HSW_PWR_WELL_STATE				(1<<30)
+#define HSW_PWR_WELL_CTL5		0x45410
+#define   HSW_PWR_WELL_ENABLE_SINGLE_STEP	(1<<31)
+#define   HSW_PWR_WELL_PWR_GATE_OVERRIDE	(1<<20)
+#define   HSW_PWR_WELL_FORCE_ON				(1<<19)
+#define HSW_PWR_WELL_CTL6		0x45414
+
+/* Per-pipe DDI Function Control */
+#define PIPE_DDI_FUNC_CTL_A			0x60400
+#define PIPE_DDI_FUNC_CTL_B			0x61400
+#define PIPE_DDI_FUNC_CTL_C			0x62400
+#define PIPE_DDI_FUNC_CTL_EDP		0x6F400
+#define DDI_FUNC_CTL(pipe) _PIPE(pipe, \
+					PIPE_DDI_FUNC_CTL_A, \
+					PIPE_DDI_FUNC_CTL_B)
+#define  PIPE_DDI_FUNC_ENABLE		(1<<31)
+/* Those bits are ignored by pipe EDP since it can only connect to DDI A */
+#define  PIPE_DDI_PORT_MASK				(0xf<<28)
+#define  PIPE_DDI_SELECT_PORT(x)		((x)<<28)
+#define  PIPE_DDI_MODE_SELECT_HDMI		(0<<24)
+#define  PIPE_DDI_MODE_SELECT_DVI		(1<<24)
+#define  PIPE_DDI_MODE_SELECT_DP_SST	(2<<24)
+#define  PIPE_DDI_MODE_SELECT_DP_MST	(3<<24)
+#define  PIPE_DDI_MODE_SELECT_FDI		(4<<24)
+#define  PIPE_DDI_BPC_8					(0<<20)
+#define  PIPE_DDI_BPC_10				(1<<20)
+#define  PIPE_DDI_BPC_6					(2<<20)
+#define  PIPE_DDI_BPC_12				(3<<20)
+#define  PIPE_DDI_BFI_ENABLE			(1<<4)
+#define  PIPE_DDI_PORT_WIDTH_X1			(0<<1)
+#define  PIPE_DDI_PORT_WIDTH_X2			(1<<1)
+#define  PIPE_DDI_PORT_WIDTH_X4			(3<<1)
+
+/* DisplayPort Transport Control */
+#define DP_TP_CTL_A			0x64040
+#define DP_TP_CTL_B			0x64140
+#define DP_TP_CTL(port) _PORT(port, \
+					DP_TP_CTL_A, \
+					DP_TP_CTL_B)
+#define  DP_TP_CTL_ENABLE		(1<<31)
+#define  DP_TP_CTL_MODE_SST	(0<<27)
+#define  DP_TP_CTL_MODE_MST	(1<<27)
+#define  DP_TP_CTL_ENHANCED_FRAME_ENABLE	(1<<18)
+#define  DP_TP_CTL_FDI_AUTOTRAIN	(1<<15)
+#define  DP_TP_CTL_LINK_TRAIN_MASK		(7<<8)
+#define  DP_TP_CTL_LINK_TRAIN_PAT1		(0<<8)
+#define  DP_TP_CTL_LINK_TRAIN_PAT2		(1<<8)
+#define  DP_TP_CTL_LINK_TRAIN_NORMAL	(3<<8)
+
+/* DisplayPort Transport Status */
+#define DP_TP_STATUS_A			0x64044
+#define DP_TP_STATUS_B			0x64144
+#define DP_TP_STATUS(port) _PORT(port, \
+					DP_TP_STATUS_A, \
+					DP_TP_STATUS_B)
+#define  DP_TP_STATUS_AUTOTRAIN_DONE	(1<<12)
+
+/* DDI Buffer Control */
+#define DDI_BUF_CTL_A				0x64000
+#define DDI_BUF_CTL_B				0x64100
+#define DDI_BUF_CTL(port) _PORT(port, \
+					DDI_BUF_CTL_A, \
+					DDI_BUF_CTL_B)
+#define  DDI_BUF_CTL_ENABLE				(1<<31)
+#define  DDI_BUF_EMP_400MV_0DB_HSW		(0<<24)   /* Sel0 */
+#define  DDI_BUF_EMP_400MV_3_5DB_HSW	(1<<24)   /* Sel1 */
+#define  DDI_BUF_EMP_400MV_6DB_HSW		(2<<24)   /* Sel2 */
+#define  DDI_BUF_EMP_400MV_9_5DB_HSW	(3<<24)   /* Sel3 */
+#define  DDI_BUF_EMP_600MV_0DB_HSW		(4<<24)   /* Sel4 */
+#define  DDI_BUF_EMP_600MV_3_5DB_HSW	(5<<24)   /* Sel5 */
+#define  DDI_BUF_EMP_600MV_6DB_HSW		(6<<24)   /* Sel6 */
+#define  DDI_BUF_EMP_800MV_0DB_HSW		(7<<24)   /* Sel7 */
+#define  DDI_BUF_EMP_800MV_3_5DB_HSW	(8<<24)   /* Sel8 */
+#define  DDI_BUF_EMP_MASK				(0xf<<24)
+#define  DDI_BUF_IS_IDLE				(1<<7)
+#define  DDI_PORT_WIDTH_X1				(0<<1)
+#define  DDI_PORT_WIDTH_X2				(1<<1)
+#define  DDI_PORT_WIDTH_X4				(3<<1)
+#define  DDI_INIT_DISPLAY_DETECTED		(1<<0)
+
+/* DDI Buffer Translations */
+#define DDI_BUF_TRANS_A				0x64E00
+#define DDI_BUF_TRANS_B				0x64E60
+#define DDI_BUF_TRANS(port) _PORT(port, \
+					DDI_BUF_TRANS_A, \
+					DDI_BUF_TRANS_B)
+
+/* Sideband Interface (SBI) is programmed indirectly, via
+ * SBI_ADDR, which contains the register offset; and SBI_DATA,
+ * which contains the payload */
+#define SBI_ADDR				0xC6000
+#define SBI_DATA				0xC6004
+#define SBI_CTL_STAT			0xC6008
+#define  SBI_CTL_OP_CRRD		(0x6<<8)
+#define  SBI_CTL_OP_CRWR		(0x7<<8)
+#define  SBI_RESPONSE_FAIL		(0x1<<1)
+#define  SBI_RESPONSE_SUCCESS	(0x0<<1)
+#define  SBI_BUSY				(0x1<<0)
+#define  SBI_READY				(0x0<<0)
+
+/* SBI offsets */
+#define  SBI_SSCDIVINTPHASE6		0x0600
+#define   SBI_SSCDIVINTPHASE_DIVSEL_MASK	((0x7f)<<1)
+#define   SBI_SSCDIVINTPHASE_DIVSEL(x)		((x)<<1)
+#define   SBI_SSCDIVINTPHASE_INCVAL_MASK	((0x7f)<<8)
+#define   SBI_SSCDIVINTPHASE_INCVAL(x)		((x)<<8)
+#define   SBI_SSCDIVINTPHASE_DIR(x)			((x)<<15)
+#define   SBI_SSCDIVINTPHASE_PROPAGATE		(1<<0)
+#define  SBI_SSCCTL					0x020c
+#define  SBI_SSCCTL6				0x060C
+#define   SBI_SSCCTL_DISABLE		(1<<0)
+#define  SBI_SSCAUXDIV6				0x0610
+#define   SBI_SSCAUXDIV_FINALDIV2SEL(x)		((x)<<4)
+#define  SBI_DBUFF0					0x2a00
+
+/* LPT PIXCLK_GATE */
+#define PIXCLK_GATE				0xC6020
+#define  PIXCLK_GATE_UNGATE		1<<0
+#define  PIXCLK_GATE_GATE		0<<0
+
+/* SPLL */
+#define SPLL_CTL				0x46020
+#define  SPLL_PLL_ENABLE		(1<<31)
+#define  SPLL_PLL_SCC			(1<<28)
+#define  SPLL_PLL_NON_SCC		(2<<28)
+#define  SPLL_PLL_FREQ_810MHz	(0<<26)
+#define  SPLL_PLL_FREQ_1350MHz	(1<<26)
+
+/* WRPLL */
+#define WRPLL_CTL1				0x46040
+#define WRPLL_CTL2				0x46060
+#define  WRPLL_PLL_ENABLE				(1<<31)
+#define  WRPLL_PLL_SELECT_SSC			(0x01<<28)
+#define  WRPLL_PLL_SELECT_NON_SCC		(0x02<<28)
+#define  WRPLL_PLL_SELECT_LCPLL_2700	(0x03<<28)
+/* WRPLL divider programming */
+#define  WRPLL_DIVIDER_REFERENCE(x)		((x)<<0)
+#define  WRPLL_DIVIDER_POST(x)			((x)<<8)
+#define  WRPLL_DIVIDER_FEEDBACK(x)		((x)<<16)
+
+/* Port clock selection */
+#define PORT_CLK_SEL_A			0x46100
+#define PORT_CLK_SEL_B			0x46104
+#define PORT_CLK_SEL(port) _PORT(port, \
+					PORT_CLK_SEL_A, \
+					PORT_CLK_SEL_B)
+#define  PORT_CLK_SEL_LCPLL_2700	(0<<29)
+#define  PORT_CLK_SEL_LCPLL_1350	(1<<29)
+#define  PORT_CLK_SEL_LCPLL_810		(2<<29)
+#define  PORT_CLK_SEL_SPLL			(3<<29)
+#define  PORT_CLK_SEL_WRPLL1		(4<<29)
+#define  PORT_CLK_SEL_WRPLL2		(5<<29)
+
+/* Pipe clock selection */
+#define PIPE_CLK_SEL_A			0x46140
+#define PIPE_CLK_SEL_B			0x46144
+#define PIPE_CLK_SEL(pipe) _PIPE(pipe, \
+					PIPE_CLK_SEL_A, \
+					PIPE_CLK_SEL_B)
+/* For each pipe, we need to select the corresponding port clock */
+#define  PIPE_CLK_SEL_DISABLED	(0x0<<29)
+#define  PIPE_CLK_SEL_PORT(x)	((x+1)<<29)
+
+/* LCPLL Control */
+#define LCPLL_CTL				0x130040
+#define  LCPLL_PLL_DISABLE		(1<<31)
+#define  LCPLL_PLL_LOCK			(1<<30)
+#define  LCPLL_CD_CLOCK_DISABLE	(1<<25)
+#define  LCPLL_CD2X_CLOCK_DISABLE	(1<<23)
+
+/* Pipe WM_LINETIME - watermark line time */
+#define PIPE_WM_LINETIME_A		0x45270
+#define PIPE_WM_LINETIME_B		0x45274
+#define PIPE_WM_LINETIME(pipe) _PIPE(pipe, \
+					PIPE_WM_LINETIME_A, \
+					PIPE_WM_LINETIME_A)
+#define   PIPE_WM_LINETIME_MASK		(0x1ff)
+#define   PIPE_WM_LINETIME_TIME(x)			((x))
+#define   PIPE_WM_LINETIME_IPS_LINETIME_MASK	(0x1ff<<16)
+#define   PIPE_WM_LINETIME_IPS_LINETIME(x)		((x)<<16)
+
+/* SFUSE_STRAP */
+#define SFUSE_STRAP				0xc2014
+#define  SFUSE_STRAP_DDIB_DETECTED	(1<<2)
+#define  SFUSE_STRAP_DDIC_DETECTED	(1<<1)
+#define  SFUSE_STRAP_DDID_DETECTED	(1<<0)
+
 #endif /* _I915_REG_H_ */
diff --git a/sys/dev/drm2/i915/i915_suspend.c b/sys/dev/drm2/i915/i915_suspend.c
index 1e219a1612a..fd31120233e 100644
--- a/sys/dev/drm2/i915/i915_suspend.c
+++ b/sys/dev/drm2/i915/i915_suspend.c
@@ -42,7 +42,7 @@ static bool i915_pipe_enabled(struct drm_device *dev, enum pipe pipe)
 		return false;
 
 	if (HAS_PCH_SPLIT(dev))
-		dpll_reg = PCH_DPLL(pipe);
+		dpll_reg = _PCH_DPLL(pipe);
 	else
 		dpll_reg = (pipe == PIPE_A) ? _DPLL_A : _DPLL_B;
 
@@ -873,22 +873,6 @@ int i915_restore_state(struct drm_device *dev)
 		I915_WRITE(IER, dev_priv->saveIER);
 		I915_WRITE(IMR, dev_priv->saveIMR);
 	}
-	DRM_UNLOCK(dev);
-
-	if (drm_core_check_feature(dev, DRIVER_MODESET))
-		intel_init_clock_gating(dev);
-
-	if (IS_IRONLAKE_M(dev)) {
-		ironlake_enable_drps(dev);
-		intel_init_emon(dev);
-	}
-
-	if (INTEL_INFO(dev)->gen >= 6) {
-		gen6_enable_rps(dev_priv);
-		gen6_update_ring_freq(dev_priv);
-	}
-
-	DRM_LOCK(dev);
 
 	/* Cache mode state */
 	I915_WRITE(CACHE_MODE_0, dev_priv->saveCACHE_MODE_0 | 0xffff0000);
diff --git a/sys/dev/drm2/i915/intel_bios.c b/sys/dev/drm2/i915/intel_bios.c
index 8bf38a5722c..04997944290 100644
--- a/sys/dev/drm2/i915/intel_bios.c
+++ b/sys/dev/drm2/i915/intel_bios.c
@@ -175,6 +175,28 @@ get_lvds_dvo_timing(const struct bdb_lvds_lfp_data *lvds_lfp_data,
 	return (const struct lvds_dvo_timing *)(entry + dvo_timing_offset);
 }
 
+/* get lvds_fp_timing entry
+ * this function may return NULL if the corresponding entry is invalid
+ */
+static const struct lvds_fp_timing *
+get_lvds_fp_timing(const struct bdb_header *bdb,
+		   const struct bdb_lvds_lfp_data *data,
+		   const struct bdb_lvds_lfp_data_ptrs *ptrs,
+		   int index)
+{
+	size_t data_ofs = (const u8 *)data - (const u8 *)bdb;
+	u16 data_size = ((const u16 *)data)[-1]; /* stored in header */
+	size_t ofs;
+
+	if (index >= DRM_ARRAY_SIZE(ptrs->ptr))
+		return NULL;
+	ofs = ptrs->ptr[index].fp_timing_offset;
+	if (ofs < data_ofs ||
+	    ofs + sizeof(struct lvds_fp_timing) > data_ofs + data_size)
+		return NULL;
+	return (const struct lvds_fp_timing *)((const u8 *)bdb + ofs);
+}
+
 /* Try to find integrated panel data */
 static void
 parse_lfp_panel_data(struct drm_i915_private *dev_priv,
@@ -184,6 +206,7 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv,
 	const struct bdb_lvds_lfp_data *lvds_lfp_data;
 	const struct bdb_lvds_lfp_data_ptrs *lvds_lfp_data_ptrs;
 	const struct lvds_dvo_timing *panel_dvo_timing;
+	const struct lvds_fp_timing *fp_timing;
 	struct drm_display_mode *panel_fixed_mode;
 	int i, downclock;
 
@@ -244,6 +267,19 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv,
 			      "Normal Clock %dKHz, downclock %dKHz\n",
 			  panel_fixed_mode->clock, 10 * downclock);
 	}
+
+	fp_timing = get_lvds_fp_timing(bdb, lvds_lfp_data,
+				       lvds_lfp_data_ptrs,
+				       lvds_options->panel_type);
+	if (fp_timing) {
+		/* check the resolution, just to be sure */
+		if (fp_timing->x_res == panel_fixed_mode->hdisplay &&
+		    fp_timing->y_res == panel_fixed_mode->vdisplay) {
+			dev_priv->bios_lvds_val = fp_timing->lvds_reg_val;
+			DRM_DEBUG_KMS("VBT initial LVDS value %x\n",
+				      dev_priv->bios_lvds_val);
+		}
+	}
 }
 
 /* Try to find sdvo panel data */
@@ -256,6 +292,11 @@ parse_sdvo_panel_data(struct drm_i915_private *dev_priv,
 	int index;
 
 	index = i915_vbt_sdvo_panel_type;
+	if (index == -2) {
+		DRM_DEBUG_KMS("Ignore SDVO panel mode from BIOS VBT tables.\n");
+		return;
+	}
+
 	if (index == -1) {
 		struct bdb_sdvo_lvds_options *sdvo_lvds_options;
 
@@ -331,11 +372,11 @@ parse_general_definitions(struct drm_i915_private *dev_priv,
 		if (block_size >= sizeof(*general)) {
 			int bus_pin = general->crt_ddc_gmbus_pin;
 			DRM_DEBUG_KMS("crt_ddc_bus_pin: %d\n", bus_pin);
-			if (bus_pin >= 1 && bus_pin <= 6)
+			if (intel_gmbus_is_port_valid(bus_pin))
 				dev_priv->crt_ddc_pin = bus_pin;
 		} else {
 			DRM_DEBUG_KMS("BDB_GD too small (%d). Invalid.\n",
-				  block_size);
+				      block_size);
 		}
 	}
 }
diff --git a/sys/dev/drm2/i915/intel_crt.c b/sys/dev/drm2/i915/intel_crt.c
index 5d96d2f22cb..c1b39eb096b 100644
--- a/sys/dev/drm2/i915/intel_crt.c
+++ b/sys/dev/drm2/i915/intel_crt.c
@@ -55,18 +55,36 @@ static struct intel_crt *intel_attached_crt(struct drm_connector *connector)
 			    struct intel_crt, base);
 }
 
-static void intel_crt_dpms(struct drm_encoder *encoder, int mode)
+static void pch_crt_dpms(struct drm_encoder *encoder, int mode)
 {
 	struct drm_device *dev = encoder->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 temp, reg;
+	u32 temp;
 
-	if (HAS_PCH_SPLIT(dev))
-		reg = PCH_ADPA;
-	else
-		reg = ADPA;
+	temp = I915_READ(PCH_ADPA);
+	temp &= ~ADPA_DAC_ENABLE;
 
-	temp = I915_READ(reg);
+	switch (mode) {
+	case DRM_MODE_DPMS_ON:
+		temp |= ADPA_DAC_ENABLE;
+		break;
+	case DRM_MODE_DPMS_STANDBY:
+	case DRM_MODE_DPMS_SUSPEND:
+	case DRM_MODE_DPMS_OFF:
+		/* Just leave port enable cleared */
+		break;
+	}
+
+	I915_WRITE(PCH_ADPA, temp);
+}
+
+static void gmch_crt_dpms(struct drm_encoder *encoder, int mode)
+{
+	struct drm_device *dev = encoder->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 temp;
+
+	temp = I915_READ(ADPA);
 	temp &= ~(ADPA_HSYNC_CNTL_DISABLE | ADPA_VSYNC_CNTL_DISABLE);
 	temp &= ~ADPA_DAC_ENABLE;
 
@@ -85,7 +103,7 @@ static void intel_crt_dpms(struct drm_encoder *encoder, int mode)
 		break;
 	}
 
-	I915_WRITE(reg, temp);
+	I915_WRITE(ADPA, temp);
 }
 
 static int intel_crt_mode_valid(struct drm_connector *connector,
@@ -111,7 +129,7 @@ static int intel_crt_mode_valid(struct drm_connector *connector,
 }
 
 static bool intel_crt_mode_fixup(struct drm_encoder *encoder,
-				 const struct drm_display_mode *mode,
+				 struct drm_display_mode *mode,
 				 struct drm_display_mode *adjusted_mode)
 {
 	return true;
@@ -279,9 +297,10 @@ static bool intel_crt_detect_ddc(struct drm_connector *connector)
 	if (intel_ddc_probe(&crt->base, dev_priv->crt_ddc_pin)) {
 		struct edid *edid;
 		bool is_digital = false;
+		device_t iic;
 
-		edid = drm_get_edid(connector,
-		    dev_priv->gmbus[dev_priv->crt_ddc_pin]);
+		iic = intel_gmbus_get_adapter(dev_priv, dev_priv->crt_ddc_pin);
+		edid = drm_get_edid(connector, iic);
 		/*
 		 * This may be a DVI-I connector with a shared DDC
 		 * link between analog and digital outputs, so we
@@ -480,15 +499,16 @@ static int intel_crt_get_modes(struct drm_connector *connector)
 	struct drm_device *dev = connector->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int ret;
+	device_t iic;
 
-	ret = intel_ddc_get_modes(connector,
-	    dev_priv->gmbus[dev_priv->crt_ddc_pin]);
+	iic = intel_gmbus_get_adapter(dev_priv, dev_priv->crt_ddc_pin);
+	ret = intel_ddc_get_modes(connector, iic);
 	if (ret || !IS_G4X(dev))
 		return ret;
 
 	/* Try to probe digital port for output in DVI-I -> VGA mode. */
-	return (intel_ddc_get_modes(connector,
-	    dev_priv->gmbus[GMBUS_PORT_DPB]));
+	iic = intel_gmbus_get_adapter(dev_priv, GMBUS_PORT_DPB);
+	return intel_ddc_get_modes(connector, iic);
 }
 
 static int intel_crt_set_property(struct drm_connector *connector,
@@ -511,12 +531,20 @@ static void intel_crt_reset(struct drm_connector *connector)
  * Routines for controlling stuff on the analog port
  */
 
-static const struct drm_encoder_helper_funcs intel_crt_helper_funcs = {
-	.dpms = intel_crt_dpms,
+static const struct drm_encoder_helper_funcs pch_encoder_funcs = {
 	.mode_fixup = intel_crt_mode_fixup,
 	.prepare = intel_encoder_prepare,
 	.commit = intel_encoder_commit,
 	.mode_set = intel_crt_mode_set,
+	.dpms = pch_crt_dpms,
+};
+
+static const struct drm_encoder_helper_funcs gmch_encoder_funcs = {
+	.mode_fixup = intel_crt_mode_fixup,
+	.prepare = intel_encoder_prepare,
+	.commit = intel_encoder_commit,
+	.mode_set = intel_crt_mode_set,
+	.dpms = gmch_crt_dpms,
 };
 
 static const struct drm_connector_funcs intel_crt_connector_funcs = {
@@ -540,7 +568,7 @@ static const struct drm_encoder_funcs intel_crt_enc_funcs = {
 
 static int intel_no_crt_dmi_callback(const struct dmi_system_id *id)
 {
-	DRM_DEBUG_KMS("Skipping CRT initialization for %s\n", id->ident);
+	DRM_INFO("Skipping CRT initialization for %s\n", id->ident);
 	return 1;
 }
 
@@ -562,6 +590,7 @@ void intel_crt_init(struct drm_device *dev)
 	struct intel_crt *crt;
 	struct intel_connector *intel_connector;
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	const struct drm_encoder_helper_funcs *encoder_helper_funcs;
 
 	/* Skip machines without VGA that falsely report hotplug events */
 	if (dmi_check_system(intel_no_crt))
@@ -584,14 +613,23 @@ void intel_crt_init(struct drm_device *dev)
 	crt->base.clone_mask = (1 << INTEL_SDVO_NON_TV_CLONE_BIT |
 				1 << INTEL_ANALOG_CLONE_BIT |
 				1 << INTEL_SDVO_LVDS_CLONE_BIT);
-	crt->base.crtc_mask = (1 << 0) | (1 << 1);
+	if (IS_HASWELL(dev))
+		crt->base.crtc_mask = (1 << 0);
+	else
+		crt->base.crtc_mask = (1 << 0) | (1 << 1);
+
 	if (IS_GEN2(dev))
 		connector->interlace_allowed = 0;
 	else
 		connector->interlace_allowed = 1;
 	connector->doublescan_allowed = 0;
 
-	drm_encoder_helper_add(&crt->base.base, &intel_crt_helper_funcs);
+	if (HAS_PCH_SPLIT(dev))
+		encoder_helper_funcs = &pch_encoder_funcs;
+	else
+		encoder_helper_funcs = &gmch_encoder_funcs;
+
+	drm_encoder_helper_add(&crt->base.base, encoder_helper_funcs);
 	drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs);
 
 #if 0
diff --git a/sys/dev/drm2/i915/intel_ddi.c b/sys/dev/drm2/i915/intel_ddi.c
new file mode 100644
index 00000000000..dc320f68fa7
--- /dev/null
+++ b/sys/dev/drm2/i915/intel_ddi.c
@@ -0,0 +1,761 @@
+/*
+ * Copyright © 2012 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Eugeni Dodonov <eugeni.dodonov@intel.com>
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <dev/drm2/drmP.h>
+#include <dev/drm2/drm.h>
+#include <dev/drm2/i915/i915_drm.h>
+#include <dev/drm2/i915/i915_drv.h>
+#include <dev/drm2/i915/intel_drv.h>
+
+/* HDMI/DVI modes ignore everything but the last 2 items. So we share
+ * them for both DP and FDI transports, allowing those ports to
+ * automatically adapt to HDMI connections as well
+ */
+static const u32 hsw_ddi_translations_dp[] = {
+	0x00FFFFFF, 0x0006000E,		/* DP parameters */
+	0x00D75FFF, 0x0005000A,
+	0x00C30FFF, 0x00040006,
+	0x80AAAFFF, 0x000B0000,
+	0x00FFFFFF, 0x0005000A,
+	0x00D75FFF, 0x000C0004,
+	0x80C30FFF, 0x000B0000,
+	0x00FFFFFF, 0x00040006,
+	0x80D75FFF, 0x000B0000,
+	0x00FFFFFF, 0x00040006		/* HDMI parameters */
+};
+
+static const u32 hsw_ddi_translations_fdi[] = {
+	0x00FFFFFF, 0x0007000E,		/* FDI parameters */
+	0x00D75FFF, 0x000F000A,
+	0x00C30FFF, 0x00060006,
+	0x00AAAFFF, 0x001E0000,
+	0x00FFFFFF, 0x000F000A,
+	0x00D75FFF, 0x00160004,
+	0x00C30FFF, 0x001E0000,
+	0x00FFFFFF, 0x00060006,
+	0x00D75FFF, 0x001E0000,
+	0x00FFFFFF, 0x00040006		/* HDMI parameters */
+};
+
+/* On Haswell, DDI port buffers must be programmed with correct values
+ * in advance. The buffer values are different for FDI and DP modes,
+ * but the HDMI/DVI fields are shared among those. So we program the DDI
+ * in either FDI or DP modes only, as HDMI connections will work with both
+ * of those
+ */
+static void intel_prepare_ddi_buffers(struct drm_device *dev, enum port port, bool use_fdi_mode)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 reg;
+	int i;
+	const u32 *ddi_translations = ((use_fdi_mode) ?
+		hsw_ddi_translations_fdi :
+		hsw_ddi_translations_dp);
+
+	DRM_DEBUG_DRIVER("Initializing DDI buffers for port %c in %s mode\n",
+			port_name(port),
+			use_fdi_mode ? "FDI" : "DP");
+
+	if (use_fdi_mode && (port != PORT_E))
+		DRM_DEBUG_KMS("Programming port %c in FDI mode, this probably will not work.\n",
+		port_name(port));
+
+	for (i=0, reg=DDI_BUF_TRANS(port); i < DRM_ARRAY_SIZE(hsw_ddi_translations_fdi); i++) {
+		I915_WRITE(reg, ddi_translations[i]);
+		reg += 4;
+	}
+}
+
+/* Program DDI buffers translations for DP. By default, program ports A-D in DP
+ * mode and port E for FDI.
+ */
+void intel_prepare_ddi(struct drm_device *dev)
+{
+	int port;
+
+	if (IS_HASWELL(dev)) {
+		for (port = PORT_A; port < PORT_E; port++)
+			intel_prepare_ddi_buffers(dev, port, false);
+
+		/* DDI E is the suggested one to work in FDI mode, so program is as such by
+		 * default. It will have to be re-programmed in case a digital DP output
+		 * will be detected on it
+		 */
+		intel_prepare_ddi_buffers(dev, PORT_E, true);
+	}
+}
+
+static const long hsw_ddi_buf_ctl_values[] = {
+	DDI_BUF_EMP_400MV_0DB_HSW,
+	DDI_BUF_EMP_400MV_3_5DB_HSW,
+	DDI_BUF_EMP_400MV_6DB_HSW,
+	DDI_BUF_EMP_400MV_9_5DB_HSW,
+	DDI_BUF_EMP_600MV_0DB_HSW,
+	DDI_BUF_EMP_600MV_3_5DB_HSW,
+	DDI_BUF_EMP_600MV_6DB_HSW,
+	DDI_BUF_EMP_800MV_0DB_HSW,
+	DDI_BUF_EMP_800MV_3_5DB_HSW
+};
+
+
+/* Starting with Haswell, different DDI ports can work in FDI mode for
+ * connection to the PCH-located connectors. For this, it is necessary to train
+ * both the DDI port and PCH receiver for the desired DDI buffer settings.
+ *
+ * The recommended port to work in FDI mode is DDI E, which we use here. Also,
+ * please note that when FDI mode is active on DDI E, it shares 2 lines with
+ * DDI A (which is used for eDP)
+ */
+
+void hsw_fdi_link_train(struct drm_crtc *crtc)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	int pipe = intel_crtc->pipe;
+	u32 reg, temp, i;
+
+	/* Configure CPU PLL, wait for warmup */
+	I915_WRITE(SPLL_CTL,
+			SPLL_PLL_ENABLE |
+			SPLL_PLL_FREQ_1350MHz |
+			SPLL_PLL_SCC);
+
+	/* Use SPLL to drive the output when in FDI mode */
+	I915_WRITE(PORT_CLK_SEL(PORT_E),
+			PORT_CLK_SEL_SPLL);
+	I915_WRITE(PIPE_CLK_SEL(pipe),
+			PIPE_CLK_SEL_PORT(PORT_E));
+
+	DELAY(20);
+
+	/* Start the training iterating through available voltages and emphasis */
+	for (i=0; i < DRM_ARRAY_SIZE(hsw_ddi_buf_ctl_values); i++) {
+		/* Configure DP_TP_CTL with auto-training */
+		I915_WRITE(DP_TP_CTL(PORT_E),
+					DP_TP_CTL_FDI_AUTOTRAIN |
+					DP_TP_CTL_ENHANCED_FRAME_ENABLE |
+					DP_TP_CTL_LINK_TRAIN_PAT1 |
+					DP_TP_CTL_ENABLE);
+
+		/* Configure and enable DDI_BUF_CTL for DDI E with next voltage */
+		temp = I915_READ(DDI_BUF_CTL(PORT_E));
+		temp = (temp & ~DDI_BUF_EMP_MASK);
+		I915_WRITE(DDI_BUF_CTL(PORT_E),
+				temp |
+				DDI_BUF_CTL_ENABLE |
+				DDI_PORT_WIDTH_X2 |
+				hsw_ddi_buf_ctl_values[i]);
+
+		DELAY(600);
+
+		/* Enable CPU FDI Receiver with auto-training */
+		reg = FDI_RX_CTL(pipe);
+		I915_WRITE(reg,
+				I915_READ(reg) |
+					FDI_LINK_TRAIN_AUTO |
+					FDI_RX_ENABLE |
+					FDI_LINK_TRAIN_PATTERN_1_CPT |
+					FDI_RX_ENHANCE_FRAME_ENABLE |
+					FDI_PORT_WIDTH_2X_LPT |
+					FDI_RX_PLL_ENABLE);
+		POSTING_READ(reg);
+		DELAY(100);
+
+		temp = I915_READ(DP_TP_STATUS(PORT_E));
+		if (temp & DP_TP_STATUS_AUTOTRAIN_DONE) {
+			DRM_DEBUG_DRIVER("BUF_CTL training done on %d step\n", i);
+
+			/* Enable normal pixel sending for FDI */
+			I915_WRITE(DP_TP_CTL(PORT_E),
+						DP_TP_CTL_FDI_AUTOTRAIN |
+						DP_TP_CTL_LINK_TRAIN_NORMAL |
+						DP_TP_CTL_ENHANCED_FRAME_ENABLE |
+						DP_TP_CTL_ENABLE);
+
+			/* Enable PIPE_DDI_FUNC_CTL for the pipe to work in FDI mode */
+			temp = I915_READ(DDI_FUNC_CTL(pipe));
+			temp &= ~PIPE_DDI_PORT_MASK;
+			temp |= PIPE_DDI_SELECT_PORT(PORT_E) |
+					PIPE_DDI_MODE_SELECT_FDI |
+					PIPE_DDI_FUNC_ENABLE |
+					PIPE_DDI_PORT_WIDTH_X2;
+			I915_WRITE(DDI_FUNC_CTL(pipe),
+					temp);
+			break;
+		} else {
+			DRM_ERROR("Error training BUF_CTL %d\n", i);
+
+			/* Disable DP_TP_CTL and FDI_RX_CTL) and retry */
+			I915_WRITE(DP_TP_CTL(PORT_E),
+					I915_READ(DP_TP_CTL(PORT_E)) &
+						~DP_TP_CTL_ENABLE);
+			I915_WRITE(FDI_RX_CTL(pipe),
+					I915_READ(FDI_RX_CTL(pipe)) &
+						~FDI_RX_PLL_ENABLE);
+			continue;
+		}
+	}
+
+	DRM_DEBUG_KMS("FDI train done.\n");
+}
+
+/* For DDI connections, it is possible to support different outputs over the
+ * same DDI port, such as HDMI or DP or even VGA via FDI. So we don't know by
+ * the time the output is detected what exactly is on the other end of it. This
+ * function aims at providing support for this detection and proper output
+ * configuration.
+ */
+void intel_ddi_init(struct drm_device *dev, enum port port)
+{
+	/* For now, we don't do any proper output detection and assume that we
+	 * handle HDMI only */
+
+	switch(port){
+	case PORT_A:
+		/* We don't handle eDP and DP yet */
+		DRM_DEBUG_DRIVER("Found digital output on DDI port A\n");
+		break;
+	/* Assume that the  ports B, C and D are working in HDMI mode for now */
+	case PORT_B:
+	case PORT_C:
+	case PORT_D:
+		intel_hdmi_init(dev, DDI_BUF_CTL(port));
+		break;
+	default:
+		DRM_DEBUG_DRIVER("No handlers defined for port %d, skipping DDI initialization\n",
+				port);
+		break;
+	}
+}
+
+/* WRPLL clock dividers */
+struct wrpll_tmds_clock {
+	u32 clock;
+	u16 p;		/* Post divider */
+	u16 n2;		/* Feedback divider */
+	u16 r2;		/* Reference divider */
+};
+
+/* Table of matching values for WRPLL clocks programming for each frequency */
+static const struct wrpll_tmds_clock wrpll_tmds_clock_table[] = {
+	{19750,	38,	25,	18},
+	{20000,	48,	32,	18},
+	{21000,	36,	21,	15},
+	{21912,	42,	29,	17},
+	{22000,	36,	22,	15},
+	{23000,	36,	23,	15},
+	{23500,	40,	40,	23},
+	{23750,	26,	16,	14},
+	{23750,	26,	16,	14},
+	{24000,	36,	24,	15},
+	{25000,	36,	25,	15},
+	{25175,	26,	40,	33},
+	{25200,	30,	21,	15},
+	{26000,	36,	26,	15},
+	{27000,	30,	21,	14},
+	{27027,	18,	100,	111},
+	{27500,	30,	29,	19},
+	{28000,	34,	30,	17},
+	{28320,	26,	30,	22},
+	{28322,	32,	42,	25},
+	{28750,	24,	23,	18},
+	{29000,	30,	29,	18},
+	{29750,	32,	30,	17},
+	{30000,	30,	25,	15},
+	{30750,	30,	41,	24},
+	{31000,	30,	31,	18},
+	{31500,	30,	28,	16},
+	{32000,	30,	32,	18},
+	{32500,	28,	32,	19},
+	{33000,	24,	22,	15},
+	{34000,	28,	30,	17},
+	{35000,	26,	32,	19},
+	{35500,	24,	30,	19},
+	{36000,	26,	26,	15},
+	{36750,	26,	46,	26},
+	{37000,	24,	23,	14},
+	{37762,	22,	40,	26},
+	{37800,	20,	21,	15},
+	{38000,	24,	27,	16},
+	{38250,	24,	34,	20},
+	{39000,	24,	26,	15},
+	{40000,	24,	32,	18},
+	{40500,	20,	21,	14},
+	{40541,	22,	147,	89},
+	{40750,	18,	19,	14},
+	{41000,	16,	17,	14},
+	{41500,	22,	44,	26},
+	{41540,	22,	44,	26},
+	{42000,	18,	21,	15},
+	{42500,	22,	45,	26},
+	{43000,	20,	43,	27},
+	{43163,	20,	24,	15},
+	{44000,	18,	22,	15},
+	{44900,	20,	108,	65},
+	{45000,	20,	25,	15},
+	{45250,	20,	52,	31},
+	{46000,	18,	23,	15},
+	{46750,	20,	45,	26},
+	{47000,	20,	40,	23},
+	{48000,	18,	24,	15},
+	{49000,	18,	49,	30},
+	{49500,	16,	22,	15},
+	{50000,	18,	25,	15},
+	{50500,	18,	32,	19},
+	{51000,	18,	34,	20},
+	{52000,	18,	26,	15},
+	{52406,	14,	34,	25},
+	{53000,	16,	22,	14},
+	{54000,	16,	24,	15},
+	{54054,	16,	173,	108},
+	{54500,	14,	24,	17},
+	{55000,	12,	22,	18},
+	{56000,	14,	45,	31},
+	{56250,	16,	25,	15},
+	{56750,	14,	25,	17},
+	{57000,	16,	27,	16},
+	{58000,	16,	43,	25},
+	{58250,	16,	38,	22},
+	{58750,	16,	40,	23},
+	{59000,	14,	26,	17},
+	{59341,	14,	40,	26},
+	{59400,	16,	44,	25},
+	{60000,	16,	32,	18},
+	{60500,	12,	39,	29},
+	{61000,	14,	49,	31},
+	{62000,	14,	37,	23},
+	{62250,	14,	42,	26},
+	{63000,	12,	21,	15},
+	{63500,	14,	28,	17},
+	{64000,	12,	27,	19},
+	{65000,	14,	32,	19},
+	{65250,	12,	29,	20},
+	{65500,	12,	32,	22},
+	{66000,	12,	22,	15},
+	{66667,	14,	38,	22},
+	{66750,	10,	21,	17},
+	{67000,	14,	33,	19},
+	{67750,	14,	58,	33},
+	{68000,	14,	30,	17},
+	{68179,	14,	46,	26},
+	{68250,	14,	46,	26},
+	{69000,	12,	23,	15},
+	{70000,	12,	28,	18},
+	{71000,	12,	30,	19},
+	{72000,	12,	24,	15},
+	{73000,	10,	23,	17},
+	{74000,	12,	23,	14},
+	{74176,	8,	100,	91},
+	{74250,	10,	22,	16},
+	{74481,	12,	43,	26},
+	{74500,	10,	29,	21},
+	{75000,	12,	25,	15},
+	{75250,	10,	39,	28},
+	{76000,	12,	27,	16},
+	{77000,	12,	53,	31},
+	{78000,	12,	26,	15},
+	{78750,	12,	28,	16},
+	{79000,	10,	38,	26},
+	{79500,	10,	28,	19},
+	{80000,	12,	32,	18},
+	{81000,	10,	21,	14},
+	{81081,	6,	100,	111},
+	{81624,	8,	29,	24},
+	{82000,	8,	17,	14},
+	{83000,	10,	40,	26},
+	{83950,	10,	28,	18},
+	{84000,	10,	28,	18},
+	{84750,	6,	16,	17},
+	{85000,	6,	17,	18},
+	{85250,	10,	30,	19},
+	{85750,	10,	27,	17},
+	{86000,	10,	43,	27},
+	{87000,	10,	29,	18},
+	{88000,	10,	44,	27},
+	{88500,	10,	41,	25},
+	{89000,	10,	28,	17},
+	{89012,	6,	90,	91},
+	{89100,	10,	33,	20},
+	{90000,	10,	25,	15},
+	{91000,	10,	32,	19},
+	{92000,	10,	46,	27},
+	{93000,	10,	31,	18},
+	{94000,	10,	40,	23},
+	{94500,	10,	28,	16},
+	{95000,	10,	44,	25},
+	{95654,	10,	39,	22},
+	{95750,	10,	39,	22},
+	{96000,	10,	32,	18},
+	{97000,	8,	23,	16},
+	{97750,	8,	42,	29},
+	{98000,	8,	45,	31},
+	{99000,	8,	22,	15},
+	{99750,	8,	34,	23},
+	{100000,	6,	20,	18},
+	{100500,	6,	19,	17},
+	{101000,	6,	37,	33},
+	{101250,	8,	21,	14},
+	{102000,	6,	17,	15},
+	{102250,	6,	25,	22},
+	{103000,	8,	29,	19},
+	{104000,	8,	37,	24},
+	{105000,	8,	28,	18},
+	{106000,	8,	22,	14},
+	{107000,	8,	46,	29},
+	{107214,	8,	27,	17},
+	{108000,	8,	24,	15},
+	{108108,	8,	173,	108},
+	{109000,	6,	23,	19},
+	{109000,	6,	23,	19},
+	{110000,	6,	22,	18},
+	{110013,	6,	22,	18},
+	{110250,	8,	49,	30},
+	{110500,	8,	36,	22},
+	{111000,	8,	23,	14},
+	{111264,	8,	150,	91},
+	{111375,	8,	33,	20},
+	{112000,	8,	63,	38},
+	{112500,	8,	25,	15},
+	{113100,	8,	57,	34},
+	{113309,	8,	42,	25},
+	{114000,	8,	27,	16},
+	{115000,	6,	23,	18},
+	{116000,	8,	43,	25},
+	{117000,	8,	26,	15},
+	{117500,	8,	40,	23},
+	{118000,	6,	38,	29},
+	{119000,	8,	30,	17},
+	{119500,	8,	46,	26},
+	{119651,	8,	39,	22},
+	{120000,	8,	32,	18},
+	{121000,	6,	39,	29},
+	{121250,	6,	31,	23},
+	{121750,	6,	23,	17},
+	{122000,	6,	42,	31},
+	{122614,	6,	30,	22},
+	{123000,	6,	41,	30},
+	{123379,	6,	37,	27},
+	{124000,	6,	51,	37},
+	{125000,	6,	25,	18},
+	{125250,	4,	13,	14},
+	{125750,	4,	27,	29},
+	{126000,	6,	21,	15},
+	{127000,	6,	24,	17},
+	{127250,	6,	41,	29},
+	{128000,	6,	27,	19},
+	{129000,	6,	43,	30},
+	{129859,	4,	25,	26},
+	{130000,	6,	26,	18},
+	{130250,	6,	42,	29},
+	{131000,	6,	32,	22},
+	{131500,	6,	38,	26},
+	{131850,	6,	41,	28},
+	{132000,	6,	22,	15},
+	{132750,	6,	28,	19},
+	{133000,	6,	34,	23},
+	{133330,	6,	37,	25},
+	{134000,	6,	61,	41},
+	{135000,	6,	21,	14},
+	{135250,	6,	167,	111},
+	{136000,	6,	62,	41},
+	{137000,	6,	35,	23},
+	{138000,	6,	23,	15},
+	{138500,	6,	40,	26},
+	{138750,	6,	37,	24},
+	{139000,	6,	34,	22},
+	{139050,	6,	34,	22},
+	{139054,	6,	34,	22},
+	{140000,	6,	28,	18},
+	{141000,	6,	36,	23},
+	{141500,	6,	22,	14},
+	{142000,	6,	30,	19},
+	{143000,	6,	27,	17},
+	{143472,	4,	17,	16},
+	{144000,	6,	24,	15},
+	{145000,	6,	29,	18},
+	{146000,	6,	47,	29},
+	{146250,	6,	26,	16},
+	{147000,	6,	49,	30},
+	{147891,	6,	23,	14},
+	{148000,	6,	23,	14},
+	{148250,	6,	28,	17},
+	{148352,	4,	100,	91},
+	{148500,	6,	33,	20},
+	{149000,	6,	48,	29},
+	{150000,	6,	25,	15},
+	{151000,	4,	19,	17},
+	{152000,	6,	27,	16},
+	{152280,	6,	44,	26},
+	{153000,	6,	34,	20},
+	{154000,	6,	53,	31},
+	{155000,	6,	31,	18},
+	{155250,	6,	50,	29},
+	{155750,	6,	45,	26},
+	{156000,	6,	26,	15},
+	{157000,	6,	61,	35},
+	{157500,	6,	28,	16},
+	{158000,	6,	65,	37},
+	{158250,	6,	44,	25},
+	{159000,	6,	53,	30},
+	{159500,	6,	39,	22},
+	{160000,	6,	32,	18},
+	{161000,	4,	31,	26},
+	{162000,	4,	18,	15},
+	{162162,	4,	131,	109},
+	{162500,	4,	53,	44},
+	{163000,	4,	29,	24},
+	{164000,	4,	17,	14},
+	{165000,	4,	22,	18},
+	{166000,	4,	32,	26},
+	{167000,	4,	26,	21},
+	{168000,	4,	46,	37},
+	{169000,	4,	104,	83},
+	{169128,	4,	64,	51},
+	{169500,	4,	39,	31},
+	{170000,	4,	34,	27},
+	{171000,	4,	19,	15},
+	{172000,	4,	51,	40},
+	{172750,	4,	32,	25},
+	{172800,	4,	32,	25},
+	{173000,	4,	41,	32},
+	{174000,	4,	49,	38},
+	{174787,	4,	22,	17},
+	{175000,	4,	35,	27},
+	{176000,	4,	30,	23},
+	{177000,	4,	38,	29},
+	{178000,	4,	29,	22},
+	{178500,	4,	37,	28},
+	{179000,	4,	53,	40},
+	{179500,	4,	73,	55},
+	{180000,	4,	20,	15},
+	{181000,	4,	55,	41},
+	{182000,	4,	31,	23},
+	{183000,	4,	42,	31},
+	{184000,	4,	30,	22},
+	{184750,	4,	26,	19},
+	{185000,	4,	37,	27},
+	{186000,	4,	51,	37},
+	{187000,	4,	36,	26},
+	{188000,	4,	32,	23},
+	{189000,	4,	21,	15},
+	{190000,	4,	38,	27},
+	{190960,	4,	41,	29},
+	{191000,	4,	41,	29},
+	{192000,	4,	27,	19},
+	{192250,	4,	37,	26},
+	{193000,	4,	20,	14},
+	{193250,	4,	53,	37},
+	{194000,	4,	23,	16},
+	{194208,	4,	23,	16},
+	{195000,	4,	26,	18},
+	{196000,	4,	45,	31},
+	{197000,	4,	35,	24},
+	{197750,	4,	41,	28},
+	{198000,	4,	22,	15},
+	{198500,	4,	25,	17},
+	{199000,	4,	28,	19},
+	{200000,	4,	37,	25},
+	{201000,	4,	61,	41},
+	{202000,	4,	112,	75},
+	{202500,	4,	21,	14},
+	{203000,	4,	146,	97},
+	{204000,	4,	62,	41},
+	{204750,	4,	44,	29},
+	{205000,	4,	38,	25},
+	{206000,	4,	29,	19},
+	{207000,	4,	23,	15},
+	{207500,	4,	40,	26},
+	{208000,	4,	37,	24},
+	{208900,	4,	48,	31},
+	{209000,	4,	48,	31},
+	{209250,	4,	31,	20},
+	{210000,	4,	28,	18},
+	{211000,	4,	25,	16},
+	{212000,	4,	22,	14},
+	{213000,	4,	30,	19},
+	{213750,	4,	38,	24},
+	{214000,	4,	46,	29},
+	{214750,	4,	35,	22},
+	{215000,	4,	43,	27},
+	{216000,	4,	24,	15},
+	{217000,	4,	37,	23},
+	{218000,	4,	42,	26},
+	{218250,	4,	42,	26},
+	{218750,	4,	34,	21},
+	{219000,	4,	47,	29},
+	{219000,	4,	47,	29},
+	{220000,	4,	44,	27},
+	{220640,	4,	49,	30},
+	{220750,	4,	36,	22},
+	{221000,	4,	36,	22},
+	{222000,	4,	23,	14},
+	{222525,	4,	28,	17},
+	{222750,	4,	33,	20},
+	{227000,	4,	37,	22},
+	{230250,	4,	29,	17},
+	{233500,	4,	38,	22},
+	{235000,	4,	40,	23},
+	{238000,	4,	30,	17},
+	{241500,	2,	17,	19},
+	{245250,	2,	20,	22},
+	{247750,	2,	22,	24},
+	{253250,	2,	15,	16},
+	{256250,	2,	18,	19},
+	{262500,	2,	31,	32},
+	{267250,	2,	66,	67},
+	{268500,	2,	94,	95},
+	{270000,	2,	14,	14},
+	{272500,	2,	77,	76},
+	{273750,	2,	57,	56},
+	{280750,	2,	24,	23},
+	{281250,	2,	23,	22},
+	{286000,	2,	17,	16},
+	{291750,	2,	26,	24},
+	{296703,	2,	56,	51},
+	{297000,	2,	22,	20},
+	{298000,	2,	21,	19},
+};
+
+void intel_ddi_mode_set(struct drm_encoder *encoder,
+				struct drm_display_mode *mode,
+				struct drm_display_mode *adjusted_mode)
+{
+	struct drm_device *dev = encoder->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_crtc *crtc = encoder->crtc;
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
+	int port = intel_hdmi->ddi_port;
+	int pipe = intel_crtc->pipe;
+	int p, n2, r2, valid=0;
+	u32 temp, i;
+
+	/* On Haswell, we need to enable the clocks and prepare DDI function to
+	 * work in HDMI mode for this pipe.
+	 */
+	DRM_DEBUG_KMS("Preparing HDMI DDI mode for Haswell on port %c, pipe %c\n", port_name(port), pipe_name(pipe));
+
+	for (i=0; i < DRM_ARRAY_SIZE(wrpll_tmds_clock_table); i++) {
+		if (crtc->mode.clock == wrpll_tmds_clock_table[i].clock) {
+			p = wrpll_tmds_clock_table[i].p;
+			n2 = wrpll_tmds_clock_table[i].n2;
+			r2 = wrpll_tmds_clock_table[i].r2;
+
+			DRM_DEBUG_KMS("WR PLL clock: found settings for %dKHz refresh rate: p=%d, n2=%d, r2=%d\n",
+					crtc->mode.clock,
+					p, n2, r2);
+
+			valid = 1;
+			break;
+		}
+	}
+
+	if (!valid) {
+		DRM_ERROR("Unable to find WR PLL clock settings for %dKHz refresh rate\n",
+				crtc->mode.clock);
+		return;
+	}
+
+	/* Enable LCPLL if disabled */
+	temp = I915_READ(LCPLL_CTL);
+	if (temp & LCPLL_PLL_DISABLE)
+		I915_WRITE(LCPLL_CTL,
+				temp & ~LCPLL_PLL_DISABLE);
+
+	/* Configure WR PLL 1, program the correct divider values for
+	 * the desired frequency and wait for warmup */
+	I915_WRITE(WRPLL_CTL1,
+			WRPLL_PLL_ENABLE |
+			WRPLL_PLL_SELECT_LCPLL_2700 |
+			WRPLL_DIVIDER_REFERENCE(r2) |
+			WRPLL_DIVIDER_FEEDBACK(n2) |
+			WRPLL_DIVIDER_POST(p));
+
+	DELAY(20);
+
+	/* Use WRPLL1 clock to drive the output to the port, and tell the pipe to use
+	 * this port for connection.
+	 */
+	I915_WRITE(PORT_CLK_SEL(port),
+			PORT_CLK_SEL_WRPLL1);
+	I915_WRITE(PIPE_CLK_SEL(pipe),
+			PIPE_CLK_SEL_PORT(port));
+
+	DELAY(20);
+
+	if (intel_hdmi->has_audio) {
+		/* Proper support for digital audio needs a new logic and a new set
+		 * of registers, so we leave it for future patch bombing.
+		 */
+		DRM_DEBUG_DRIVER("HDMI audio on pipe %c not yet supported on DDI\n",
+				 pipe_name(intel_crtc->pipe));
+	}
+
+	/* Enable PIPE_DDI_FUNC_CTL for the pipe to work in HDMI mode */
+	temp = I915_READ(DDI_FUNC_CTL(pipe));
+	temp &= ~PIPE_DDI_PORT_MASK;
+	temp &= ~PIPE_DDI_BPC_12;
+	temp |= PIPE_DDI_SELECT_PORT(port) |
+			PIPE_DDI_MODE_SELECT_HDMI |
+			((intel_crtc->bpp > 24) ?
+				PIPE_DDI_BPC_12 :
+				PIPE_DDI_BPC_8) |
+			PIPE_DDI_FUNC_ENABLE;
+
+	I915_WRITE(DDI_FUNC_CTL(pipe), temp);
+
+	intel_hdmi_set_avi_infoframe(encoder, adjusted_mode);
+	intel_hdmi_set_spd_infoframe(encoder);
+}
+
+void intel_ddi_dpms(struct drm_encoder *encoder, int mode)
+{
+	struct drm_device *dev = encoder->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
+	int port = intel_hdmi->ddi_port;
+	u32 temp;
+
+	temp = I915_READ(DDI_BUF_CTL(port));
+
+	if (mode != DRM_MODE_DPMS_ON) {
+		temp &= ~DDI_BUF_CTL_ENABLE;
+	} else {
+		temp |= DDI_BUF_CTL_ENABLE;
+	}
+
+	/* Enable DDI_BUF_CTL. In HDMI/DVI mode, the port width,
+	 * and swing/emphasis values are ignored so nothing special needs
+	 * to be done besides enabling the port.
+	 */
+	I915_WRITE(DDI_BUF_CTL(port),
+			temp);
+}
diff --git a/sys/dev/drm2/i915/intel_display.c b/sys/dev/drm2/i915/intel_display.c
index ccff1e42f2b..86c3a4de54c 100644
--- a/sys/dev/drm2/i915/intel_display.c
+++ b/sys/dev/drm2/i915/intel_display.c
@@ -35,13 +35,11 @@ __FBSDID("$FreeBSD$");
 #include <dev/drm2/drm_edid.h>
 #include <dev/drm2/drm_dp_helper.h>
 #include <dev/drm2/drm_crtc_helper.h>
-#include <sys/kdb.h>
 #include <sys/limits.h>
 
 #define HAS_eDP (intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))
 
 bool intel_pipe_has_type(struct drm_crtc *crtc, int type);
-static void intel_update_watermarks(struct drm_device *dev);
 static void intel_increase_pllclock(struct drm_crtc *crtc);
 static void intel_crtc_update_cursor(struct drm_crtc *crtc, bool on);
 
@@ -357,6 +355,110 @@ static const intel_limit_t intel_limits_ironlake_display_port = {
 	.find_pll = intel_find_pll_ironlake_dp,
 };
 
+u32 intel_dpio_read(struct drm_i915_private *dev_priv, int reg)
+{
+	u32 val = 0;
+
+	mtx_lock(&dev_priv->dpio_lock);
+	if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) {
+		DRM_ERROR("DPIO idle wait timed out\n");
+		goto out_unlock;
+	}
+
+	I915_WRITE(DPIO_REG, reg);
+	I915_WRITE(DPIO_PKT, DPIO_RID | DPIO_OP_READ | DPIO_PORTID |
+		   DPIO_BYTE);
+	if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) {
+		DRM_ERROR("DPIO read wait timed out\n");
+		goto out_unlock;
+	}
+	val = I915_READ(DPIO_DATA);
+
+out_unlock:
+	mtx_unlock(&dev_priv->dpio_lock);
+	return val;
+}
+
+#if 0
+static void intel_dpio_write(struct drm_i915_private *dev_priv, int reg,
+			     u32 val)
+{
+
+	mtx_lock(&dev_priv->dpio_lock);
+	if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100)) {
+		DRM_ERROR("DPIO idle wait timed out\n");
+		goto out_unlock;
+	}
+
+	I915_WRITE(DPIO_DATA, val);
+	I915_WRITE(DPIO_REG, reg);
+	I915_WRITE(DPIO_PKT, DPIO_RID | DPIO_OP_WRITE | DPIO_PORTID |
+		   DPIO_BYTE);
+	if (wait_for_atomic_us((I915_READ(DPIO_PKT) & DPIO_BUSY) == 0, 100))
+		DRM_ERROR("DPIO write wait timed out\n");
+
+out_unlock:
+	mtx_unlock(&dev_priv->dpio_lock);
+}
+#endif
+
+static void vlv_init_dpio(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	/* Reset the DPIO config */
+	I915_WRITE(DPIO_CTL, 0);
+	POSTING_READ(DPIO_CTL);
+	I915_WRITE(DPIO_CTL, 1);
+	POSTING_READ(DPIO_CTL);
+}
+
+static int intel_dual_link_lvds_callback(const struct dmi_system_id *id)
+{
+	DRM_INFO("Forcing lvds to dual link mode on %s\n", id->ident);
+	return 1;
+}
+
+static const struct dmi_system_id intel_dual_link_lvds[] = {
+	{
+		.callback = intel_dual_link_lvds_callback,
+		.ident = "Apple MacBook Pro (Core i5/i7 Series)",
+		.matches = {
+			DMI_MATCH(DMI_SYS_VENDOR, "Apple Inc."),
+			DMI_MATCH(DMI_PRODUCT_NAME, "MacBookPro8,2"),
+		},
+	},
+	{ }	/* terminating entry */
+};
+
+static bool is_dual_link_lvds(struct drm_i915_private *dev_priv,
+			      unsigned int reg)
+{
+	unsigned int val;
+
+	/* use the module option value if specified */
+	if (i915_lvds_channel_mode > 0)
+		return i915_lvds_channel_mode == 2;
+
+	if (dmi_check_system(intel_dual_link_lvds))
+		return true;
+
+	if (dev_priv->lvds_val)
+		val = dev_priv->lvds_val;
+	else {
+		/* BIOS should set the proper LVDS register value at boot, but
+		 * in reality, it doesn't set the value when the lid is closed;
+		 * we need to check "the value to be set" in VBT when LVDS
+		 * register is uninitialized.
+		 */
+		val = I915_READ(reg);
+		if (!(val & ~LVDS_DETECTED))
+			val = dev_priv->bios_lvds_val;
+		dev_priv->lvds_val = val;
+	}
+	return (val & LVDS_CLKB_POWER_MASK) == LVDS_CLKB_POWER_UP;
+}
+
 static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc,
 						int refclk)
 {
@@ -365,8 +467,7 @@ static const intel_limit_t *intel_ironlake_limit(struct drm_crtc *crtc,
 	const intel_limit_t *limit;
 
 	if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
-		if ((I915_READ(PCH_LVDS) & LVDS_CLKB_POWER_MASK) ==
-		    LVDS_CLKB_POWER_UP) {
+		if (is_dual_link_lvds(dev_priv, PCH_LVDS)) {
 			/* LVDS dual channel */
 			if (refclk == 100000)
 				limit = &intel_limits_ironlake_dual_lvds_100m;
@@ -394,8 +495,7 @@ static const intel_limit_t *intel_g4x_limit(struct drm_crtc *crtc)
 	const intel_limit_t *limit;
 
 	if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
-		if ((I915_READ(LVDS) & LVDS_CLKB_POWER_MASK) ==
-		    LVDS_CLKB_POWER_UP)
+		if (is_dual_link_lvds(dev_priv, LVDS))
 			/* LVDS with dual channel */
 			limit = &intel_limits_g4x_dual_channel_lvds;
 		else
@@ -533,8 +633,7 @@ intel_find_best_PLL(const intel_limit_t *limit, struct drm_crtc *crtc,
 		 * reliably set up different single/dual channel state, if we
 		 * even can.
 		 */
-		if ((I915_READ(LVDS) & LVDS_CLKB_POWER_MASK) ==
-		    LVDS_CLKB_POWER_UP)
+		if (is_dual_link_lvds(dev_priv, LVDS))
 			clock.p2 = limit->p2.p2_fast;
 		else
 			clock.p2 = limit->p2.p2_slow;
@@ -703,6 +802,17 @@ intel_find_pll_g4x_dp(const intel_limit_t *limit, struct drm_crtc *crtc,
 	return true;
 }
 
+static void ironlake_wait_for_vblank(struct drm_device *dev, int pipe)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 frame, frame_reg = PIPEFRAME(pipe);
+
+	frame = I915_READ(frame_reg);
+
+	if (wait_for(I915_READ_NOTRACE(frame_reg) != frame, 50))
+		DRM_DEBUG_KMS("vblank wait timed out\n");
+}
+
 /**
  * intel_wait_for_vblank - wait for vblank on a given pipe
  * @dev: drm device
@@ -716,6 +826,11 @@ void intel_wait_for_vblank(struct drm_device *dev, int pipe)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	int pipestat_reg = PIPESTAT(pipe);
 
+	if (INTEL_INFO(dev)->gen >= 5) {
+		ironlake_wait_for_vblank(dev, pipe);
+		return;
+	}
+
 	/* Clear existing vblank status. Note this will clear any other
 	 * sticky status fields as well.
 	 *
@@ -769,15 +884,20 @@ void intel_wait_for_pipe_off(struct drm_device *dev, int pipe)
 		    1, "915pip"))
 			DRM_DEBUG_KMS("pipe_off wait timed out\n");
 	} else {
-		u32 last_line;
+		u32 last_line, line_mask;
 		int reg = PIPEDSL(pipe);
 		unsigned long timeout = jiffies + msecs_to_jiffies(100);
 
+		if (IS_GEN2(dev))
+			line_mask = DSL_LINEMASK_GEN2;
+		else
+			line_mask = DSL_LINEMASK_GEN3;
+
 		/* Wait for the display line to settle */
 		do {
-			last_line = I915_READ(reg) & DSL_LINEMASK;
+			last_line = I915_READ(reg) & line_mask;
 			DELAY(5000);
-		} while (((I915_READ(reg) & DSL_LINEMASK) != last_line) &&
+		} while (((I915_READ(reg) & line_mask) != last_line) &&
 			 time_after(timeout, jiffies));
 		if (time_after(jiffies, timeout))
 			DRM_DEBUG_KMS("pipe_off wait timed out\n");
@@ -809,26 +929,33 @@ static void assert_pll(struct drm_i915_private *dev_priv,
 
 /* For ILK+ */
 static void assert_pch_pll(struct drm_i915_private *dev_priv,
-			   enum pipe pipe, bool state)
+			   struct intel_crtc *intel_crtc, bool state)
 {
 	int reg;
 	u32 val;
 	bool cur_state;
 
+	if (HAS_PCH_LPT(dev_priv->dev)) {
+		DRM_DEBUG_DRIVER("LPT detected: skipping PCH PLL test\n");
+		return;
+	}
+
+	if (!intel_crtc->pch_pll) {
+		printf("asserting PCH PLL enabled with no PLL\n");
+		return;
+	}
+
 	if (HAS_PCH_CPT(dev_priv->dev)) {
 		u32 pch_dpll;
 
 		pch_dpll = I915_READ(PCH_DPLL_SEL);
 
 		/* Make sure the selected PLL is enabled to the transcoder */
-		KASSERT(((pch_dpll >> (4 * pipe)) & 8) != 0,
-		    ("transcoder %d PLL not enabled\n", pipe));
-
-		/* Convert the transcoder pipe number to a pll pipe number */
-		pipe = (pch_dpll >> (4 * pipe)) & 1;
+		KASSERT(((pch_dpll >> (4 * intel_crtc->pipe)) & 8) != 0,
+		    ("transcoder %d PLL not enabled\n", intel_crtc->pipe));
 	}
 
-	reg = PCH_DPLL(pipe);
+	reg = intel_crtc->pch_pll->pll_reg;
 	val = I915_READ(reg);
 	cur_state = !!(val & DPLL_VCO_ENABLE);
 	if (cur_state != state)
@@ -845,9 +972,16 @@ static void assert_fdi_tx(struct drm_i915_private *dev_priv,
 	u32 val;
 	bool cur_state;
 
-	reg = FDI_TX_CTL(pipe);
-	val = I915_READ(reg);
-	cur_state = !!(val & FDI_TX_ENABLE);
+	if (IS_HASWELL(dev_priv->dev)) {
+		/* On Haswell, DDI is used instead of FDI_TX_CTL */
+		reg = DDI_FUNC_CTL(pipe);
+		val = I915_READ(reg);
+		cur_state = !!(val & PIPE_DDI_FUNC_ENABLE);
+	} else {
+		reg = FDI_TX_CTL(pipe);
+		val = I915_READ(reg);
+		cur_state = !!(val & FDI_TX_ENABLE);
+	}
 	if (cur_state != state)
 		printf("FDI TX state assertion failure (expected %s, current %s)\n",
 		    state_string(state), state_string(cur_state));
@@ -862,9 +996,14 @@ static void assert_fdi_rx(struct drm_i915_private *dev_priv,
 	u32 val;
 	bool cur_state;
 
-	reg = FDI_RX_CTL(pipe);
-	val = I915_READ(reg);
-	cur_state = !!(val & FDI_RX_ENABLE);
+	if (IS_HASWELL(dev_priv->dev) && pipe > 0) {
+			DRM_ERROR("Attempting to enable FDI_RX on Haswell pipe > 0\n");
+			return;
+	} else {
+		reg = FDI_RX_CTL(pipe);
+		val = I915_READ(reg);
+		cur_state = !!(val & FDI_RX_ENABLE);
+	}
 	if (cur_state != state)
 		printf("FDI RX state assertion failure (expected %s, current %s)\n",
 		    state_string(state), state_string(cur_state));
@@ -882,6 +1021,10 @@ static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
 	if (dev_priv->info->gen == 5)
 		return;
 
+	/* On Haswell, DDI ports are responsible for the FDI PLL setup */
+	if (IS_HASWELL(dev_priv->dev))
+		return;
+
 	reg = FDI_TX_CTL(pipe);
 	val = I915_READ(reg);
 	if (!(val & FDI_TX_PLL_ENABLE))
@@ -894,6 +1037,10 @@ static void assert_fdi_rx_pll_enabled(struct drm_i915_private *dev_priv,
 	int reg;
 	u32 val;
 
+	if (IS_HASWELL(dev_priv->dev) && pipe > 0) {
+		DRM_ERROR("Attempting to enable FDI on Haswell with pipe > 0\n");
+		return;
+	}
 	reg = FDI_RX_CTL(pipe);
 	val = I915_READ(reg);
 	if (!(val & FDI_RX_PLL_ENABLE))
@@ -1000,6 +1147,11 @@ static void assert_pch_refclk_enabled(struct drm_i915_private *dev_priv)
 	u32 val;
 	bool enabled;
 
+	if (HAS_PCH_LPT(dev_priv->dev)) {
+		DRM_DEBUG_DRIVER("LPT does not has PCH refclk, skipping check\n");
+		return;
+	}
+
 	val = I915_READ(PCH_DREF_CONTROL);
 	enabled = !!(val & (DREF_SSC_SOURCE_MASK | DREF_NONSPREAD_SOURCE_MASK |
 			    DREF_SUPERSPREAD_SOURCE_MASK));
@@ -1199,6 +1351,68 @@ static void intel_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
 	POSTING_READ(reg);
 }
 
+/* SBI access */
+static void
+intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value)
+{
+
+	mtx_lock(&dev_priv->dpio_lock);
+	if (wait_for((I915_READ(SBI_CTL_STAT) & SBI_READY) == 0,
+				100)) {
+		DRM_ERROR("timeout waiting for SBI to become ready\n");
+		goto out_unlock;
+	}
+
+	I915_WRITE(SBI_ADDR,
+			(reg << 16));
+	I915_WRITE(SBI_DATA,
+			value);
+	I915_WRITE(SBI_CTL_STAT,
+			SBI_BUSY |
+			SBI_CTL_OP_CRWR);
+
+	if (wait_for((I915_READ(SBI_CTL_STAT) & (SBI_READY | SBI_RESPONSE_SUCCESS)) == 0,
+				100)) {
+		DRM_ERROR("timeout waiting for SBI to complete write transaction\n");
+		goto out_unlock;
+	}
+
+out_unlock:
+	mtx_unlock(&dev_priv->dpio_lock);
+}
+
+static u32
+intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg)
+{
+	u32 value;
+
+	value = 0;
+	mtx_lock(&dev_priv->dpio_lock);
+	if (wait_for((I915_READ(SBI_CTL_STAT) & SBI_READY) == 0,
+				100)) {
+		DRM_ERROR("timeout waiting for SBI to become ready\n");
+		goto out_unlock;
+	}
+
+	I915_WRITE(SBI_ADDR,
+			(reg << 16));
+	I915_WRITE(SBI_CTL_STAT,
+			SBI_BUSY |
+			SBI_CTL_OP_CRRD);
+
+	if (wait_for((I915_READ(SBI_CTL_STAT) & (SBI_READY | SBI_RESPONSE_SUCCESS)) == 0,
+				100)) {
+		DRM_ERROR("timeout waiting for SBI to complete read transaction\n");
+		goto out_unlock;
+	}
+
+	value = I915_READ(SBI_DATA);
+
+out_unlock:
+	mtx_unlock(&dev_priv->dpio_lock);
+	return value;
+}
+
 /**
  * intel_enable_pch_pll - enable PCH PLL
  * @dev_priv: i915 private structure
@@ -1207,60 +1421,93 @@ static void intel_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
  * The PCH PLL needs to be enabled before the PCH transcoder, since it
  * drives the transcoder clock.
  */
-static void intel_enable_pch_pll(struct drm_i915_private *dev_priv,
-				 enum pipe pipe)
+static void intel_enable_pch_pll(struct intel_crtc *intel_crtc)
 {
+	struct drm_i915_private *dev_priv = intel_crtc->base.dev->dev_private;
+	struct intel_pch_pll *pll;
 	int reg;
 	u32 val;
 
-	if (pipe > 1)
+	/* PCH PLLs only available on ILK, SNB and IVB */
+	KASSERT(dev_priv->info->gen >= 5, ("Wrong device gen"));
+	pll = intel_crtc->pch_pll;
+	if (pll == NULL)
 		return;
 
-	/* PCH only available on ILK+ */
-	KASSERT(dev_priv->info->gen >= 5, ("Wrong device gen"));
+	if (pll->refcount == 0) {
+		DRM_DEBUG_KMS("pll->refcount == 0\n");
+		return;
+	}
+
+	DRM_DEBUG_KMS("enable PCH PLL %x (active %d, on? %d)for crtc %d\n",
+		      pll->pll_reg, pll->active, pll->on,
+		      intel_crtc->base.base.id);
 
 	/* PCH refclock must be enabled first */
 	assert_pch_refclk_enabled(dev_priv);
 
-	reg = PCH_DPLL(pipe);
+	if (pll->active++ && pll->on) {
+		assert_pch_pll_enabled(dev_priv, intel_crtc);
+		return;
+	}
+
+	DRM_DEBUG_KMS("enabling PCH PLL %x\n", pll->pll_reg);
+
+	reg = pll->pll_reg;
 	val = I915_READ(reg);
 	val |= DPLL_VCO_ENABLE;
 	I915_WRITE(reg, val);
 	POSTING_READ(reg);
 	DELAY(200);
+
+	pll->on = true;
 }
 
-static void intel_disable_pch_pll(struct drm_i915_private *dev_priv,
-				  enum pipe pipe)
+static void intel_disable_pch_pll(struct intel_crtc *intel_crtc)
 {
+	struct drm_i915_private *dev_priv = intel_crtc->base.dev->dev_private;
+	struct intel_pch_pll *pll = intel_crtc->pch_pll;
 	int reg;
-	u32 val, pll_mask = TRANSC_DPLL_ENABLE | TRANSC_DPLLB_SEL,
-		pll_sel = TRANSC_DPLL_ENABLE;
-
-	if (pipe > 1)
-		return;
+	u32 val;
 
 	/* PCH only available on ILK+ */
 	KASSERT(dev_priv->info->gen >= 5, ("Wrong device gen"));
-
-	/* Make sure transcoder isn't still depending on us */
-	assert_transcoder_disabled(dev_priv, pipe);
-
-	if (pipe == 0)
-		pll_sel |= TRANSC_DPLLA_SEL;
-	else if (pipe == 1)
-		pll_sel |= TRANSC_DPLLB_SEL;
-
-
-	if ((I915_READ(PCH_DPLL_SEL) & pll_mask) == pll_sel)
+	if (pll == NULL)
 		return;
 
-	reg = PCH_DPLL(pipe);
+	if (pll->refcount == 0) {
+		DRM_DEBUG_KMS("pll->refcount == 0\n");
+		return;
+	}
+
+	DRM_DEBUG_KMS("disable PCH PLL %x (active %d, on? %d) for crtc %d\n",
+		      pll->pll_reg, pll->active, pll->on,
+		      intel_crtc->base.base.id);
+
+	if (pll->active == 0) {
+		DRM_DEBUG_KMS("pll->active == 0\n");
+		assert_pch_pll_disabled(dev_priv, intel_crtc);
+		return;
+	}
+
+	if (--pll->active) {
+		assert_pch_pll_enabled(dev_priv, intel_crtc);
+		return;
+	}
+
+	DRM_DEBUG_KMS("disabling PCH PLL %x\n", pll->pll_reg);
+
+	/* Make sure transcoder isn't still depending on us */
+	assert_transcoder_disabled(dev_priv, intel_crtc->pipe);
+ 
+	reg = pll->pll_reg;
 	val = I915_READ(reg);
 	val &= ~DPLL_VCO_ENABLE;
 	I915_WRITE(reg, val);
 	POSTING_READ(reg);
 	DELAY(200);
+
+	pll->on = false;
 }
 
 static void intel_enable_transcoder(struct drm_i915_private *dev_priv,
@@ -1274,17 +1521,19 @@ static void intel_enable_transcoder(struct drm_i915_private *dev_priv,
 	KASSERT(dev_priv->info->gen >= 5, ("Wrong device gen"));
 
 	/* Make sure PCH DPLL is enabled */
-	assert_pch_pll_enabled(dev_priv, pipe);
+	assert_pch_pll_enabled(dev_priv, to_intel_crtc(crtc));
 
 	/* FDI must be feeding us bits for PCH ports */
 	assert_fdi_tx_enabled(dev_priv, pipe);
 	assert_fdi_rx_enabled(dev_priv, pipe);
 
-
+	if (IS_HASWELL(dev_priv->dev) && pipe > 0) {
+		DRM_ERROR("Attempting to enable transcoder on Haswell with pipe > 0\n");
+		return;
+	}
 	reg = TRANSCONF(pipe);
 	val = I915_READ(reg);
 	pipeconf_val = I915_READ(PIPECONF(pipe));
-
 	if (HAS_PCH_IBX(dev_priv->dev)) {
 		/*
 		 * make the BPC in transcoder be consistent with
@@ -1420,7 +1669,7 @@ static void intel_disable_pipe(struct drm_i915_private *dev_priv,
  * Plane regs are double buffered, going from enabled->disabled needs a
  * trigger in order to latch.  The display address reg provides this.
  */
-static void intel_flush_display_plane(struct drm_i915_private *dev_priv,
+void intel_flush_display_plane(struct drm_i915_private *dev_priv,
 				      enum plane plane)
 {
 	I915_WRITE(DSPADDR(plane), I915_READ(DSPADDR(plane)));
@@ -1531,486 +1780,6 @@ static void intel_disable_pch_ports(struct drm_i915_private *dev_priv,
 	disable_pch_hdmi(dev_priv, pipe, HDMID);
 }
 
-static void i8xx_disable_fbc(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 fbc_ctl;
-
-	/* Disable compression */
-	fbc_ctl = I915_READ(FBC_CONTROL);
-	if ((fbc_ctl & FBC_CTL_EN) == 0)
-		return;
-
-	fbc_ctl &= ~FBC_CTL_EN;
-	I915_WRITE(FBC_CONTROL, fbc_ctl);
-
-	/* Wait for compressing bit to clear */
-	if (_intel_wait_for(dev,
-	    (I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10,
-	    1, "915fbd")) {
-		DRM_DEBUG_KMS("FBC idle timed out\n");
-		return;
-	}
-
-	DRM_DEBUG_KMS("disabled FBC\n");
-}
-
-static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
-{
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_framebuffer *fb = crtc->fb;
-	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
-	struct drm_i915_gem_object *obj = intel_fb->obj;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	int cfb_pitch;
-	int plane, i;
-	u32 fbc_ctl, fbc_ctl2;
-
-	cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
-	if (fb->pitches[0] < cfb_pitch)
-		cfb_pitch = fb->pitches[0];
-
-	/* FBC_CTL wants 64B units */
-	cfb_pitch = (cfb_pitch / 64) - 1;
-	plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
-
-	/* Clear old tags */
-	for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
-		I915_WRITE(FBC_TAG + (i * 4), 0);
-
-	/* Set it up... */
-	fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE;
-	fbc_ctl2 |= plane;
-	I915_WRITE(FBC_CONTROL2, fbc_ctl2);
-	I915_WRITE(FBC_FENCE_OFF, crtc->y);
-
-	/* enable it... */
-	fbc_ctl = FBC_CTL_EN | FBC_CTL_PERIODIC;
-	if (IS_I945GM(dev))
-		fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
-	fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
-	fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
-	fbc_ctl |= obj->fence_reg;
-	I915_WRITE(FBC_CONTROL, fbc_ctl);
-
-	DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %d, ",
-		      cfb_pitch, crtc->y, intel_crtc->plane);
-}
-
-static bool i8xx_fbc_enabled(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
-}
-
-static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
-{
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_framebuffer *fb = crtc->fb;
-	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
-	struct drm_i915_gem_object *obj = intel_fb->obj;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
-	unsigned long stall_watermark = 200;
-	u32 dpfc_ctl;
-
-	dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
-	dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg;
-	I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
-
-	I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
-		   (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
-		   (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
-	I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
-
-	/* enable it... */
-	I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) | DPFC_CTL_EN);
-
-	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
-}
-
-static void g4x_disable_fbc(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 dpfc_ctl;
-
-	/* Disable compression */
-	dpfc_ctl = I915_READ(DPFC_CONTROL);
-	if (dpfc_ctl & DPFC_CTL_EN) {
-		dpfc_ctl &= ~DPFC_CTL_EN;
-		I915_WRITE(DPFC_CONTROL, dpfc_ctl);
-
-		DRM_DEBUG_KMS("disabled FBC\n");
-	}
-}
-
-static bool g4x_fbc_enabled(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
-}
-
-static void sandybridge_blit_fbc_update(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 blt_ecoskpd;
-
-	/* Make sure blitter notifies FBC of writes */
-	gen6_gt_force_wake_get(dev_priv);
-	blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
-	blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY <<
-		GEN6_BLITTER_LOCK_SHIFT;
-	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
-	blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY;
-	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
-	blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
-			 GEN6_BLITTER_LOCK_SHIFT);
-	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
-	POSTING_READ(GEN6_BLITTER_ECOSKPD);
-	gen6_gt_force_wake_put(dev_priv);
-}
-
-static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
-{
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_framebuffer *fb = crtc->fb;
-	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
-	struct drm_i915_gem_object *obj = intel_fb->obj;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
-	unsigned long stall_watermark = 200;
-	u32 dpfc_ctl;
-
-	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
-	dpfc_ctl &= DPFC_RESERVED;
-	dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X);
-	/* Set persistent mode for front-buffer rendering, ala X. */
-	dpfc_ctl |= DPFC_CTL_PERSISTENT_MODE;
-	dpfc_ctl |= (DPFC_CTL_FENCE_EN | obj->fence_reg);
-	I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
-
-	I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
-		   (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
-		   (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
-	I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
-	I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset | ILK_FBC_RT_VALID);
-	/* enable it... */
-	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
-
-	if (IS_GEN6(dev)) {
-		I915_WRITE(SNB_DPFC_CTL_SA,
-			   SNB_CPU_FENCE_ENABLE | obj->fence_reg);
-		I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
-		sandybridge_blit_fbc_update(dev);
-	}
-
-	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
-}
-
-static void ironlake_disable_fbc(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 dpfc_ctl;
-
-	/* Disable compression */
-	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
-	if (dpfc_ctl & DPFC_CTL_EN) {
-		dpfc_ctl &= ~DPFC_CTL_EN;
-		I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
-
-		DRM_DEBUG_KMS("disabled FBC\n");
-	}
-}
-
-static bool ironlake_fbc_enabled(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
-}
-
-bool intel_fbc_enabled(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	if (!dev_priv->display.fbc_enabled)
-		return false;
-
-	return dev_priv->display.fbc_enabled(dev);
-}
-
-static void intel_fbc_work_fn(void *arg, int pending)
-{
-	struct intel_fbc_work *work = arg;
-	struct drm_device *dev = work->crtc->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	DRM_LOCK(dev);
-	if (work == dev_priv->fbc_work) {
-		/* Double check that we haven't switched fb without cancelling
-		 * the prior work.
-		 */
-		if (work->crtc->fb == work->fb) {
-			dev_priv->display.enable_fbc(work->crtc,
-						     work->interval);
-
-			dev_priv->cfb_plane = to_intel_crtc(work->crtc)->plane;
-			dev_priv->cfb_fb = work->crtc->fb->base.id;
-			dev_priv->cfb_y = work->crtc->y;
-		}
-
-		dev_priv->fbc_work = NULL;
-	}
-	DRM_UNLOCK(dev);
-
-	free(work, DRM_MEM_KMS);
-}
-
-static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
-{
-	u_int pending;
-
-	if (dev_priv->fbc_work == NULL)
-		return;
-
-	DRM_DEBUG_KMS("cancelling pending FBC enable\n");
-
-	/* Synchronisation is provided by struct_mutex and checking of
-	 * dev_priv->fbc_work, so we can perform the cancellation
-	 * entirely asynchronously.
-	 */
-	if (taskqueue_cancel_timeout(dev_priv->tq, &dev_priv->fbc_work->task,
-	    &pending) == 0)
-		/* tasklet was killed before being run, clean up */
-		free(dev_priv->fbc_work, DRM_MEM_KMS);
-
-	/* Mark the work as no longer wanted so that if it does
-	 * wake-up (because the work was already running and waiting
-	 * for our mutex), it will discover that is no longer
-	 * necessary to run.
-	 */
-	dev_priv->fbc_work = NULL;
-}
-
-static void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
-{
-	struct intel_fbc_work *work;
-	struct drm_device *dev = crtc->dev;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	if (!dev_priv->display.enable_fbc)
-		return;
-
-	intel_cancel_fbc_work(dev_priv);
-
-	work = malloc(sizeof(*work), DRM_MEM_KMS, M_WAITOK | M_ZERO);
-	work->crtc = crtc;
-	work->fb = crtc->fb;
-	work->interval = interval;
-	TIMEOUT_TASK_INIT(dev_priv->tq, &work->task, 0, intel_fbc_work_fn,
-	    work);
-
-	dev_priv->fbc_work = work;
-
-	DRM_DEBUG_KMS("scheduling delayed FBC enable\n");
-
-	/* Delay the actual enabling to let pageflipping cease and the
-	 * display to settle before starting the compression. Note that
-	 * this delay also serves a second purpose: it allows for a
-	 * vblank to pass after disabling the FBC before we attempt
-	 * to modify the control registers.
-	 *
-	 * A more complicated solution would involve tracking vblanks
-	 * following the termination of the page-flipping sequence
-	 * and indeed performing the enable as a co-routine and not
-	 * waiting synchronously upon the vblank.
-	 */
-	taskqueue_enqueue_timeout(dev_priv->tq, &work->task,
-	    msecs_to_jiffies(50));
-}
-
-void intel_disable_fbc(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	intel_cancel_fbc_work(dev_priv);
-
-	if (!dev_priv->display.disable_fbc)
-		return;
-
-	dev_priv->display.disable_fbc(dev);
-	dev_priv->cfb_plane = -1;
-}
-
-/**
- * intel_update_fbc - enable/disable FBC as needed
- * @dev: the drm_device
- *
- * Set up the framebuffer compression hardware at mode set time.  We
- * enable it if possible:
- *   - plane A only (on pre-965)
- *   - no pixel mulitply/line duplication
- *   - no alpha buffer discard
- *   - no dual wide
- *   - framebuffer <= 2048 in width, 1536 in height
- *
- * We can't assume that any compression will take place (worst case),
- * so the compressed buffer has to be the same size as the uncompressed
- * one.  It also must reside (along with the line length buffer) in
- * stolen memory.
- *
- * We need to enable/disable FBC on a global basis.
- */
-static void intel_update_fbc(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_crtc *crtc = NULL, *tmp_crtc;
-	struct intel_crtc *intel_crtc;
-	struct drm_framebuffer *fb;
-	struct intel_framebuffer *intel_fb;
-	struct drm_i915_gem_object *obj;
-	int enable_fbc;
-
-	DRM_DEBUG_KMS("\n");
-
-	if (!i915_powersave)
-		return;
-
-	if (!I915_HAS_FBC(dev))
-		return;
-
-	/*
-	 * If FBC is already on, we just have to verify that we can
-	 * keep it that way...
-	 * Need to disable if:
-	 *   - more than one pipe is active
-	 *   - changing FBC params (stride, fence, mode)
-	 *   - new fb is too large to fit in compressed buffer
-	 *   - going to an unsupported config (interlace, pixel multiply, etc.)
-	 */
-	list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
-		if (tmp_crtc->enabled && tmp_crtc->fb) {
-			if (crtc) {
-				DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
-				dev_priv->no_fbc_reason = FBC_MULTIPLE_PIPES;
-				goto out_disable;
-			}
-			crtc = tmp_crtc;
-		}
-	}
-
-	if (!crtc || crtc->fb == NULL) {
-		DRM_DEBUG_KMS("no output, disabling\n");
-		dev_priv->no_fbc_reason = FBC_NO_OUTPUT;
-		goto out_disable;
-	}
-
-	intel_crtc = to_intel_crtc(crtc);
-	fb = crtc->fb;
-	intel_fb = to_intel_framebuffer(fb);
-	obj = intel_fb->obj;
-
-	enable_fbc = i915_enable_fbc;
-	if (enable_fbc < 0) {
-		DRM_DEBUG_KMS("fbc set to per-chip default\n");
-		enable_fbc = 1;
-		if (INTEL_INFO(dev)->gen <= 6)
-			enable_fbc = 0;
-	}
-	if (!enable_fbc) {
-		DRM_DEBUG_KMS("fbc disabled per module param\n");
-		dev_priv->no_fbc_reason = FBC_MODULE_PARAM;
-		goto out_disable;
-	}
-	if (intel_fb->obj->base.size > dev_priv->cfb_size) {
-		DRM_DEBUG_KMS("framebuffer too large, disabling "
-			      "compression\n");
-		dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
-		goto out_disable;
-	}
-	if ((crtc->mode.flags & DRM_MODE_FLAG_INTERLACE) ||
-	    (crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN)) {
-		DRM_DEBUG_KMS("mode incompatible with compression, "
-			      "disabling\n");
-		dev_priv->no_fbc_reason = FBC_UNSUPPORTED_MODE;
-		goto out_disable;
-	}
-	if ((crtc->mode.hdisplay > 2048) ||
-	    (crtc->mode.vdisplay > 1536)) {
-		DRM_DEBUG_KMS("mode too large for compression, disabling\n");
-		dev_priv->no_fbc_reason = FBC_MODE_TOO_LARGE;
-		goto out_disable;
-	}
-	if ((IS_I915GM(dev) || IS_I945GM(dev)) && intel_crtc->plane != 0) {
-		DRM_DEBUG_KMS("plane not 0, disabling compression\n");
-		dev_priv->no_fbc_reason = FBC_BAD_PLANE;
-		goto out_disable;
-	}
-	if (obj->tiling_mode != I915_TILING_X ||
-	    obj->fence_reg == I915_FENCE_REG_NONE) {
-		DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
-		dev_priv->no_fbc_reason = FBC_NOT_TILED;
-		goto out_disable;
-	}
-
-	/* If the kernel debugger is active, always disable compression */
-	if (kdb_active)
-		goto out_disable;
-
-	/* If the scanout has not changed, don't modify the FBC settings.
-	 * Note that we make the fundamental assumption that the fb->obj
-	 * cannot be unpinned (and have its GTT offset and fence revoked)
-	 * without first being decoupled from the scanout and FBC disabled.
-	 */
-	if (dev_priv->cfb_plane == intel_crtc->plane &&
-	    dev_priv->cfb_fb == fb->base.id &&
-	    dev_priv->cfb_y == crtc->y)
-		return;
-
-	if (intel_fbc_enabled(dev)) {
-		/* We update FBC along two paths, after changing fb/crtc
-		 * configuration (modeswitching) and after page-flipping
-		 * finishes. For the latter, we know that not only did
-		 * we disable the FBC at the start of the page-flip
-		 * sequence, but also more than one vblank has passed.
-		 *
-		 * For the former case of modeswitching, it is possible
-		 * to switch between two FBC valid configurations
-		 * instantaneously so we do need to disable the FBC
-		 * before we can modify its control registers. We also
-		 * have to wait for the next vblank for that to take
-		 * effect. However, since we delay enabling FBC we can
-		 * assume that a vblank has passed since disabling and
-		 * that we can safely alter the registers in the deferred
-		 * callback.
-		 *
-		 * In the scenario that we go from a valid to invalid
-		 * and then back to valid FBC configuration we have
-		 * no strict enforcement that a vblank occurred since
-		 * disabling the FBC. However, along all current pipe
-		 * disabling paths we do need to wait for a vblank at
-		 * some point. And we wait before enabling FBC anyway.
-		 */
-		DRM_DEBUG_KMS("disabling active FBC for update\n");
-		intel_disable_fbc(dev);
-	}
-
-	intel_enable_fbc(crtc, 500);
-	return;
-
-out_disable:
-	/* Multiple disables should be harmless */
-	if (intel_fbc_enabled(dev)) {
-		DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
-		intel_disable_fbc(dev);
-	}
-}
-
 int
 intel_pin_and_fence_fb_obj(struct drm_device *dev,
 			   struct drm_i915_gem_object *obj,
@@ -2052,19 +1821,17 @@ intel_pin_and_fence_fb_obj(struct drm_device *dev,
 	 * framebuffer compression.  For simplicity, we always install
 	 * a fence as the cost is not that onerous.
 	 */
-	if (obj->tiling_mode != I915_TILING_NONE) {
-		ret = i915_gem_object_get_fence(obj, pipelined);
-		if (ret)
-			goto err_unpin;
+	ret = i915_gem_object_get_fence(obj);
+	if (ret)
+		goto err_unpin;
 
-		i915_gem_object_pin_fence(obj);
-	}
+	i915_gem_object_pin_fence(obj);
 
 	dev_priv->mm.interruptible = true;
 	return 0;
 
 err_unpin:
-	i915_gem_object_unpin(obj);
+	i915_gem_object_unpin_from_display_plane(obj);
 err_interruptible:
 	dev_priv->mm.interruptible = true;
 	return ret;
@@ -2073,7 +1840,7 @@ err_interruptible:
 void intel_unpin_fb_obj(struct drm_i915_gem_object *obj)
 {
 	i915_gem_object_unpin_fence(obj);
-	i915_gem_object_unpin(obj);
+	i915_gem_object_unpin_from_display_plane(obj);
 }
 
 static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb,
@@ -2139,7 +1906,7 @@ static int i9xx_update_plane(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 		      Start, Offset, x, y, fb->pitches[0]);
 	I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
 	if (INTEL_INFO(dev)->gen >= 4) {
-		I915_WRITE(DSPSURF(plane), Start);
+		I915_MODIFY_DISPBASE(DSPSURF(plane), Start);
 		I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
 		I915_WRITE(DSPADDR(plane), Offset);
 	} else
@@ -2224,7 +1991,7 @@ static int ironlake_update_plane(struct drm_crtc *crtc,
 	DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
 		      Start, Offset, x, y, fb->pitches[0]);
 	I915_WRITE(DSPSTRIDE(plane), fb->pitches[0]);
-	I915_WRITE(DSPSURF(plane), Start);
+	I915_MODIFY_DISPBASE(DSPSURF(plane), Start);
 	I915_WRITE(DSPTILEOFF(plane), (y << 16) | x);
 	I915_WRITE(DSPADDR(plane), Offset);
 	POSTING_READ(reg);
@@ -2239,16 +2006,12 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 {
 	struct drm_device *dev = crtc->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	int ret;
 
-	ret = dev_priv->display.update_plane(crtc, fb, x, y);
-	if (ret)
-		return ret;
-
-	intel_update_fbc(dev);
+	if (dev_priv->display.disable_fbc)
+		dev_priv->display.disable_fbc(dev);
 	intel_increase_pllclock(crtc);
 
-	return 0;
+	return dev_priv->display.update_plane(crtc, fb, x, y);
 }
 
 static int
@@ -2287,10 +2050,9 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
 		    struct drm_framebuffer *old_fb)
 {
 	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
 #if 0
 	struct drm_i915_master_private *master_priv;
-#else
-	drm_i915_private_t *dev_priv = dev->dev_private;
 #endif
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int ret;
@@ -2301,16 +2063,10 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
 		return 0;
 	}
 
-	switch (intel_crtc->plane) {
-	case 0:
-	case 1:
-		break;
-	case 2:
-		if (IS_IVYBRIDGE(dev))
-			break;
-		/* fall through otherwise */
-	default:
-		DRM_ERROR("no plane for crtc\n");
+	if(intel_crtc->plane > dev_priv->num_pipe) {
+		DRM_ERROR("no plane for crtc: plane %d, num_pipes %d\n",
+				intel_crtc->plane,
+				dev_priv->num_pipe);
 		return -EINVAL;
 	}
 
@@ -2327,8 +2083,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
 	if (old_fb)
 		intel_finish_fb(old_fb);
 
-	ret = intel_pipe_set_base_atomic(crtc, crtc->fb, x, y,
-					 LEAVE_ATOMIC_MODE_SET);
+	ret = dev_priv->display.update_plane(crtc, crtc->fb, x, y);
 	if (ret) {
 		intel_unpin_fb_obj(to_intel_framebuffer(crtc->fb)->obj);
 		DRM_UNLOCK(dev);
@@ -2341,6 +2096,7 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
 		intel_unpin_fb_obj(to_intel_framebuffer(old_fb)->obj);
 	}
 
+	intel_update_fbc(dev);
 	DRM_UNLOCK(dev);
 
 #if 0
@@ -2576,7 +2332,7 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe;
-	u32 reg, temp, i;
+	u32 reg, temp, i, retry;
 
 	/* Train 1: umask FDI RX Interrupt symbol_lock and bit_lock bit
 	   for train result */
@@ -2628,15 +2384,20 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
 		POSTING_READ(reg);
 		DELAY(500);
 
-		reg = FDI_RX_IIR(pipe);
-		temp = I915_READ(reg);
-		DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
+		for (retry = 0; retry < 5; retry++) {
+			reg = FDI_RX_IIR(pipe);
+			temp = I915_READ(reg);
+			DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
 
-		if (temp & FDI_RX_BIT_LOCK) {
-			I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
-			DRM_DEBUG_KMS("FDI train 1 done.\n");
-			break;
+			if (temp & FDI_RX_BIT_LOCK) {
+				I915_WRITE(reg, temp | FDI_RX_BIT_LOCK);
+				DRM_DEBUG_KMS("FDI train 1 done.\n");
+				break;
+			}
+			DELAY(50);
 		}
+		if (retry < 5)
+			break;
 	}
 	if (i == 4)
 		DRM_ERROR("FDI train 1 fail!\n");
@@ -2677,15 +2438,20 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
 		POSTING_READ(reg);
 		DELAY(500);
 
-		reg = FDI_RX_IIR(pipe);
-		temp = I915_READ(reg);
-		DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
+		for (retry = 0; retry < 5; retry++) {
+			reg = FDI_RX_IIR(pipe);
+			temp = I915_READ(reg);
+			DRM_DEBUG_KMS("FDI_RX_IIR 0x%x\n", temp);
 
-		if (temp & FDI_RX_SYMBOL_LOCK) {
-			I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
-			DRM_DEBUG_KMS("FDI train 2 done.\n");
-			break;
+			if (temp & FDI_RX_SYMBOL_LOCK) {
+				I915_WRITE(reg, temp | FDI_RX_SYMBOL_LOCK);
+				DRM_DEBUG_KMS("FDI train 2 done.\n");
+				break;
+			}
+			DELAY(50);
 		}
+		if (retry < 5)
+			break;
 	}
 	if (i == 4)
 		DRM_ERROR("FDI train 2 fail!\n");
@@ -2834,15 +2600,19 @@ static void ironlake_fdi_pll_enable(struct drm_crtc *crtc)
 	POSTING_READ(reg);
 	DELAY(200);
 
-	/* Enable CPU FDI TX PLL, always on for Ironlake */
-	reg = FDI_TX_CTL(pipe);
-	temp = I915_READ(reg);
-	if ((temp & FDI_TX_PLL_ENABLE) == 0) {
-		I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
+	/* On Haswell, the PLL configuration for ports and pipes is handled
+	 * separately, as part of DDI setup */
+	if (!IS_HASWELL(dev)) {
+		/* Enable CPU FDI TX PLL, always on for Ironlake */
+		reg = FDI_TX_CTL(pipe);
+		temp = I915_READ(reg);
+		if ((temp & FDI_TX_PLL_ENABLE) == 0) {
+			I915_WRITE(reg, temp | FDI_TX_PLL_ENABLE);
 
-		POSTING_READ(reg);
-		DELAY(100);
-	}
+			POSTING_READ(reg);
+			DELAY(100);
+		}
+ 	}
 }
 
 static void cpt_phase_pointer_disable(struct drm_device *dev, int pipe)
@@ -2915,42 +2685,16 @@ static void ironlake_fdi_disable(struct drm_crtc *crtc)
 	DELAY(100);
 }
 
-/*
- * When we disable a pipe, we need to clear any pending scanline wait events
- * to avoid hanging the ring, which we assume we are waiting on.
- */
-static void intel_clear_scanline_wait(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct intel_ring_buffer *ring;
-	u32 tmp;
-
-	if (IS_GEN2(dev))
-		/* Can't break the hang on i8xx */
-		return;
-
-	ring = LP_RING(dev_priv);
-	tmp = I915_READ_CTL(ring);
-	if (tmp & RING_WAIT)
-		I915_WRITE_CTL(ring, tmp);
-}
-
 static void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
 {
-	struct drm_i915_gem_object *obj;
-	struct drm_i915_private *dev_priv;
-	struct drm_device *dev;
+	struct drm_device *dev = crtc->dev;
 
 	if (crtc->fb == NULL)
 		return;
 
-	obj = to_intel_framebuffer(crtc->fb)->obj;
-	dev = crtc->dev;
-	dev_priv = dev->dev_private;
-	mtx_lock(&dev->event_lock);
-	while (atomic_load_acq_int(&obj->pending_flip) != 0)
-		msleep(&obj->pending_flip, &dev->event_lock, 0, "915wfl", 0);
-	mtx_unlock(&dev->event_lock);
+	DRM_LOCK(dev);
+	intel_finish_fb(crtc->fb);
+	DRM_UNLOCK(dev);
 }
 
 static bool intel_crtc_driving_pch(struct drm_crtc *crtc)
@@ -2967,6 +2711,23 @@ static bool intel_crtc_driving_pch(struct drm_crtc *crtc)
 		if (encoder->base.crtc != crtc)
 			continue;
 
+		/* On Haswell, LPT PCH handles the VGA connection via FDI, and Haswell
+		 * CPU handles all others */
+		if (IS_HASWELL(dev)) {
+			/* It is still unclear how this will work on PPT, so throw up a warning */
+			if (!HAS_PCH_LPT(dev))
+				DRM_DEBUG_KMS("Haswell: PPT\n");
+
+			if (encoder->type == DRM_MODE_ENCODER_DAC) {
+				DRM_DEBUG_KMS("Haswell detected DAC encoder, assuming is PCH\n");
+				return true;
+			} else {
+				DRM_DEBUG_KMS("Haswell detected encoder %d, assuming is CPU\n",
+						encoder->type);
+				return false;
+			}
+		}
+
 		switch (encoder->type) {
 		case INTEL_OUTPUT_EDP:
 			if (!intel_encoder_is_pch_edp(&encoder->base))
@@ -2978,6 +2739,99 @@ static bool intel_crtc_driving_pch(struct drm_crtc *crtc)
 	return true;
 }
 
+/* Program iCLKIP clock to the desired frequency */
+static void lpt_program_iclkip(struct drm_crtc *crtc)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 divsel, phaseinc, auxdiv, phasedir = 0;
+	u32 temp;
+
+	/* It is necessary to ungate the pixclk gate prior to programming
+	 * the divisors, and gate it back when it is done.
+	 */
+	I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_GATE);
+
+	/* Disable SSCCTL */
+	intel_sbi_write(dev_priv, SBI_SSCCTL6,
+				intel_sbi_read(dev_priv, SBI_SSCCTL6) |
+					SBI_SSCCTL_DISABLE);
+
+	/* 20MHz is a corner case which is out of range for the 7-bit divisor */
+	if (crtc->mode.clock == 20000) {
+		auxdiv = 1;
+		divsel = 0x41;
+		phaseinc = 0x20;
+	} else {
+		/* The iCLK virtual clock root frequency is in MHz,
+		 * but the crtc->mode.clock in in KHz. To get the divisors,
+		 * it is necessary to divide one by another, so we
+		 * convert the virtual clock precision to KHz here for higher
+		 * precision.
+		 */
+		u32 iclk_virtual_root_freq = 172800 * 1000;
+		u32 iclk_pi_range = 64;
+		u32 desired_divisor, msb_divisor_value, pi_value;
+
+		desired_divisor = (iclk_virtual_root_freq / crtc->mode.clock);
+		msb_divisor_value = desired_divisor / iclk_pi_range;
+		pi_value = desired_divisor % iclk_pi_range;
+
+		auxdiv = 0;
+		divsel = msb_divisor_value - 2;
+		phaseinc = pi_value;
+	}
+
+	/* This should not happen with any sane values */
+	if ((SBI_SSCDIVINTPHASE_DIVSEL(divsel) &
+	     ~SBI_SSCDIVINTPHASE_DIVSEL_MASK))
+		DRM_DEBUG_KMS("DIVSEL_MASK");
+	if ((SBI_SSCDIVINTPHASE_DIR(phasedir) &
+	     ~SBI_SSCDIVINTPHASE_INCVAL_MASK))
+		DRM_DEBUG_KMS("INCVAL_MASK");
+
+	DRM_DEBUG_KMS("iCLKIP clock: found settings for %dKHz refresh rate: auxdiv=%x, divsel=%x, phasedir=%x, phaseinc=%x\n",
+			crtc->mode.clock,
+			auxdiv,
+			divsel,
+			phasedir,
+			phaseinc);
+
+	/* Program SSCDIVINTPHASE6 */
+	temp = intel_sbi_read(dev_priv, SBI_SSCDIVINTPHASE6);
+	temp &= ~SBI_SSCDIVINTPHASE_DIVSEL_MASK;
+	temp |= SBI_SSCDIVINTPHASE_DIVSEL(divsel);
+	temp &= ~SBI_SSCDIVINTPHASE_INCVAL_MASK;
+	temp |= SBI_SSCDIVINTPHASE_INCVAL(phaseinc);
+	temp |= SBI_SSCDIVINTPHASE_DIR(phasedir);
+	temp |= SBI_SSCDIVINTPHASE_PROPAGATE;
+
+	intel_sbi_write(dev_priv,
+			SBI_SSCDIVINTPHASE6,
+			temp);
+
+	/* Program SSCAUXDIV */
+	temp = intel_sbi_read(dev_priv, SBI_SSCAUXDIV6);
+	temp &= ~SBI_SSCAUXDIV_FINALDIV2SEL(1);
+	temp |= SBI_SSCAUXDIV_FINALDIV2SEL(auxdiv);
+	intel_sbi_write(dev_priv,
+			SBI_SSCAUXDIV6,
+			temp);
+
+
+	/* Enable modulator and associated divider */
+	temp = intel_sbi_read(dev_priv, SBI_SSCCTL6);
+	temp &= ~SBI_SSCCTL_DISABLE;
+	intel_sbi_write(dev_priv,
+			SBI_SSCCTL6,
+			temp);
+
+	/* Wait for initialization time */
+	DELAY(24);
+
+	I915_WRITE(PIXCLK_GATE, PIXCLK_GATE_UNGATE);
+}
+
 /*
  * Enable PCH resources required for PCH ports:
  *   - PCH PLLs
@@ -2992,29 +2846,41 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	int pipe = intel_crtc->pipe;
-	u32 reg, temp, transc_sel;
+	u32 reg, temp;
+
+	assert_transcoder_disabled(dev_priv, pipe);
 
 	/* For PCH output, training FDI link */
 	dev_priv->display.fdi_link_train(crtc);
 
-	intel_enable_pch_pll(dev_priv, pipe);
+	intel_enable_pch_pll(intel_crtc);
 
-	if (HAS_PCH_CPT(dev)) {
-		transc_sel = intel_crtc->use_pll_a ? TRANSC_DPLLA_SEL :
-			TRANSC_DPLLB_SEL;
+	if (HAS_PCH_LPT(dev)) {
+		DRM_DEBUG_KMS("LPT detected: programming iCLKIP\n");
+		lpt_program_iclkip(crtc);
+	} else if (HAS_PCH_CPT(dev)) {
+		u32 sel;
 
-		/* Be sure PCH DPLL SEL is set */
 		temp = I915_READ(PCH_DPLL_SEL);
-		if (pipe == 0) {
-			temp &= ~(TRANSA_DPLLB_SEL);
-			temp |= (TRANSA_DPLL_ENABLE | TRANSA_DPLLA_SEL);
-		} else if (pipe == 1) {
-			temp &= ~(TRANSB_DPLLB_SEL);
-			temp |= (TRANSB_DPLL_ENABLE | TRANSB_DPLLB_SEL);
-		} else if (pipe == 2) {
-			temp &= ~(TRANSC_DPLLB_SEL);
-			temp |= (TRANSC_DPLL_ENABLE | transc_sel);
+		switch (pipe) {
+		default:
+		case 0:
+			temp |= TRANSA_DPLL_ENABLE;
+			sel = TRANSA_DPLLB_SEL;
+			break;
+		case 1:
+			temp |= TRANSB_DPLL_ENABLE;
+			sel = TRANSB_DPLLB_SEL;
+			break;
+		case 2:
+			temp |= TRANSC_DPLL_ENABLE;
+			sel = TRANSC_DPLLB_SEL;
+			break;
 		}
+		if (intel_crtc->pch_pll->pll_reg == _PCH_DPLL_B)
+			temp |= sel;
+		else
+			temp &= ~sel;
 		I915_WRITE(PCH_DPLL_SEL, temp);
 	}
 
@@ -3029,7 +2895,8 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
 	I915_WRITE(TRANS_VSYNC(pipe),  I915_READ(VSYNC(pipe)));
 	I915_WRITE(TRANS_VSYNCSHIFT(pipe),  I915_READ(VSYNCSHIFT(pipe)));
 
-	intel_fdi_normal_train(crtc);
+	if (!IS_HASWELL(dev))
+		intel_fdi_normal_train(crtc);
 
 	/* For PCH DP, enable TRANS_DP_CTL */
 	if (HAS_PCH_CPT(dev) &&
@@ -3072,6 +2939,93 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
 	intel_enable_transcoder(dev_priv, pipe);
 }
 
+static void intel_put_pch_pll(struct intel_crtc *intel_crtc)
+{
+	struct intel_pch_pll *pll = intel_crtc->pch_pll;
+
+	if (pll == NULL)
+		return;
+
+	if (pll->refcount == 0) {
+		printf("bad PCH PLL refcount\n");
+		return;
+	}
+
+	--pll->refcount;
+	intel_crtc->pch_pll = NULL;
+}
+
+static struct intel_pch_pll *intel_get_pch_pll(struct intel_crtc *intel_crtc, u32 dpll, u32 fp)
+{
+	struct drm_i915_private *dev_priv = intel_crtc->base.dev->dev_private;
+	struct intel_pch_pll *pll;
+	int i;
+
+	pll = intel_crtc->pch_pll;
+	if (pll) {
+		DRM_DEBUG_KMS("CRTC:%d reusing existing PCH PLL %x\n",
+			      intel_crtc->base.base.id, pll->pll_reg);
+		goto prepare;
+	}
+
+	if (HAS_PCH_IBX(dev_priv->dev)) {
+		/* Ironlake PCH has a fixed PLL->PCH pipe mapping. */
+		i = intel_crtc->pipe;
+		pll = &dev_priv->pch_plls[i];
+
+		DRM_DEBUG_KMS("CRTC:%d using pre-allocated PCH PLL %x\n",
+			      intel_crtc->base.base.id, pll->pll_reg);
+
+		goto found;
+	}
+
+	for (i = 0; i < dev_priv->num_pch_pll; i++) {
+		pll = &dev_priv->pch_plls[i];
+
+		/* Only want to check enabled timings first */
+		if (pll->refcount == 0)
+			continue;
+
+		if (dpll == (I915_READ(pll->pll_reg) & 0x7fffffff) &&
+		    fp == I915_READ(pll->fp0_reg)) {
+			DRM_DEBUG_KMS("CRTC:%d sharing existing PCH PLL %x (refcount %d, ative %d)\n",
+				      intel_crtc->base.base.id,
+				      pll->pll_reg, pll->refcount, pll->active);
+
+			goto found;
+		}
+	}
+
+	/* Ok no matching timings, maybe there's a free one? */
+	for (i = 0; i < dev_priv->num_pch_pll; i++) { /* XXXKIB: HACK */
+		pll = &dev_priv->pch_plls[i];
+		if (pll->refcount == 0) {
+			DRM_DEBUG_KMS("CRTC:%d allocated PCH PLL %x\n",
+				      intel_crtc->base.base.id, pll->pll_reg);
+			goto found;
+		}
+	}
+
+	return NULL;
+
+found:
+	intel_crtc->pch_pll = pll;
+	pll->refcount++;
+	DRM_DEBUG_DRIVER("using pll %d for pipe %d\n", i, intel_crtc->pipe);
+prepare: /* separate function? */
+	DRM_DEBUG_DRIVER("switching PLL %x off\n", pll->pll_reg);
+
+	/* Wait for the clocks to stabilize before rewriting the regs */
+	I915_WRITE(pll->pll_reg, dpll & ~DPLL_VCO_ENABLE);
+	POSTING_READ(pll->pll_reg);
+	DELAY(150);
+
+	I915_WRITE(pll->fp0_reg, fp);
+	I915_WRITE(pll->pll_reg, dpll & ~DPLL_VCO_ENABLE);
+	pll->on = false;
+	return pll;
+}
+
 void intel_cpt_verify_modeset(struct drm_device *dev, int pipe)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
@@ -3213,8 +3167,7 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 	}
 
 	/* disable PCH DPLL */
-	if (!intel_crtc->no_pll)
-		intel_disable_pch_pll(dev_priv, pipe);
+	intel_disable_pch_pll(intel_crtc);
 
 	/* Switch from PCDclk to Rawclk */
 	reg = FDI_RX_CTL(pipe);
@@ -3242,7 +3195,6 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 
 	DRM_LOCK(dev);
 	intel_update_fbc(dev);
-	intel_clear_scanline_wait(dev);
 	DRM_UNLOCK(dev);
 }
 
@@ -3270,6 +3222,12 @@ static void ironlake_crtc_dpms(struct drm_crtc *crtc, int mode)
 	}
 }
 
+static void ironlake_crtc_off(struct drm_crtc *crtc)
+{
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	intel_put_pch_pll(intel_crtc);
+}
+
 static void intel_crtc_dpms_overlay(struct intel_crtc *intel_crtc, bool enable)
 {
 	if (!enable && intel_crtc->overlay) {
@@ -3341,7 +3299,6 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
 	intel_crtc->active = false;
 	intel_update_fbc(dev);
 	intel_update_watermarks(dev);
-	intel_clear_scanline_wait(dev);
 }
 
 static void i9xx_crtc_dpms(struct drm_crtc *crtc, int mode)
@@ -3361,6 +3318,10 @@ static void i9xx_crtc_dpms(struct drm_crtc *crtc, int mode)
 	}
 }
 
+static void i9xx_crtc_off(struct drm_crtc *crtc)
+{
+}
+
 /**
  * Sets the power management mode of the pipe and plane.
  */
@@ -3425,26 +3386,12 @@ static void intel_crtc_disable(struct drm_crtc *crtc)
 {
 	struct drm_crtc_helper_funcs *crtc_funcs = crtc->helper_private;
 	struct drm_device *dev = crtc->dev;
-
-	/* Flush any pending WAITs before we disable the pipe. Note that
-	 * we need to drop the struct_mutex in order to acquire it again
-	 * during the lowlevel dpms routines around a couple of the
-	 * operations. It does not look trivial nor desirable to move
-	 * that locking higher. So instead we leave a window for the
-	 * submission of further commands on the fb before we can actually
-	 * disable it. This race with userspace exists anyway, and we can
-	 * only rely on the pipe being disabled by userspace after it
-	 * receives the hotplug notification and has flushed any pending
-	 * batches.
-	 */
-	if (crtc->fb) {
-		DRM_LOCK(dev);
-		intel_finish_fb(crtc->fb);
-		DRM_UNLOCK(dev);
-	}
+	struct drm_i915_private *dev_priv = dev->dev_private;
 
 	crtc_funcs->dpms(crtc, DRM_MODE_DPMS_OFF);
- 	assert_plane_disabled(dev->dev_private, to_intel_crtc(crtc)->plane);
+	dev_priv->display.off(crtc);
+
+	assert_plane_disabled(dev->dev_private, to_intel_crtc(crtc)->plane);
 	assert_pipe_disabled(dev->dev_private, to_intel_crtc(crtc)->pipe);
 
 	if (crtc->fb) {
@@ -3493,8 +3440,7 @@ void intel_encoder_commit(struct drm_encoder *encoder)
 {
 	struct drm_encoder_helper_funcs *encoder_funcs = encoder->helper_private;
 	struct drm_device *dev = encoder->dev;
-	struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
-	struct intel_crtc *intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
+	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
 
 	/* lvds has its own version of commit see intel_lvds_commit */
 	encoder_funcs->dpms(encoder, DRM_MODE_DPMS_ON);
@@ -3532,6 +3478,11 @@ static bool intel_crtc_mode_fixup(struct drm_crtc *crtc,
 	return true;
 }
 
+static int valleyview_get_display_clock_speed(struct drm_device *dev)
+{
+	return 400000; /* FIXME */
+}
+
 static int i945_get_display_clock_speed(struct drm_device *dev)
 {
 	return 400000;
@@ -3629,1341 +3580,6 @@ ironlake_compute_m_n(int bits_per_pixel, int nlanes, int pixel_clock,
 	fdi_reduce_ratio(&m_n->link_m, &m_n->link_n);
 }
 
-
-struct intel_watermark_params {
-	unsigned long fifo_size;
-	unsigned long max_wm;
-	unsigned long default_wm;
-	unsigned long guard_size;
-	unsigned long cacheline_size;
-};
-
-/* Pineview has different values for various configs */
-static const struct intel_watermark_params pineview_display_wm = {
-	PINEVIEW_DISPLAY_FIFO,
-	PINEVIEW_MAX_WM,
-	PINEVIEW_DFT_WM,
-	PINEVIEW_GUARD_WM,
-	PINEVIEW_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params pineview_display_hplloff_wm = {
-	PINEVIEW_DISPLAY_FIFO,
-	PINEVIEW_MAX_WM,
-	PINEVIEW_DFT_HPLLOFF_WM,
-	PINEVIEW_GUARD_WM,
-	PINEVIEW_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params pineview_cursor_wm = {
-	PINEVIEW_CURSOR_FIFO,
-	PINEVIEW_CURSOR_MAX_WM,
-	PINEVIEW_CURSOR_DFT_WM,
-	PINEVIEW_CURSOR_GUARD_WM,
-	PINEVIEW_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
-	PINEVIEW_CURSOR_FIFO,
-	PINEVIEW_CURSOR_MAX_WM,
-	PINEVIEW_CURSOR_DFT_WM,
-	PINEVIEW_CURSOR_GUARD_WM,
-	PINEVIEW_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params g4x_wm_info = {
-	G4X_FIFO_SIZE,
-	G4X_MAX_WM,
-	G4X_MAX_WM,
-	2,
-	G4X_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params g4x_cursor_wm_info = {
-	I965_CURSOR_FIFO,
-	I965_CURSOR_MAX_WM,
-	I965_CURSOR_DFT_WM,
-	2,
-	G4X_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params i965_cursor_wm_info = {
-	I965_CURSOR_FIFO,
-	I965_CURSOR_MAX_WM,
-	I965_CURSOR_DFT_WM,
-	2,
-	I915_FIFO_LINE_SIZE,
-};
-static const struct intel_watermark_params i945_wm_info = {
-	I945_FIFO_SIZE,
-	I915_MAX_WM,
-	1,
-	2,
-	I915_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params i915_wm_info = {
-	I915_FIFO_SIZE,
-	I915_MAX_WM,
-	1,
-	2,
-	I915_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params i855_wm_info = {
-	I855GM_FIFO_SIZE,
-	I915_MAX_WM,
-	1,
-	2,
-	I830_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params i830_wm_info = {
-	I830_FIFO_SIZE,
-	I915_MAX_WM,
-	1,
-	2,
-	I830_FIFO_LINE_SIZE
-};
-
-static const struct intel_watermark_params ironlake_display_wm_info = {
-	ILK_DISPLAY_FIFO,
-	ILK_DISPLAY_MAXWM,
-	ILK_DISPLAY_DFTWM,
-	2,
-	ILK_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params ironlake_cursor_wm_info = {
-	ILK_CURSOR_FIFO,
-	ILK_CURSOR_MAXWM,
-	ILK_CURSOR_DFTWM,
-	2,
-	ILK_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params ironlake_display_srwm_info = {
-	ILK_DISPLAY_SR_FIFO,
-	ILK_DISPLAY_MAX_SRWM,
-	ILK_DISPLAY_DFT_SRWM,
-	2,
-	ILK_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params ironlake_cursor_srwm_info = {
-	ILK_CURSOR_SR_FIFO,
-	ILK_CURSOR_MAX_SRWM,
-	ILK_CURSOR_DFT_SRWM,
-	2,
-	ILK_FIFO_LINE_SIZE
-};
-
-static const struct intel_watermark_params sandybridge_display_wm_info = {
-	SNB_DISPLAY_FIFO,
-	SNB_DISPLAY_MAXWM,
-	SNB_DISPLAY_DFTWM,
-	2,
-	SNB_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params sandybridge_cursor_wm_info = {
-	SNB_CURSOR_FIFO,
-	SNB_CURSOR_MAXWM,
-	SNB_CURSOR_DFTWM,
-	2,
-	SNB_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params sandybridge_display_srwm_info = {
-	SNB_DISPLAY_SR_FIFO,
-	SNB_DISPLAY_MAX_SRWM,
-	SNB_DISPLAY_DFT_SRWM,
-	2,
-	SNB_FIFO_LINE_SIZE
-};
-static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
-	SNB_CURSOR_SR_FIFO,
-	SNB_CURSOR_MAX_SRWM,
-	SNB_CURSOR_DFT_SRWM,
-	2,
-	SNB_FIFO_LINE_SIZE
-};
-
-
-/**
- * intel_calculate_wm - calculate watermark level
- * @clock_in_khz: pixel clock
- * @wm: chip FIFO params
- * @pixel_size: display pixel size
- * @latency_ns: memory latency for the platform
- *
- * Calculate the watermark level (the level at which the display plane will
- * start fetching from memory again).  Each chip has a different display
- * FIFO size and allocation, so the caller needs to figure that out and pass
- * in the correct intel_watermark_params structure.
- *
- * As the pixel clock runs, the FIFO will be drained at a rate that depends
- * on the pixel size.  When it reaches the watermark level, it'll start
- * fetching FIFO line sized based chunks from memory until the FIFO fills
- * past the watermark point.  If the FIFO drains completely, a FIFO underrun
- * will occur, and a display engine hang could result.
- */
-static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
-					const struct intel_watermark_params *wm,
-					int fifo_size,
-					int pixel_size,
-					unsigned long latency_ns)
-{
-	long entries_required, wm_size;
-
-	/*
-	 * Note: we need to make sure we don't overflow for various clock &
-	 * latency values.
-	 * clocks go from a few thousand to several hundred thousand.
-	 * latency is usually a few thousand
-	 */
-	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
-		1000;
-	entries_required = howmany(entries_required, wm->cacheline_size);
-
-	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
-
-	wm_size = fifo_size - (entries_required + wm->guard_size);
-
-	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
-
-	/* Don't promote wm_size to unsigned... */
-	if (wm_size > (long)wm->max_wm)
-		wm_size = wm->max_wm;
-	if (wm_size <= 0)
-		wm_size = wm->default_wm;
-	return wm_size;
-}
-
-struct cxsr_latency {
-	int is_desktop;
-	int is_ddr3;
-	unsigned long fsb_freq;
-	unsigned long mem_freq;
-	unsigned long display_sr;
-	unsigned long display_hpll_disable;
-	unsigned long cursor_sr;
-	unsigned long cursor_hpll_disable;
-};
-
-static const struct cxsr_latency cxsr_latency_table[] = {
-	{1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
-	{1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
-	{1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
-	{1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
-	{1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
-
-	{1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
-	{1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
-	{1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
-	{1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
-	{1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
-
-	{1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
-	{1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
-	{1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
-	{1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
-	{1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
-
-	{0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
-	{0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
-	{0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
-	{0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
-	{0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
-
-	{0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
-	{0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
-	{0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
-	{0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
-	{0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
-
-	{0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
-	{0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
-	{0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
-	{0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
-	{0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
-};
-
-static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
-							 int is_ddr3,
-							 int fsb,
-							 int mem)
-{
-	const struct cxsr_latency *latency;
-	int i;
-
-	if (fsb == 0 || mem == 0)
-		return NULL;
-
-	for (i = 0; i < DRM_ARRAY_SIZE(cxsr_latency_table); i++) {
-		latency = &cxsr_latency_table[i];
-		if (is_desktop == latency->is_desktop &&
-		    is_ddr3 == latency->is_ddr3 &&
-		    fsb == latency->fsb_freq && mem == latency->mem_freq)
-			return latency;
-	}
-
-	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
-
-	return NULL;
-}
-
-static void pineview_disable_cxsr(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	/* deactivate cxsr */
-	I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
-}
-
-/*
- * Latency for FIFO fetches is dependent on several factors:
- *   - memory configuration (speed, channels)
- *   - chipset
- *   - current MCH state
- * It can be fairly high in some situations, so here we assume a fairly
- * pessimal value.  It's a tradeoff between extra memory fetches (if we
- * set this value too high, the FIFO will fetch frequently to stay full)
- * and power consumption (set it too low to save power and we might see
- * FIFO underruns and display "flicker").
- *
- * A value of 5us seems to be a good balance; safe for very low end
- * platforms but not overly aggressive on lower latency configs.
- */
-static const int latency_ns = 5000;
-
-static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	uint32_t dsparb = I915_READ(DSPARB);
-	int size;
-
-	size = dsparb & 0x7f;
-	if (plane)
-		size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
-
-	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
-		      plane ? "B" : "A", size);
-
-	return size;
-}
-
-static int i85x_get_fifo_size(struct drm_device *dev, int plane)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	uint32_t dsparb = I915_READ(DSPARB);
-	int size;
-
-	size = dsparb & 0x1ff;
-	if (plane)
-		size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
-	size >>= 1; /* Convert to cachelines */
-
-	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
-		      plane ? "B" : "A", size);
-
-	return size;
-}
-
-static int i845_get_fifo_size(struct drm_device *dev, int plane)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	uint32_t dsparb = I915_READ(DSPARB);
-	int size;
-
-	size = dsparb & 0x7f;
-	size >>= 2; /* Convert to cachelines */
-
-	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
-		      plane ? "B" : "A",
-		      size);
-
-	return size;
-}
-
-static int i830_get_fifo_size(struct drm_device *dev, int plane)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	uint32_t dsparb = I915_READ(DSPARB);
-	int size;
-
-	size = dsparb & 0x7f;
-	size >>= 1; /* Convert to cachelines */
-
-	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
-		      plane ? "B" : "A", size);
-
-	return size;
-}
-
-static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
-{
-	struct drm_crtc *crtc, *enabled = NULL;
-
-	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-		if (crtc->enabled && crtc->fb) {
-			if (enabled)
-				return NULL;
-			enabled = crtc;
-		}
-	}
-
-	return enabled;
-}
-
-static void pineview_update_wm(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_crtc *crtc;
-	const struct cxsr_latency *latency;
-	u32 reg;
-	unsigned long wm;
-
-	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
-					 dev_priv->fsb_freq, dev_priv->mem_freq);
-	if (!latency) {
-		DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
-		pineview_disable_cxsr(dev);
-		return;
-	}
-
-	crtc = single_enabled_crtc(dev);
-	if (crtc) {
-		int clock = crtc->mode.clock;
-		int pixel_size = crtc->fb->bits_per_pixel / 8;
-
-		/* Display SR */
-		wm = intel_calculate_wm(clock, &pineview_display_wm,
-					pineview_display_wm.fifo_size,
-					pixel_size, latency->display_sr);
-		reg = I915_READ(DSPFW1);
-		reg &= ~DSPFW_SR_MASK;
-		reg |= wm << DSPFW_SR_SHIFT;
-		I915_WRITE(DSPFW1, reg);
-		DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
-
-		/* cursor SR */
-		wm = intel_calculate_wm(clock, &pineview_cursor_wm,
-					pineview_display_wm.fifo_size,
-					pixel_size, latency->cursor_sr);
-		reg = I915_READ(DSPFW3);
-		reg &= ~DSPFW_CURSOR_SR_MASK;
-		reg |= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
-		I915_WRITE(DSPFW3, reg);
-
-		/* Display HPLL off SR */
-		wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
-					pineview_display_hplloff_wm.fifo_size,
-					pixel_size, latency->display_hpll_disable);
-		reg = I915_READ(DSPFW3);
-		reg &= ~DSPFW_HPLL_SR_MASK;
-		reg |= wm & DSPFW_HPLL_SR_MASK;
-		I915_WRITE(DSPFW3, reg);
-
-		/* cursor HPLL off SR */
-		wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
-					pineview_display_hplloff_wm.fifo_size,
-					pixel_size, latency->cursor_hpll_disable);
-		reg = I915_READ(DSPFW3);
-		reg &= ~DSPFW_HPLL_CURSOR_MASK;
-		reg |= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
-		I915_WRITE(DSPFW3, reg);
-		DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
-
-		/* activate cxsr */
-		I915_WRITE(DSPFW3,
-			   I915_READ(DSPFW3) | PINEVIEW_SELF_REFRESH_EN);
-		DRM_DEBUG_KMS("Self-refresh is enabled\n");
-	} else {
-		pineview_disable_cxsr(dev);
-		DRM_DEBUG_KMS("Self-refresh is disabled\n");
-	}
-}
-
-static bool g4x_compute_wm0(struct drm_device *dev,
-			    int plane,
-			    const struct intel_watermark_params *display,
-			    int display_latency_ns,
-			    const struct intel_watermark_params *cursor,
-			    int cursor_latency_ns,
-			    int *plane_wm,
-			    int *cursor_wm)
-{
-	struct drm_crtc *crtc;
-	int htotal, hdisplay, clock, pixel_size;
-	int line_time_us, line_count;
-	int entries, tlb_miss;
-
-	crtc = intel_get_crtc_for_plane(dev, plane);
-	if (crtc->fb == NULL || !crtc->enabled) {
-		*cursor_wm = cursor->guard_size;
-		*plane_wm = display->guard_size;
-		return false;
-	}
-
-	htotal = crtc->mode.htotal;
-	hdisplay = crtc->mode.hdisplay;
-	clock = crtc->mode.clock;
-	pixel_size = crtc->fb->bits_per_pixel / 8;
-
-	/* Use the small buffer method to calculate plane watermark */
-	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
-	tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
-	if (tlb_miss > 0)
-		entries += tlb_miss;
-	entries = howmany(entries, display->cacheline_size);
-	*plane_wm = entries + display->guard_size;
-	if (*plane_wm > (int)display->max_wm)
-		*plane_wm = display->max_wm;
-
-	/* Use the large buffer method to calculate cursor watermark */
-	line_time_us = ((htotal * 1000) / clock);
-	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
-	entries = line_count * 64 * pixel_size;
-	tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
-	if (tlb_miss > 0)
-		entries += tlb_miss;
-	entries = howmany(entries, cursor->cacheline_size);
-	*cursor_wm = entries + cursor->guard_size;
-	if (*cursor_wm > (int)cursor->max_wm)
-		*cursor_wm = (int)cursor->max_wm;
-
-	return true;
-}
-
-/*
- * Check the wm result.
- *
- * If any calculated watermark values is larger than the maximum value that
- * can be programmed into the associated watermark register, that watermark
- * must be disabled.
- */
-static bool g4x_check_srwm(struct drm_device *dev,
-			   int display_wm, int cursor_wm,
-			   const struct intel_watermark_params *display,
-			   const struct intel_watermark_params *cursor)
-{
-	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
-		      display_wm, cursor_wm);
-
-	if (display_wm > display->max_wm) {
-		DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
-			      display_wm, display->max_wm);
-		return false;
-	}
-
-	if (cursor_wm > cursor->max_wm) {
-		DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
-			      cursor_wm, cursor->max_wm);
-		return false;
-	}
-
-	if (!(display_wm || cursor_wm)) {
-		DRM_DEBUG_KMS("SR latency is 0, disabling\n");
-		return false;
-	}
-
-	return true;
-}
-
-static bool g4x_compute_srwm(struct drm_device *dev,
-			     int plane,
-			     int latency_ns,
-			     const struct intel_watermark_params *display,
-			     const struct intel_watermark_params *cursor,
-			     int *display_wm, int *cursor_wm)
-{
-	struct drm_crtc *crtc;
-	int hdisplay, htotal, pixel_size, clock;
-	unsigned long line_time_us;
-	int line_count, line_size;
-	int small, large;
-	int entries;
-
-	if (!latency_ns) {
-		*display_wm = *cursor_wm = 0;
-		return false;
-	}
-
-	crtc = intel_get_crtc_for_plane(dev, plane);
-	hdisplay = crtc->mode.hdisplay;
-	htotal = crtc->mode.htotal;
-	clock = crtc->mode.clock;
-	pixel_size = crtc->fb->bits_per_pixel / 8;
-
-	line_time_us = (htotal * 1000) / clock;
-	line_count = (latency_ns / line_time_us + 1000) / 1000;
-	line_size = hdisplay * pixel_size;
-
-	/* Use the minimum of the small and large buffer method for primary */
-	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
-	large = line_count * line_size;
-
-	entries = howmany(min(small, large), display->cacheline_size);
-	*display_wm = entries + display->guard_size;
-
-	/* calculate the self-refresh watermark for display cursor */
-	entries = line_count * pixel_size * 64;
-	entries = howmany(entries, cursor->cacheline_size);
-	*cursor_wm = entries + cursor->guard_size;
-
-	return g4x_check_srwm(dev,
-			      *display_wm, *cursor_wm,
-			      display, cursor);
-}
-
-#define single_plane_enabled(mask) ((mask) != 0 && powerof2(mask))
-
-static void g4x_update_wm(struct drm_device *dev)
-{
-	static const int sr_latency_ns = 12000;
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
-	int plane_sr, cursor_sr;
-	unsigned int enabled = 0;
-
-	if (g4x_compute_wm0(dev, 0,
-			    &g4x_wm_info, latency_ns,
-			    &g4x_cursor_wm_info, latency_ns,
-			    &planea_wm, &cursora_wm))
-		enabled |= 1;
-
-	if (g4x_compute_wm0(dev, 1,
-			    &g4x_wm_info, latency_ns,
-			    &g4x_cursor_wm_info, latency_ns,
-			    &planeb_wm, &cursorb_wm))
-		enabled |= 2;
-
-	plane_sr = cursor_sr = 0;
-	if (single_plane_enabled(enabled) &&
-	    g4x_compute_srwm(dev, ffs(enabled) - 1,
-			     sr_latency_ns,
-			     &g4x_wm_info,
-			     &g4x_cursor_wm_info,
-			     &plane_sr, &cursor_sr))
-		I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
-	else
-		I915_WRITE(FW_BLC_SELF,
-			   I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
-
-	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
-		      planea_wm, cursora_wm,
-		      planeb_wm, cursorb_wm,
-		      plane_sr, cursor_sr);
-
-	I915_WRITE(DSPFW1,
-		   (plane_sr << DSPFW_SR_SHIFT) |
-		   (cursorb_wm << DSPFW_CURSORB_SHIFT) |
-		   (planeb_wm << DSPFW_PLANEB_SHIFT) |
-		   planea_wm);
-	I915_WRITE(DSPFW2,
-		   (I915_READ(DSPFW2) & DSPFW_CURSORA_MASK) |
-		   (cursora_wm << DSPFW_CURSORA_SHIFT));
-	/* HPLL off in SR has some issues on G4x... disable it */
-	I915_WRITE(DSPFW3,
-		   (I915_READ(DSPFW3) & ~DSPFW_HPLL_SR_EN) |
-		   (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
-}
-
-static void i965_update_wm(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_crtc *crtc;
-	int srwm = 1;
-	int cursor_sr = 16;
-
-	/* Calc sr entries for one plane configs */
-	crtc = single_enabled_crtc(dev);
-	if (crtc) {
-		/* self-refresh has much higher latency */
-		static const int sr_latency_ns = 12000;
-		int clock = crtc->mode.clock;
-		int htotal = crtc->mode.htotal;
-		int hdisplay = crtc->mode.hdisplay;
-		int pixel_size = crtc->fb->bits_per_pixel / 8;
-		unsigned long line_time_us;
-		int entries;
-
-		line_time_us = ((htotal * 1000) / clock);
-
-		/* Use ns/us then divide to preserve precision */
-		entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
-			pixel_size * hdisplay;
-		entries = howmany(entries, I915_FIFO_LINE_SIZE);
-		srwm = I965_FIFO_SIZE - entries;
-		if (srwm < 0)
-			srwm = 1;
-		srwm &= 0x1ff;
-		DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
-			      entries, srwm);
-
-		entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
-			pixel_size * 64;
-		entries = howmany(entries, i965_cursor_wm_info.cacheline_size);
-		cursor_sr = i965_cursor_wm_info.fifo_size -
-			(entries + i965_cursor_wm_info.guard_size);
-
-		if (cursor_sr > i965_cursor_wm_info.max_wm)
-			cursor_sr = i965_cursor_wm_info.max_wm;
-
-		DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
-			      "cursor %d\n", srwm, cursor_sr);
-
-		if (IS_CRESTLINE(dev))
-			I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
-	} else {
-		/* Turn off self refresh if both pipes are enabled */
-		if (IS_CRESTLINE(dev))
-			I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
-				   & ~FW_BLC_SELF_EN);
-	}
-
-	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
-		      srwm);
-
-	/* 965 has limitations... */
-	I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
-		   (8 << 16) | (8 << 8) | (8 << 0));
-	I915_WRITE(DSPFW2, (8 << 8) | (8 << 0));
-	/* update cursor SR watermark */
-	I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
-}
-
-static void i9xx_update_wm(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	const struct intel_watermark_params *wm_info;
-	uint32_t fwater_lo;
-	uint32_t fwater_hi;
-	int cwm, srwm = 1;
-	int fifo_size;
-	int planea_wm, planeb_wm;
-	struct drm_crtc *crtc, *enabled = NULL;
-
-	if (IS_I945GM(dev))
-		wm_info = &i945_wm_info;
-	else if (!IS_GEN2(dev))
-		wm_info = &i915_wm_info;
-	else
-		wm_info = &i855_wm_info;
-
-	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
-	crtc = intel_get_crtc_for_plane(dev, 0);
-	if (crtc->enabled && crtc->fb) {
-		planea_wm = intel_calculate_wm(crtc->mode.clock,
-					       wm_info, fifo_size,
-					       crtc->fb->bits_per_pixel / 8,
-					       latency_ns);
-		enabled = crtc;
-	} else
-		planea_wm = fifo_size - wm_info->guard_size;
-
-	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
-	crtc = intel_get_crtc_for_plane(dev, 1);
-	if (crtc->enabled && crtc->fb) {
-		planeb_wm = intel_calculate_wm(crtc->mode.clock,
-					       wm_info, fifo_size,
-					       crtc->fb->bits_per_pixel / 8,
-					       latency_ns);
-		if (enabled == NULL)
-			enabled = crtc;
-		else
-			enabled = NULL;
-	} else
-		planeb_wm = fifo_size - wm_info->guard_size;
-
-	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
-
-	/*
-	 * Overlay gets an aggressive default since video jitter is bad.
-	 */
-	cwm = 2;
-
-	/* Play safe and disable self-refresh before adjusting watermarks. */
-	if (IS_I945G(dev) || IS_I945GM(dev))
-		I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK | 0);
-	else if (IS_I915GM(dev))
-		I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
-
-	/* Calc sr entries for one plane configs */
-	if (HAS_FW_BLC(dev) && enabled) {
-		/* self-refresh has much higher latency */
-		static const int sr_latency_ns = 6000;
-		int clock = enabled->mode.clock;
-		int htotal = enabled->mode.htotal;
-		int hdisplay = enabled->mode.hdisplay;
-		int pixel_size = enabled->fb->bits_per_pixel / 8;
-		unsigned long line_time_us;
-		int entries;
-
-		line_time_us = (htotal * 1000) / clock;
-
-		/* Use ns/us then divide to preserve precision */
-		entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
-			pixel_size * hdisplay;
-		entries = howmany(entries, wm_info->cacheline_size);
-		DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
-		srwm = wm_info->fifo_size - entries;
-		if (srwm < 0)
-			srwm = 1;
-
-		if (IS_I945G(dev) || IS_I945GM(dev))
-			I915_WRITE(FW_BLC_SELF,
-				   FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
-		else if (IS_I915GM(dev))
-			I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
-	}
-
-	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
-		      planea_wm, planeb_wm, cwm, srwm);
-
-	fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
-	fwater_hi = (cwm & 0x1f);
-
-	/* Set request length to 8 cachelines per fetch */
-	fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
-	fwater_hi = fwater_hi | (1 << 8);
-
-	I915_WRITE(FW_BLC, fwater_lo);
-	I915_WRITE(FW_BLC2, fwater_hi);
-
-	if (HAS_FW_BLC(dev)) {
-		if (enabled) {
-			if (IS_I945G(dev) || IS_I945GM(dev))
-				I915_WRITE(FW_BLC_SELF,
-					   FW_BLC_SELF_EN_MASK | FW_BLC_SELF_EN);
-			else if (IS_I915GM(dev))
-				I915_WRITE(INSTPM, I915_READ(INSTPM) | INSTPM_SELF_EN);
-			DRM_DEBUG_KMS("memory self refresh enabled\n");
-		} else
-			DRM_DEBUG_KMS("memory self refresh disabled\n");
-	}
-}
-
-static void i830_update_wm(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_crtc *crtc;
-	uint32_t fwater_lo;
-	int planea_wm;
-
-	crtc = single_enabled_crtc(dev);
-	if (crtc == NULL)
-		return;
-
-	planea_wm = intel_calculate_wm(crtc->mode.clock, &i830_wm_info,
-				       dev_priv->display.get_fifo_size(dev, 0),
-				       crtc->fb->bits_per_pixel / 8,
-				       latency_ns);
-	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
-	fwater_lo |= (3<<8) | planea_wm;
-
-	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
-
-	I915_WRITE(FW_BLC, fwater_lo);
-}
-
-#define ILK_LP0_PLANE_LATENCY		700
-#define ILK_LP0_CURSOR_LATENCY		1300
-
-/*
- * Check the wm result.
- *
- * If any calculated watermark values is larger than the maximum value that
- * can be programmed into the associated watermark register, that watermark
- * must be disabled.
- */
-static bool ironlake_check_srwm(struct drm_device *dev, int level,
-				int fbc_wm, int display_wm, int cursor_wm,
-				const struct intel_watermark_params *display,
-				const struct intel_watermark_params *cursor)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
-		      " cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
-
-	if (fbc_wm > SNB_FBC_MAX_SRWM) {
-		DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
-			      fbc_wm, SNB_FBC_MAX_SRWM, level);
-
-		/* fbc has it's own way to disable FBC WM */
-		I915_WRITE(DISP_ARB_CTL,
-			   I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS);
-		return false;
-	}
-
-	if (display_wm > display->max_wm) {
-		DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
-			      display_wm, SNB_DISPLAY_MAX_SRWM, level);
-		return false;
-	}
-
-	if (cursor_wm > cursor->max_wm) {
-		DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
-			      cursor_wm, SNB_CURSOR_MAX_SRWM, level);
-		return false;
-	}
-
-	if (!(fbc_wm || display_wm || cursor_wm)) {
-		DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
-		return false;
-	}
-
-	return true;
-}
-
-/*
- * Compute watermark values of WM[1-3],
- */
-static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
-				  int latency_ns,
-				  const struct intel_watermark_params *display,
-				  const struct intel_watermark_params *cursor,
-				  int *fbc_wm, int *display_wm, int *cursor_wm)
-{
-	struct drm_crtc *crtc;
-	unsigned long line_time_us;
-	int hdisplay, htotal, pixel_size, clock;
-	int line_count, line_size;
-	int small, large;
-	int entries;
-
-	if (!latency_ns) {
-		*fbc_wm = *display_wm = *cursor_wm = 0;
-		return false;
-	}
-
-	crtc = intel_get_crtc_for_plane(dev, plane);
-	hdisplay = crtc->mode.hdisplay;
-	htotal = crtc->mode.htotal;
-	clock = crtc->mode.clock;
-	pixel_size = crtc->fb->bits_per_pixel / 8;
-
-	line_time_us = (htotal * 1000) / clock;
-	line_count = (latency_ns / line_time_us + 1000) / 1000;
-	line_size = hdisplay * pixel_size;
-
-	/* Use the minimum of the small and large buffer method for primary */
-	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
-	large = line_count * line_size;
-
-	entries = howmany(min(small, large), display->cacheline_size);
-	*display_wm = entries + display->guard_size;
-
-	/*
-	 * Spec says:
-	 * FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
-	 */
-	*fbc_wm = howmany(*display_wm * 64, line_size) + 2;
-
-	/* calculate the self-refresh watermark for display cursor */
-	entries = line_count * pixel_size * 64;
-	entries = howmany(entries, cursor->cacheline_size);
-	*cursor_wm = entries + cursor->guard_size;
-
-	return ironlake_check_srwm(dev, level,
-				   *fbc_wm, *display_wm, *cursor_wm,
-				   display, cursor);
-}
-
-static void ironlake_update_wm(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	int fbc_wm, plane_wm, cursor_wm;
-	unsigned int enabled;
-
-	enabled = 0;
-	if (g4x_compute_wm0(dev, 0,
-			    &ironlake_display_wm_info,
-			    ILK_LP0_PLANE_LATENCY,
-			    &ironlake_cursor_wm_info,
-			    ILK_LP0_CURSOR_LATENCY,
-			    &plane_wm, &cursor_wm)) {
-		I915_WRITE(WM0_PIPEA_ILK,
-			   (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
-		DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
-			      " plane %d, " "cursor: %d\n",
-			      plane_wm, cursor_wm);
-		enabled |= 1;
-	}
-
-	if (g4x_compute_wm0(dev, 1,
-			    &ironlake_display_wm_info,
-			    ILK_LP0_PLANE_LATENCY,
-			    &ironlake_cursor_wm_info,
-			    ILK_LP0_CURSOR_LATENCY,
-			    &plane_wm, &cursor_wm)) {
-		I915_WRITE(WM0_PIPEB_ILK,
-			   (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
-		DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
-			      " plane %d, cursor: %d\n",
-			      plane_wm, cursor_wm);
-		enabled |= 2;
-	}
-
-	/*
-	 * Calculate and update the self-refresh watermark only when one
-	 * display plane is used.
-	 */
-	I915_WRITE(WM3_LP_ILK, 0);
-	I915_WRITE(WM2_LP_ILK, 0);
-	I915_WRITE(WM1_LP_ILK, 0);
-
-	if (!single_plane_enabled(enabled))
-		return;
-	enabled = ffs(enabled) - 1;
-
-	/* WM1 */
-	if (!ironlake_compute_srwm(dev, 1, enabled,
-				   ILK_READ_WM1_LATENCY() * 500,
-				   &ironlake_display_srwm_info,
-				   &ironlake_cursor_srwm_info,
-				   &fbc_wm, &plane_wm, &cursor_wm))
-		return;
-
-	I915_WRITE(WM1_LP_ILK,
-		   WM1_LP_SR_EN |
-		   (ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
-		   (fbc_wm << WM1_LP_FBC_SHIFT) |
-		   (plane_wm << WM1_LP_SR_SHIFT) |
-		   cursor_wm);
-
-	/* WM2 */
-	if (!ironlake_compute_srwm(dev, 2, enabled,
-				   ILK_READ_WM2_LATENCY() * 500,
-				   &ironlake_display_srwm_info,
-				   &ironlake_cursor_srwm_info,
-				   &fbc_wm, &plane_wm, &cursor_wm))
-		return;
-
-	I915_WRITE(WM2_LP_ILK,
-		   WM2_LP_EN |
-		   (ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
-		   (fbc_wm << WM1_LP_FBC_SHIFT) |
-		   (plane_wm << WM1_LP_SR_SHIFT) |
-		   cursor_wm);
-
-	/*
-	 * WM3 is unsupported on ILK, probably because we don't have latency
-	 * data for that power state
-	 */
-}
-
-void sandybridge_update_wm(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	int latency = SNB_READ_WM0_LATENCY() * 100;	/* In unit 0.1us */
-	u32 val;
-	int fbc_wm, plane_wm, cursor_wm;
-	unsigned int enabled;
-
-	enabled = 0;
-	if (g4x_compute_wm0(dev, 0,
-			    &sandybridge_display_wm_info, latency,
-			    &sandybridge_cursor_wm_info, latency,
-			    &plane_wm, &cursor_wm)) {
-		val = I915_READ(WM0_PIPEA_ILK);
-		val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
-		I915_WRITE(WM0_PIPEA_ILK, val |
-			   ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
-		DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
-			      " plane %d, " "cursor: %d\n",
-			      plane_wm, cursor_wm);
-		enabled |= 1;
-	}
-
-	if (g4x_compute_wm0(dev, 1,
-			    &sandybridge_display_wm_info, latency,
-			    &sandybridge_cursor_wm_info, latency,
-			    &plane_wm, &cursor_wm)) {
-		val = I915_READ(WM0_PIPEB_ILK);
-		val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
-		I915_WRITE(WM0_PIPEB_ILK, val |
-			   ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
-		DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
-			      " plane %d, cursor: %d\n",
-			      plane_wm, cursor_wm);
-		enabled |= 2;
-	}
-
-	/* IVB has 3 pipes */
-	if (IS_IVYBRIDGE(dev) &&
-	    g4x_compute_wm0(dev, 2,
-			    &sandybridge_display_wm_info, latency,
-			    &sandybridge_cursor_wm_info, latency,
-			    &plane_wm, &cursor_wm)) {
-		val = I915_READ(WM0_PIPEC_IVB);
-		val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
-		I915_WRITE(WM0_PIPEC_IVB, val |
-			   ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
-		DRM_DEBUG_KMS("FIFO watermarks For pipe C -"
-			      " plane %d, cursor: %d\n",
-			      plane_wm, cursor_wm);
-		enabled |= 3;
-	}
-
-	/*
-	 * Calculate and update the self-refresh watermark only when one
-	 * display plane is used.
-	 *
-	 * SNB support 3 levels of watermark.
-	 *
-	 * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
-	 * and disabled in the descending order
-	 *
-	 */
-	I915_WRITE(WM3_LP_ILK, 0);
-	I915_WRITE(WM2_LP_ILK, 0);
-	I915_WRITE(WM1_LP_ILK, 0);
-
-	if (!single_plane_enabled(enabled) ||
-	    dev_priv->sprite_scaling_enabled)
-		return;
-	enabled = ffs(enabled) - 1;
-
-	/* WM1 */
-	if (!ironlake_compute_srwm(dev, 1, enabled,
-				   SNB_READ_WM1_LATENCY() * 500,
-				   &sandybridge_display_srwm_info,
-				   &sandybridge_cursor_srwm_info,
-				   &fbc_wm, &plane_wm, &cursor_wm))
-		return;
-
-	I915_WRITE(WM1_LP_ILK,
-		   WM1_LP_SR_EN |
-		   (SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
-		   (fbc_wm << WM1_LP_FBC_SHIFT) |
-		   (plane_wm << WM1_LP_SR_SHIFT) |
-		   cursor_wm);
-
-	/* WM2 */
-	if (!ironlake_compute_srwm(dev, 2, enabled,
-				   SNB_READ_WM2_LATENCY() * 500,
-				   &sandybridge_display_srwm_info,
-				   &sandybridge_cursor_srwm_info,
-				   &fbc_wm, &plane_wm, &cursor_wm))
-		return;
-
-	I915_WRITE(WM2_LP_ILK,
-		   WM2_LP_EN |
-		   (SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
-		   (fbc_wm << WM1_LP_FBC_SHIFT) |
-		   (plane_wm << WM1_LP_SR_SHIFT) |
-		   cursor_wm);
-
-	/* WM3 */
-	if (!ironlake_compute_srwm(dev, 3, enabled,
-				   SNB_READ_WM3_LATENCY() * 500,
-				   &sandybridge_display_srwm_info,
-				   &sandybridge_cursor_srwm_info,
-				   &fbc_wm, &plane_wm, &cursor_wm))
-		return;
-
-	I915_WRITE(WM3_LP_ILK,
-		   WM3_LP_EN |
-		   (SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) |
-		   (fbc_wm << WM1_LP_FBC_SHIFT) |
-		   (plane_wm << WM1_LP_SR_SHIFT) |
-		   cursor_wm);
-}
-
-static bool
-sandybridge_compute_sprite_wm(struct drm_device *dev, int plane,
-			      uint32_t sprite_width, int pixel_size,
-			      const struct intel_watermark_params *display,
-			      int display_latency_ns, int *sprite_wm)
-{
-	struct drm_crtc *crtc;
-	int clock;
-	int entries, tlb_miss;
-
-	crtc = intel_get_crtc_for_plane(dev, plane);
-	if (crtc->fb == NULL || !crtc->enabled) {
-		*sprite_wm = display->guard_size;
-		return false;
-	}
-
-	clock = crtc->mode.clock;
-
-	/* Use the small buffer method to calculate the sprite watermark */
-	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
-	tlb_miss = display->fifo_size*display->cacheline_size -
-		sprite_width * 8;
-	if (tlb_miss > 0)
-		entries += tlb_miss;
-	entries = howmany(entries, display->cacheline_size);
-	*sprite_wm = entries + display->guard_size;
-	if (*sprite_wm > (int)display->max_wm)
-		*sprite_wm = display->max_wm;
-
-	return true;
-}
-
-static bool
-sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane,
-				uint32_t sprite_width, int pixel_size,
-				const struct intel_watermark_params *display,
-				int latency_ns, int *sprite_wm)
-{
-	struct drm_crtc *crtc;
-	unsigned long line_time_us;
-	int clock;
-	int line_count, line_size;
-	int small, large;
-	int entries;
-
-	if (!latency_ns) {
-		*sprite_wm = 0;
-		return false;
-	}
-
-	crtc = intel_get_crtc_for_plane(dev, plane);
-	clock = crtc->mode.clock;
-	if (!clock) {
-		*sprite_wm = 0;
-		return false;
-	}
-
-	line_time_us = (sprite_width * 1000) / clock;
-	if (!line_time_us) {
-		*sprite_wm = 0;
-		return false;
-	}
-
-	line_count = (latency_ns / line_time_us + 1000) / 1000;
-	line_size = sprite_width * pixel_size;
-
-	/* Use the minimum of the small and large buffer method for primary */
-	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
-	large = line_count * line_size;
-
-	entries = howmany(min(small, large), display->cacheline_size);
-	*sprite_wm = entries + display->guard_size;
-
-	return *sprite_wm > 0x3ff ? false : true;
-}
-
-static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
-					 uint32_t sprite_width, int pixel_size)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	int latency = SNB_READ_WM0_LATENCY() * 100;	/* In unit 0.1us */
-	u32 val;
-	int sprite_wm, reg;
-	int ret;
-
-	switch (pipe) {
-	case 0:
-		reg = WM0_PIPEA_ILK;
-		break;
-	case 1:
-		reg = WM0_PIPEB_ILK;
-		break;
-	case 2:
-		reg = WM0_PIPEC_IVB;
-		break;
-	default:
-		return; /* bad pipe */
-	}
-
-	ret = sandybridge_compute_sprite_wm(dev, pipe, sprite_width, pixel_size,
-					    &sandybridge_display_wm_info,
-					    latency, &sprite_wm);
-	if (!ret) {
-		DRM_DEBUG_KMS("failed to compute sprite wm for pipe %d\n",
-			      pipe);
-		return;
-	}
-
-	val = I915_READ(reg);
-	val &= ~WM0_PIPE_SPRITE_MASK;
-	I915_WRITE(reg, val | (sprite_wm << WM0_PIPE_SPRITE_SHIFT));
-	DRM_DEBUG_KMS("sprite watermarks For pipe %d - %d\n", pipe, sprite_wm);
-
-
-	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
-					      pixel_size,
-					      &sandybridge_display_srwm_info,
-					      SNB_READ_WM1_LATENCY() * 500,
-					      &sprite_wm);
-	if (!ret) {
-		DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %d\n",
-			      pipe);
-		return;
-	}
-	I915_WRITE(WM1S_LP_ILK, sprite_wm);
-
-	/* Only IVB has two more LP watermarks for sprite */
-	if (!IS_IVYBRIDGE(dev))
-		return;
-
-	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
-					      pixel_size,
-					      &sandybridge_display_srwm_info,
-					      SNB_READ_WM2_LATENCY() * 500,
-					      &sprite_wm);
-	if (!ret) {
-		DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %d\n",
-			      pipe);
-		return;
-	}
-	I915_WRITE(WM2S_LP_IVB, sprite_wm);
-
-	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
-					      pixel_size,
-					      &sandybridge_display_srwm_info,
-					      SNB_READ_WM3_LATENCY() * 500,
-					      &sprite_wm);
-	if (!ret) {
-		DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %d\n",
-			      pipe);
-		return;
-	}
-	I915_WRITE(WM3S_LP_IVB, sprite_wm);
-}
-
-/**
- * intel_update_watermarks - update FIFO watermark values based on current modes
- *
- * Calculate watermark values for the various WM regs based on current mode
- * and plane configuration.
- *
- * There are several cases to deal with here:
- *   - normal (i.e. non-self-refresh)
- *   - self-refresh (SR) mode
- *   - lines are large relative to FIFO size (buffer can hold up to 2)
- *   - lines are small relative to FIFO size (buffer can hold more than 2
- *     lines), so need to account for TLB latency
- *
- *   The normal calculation is:
- *     watermark = dotclock * bytes per pixel * latency
- *   where latency is platform & configuration dependent (we assume pessimal
- *   values here).
- *
- *   The SR calculation is:
- *     watermark = (trunc(latency/line time)+1) * surface width *
- *       bytes per pixel
- *   where
- *     line time = htotal / dotclock
- *     surface width = hdisplay for normal plane and 64 for cursor
- *   and latency is assumed to be high, as above.
- *
- * The final value programmed to the register should always be rounded up,
- * and include an extra 2 entries to account for clock crossings.
- *
- * We don't use the sprite, so we can ignore that.  And on Crestline we have
- * to set the non-SR watermarks to 8.
- */
-static void intel_update_watermarks(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	if (dev_priv->display.update_wm)
-		dev_priv->display.update_wm(dev);
-}
-
-void intel_update_sprite_watermarks(struct drm_device *dev, int pipe,
-				    uint32_t sprite_width, int pixel_size)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	if (dev_priv->display.update_sprite_wm)
-		dev_priv->display.update_sprite_wm(dev, pipe, sprite_width,
-						   pixel_size);
-}
-
 static inline bool intel_panel_use_ssc(struct drm_i915_private *dev_priv)
 {
 	if (i915_panel_use_ssc >= 0)
@@ -5187,6 +3803,222 @@ static void i9xx_update_pll_dividers(struct drm_crtc *crtc,
 	}
 }
 
+static void intel_update_lvds(struct drm_crtc *crtc, intel_clock_t *clock,
+			      struct drm_display_mode *adjusted_mode)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	int pipe = intel_crtc->pipe;
+	u32 temp;
+
+	temp = I915_READ(LVDS);
+	temp |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP;
+	if (pipe == 1) {
+		temp |= LVDS_PIPEB_SELECT;
+	} else {
+		temp &= ~LVDS_PIPEB_SELECT;
+	}
+	/* set the corresponsding LVDS_BORDER bit */
+	temp |= dev_priv->lvds_border_bits;
+	/* Set the B0-B3 data pairs corresponding to whether we're going to
+	 * set the DPLLs for dual-channel mode or not.
+	 */
+	if (clock->p2 == 7)
+		temp |= LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP;
+	else
+		temp &= ~(LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP);
+
+	/* It would be nice to set 24 vs 18-bit mode (LVDS_A3_POWER_UP)
+	 * appropriately here, but we need to look more thoroughly into how
+	 * panels behave in the two modes.
+	 */
+	/* set the dithering flag on LVDS as needed */
+	if (INTEL_INFO(dev)->gen >= 4) {
+		if (dev_priv->lvds_dither)
+			temp |= LVDS_ENABLE_DITHER;
+		else
+			temp &= ~LVDS_ENABLE_DITHER;
+	}
+	temp &= ~(LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY);
+	if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC)
+		temp |= LVDS_HSYNC_POLARITY;
+	if (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC)
+		temp |= LVDS_VSYNC_POLARITY;
+	I915_WRITE(LVDS, temp);
+}
+
+static void i9xx_update_pll(struct drm_crtc *crtc,
+			    struct drm_display_mode *mode,
+			    struct drm_display_mode *adjusted_mode,
+			    intel_clock_t *clock, intel_clock_t *reduced_clock,
+			    int num_connectors)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	int pipe = intel_crtc->pipe;
+	u32 dpll;
+	bool is_sdvo;
+
+	is_sdvo = intel_pipe_has_type(crtc, INTEL_OUTPUT_SDVO) ||
+		intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI);
+
+	dpll = DPLL_VGA_MODE_DIS;
+
+	if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
+		dpll |= DPLLB_MODE_LVDS;
+	else
+		dpll |= DPLLB_MODE_DAC_SERIAL;
+	if (is_sdvo) {
+		int pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
+		if (pixel_multiplier > 1) {
+			if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
+				dpll |= (pixel_multiplier - 1) << SDVO_MULTIPLIER_SHIFT_HIRES;
+		}
+		dpll |= DPLL_DVO_HIGH_SPEED;
+	}
+	if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT))
+		dpll |= DPLL_DVO_HIGH_SPEED;
+
+	/* compute bitmask from p1 value */
+	if (IS_PINEVIEW(dev))
+		dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
+	else {
+		dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
+		if (IS_G4X(dev) && reduced_clock)
+			dpll |= (1 << (reduced_clock->p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
+	}
+	switch (clock->p2) {
+	case 5:
+		dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
+		break;
+	case 7:
+		dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
+		break;
+	case 10:
+		dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
+		break;
+	case 14:
+		dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
+		break;
+	}
+	if (INTEL_INFO(dev)->gen >= 4)
+		dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
+
+	if (is_sdvo && intel_pipe_has_type(crtc, INTEL_OUTPUT_TVOUT))
+		dpll |= PLL_REF_INPUT_TVCLKINBC;
+	else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_TVOUT))
+		/* XXX: just matching BIOS for now */
+		/*	dpll |= PLL_REF_INPUT_TVCLKINBC; */
+		dpll |= 3;
+	else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) &&
+		 intel_panel_use_ssc(dev_priv) && num_connectors < 2)
+		dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
+	else
+		dpll |= PLL_REF_INPUT_DREFCLK;
+
+	dpll |= DPLL_VCO_ENABLE;
+	I915_WRITE(DPLL(pipe), dpll & ~DPLL_VCO_ENABLE);
+	POSTING_READ(DPLL(pipe));
+	DELAY(150);
+
+	/* The LVDS pin pair needs to be on before the DPLLs are enabled.
+	 * This is an exception to the general rule that mode_set doesn't turn
+	 * things on.
+	 */
+	if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
+		intel_update_lvds(crtc, clock, adjusted_mode);
+
+	if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT))
+		intel_dp_set_m_n(crtc, mode, adjusted_mode);
+
+	I915_WRITE(DPLL(pipe), dpll);
+
+	/* Wait for the clocks to stabilize. */
+	POSTING_READ(DPLL(pipe));
+	DELAY(150);
+
+	if (INTEL_INFO(dev)->gen >= 4) {
+		u32 temp = 0;
+		if (is_sdvo) {
+			temp = intel_mode_get_pixel_multiplier(adjusted_mode);
+			if (temp > 1)
+				temp = (temp - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
+			else
+				temp = 0;
+		}
+		I915_WRITE(DPLL_MD(pipe), temp);
+	} else {
+		/* The pixel multiplier can only be updated once the
+		 * DPLL is enabled and the clocks are stable.
+		 *
+		 * So write it again.
+		 */
+		I915_WRITE(DPLL(pipe), dpll);
+	}
+}
+
+static void i8xx_update_pll(struct drm_crtc *crtc,
+			    struct drm_display_mode *adjusted_mode,
+			    intel_clock_t *clock,
+			    int num_connectors)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	int pipe = intel_crtc->pipe;
+	u32 dpll;
+
+	dpll = DPLL_VGA_MODE_DIS;
+
+	if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS)) {
+		dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
+	} else {
+		if (clock->p1 == 2)
+			dpll |= PLL_P1_DIVIDE_BY_TWO;
+		else
+			dpll |= (clock->p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
+		if (clock->p2 == 4)
+			dpll |= PLL_P2_DIVIDE_BY_4;
+	}
+
+	if (intel_pipe_has_type(crtc, INTEL_OUTPUT_TVOUT))
+		/* XXX: just matching BIOS for now */
+		/*	dpll |= PLL_REF_INPUT_TVCLKINBC; */
+		dpll |= 3;
+	else if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS) &&
+		 intel_panel_use_ssc(dev_priv) && num_connectors < 2)
+		dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
+	else
+		dpll |= PLL_REF_INPUT_DREFCLK;
+
+	dpll |= DPLL_VCO_ENABLE;
+	I915_WRITE(DPLL(pipe), dpll & ~DPLL_VCO_ENABLE);
+	POSTING_READ(DPLL(pipe));
+	DELAY(150);
+
+	I915_WRITE(DPLL(pipe), dpll);
+
+	/* Wait for the clocks to stabilize. */
+	POSTING_READ(DPLL(pipe));
+	DELAY(150);
+
+	/* The LVDS pin pair needs to be on before the DPLLs are enabled.
+	 * This is an exception to the general rule that mode_set doesn't turn
+	 * things on.
+	 */
+	if (intel_pipe_has_type(crtc, INTEL_OUTPUT_LVDS))
+		intel_update_lvds(crtc, clock, adjusted_mode);
+
+	/* The pixel multiplier can only be updated once the
+	 * DPLL is enabled and the clocks are stable.
+	 *
+	 * So write it again.
+	 */
+	I915_WRITE(DPLL(pipe), dpll);
+}
+
 static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
 			      struct drm_display_mode *mode,
 			      struct drm_display_mode *adjusted_mode,
@@ -5200,15 +4032,13 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
 	int plane = intel_crtc->plane;
 	int refclk, num_connectors = 0;
 	intel_clock_t clock, reduced_clock;
-	u32 dpll, dspcntr, pipeconf, vsyncshift;
-	bool ok, has_reduced_clock = false, is_sdvo = false, is_dvo = false;
-	bool is_crt = false, is_lvds = false, is_tv = false, is_dp = false;
+	u32 dspcntr, pipeconf, vsyncshift;
+	bool ok, has_reduced_clock = false, is_sdvo = false;
+	bool is_lvds = false, is_tv = false, is_dp = false;
 	struct drm_mode_config *mode_config = &dev->mode_config;
 	struct intel_encoder *encoder;
 	const intel_limit_t *limit;
 	int ret;
-	u32 temp;
-	u32 lvds_sync = 0;
 
 	list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
 		if (encoder->base.crtc != crtc)
@@ -5224,15 +4054,9 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
 			if (encoder->needs_tv_clock)
 				is_tv = true;
 			break;
-		case INTEL_OUTPUT_DVO:
-			is_dvo = true;
-			break;
 		case INTEL_OUTPUT_TVOUT:
 			is_tv = true;
 			break;
-		case INTEL_OUTPUT_ANALOG:
-			is_crt = true;
-			break;
 		case INTEL_OUTPUT_DISPLAYPORT:
 			is_dp = true;
 			break;
@@ -5279,71 +4103,12 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
 	i9xx_update_pll_dividers(crtc, &clock, has_reduced_clock ?
 				 &reduced_clock : NULL);
 
-	dpll = DPLL_VGA_MODE_DIS;
-
-	if (!IS_GEN2(dev)) {
-		if (is_lvds)
-			dpll |= DPLLB_MODE_LVDS;
-		else
-			dpll |= DPLLB_MODE_DAC_SERIAL;
-		if (is_sdvo) {
-			int pixel_multiplier = intel_mode_get_pixel_multiplier(adjusted_mode);
-			if (pixel_multiplier > 1) {
-				if (IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
-					dpll |= (pixel_multiplier - 1) << SDVO_MULTIPLIER_SHIFT_HIRES;
-			}
-			dpll |= DPLL_DVO_HIGH_SPEED;
-		}
-		if (is_dp)
-			dpll |= DPLL_DVO_HIGH_SPEED;
-
-		/* compute bitmask from p1 value */
-		if (IS_PINEVIEW(dev))
-			dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT_PINEVIEW;
-		else {
-			dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
-			if (IS_G4X(dev) && has_reduced_clock)
-				dpll |= (1 << (reduced_clock.p1 - 1)) << DPLL_FPA1_P1_POST_DIV_SHIFT;
-		}
-		switch (clock.p2) {
-		case 5:
-			dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_5;
-			break;
-		case 7:
-			dpll |= DPLLB_LVDS_P2_CLOCK_DIV_7;
-			break;
-		case 10:
-			dpll |= DPLL_DAC_SERIAL_P2_CLOCK_DIV_10;
-			break;
-		case 14:
-			dpll |= DPLLB_LVDS_P2_CLOCK_DIV_14;
-			break;
-		}
-		if (INTEL_INFO(dev)->gen >= 4)
-			dpll |= (6 << PLL_LOAD_PULSE_PHASE_SHIFT);
-	} else {
-		if (is_lvds) {
-			dpll |= (1 << (clock.p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
-		} else {
-			if (clock.p1 == 2)
-				dpll |= PLL_P1_DIVIDE_BY_TWO;
-			else
-				dpll |= (clock.p1 - 2) << DPLL_FPA01_P1_POST_DIV_SHIFT;
-			if (clock.p2 == 4)
-				dpll |= PLL_P2_DIVIDE_BY_4;
-		}
-	}
-
-	if (is_sdvo && is_tv)
-		dpll |= PLL_REF_INPUT_TVCLKINBC;
-	else if (is_tv)
-		/* XXX: just matching BIOS for now */
-		/*	dpll |= PLL_REF_INPUT_TVCLKINBC; */
-		dpll |= 3;
-	else if (is_lvds && intel_panel_use_ssc(dev_priv) && num_connectors < 2)
-		dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
+	if (IS_GEN2(dev))
+		i8xx_update_pll(crtc, adjusted_mode, &clock, num_connectors);
 	else
-		dpll |= PLL_REF_INPUT_DREFCLK;
+		i9xx_update_pll(crtc, mode, adjusted_mode, &clock,
+				has_reduced_clock ? &reduced_clock : NULL,
+				num_connectors);
 
 	/* setup pipeconf */
 	pipeconf = I915_READ(PIPECONF(pipe));
@@ -5380,97 +4145,9 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
 		}
 	}
 
-	dpll |= DPLL_VCO_ENABLE;
-
 	DRM_DEBUG_KMS("Mode for pipe %c:\n", pipe == 0 ? 'A' : 'B');
 	drm_mode_debug_printmodeline(mode);
 
-	I915_WRITE(DPLL(pipe), dpll & ~DPLL_VCO_ENABLE);
-
-	POSTING_READ(DPLL(pipe));
-	DELAY(150);
-
-	/* The LVDS pin pair needs to be on before the DPLLs are enabled.
-	 * This is an exception to the general rule that mode_set doesn't turn
-	 * things on.
-	 */
-	if (is_lvds) {
-		temp = I915_READ(LVDS);
-		temp |= LVDS_PORT_EN | LVDS_A0A2_CLKA_POWER_UP;
-		if (pipe == 1) {
-			temp |= LVDS_PIPEB_SELECT;
-		} else {
-			temp &= ~LVDS_PIPEB_SELECT;
-		}
-		/* set the corresponsding LVDS_BORDER bit */
-		temp |= dev_priv->lvds_border_bits;
-		/* Set the B0-B3 data pairs corresponding to whether we're going to
-		 * set the DPLLs for dual-channel mode or not.
-		 */
-		if (clock.p2 == 7)
-			temp |= LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP;
-		else
-			temp &= ~(LVDS_B0B3_POWER_UP | LVDS_CLKB_POWER_UP);
-
-		/* It would be nice to set 24 vs 18-bit mode (LVDS_A3_POWER_UP)
-		 * appropriately here, but we need to look more thoroughly into how
-		 * panels behave in the two modes.
-		 */
-		/* set the dithering flag on LVDS as needed */
-		if (INTEL_INFO(dev)->gen >= 4) {
-			if (dev_priv->lvds_dither)
-				temp |= LVDS_ENABLE_DITHER;
-			else
-				temp &= ~LVDS_ENABLE_DITHER;
-		}
-		if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC)
-			lvds_sync |= LVDS_HSYNC_POLARITY;
-		if (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC)
-			lvds_sync |= LVDS_VSYNC_POLARITY;
-		if ((temp & (LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY))
-		    != lvds_sync) {
-			char flags[2] = "-+";
-			DRM_INFO("Changing LVDS panel from "
-				 "(%chsync, %cvsync) to (%chsync, %cvsync)\n",
-				 flags[!(temp & LVDS_HSYNC_POLARITY)],
-				 flags[!(temp & LVDS_VSYNC_POLARITY)],
-				 flags[!(lvds_sync & LVDS_HSYNC_POLARITY)],
-				 flags[!(lvds_sync & LVDS_VSYNC_POLARITY)]);
-			temp &= ~(LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY);
-			temp |= lvds_sync;
-		}
-		I915_WRITE(LVDS, temp);
-	}
-
-	if (is_dp) {
-		intel_dp_set_m_n(crtc, mode, adjusted_mode);
-	}
-
-	I915_WRITE(DPLL(pipe), dpll);
-
-	/* Wait for the clocks to stabilize. */
-	POSTING_READ(DPLL(pipe));
-	DELAY(150);
-
-	if (INTEL_INFO(dev)->gen >= 4) {
-		temp = 0;
-		if (is_sdvo) {
-			temp = intel_mode_get_pixel_multiplier(adjusted_mode);
-			if (temp > 1)
-				temp = (temp - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
-			else
-				temp = 0;
-		}
-		I915_WRITE(DPLL_MD(pipe), temp);
-	} else {
-		/* The pixel multiplier can only be updated once the
-		 * DPLL is enabled and the clocks are stable.
-		 *
-		 * So write it again.
-		 */
-		I915_WRITE(DPLL(pipe), dpll);
-	}
-
 	if (HAS_PIPE_CXSR(dev)) {
 		if (intel_crtc->lowfreq_avail) {
 			DRM_DEBUG_KMS("enabling CxSR downclocking\n");
@@ -5536,7 +4213,6 @@ static int i9xx_crtc_mode_set(struct drm_crtc *crtc,
 
 	I915_WRITE(DSPCNTR(plane), dspcntr);
 	POSTING_READ(DSPCNTR(plane));
-	intel_enable_plane(dev_priv, plane, pipe);
 
 	ret = intel_pipe_set_base(crtc, x, y, old_fb);
 
@@ -5712,17 +4388,16 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 	u32 dpll, fp = 0, fp2 = 0, dspcntr, pipeconf;
 	bool ok, has_reduced_clock = false, is_sdvo = false;
 	bool is_crt = false, is_lvds = false, is_tv = false, is_dp = false;
-	struct intel_encoder *has_edp_encoder = NULL;
 	struct drm_mode_config *mode_config = &dev->mode_config;
-	struct intel_encoder *encoder;
+	struct intel_encoder *encoder, *edp_encoder = NULL;
 	const intel_limit_t *limit;
 	int ret;
 	struct fdi_m_n m_n = {0};
 	u32 temp;
-	u32 lvds_sync = 0;
 	int target_clock, pixel_multiplier, lane, link_bw, factor;
 	unsigned int pipe_bpp;
 	bool dither;
+	bool is_cpu_edp = false, is_pch_edp = false;
 
 	list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
 		if (encoder->base.crtc != crtc)
@@ -5748,7 +4423,12 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 			is_dp = true;
 			break;
 		case INTEL_OUTPUT_EDP:
-			has_edp_encoder = encoder;
+			is_dp = true;
+			if (intel_encoder_is_pch_edp(&encoder->base))
+				is_pch_edp = true;
+			else
+				is_cpu_edp = true;
+			edp_encoder = encoder;
 			break;
 		}
 
@@ -5811,15 +4491,13 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 	lane = 0;
 	/* CPU eDP doesn't require FDI link, so just set DP M/N
 	   according to current link config */
-	if (has_edp_encoder &&
-	    !intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
+	if (is_cpu_edp) {
 		target_clock = mode->clock;
-		intel_edp_link_config(has_edp_encoder,
-				      &lane, &link_bw);
+		intel_edp_link_config(edp_encoder, &lane, &link_bw);
 	} else {
 		/* [e]DP over FDI requires target mode clock
 		   instead of link clock */
-		if (is_dp || intel_encoder_is_pch_edp(&has_edp_encoder->base))
+		if (is_dp)
 			target_clock = mode->clock;
 		else
 			target_clock = adjusted_mode->clock;
@@ -5910,7 +4588,7 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 		}
 		dpll |= DPLL_DVO_HIGH_SPEED;
 	}
-	if (is_dp || intel_encoder_is_pch_edp(&has_edp_encoder->base))
+	if (is_dp && !is_cpu_edp)
 		dpll |= DPLL_DVO_HIGH_SPEED;
 
 	/* compute bitmask from p1 value */
@@ -5949,34 +4627,25 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 
 	/* Set up the display plane register */
 	dspcntr = DISPPLANE_GAMMA_ENABLE;
-
 	DRM_DEBUG_KMS("Mode for pipe %d:\n", pipe);
 	drm_mode_debug_printmodeline(mode);
 
-	/* PCH eDP needs FDI, but CPU eDP does not */
-	if (!intel_crtc->no_pll) {
-		if (!has_edp_encoder ||
-		    intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
-			I915_WRITE(PCH_FP0(pipe), fp);
-			I915_WRITE(PCH_DPLL(pipe), dpll & ~DPLL_VCO_ENABLE);
+	/* CPU eDP is the only output that doesn't need a PCH PLL of its own on
+	 * pre-Haswell/LPT generation */
+	if (HAS_PCH_LPT(dev)) {
+		DRM_DEBUG_KMS("LPT detected: no PLL for pipe %d necessary\n",
+				pipe);
+	} else if (!is_cpu_edp) {
+		struct intel_pch_pll *pll;
 
-			POSTING_READ(PCH_DPLL(pipe));
-			DELAY(150);
-		}
-	} else {
-		if (dpll == (I915_READ(PCH_DPLL(0)) & 0x7fffffff) &&
-		    fp == I915_READ(PCH_FP0(0))) {
-			intel_crtc->use_pll_a = true;
-			DRM_DEBUG_KMS("using pipe a dpll\n");
-		} else if (dpll == (I915_READ(PCH_DPLL(1)) & 0x7fffffff) &&
-			   fp == I915_READ(PCH_FP0(1))) {
-			intel_crtc->use_pll_a = false;
-			DRM_DEBUG_KMS("using pipe b dpll\n");
-		} else {
-			DRM_DEBUG_KMS("no matching PLL configuration for pipe 2\n");
-			return -EINVAL;
-		}
-	}
+		pll = intel_get_pch_pll(intel_crtc, dpll, fp);
+		if (pll == NULL) {
+			DRM_DEBUG_DRIVER("failed to find PLL for pipe %d\n",
+					 pipe);
+ 			return -EINVAL;
+ 		}
+	} else
+		intel_put_pch_pll(intel_crtc);
 
 	/* The LVDS pin pair needs to be on before the DPLLs are enabled.
 	 * This is an exception to the general rule that mode_set doesn't turn
@@ -6009,22 +4678,11 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 		 * appropriately here, but we need to look more thoroughly into how
 		 * panels behave in the two modes.
 		 */
+		temp &= ~(LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY);
 		if (adjusted_mode->flags & DRM_MODE_FLAG_NHSYNC)
-			lvds_sync |= LVDS_HSYNC_POLARITY;
+			temp |= LVDS_HSYNC_POLARITY;
 		if (adjusted_mode->flags & DRM_MODE_FLAG_NVSYNC)
-			lvds_sync |= LVDS_VSYNC_POLARITY;
-		if ((temp & (LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY))
-		    != lvds_sync) {
-			char flags[2] = "-+";
-			DRM_INFO("Changing LVDS panel from "
-				 "(%chsync, %cvsync) to (%chsync, %cvsync)\n",
-				 flags[!(temp & LVDS_HSYNC_POLARITY)],
-				 flags[!(temp & LVDS_VSYNC_POLARITY)],
-				 flags[!(lvds_sync & LVDS_HSYNC_POLARITY)],
-				 flags[!(lvds_sync & LVDS_VSYNC_POLARITY)]);
-			temp &= ~(LVDS_HSYNC_POLARITY | LVDS_VSYNC_POLARITY);
-			temp |= lvds_sync;
-		}
+			temp |= LVDS_VSYNC_POLARITY;
 		I915_WRITE(PCH_LVDS, temp);
 	}
 
@@ -6034,7 +4692,7 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 		pipeconf |= PIPECONF_DITHER_EN;
 		pipeconf |= PIPECONF_DITHER_TYPE_SP;
 	}
-	if (is_dp || intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
+	if (is_dp && !is_cpu_edp) {
 		intel_dp_set_m_n(crtc, mode, adjusted_mode);
 	} else {
 		/* For non-DP output, clear any trans DP clock recovery setting.*/
@@ -6044,13 +4702,11 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 		I915_WRITE(TRANSDPLINK_N1(pipe), 0);
 	}
 
-	if (!intel_crtc->no_pll &&
-	    (!has_edp_encoder ||
-	     intel_encoder_is_pch_edp(&has_edp_encoder->base))) {
-		I915_WRITE(PCH_DPLL(pipe), dpll);
+	if (intel_crtc->pch_pll) {
+		I915_WRITE(intel_crtc->pch_pll->pll_reg, dpll);
 
 		/* Wait for the clocks to stabilize. */
-		POSTING_READ(PCH_DPLL(pipe));
+		POSTING_READ(intel_crtc->pch_pll->pll_reg);
 		DELAY(150);
 
 		/* The pixel multiplier can only be updated once the
@@ -6058,20 +4714,20 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 		 *
 		 * So write it again.
 		 */
-		I915_WRITE(PCH_DPLL(pipe), dpll);
+		I915_WRITE(intel_crtc->pch_pll->pll_reg, dpll);
 	}
 
 	intel_crtc->lowfreq_avail = false;
-	if (!intel_crtc->no_pll) {
+	if (intel_crtc->pch_pll) {
 		if (is_lvds && has_reduced_clock && i915_powersave) {
-			I915_WRITE(PCH_FP1(pipe), fp2);
+			I915_WRITE(intel_crtc->pch_pll->fp1_reg, fp2);
 			intel_crtc->lowfreq_avail = true;
 			if (HAS_PIPE_CXSR(dev)) {
 				DRM_DEBUG_KMS("enabling CxSR downclocking\n");
 				pipeconf |= PIPECONF_CXSR_DOWNCLOCK;
 			}
 		} else {
-			I915_WRITE(PCH_FP1(pipe), fp);
+			I915_WRITE(intel_crtc->pch_pll->fp1_reg, fp);
 			if (HAS_PIPE_CXSR(dev)) {
 				DRM_DEBUG_KMS("disabling CxSR downclocking\n");
 				pipeconf &= ~PIPECONF_CXSR_DOWNCLOCK;
@@ -6124,10 +4780,8 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 	I915_WRITE(PIPE_LINK_M1(pipe), m_n.link_m);
 	I915_WRITE(PIPE_LINK_N1(pipe), m_n.link_n);
 
-	if (has_edp_encoder &&
-	    !intel_encoder_is_pch_edp(&has_edp_encoder->base)) {
+	if (is_cpu_edp)
 		ironlake_set_pll_edp(crtc, adjusted_mode->clock);
-	}
 
 	I915_WRITE(PIPECONF(pipe), pipeconf);
 	POSTING_READ(PIPECONF(pipe));
@@ -6141,6 +4795,8 @@ static int ironlake_crtc_mode_set(struct drm_crtc *crtc,
 
 	intel_update_watermarks(dev);
 
+	intel_update_linetime_watermarks(dev, pipe, adjusted_mode);
+
 	return ret;
 }
 
@@ -6499,7 +5155,7 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
 	if (!visible && !intel_crtc->cursor_visible)
 		return;
 
-	if (IS_IVYBRIDGE(dev)) {
+	if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
 		I915_WRITE(CURPOS_IVB(pipe), pos);
 		ivb_update_cursor(crtc, base);
 	} else {
@@ -6509,9 +5165,6 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
 		else
 			i9xx_update_cursor(crtc, base);
 	}
-
-	if (visible)
-		intel_mark_busy(dev, to_intel_framebuffer(crtc->fb)->obj);
 }
 
 static int intel_crtc_cursor_set(struct drm_crtc *crtc,
@@ -6596,7 +5249,7 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
 			if (intel_crtc->cursor_bo != obj)
 				i915_gem_detach_phys_object(dev, intel_crtc->cursor_bo);
 		} else
-			i915_gem_object_unpin(intel_crtc->cursor_bo);
+			i915_gem_object_unpin_from_display_plane(intel_crtc->cursor_bo);
 		drm_gem_object_unreference(&intel_crtc->cursor_bo->base);
 	}
 
@@ -6611,7 +5264,7 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
 
 	return 0;
 fail_unpin:
-	i915_gem_object_unpin(obj);
+	i915_gem_object_unpin_from_display_plane(obj);
 fail_locked:
 	DRM_UNLOCK(dev);
 fail:
@@ -7039,7 +5692,6 @@ struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
 	mode->vsync_end = ((vsync & 0xffff0000) >> 16) + 1;
 
 	drm_mode_set_name(mode);
-	drm_mode_set_crtcinfo(mode, 0);
 
 	return mode;
 }
@@ -7206,12 +5858,16 @@ void intel_mark_busy(struct drm_device *dev, struct drm_i915_gem_object *obj)
 	if (!drm_core_check_feature(dev, DRIVER_MODESET))
 		return;
 
-	if (!dev_priv->busy)
+	if (!dev_priv->busy) {
+		intel_sanitize_pm(dev);
 		dev_priv->busy = true;
-	else
+	} else
 		callout_reset(&dev_priv->idle_callout, GPU_IDLE_TIMEOUT,
 		    intel_gpu_idle_timer, dev);
 
+	if (obj == NULL)
+		return;
+
 	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
 		if (!crtc->fb)
 			continue;
@@ -7384,18 +6040,19 @@ static int intel_gen2_queue_flip(struct drm_device *dev,
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	unsigned long offset;
 	u32 flip_mask;
+	struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
 	int ret;
 
-	ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
+	ret = intel_pin_and_fence_fb_obj(dev, obj, ring);
 	if (ret)
-		goto out;
+		goto err;
 
 	/* Offset into the new buffer for cases of shared fbs between CRTCs */
 	offset = crtc->y * fb->pitches[0] + crtc->x * fb->bits_per_pixel/8;
 
-	ret = BEGIN_LP_RING(6);
+	ret = intel_ring_begin(ring, 6);
 	if (ret)
-		goto out;
+		goto err_unpin;
 
 	/* Can't queue multiple flips, so wait for the previous
 	 * one to finish before executing the next.
@@ -7404,15 +6061,19 @@ static int intel_gen2_queue_flip(struct drm_device *dev,
 		flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
 	else
 		flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
-	OUT_RING(MI_WAIT_FOR_EVENT | flip_mask);
-	OUT_RING(MI_NOOP);
-	OUT_RING(MI_DISPLAY_FLIP |
-		 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
-	OUT_RING(fb->pitches[0]);
-	OUT_RING(obj->gtt_offset + offset);
-	OUT_RING(0); /* aux display base address, unused */
-	ADVANCE_LP_RING();
-out:
+	intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_emit(ring, MI_DISPLAY_FLIP |
+			MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
+	intel_ring_emit(ring, fb->pitches[0]);
+	intel_ring_emit(ring, obj->gtt_offset + offset);
+	intel_ring_emit(ring, 0); /* aux display base address, unused */
+	intel_ring_advance(ring);
+	return 0;
+
+err_unpin:
+	intel_unpin_fb_obj(obj);
+err:
 	return ret;
 }
 
@@ -7425,33 +6086,38 @@ static int intel_gen3_queue_flip(struct drm_device *dev,
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	unsigned long offset;
 	u32 flip_mask;
+	struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
 	int ret;
 
-	ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
+	ret = intel_pin_and_fence_fb_obj(dev, obj, ring);
 	if (ret)
-		goto out;
+		goto err;
 
 	/* Offset into the new buffer for cases of shared fbs between CRTCs */
 	offset = crtc->y * fb->pitches[0] + crtc->x * fb->bits_per_pixel/8;
 
-	ret = BEGIN_LP_RING(6);
+	ret = intel_ring_begin(ring, 6);
 	if (ret)
-		goto out;
+		goto err_unpin;
 
 	if (intel_crtc->plane)
 		flip_mask = MI_WAIT_FOR_PLANE_B_FLIP;
 	else
 		flip_mask = MI_WAIT_FOR_PLANE_A_FLIP;
-	OUT_RING(MI_WAIT_FOR_EVENT | flip_mask);
-	OUT_RING(MI_NOOP);
-	OUT_RING(MI_DISPLAY_FLIP_I915 |
-		 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
-	OUT_RING(fb->pitches[0]);
-	OUT_RING(obj->gtt_offset + offset);
-	OUT_RING(MI_NOOP);
+	intel_ring_emit(ring, MI_WAIT_FOR_EVENT | flip_mask);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 |
+			MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
+	intel_ring_emit(ring, fb->pitches[0]);
+	intel_ring_emit(ring, obj->gtt_offset + offset);
+	intel_ring_emit(ring, MI_NOOP);
 
-	ADVANCE_LP_RING();
-out:
+	intel_ring_advance(ring);
+	return 0;
+
+err_unpin:
+	intel_unpin_fb_obj(obj);
+err:
 	return ret;
 }
 
@@ -7463,24 +6129,25 @@ static int intel_gen4_queue_flip(struct drm_device *dev,
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 	uint32_t pf, pipesrc;
+	struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
 	int ret;
 
-	ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
+	ret = intel_pin_and_fence_fb_obj(dev, obj, ring);
 	if (ret)
-		goto out;
+		goto err;
 
-	ret = BEGIN_LP_RING(4);
+	ret = intel_ring_begin(ring, 4);
 	if (ret)
-		goto out;
+		goto err_unpin;
 
 	/* i965+ uses the linear or tiled offsets from the
 	 * Display Registers (which do not change across a page-flip)
 	 * so we need only reprogram the base address.
 	 */
-	OUT_RING(MI_DISPLAY_FLIP |
-		 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
-	OUT_RING(fb->pitches[0]);
-	OUT_RING(obj->gtt_offset | obj->tiling_mode);
+	intel_ring_emit(ring, MI_DISPLAY_FLIP |
+			MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
+	intel_ring_emit(ring, fb->pitches[0]);
+	intel_ring_emit(ring, obj->gtt_offset | obj->tiling_mode);
 
 	/* XXX Enabling the panel-fitter across page-flip is so far
 	 * untested on non-native modes, so ignore it for now.
@@ -7488,9 +6155,13 @@ static int intel_gen4_queue_flip(struct drm_device *dev,
 	 */
 	pf = 0;
 	pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
-	OUT_RING(pf | pipesrc);
-	ADVANCE_LP_RING();
-out:
+	intel_ring_emit(ring, pf | pipesrc);
+	intel_ring_advance(ring);
+	return 0;
+
+err_unpin:
+	intel_unpin_fb_obj(obj);
+err:
 	return ret;
 }
 
@@ -7501,21 +6172,22 @@ static int intel_gen6_queue_flip(struct drm_device *dev,
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
 	uint32_t pf, pipesrc;
 	int ret;
 
-	ret = intel_pin_and_fence_fb_obj(dev, obj, LP_RING(dev_priv));
+	ret = intel_pin_and_fence_fb_obj(dev, obj, ring);
 	if (ret)
-		goto out;
+		goto err;
 
-	ret = BEGIN_LP_RING(4);
+	ret = intel_ring_begin(ring, 4);
 	if (ret)
-		goto out;
+		goto err_unpin;
 
-	OUT_RING(MI_DISPLAY_FLIP |
-		 MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
-	OUT_RING(fb->pitches[0] | obj->tiling_mode);
-	OUT_RING(obj->gtt_offset);
+	intel_ring_emit(ring, MI_DISPLAY_FLIP |
+			MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
+	intel_ring_emit(ring, fb->pitches[0] | obj->tiling_mode);
+	intel_ring_emit(ring, obj->gtt_offset);
 
 	/* Contrary to the suggestions in the documentation,
 	 * "Enable Panel Fitter" does not seem to be required when page
@@ -7525,9 +6197,13 @@ static int intel_gen6_queue_flip(struct drm_device *dev,
 	 */
 	pf = 0;
 	pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
-	OUT_RING(pf | pipesrc);
-	ADVANCE_LP_RING();
-out:
+	intel_ring_emit(ring, pf | pipesrc);
+	intel_ring_advance(ring);
+	return 0;
+
+err_unpin:
+	intel_unpin_fb_obj(obj);
+err:
 	return ret;
 }
 
@@ -7549,18 +6225,22 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
 
 	ret = intel_pin_and_fence_fb_obj(dev, obj, ring);
 	if (ret)
-		goto out;
+		goto err;
 
 	ret = intel_ring_begin(ring, 4);
 	if (ret)
-		goto out;
+		goto err_unpin;
 
 	intel_ring_emit(ring, MI_DISPLAY_FLIP_I915 | (intel_crtc->plane << 19));
 	intel_ring_emit(ring, (fb->pitches[0] | obj->tiling_mode));
 	intel_ring_emit(ring, (obj->gtt_offset));
 	intel_ring_emit(ring, (MI_NOOP));
 	intel_ring_advance(ring);
-out:
+	return 0;
+
+err_unpin:
+	intel_unpin_fb_obj(obj);
+err:
 	return ret;
 }
 
@@ -7633,6 +6313,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 	if (ret)
 		goto cleanup_pending;
 	intel_disable_fbc(dev);
+	intel_mark_busy(dev, obj);
 	DRM_UNLOCK(dev);
 
 	CTR2(KTR_DRM, "i915_flip_request %d %p", intel_crtc->plane, obj);
@@ -7661,10 +6342,11 @@ static void intel_sanitize_modesetting(struct drm_device *dev,
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 reg, val;
+	int i;
 
 	/* Clear any frame start delays used for debugging left by the BIOS */
-	for_each_pipe(pipe) {
-		reg = PIPECONF(pipe);
+	for_each_pipe(i) {
+		reg = PIPECONF(i);
 		I915_WRITE(reg, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
 	}
 
@@ -7734,6 +6416,23 @@ static const struct drm_crtc_funcs intel_crtc_funcs = {
 	.page_flip = intel_crtc_page_flip,
 };
 
+static void intel_pch_pll_init(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	int i;
+
+	if (dev_priv->num_pch_pll == 0) {
+		DRM_DEBUG_KMS("No PCH PLLs on this hardware, skipping initialisation\n");
+		return;
+	}
+
+	for (i = 0; i < dev_priv->num_pch_pll; i++) {
+		dev_priv->pch_plls[i].pll_reg = _PCH_DPLL(i);
+		dev_priv->pch_plls[i].fp0_reg = _PCH_FP0(i);
+		dev_priv->pch_plls[i].fp1_reg = _PCH_FP1(i);
+	}
+}
+
 static void intel_crtc_init(struct drm_device *dev, int pipe)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
@@ -7772,8 +6471,6 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
 	intel_crtc->bpp = 24; /* default for pre-Ironlake */
 
 	if (HAS_PCH_SPLIT(dev)) {
-		if (pipe == 2 && IS_IVYBRIDGE(dev))
-			intel_crtc->no_pll = true;
 		intel_helper_funcs.prepare = ironlake_crtc_prepare;
 		intel_helper_funcs.commit = ironlake_crtc_commit;
 	} else {
@@ -7791,15 +6488,12 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
 int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
 				struct drm_file *file)
 {
-	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_i915_get_pipe_from_crtc_id *pipe_from_crtc_id = data;
 	struct drm_mode_object *drmmode_obj;
 	struct intel_crtc *crtc;
 
-	if (!dev_priv) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
-	}
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
 
 	drmmode_obj = drm_mode_object_find(dev, pipe_from_crtc_id->crtc_id,
 			DRM_MODE_OBJECT_CRTC);
@@ -7872,7 +6566,26 @@ static void intel_setup_outputs(struct drm_device *dev)
 
 	intel_crt_init(dev);
 
-	if (HAS_PCH_SPLIT(dev)) {
+	if (IS_HASWELL(dev)) {
+		int found;
+
+		/* Haswell uses DDI functions to detect digital outputs */
+		found = I915_READ(DDI_BUF_CTL_A) & DDI_INIT_DISPLAY_DETECTED;
+		/* DDI A only supports eDP */
+		if (found)
+			intel_ddi_init(dev, PORT_A);
+
+		/* DDI B, C and D detection is indicated by the SFUSE_STRAP
+		 * register */
+		found = I915_READ(SFUSE_STRAP);
+
+		if (found & SFUSE_STRAP_DDIB_DETECTED)
+			intel_ddi_init(dev, PORT_B);
+		if (found & SFUSE_STRAP_DDIC_DETECTED)
+			intel_ddi_init(dev, PORT_C);
+		if (found & SFUSE_STRAP_DDID_DETECTED)
+			intel_ddi_init(dev, PORT_D);
+	} else if (HAS_PCH_SPLIT(dev)) {
 		int found;
 
 		DRM_DEBUG_KMS(
@@ -7887,7 +6600,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 
 		if (I915_READ(HDMIB) & PORT_DETECTED) {
 			/* PCH SDVOB multiplex with HDMIB */
-			found = intel_sdvo_init(dev, PCH_SDVOB);
+			found = intel_sdvo_init(dev, PCH_SDVOB, true);
 			if (!found)
 				intel_hdmi_init(dev, HDMIB);
 			if (!found && (I915_READ(PCH_DP_B) & DP_DETECTED))
@@ -7911,7 +6624,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 
 		if (I915_READ(SDVOB) & SDVO_DETECTED) {
 			DRM_DEBUG_KMS("probing SDVOB\n");
-			found = intel_sdvo_init(dev, SDVOB);
+			found = intel_sdvo_init(dev, SDVOB, true);
 			if (!found && SUPPORTS_INTEGRATED_HDMI(dev)) {
 				DRM_DEBUG_KMS("probing HDMI on SDVOB\n");
 				intel_hdmi_init(dev, SDVOB);
@@ -7927,7 +6640,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 
 		if (I915_READ(SDVOB) & SDVO_DETECTED) {
 			DRM_DEBUG_KMS("probing SDVOC\n");
-			found = intel_sdvo_init(dev, SDVOC);
+			found = intel_sdvo_init(dev, SDVOC, false);
 		}
 
 		if (!found && (I915_READ(SDVOC) & SDVO_DETECTED)) {
@@ -8061,905 +6774,6 @@ static const struct drm_mode_config_funcs intel_mode_funcs = {
 	.output_poll_changed = intel_fb_output_poll_changed,
 };
 
-static struct drm_i915_gem_object *
-intel_alloc_context_page(struct drm_device *dev)
-{
-	struct drm_i915_gem_object *ctx;
-	int ret;
-
-	DRM_LOCK_ASSERT(dev);
-
-	ctx = i915_gem_alloc_object(dev, 4096);
-	if (!ctx) {
-		DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
-		return NULL;
-	}
-
-	ret = i915_gem_object_pin(ctx, 4096, true);
-	if (ret) {
-		DRM_ERROR("failed to pin power context: %d\n", ret);
-		goto err_unref;
-	}
-
-	ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
-	if (ret) {
-		DRM_ERROR("failed to set-domain on power context: %d\n", ret);
-		goto err_unpin;
-	}
-
-	return ctx;
-
-err_unpin:
-	i915_gem_object_unpin(ctx);
-err_unref:
-	drm_gem_object_unreference(&ctx->base);
-	DRM_UNLOCK(dev);
-	return NULL;
-}
-
-bool ironlake_set_drps(struct drm_device *dev, u8 val)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u16 rgvswctl;
-
-	rgvswctl = I915_READ16(MEMSWCTL);
-	if (rgvswctl & MEMCTL_CMD_STS) {
-		DRM_DEBUG("gpu busy, RCS change rejected\n");
-		return false; /* still busy with another command */
-	}
-
-	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
-		(val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
-	I915_WRITE16(MEMSWCTL, rgvswctl);
-	POSTING_READ16(MEMSWCTL);
-
-	rgvswctl |= MEMCTL_CMD_STS;
-	I915_WRITE16(MEMSWCTL, rgvswctl);
-
-	return true;
-}
-
-void ironlake_enable_drps(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 rgvmodectl = I915_READ(MEMMODECTL);
-	u8 fmax, fmin, fstart, vstart;
-
-	/* Enable temp reporting */
-	I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
-	I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
-
-	/* 100ms RC evaluation intervals */
-	I915_WRITE(RCUPEI, 100000);
-	I915_WRITE(RCDNEI, 100000);
-
-	/* Set max/min thresholds to 90ms and 80ms respectively */
-	I915_WRITE(RCBMAXAVG, 90000);
-	I915_WRITE(RCBMINAVG, 80000);
-
-	I915_WRITE(MEMIHYST, 1);
-
-	/* Set up min, max, and cur for interrupt handling */
-	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
-	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
-	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
-		MEMMODE_FSTART_SHIFT;
-
-	vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
-		PXVFREQ_PX_SHIFT;
-
-	dev_priv->fmax = fmax; /* IPS callback will increase this */
-	dev_priv->fstart = fstart;
-
-	dev_priv->max_delay = fstart;
-	dev_priv->min_delay = fmin;
-	dev_priv->cur_delay = fstart;
-
-	DRM_DEBUG("fmax: %d, fmin: %d, fstart: %d\n",
-			 fmax, fmin, fstart);
-
-	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
-
-	/*
-	 * Interrupts will be enabled in ironlake_irq_postinstall
-	 */
-
-	I915_WRITE(VIDSTART, vstart);
-	POSTING_READ(VIDSTART);
-
-	rgvmodectl |= MEMMODE_SWMODE_EN;
-	I915_WRITE(MEMMODECTL, rgvmodectl);
-
-	if (_intel_wait_for(dev,
-	    (I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10,
-	    1, "915per"))
-		DRM_ERROR("stuck trying to change perf mode\n");
-	pause("915dsp", 1);
-
-	ironlake_set_drps(dev, fstart);
-
-	dev_priv->last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
-		I915_READ(0x112e0);
-	dev_priv->last_time1 = jiffies_to_msecs(jiffies);
-	dev_priv->last_count2 = I915_READ(0x112f4);
-	nanotime(&dev_priv->last_time2);
-}
-
-void ironlake_disable_drps(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u16 rgvswctl = I915_READ16(MEMSWCTL);
-
-	/* Ack interrupts, disable EFC interrupt */
-	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
-	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
-	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
-	I915_WRITE(DEIIR, DE_PCU_EVENT);
-	I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
-
-	/* Go back to the starting frequency */
-	ironlake_set_drps(dev, dev_priv->fstart);
-	pause("915dsp", 1);
-	rgvswctl |= MEMCTL_CMD_STS;
-	I915_WRITE(MEMSWCTL, rgvswctl);
-	pause("915dsp", 1);
-
-}
-
-void gen6_set_rps(struct drm_device *dev, u8 val)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 swreq;
-
-	swreq = (val & 0x3ff) << 25;
-	I915_WRITE(GEN6_RPNSWREQ, swreq);
-}
-
-void gen6_disable_rps(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
-	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
-	I915_WRITE(GEN6_PMIER, 0);
-	/* Complete PM interrupt masking here doesn't race with the rps work
-	 * item again unmasking PM interrupts because that is using a different
-	 * register (PMIMR) to mask PM interrupts. The only risk is in leaving
-	 * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
-
-	mtx_lock(&dev_priv->rps_lock);
-	dev_priv->pm_iir = 0;
-	mtx_unlock(&dev_priv->rps_lock);
-
-	I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
-}
-
-static unsigned long intel_pxfreq(u32 vidfreq)
-{
-	unsigned long freq;
-	int div = (vidfreq & 0x3f0000) >> 16;
-	int post = (vidfreq & 0x3000) >> 12;
-	int pre = (vidfreq & 0x7);
-
-	if (!pre)
-		return 0;
-
-	freq = ((div * 133333) / ((1<<post) * pre));
-
-	return freq;
-}
-
-void intel_init_emon(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 lcfuse;
-	u8 pxw[16];
-	int i;
-
-	/* Disable to program */
-	I915_WRITE(ECR, 0);
-	POSTING_READ(ECR);
-
-	/* Program energy weights for various events */
-	I915_WRITE(SDEW, 0x15040d00);
-	I915_WRITE(CSIEW0, 0x007f0000);
-	I915_WRITE(CSIEW1, 0x1e220004);
-	I915_WRITE(CSIEW2, 0x04000004);
-
-	for (i = 0; i < 5; i++)
-		I915_WRITE(PEW + (i * 4), 0);
-	for (i = 0; i < 3; i++)
-		I915_WRITE(DEW + (i * 4), 0);
-
-	/* Program P-state weights to account for frequency power adjustment */
-	for (i = 0; i < 16; i++) {
-		u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
-		unsigned long freq = intel_pxfreq(pxvidfreq);
-		unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
-			PXVFREQ_PX_SHIFT;
-		unsigned long val;
-
-		val = vid * vid;
-		val *= (freq / 1000);
-		val *= 255;
-		val /= (127*127*900);
-		if (val > 0xff)
-			DRM_ERROR("bad pxval: %ld\n", val);
-		pxw[i] = val;
-	}
-	/* Render standby states get 0 weight */
-	pxw[14] = 0;
-	pxw[15] = 0;
-
-	for (i = 0; i < 4; i++) {
-		u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
-			(pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
-		I915_WRITE(PXW + (i * 4), val);
-	}
-
-	/* Adjust magic regs to magic values (more experimental results) */
-	I915_WRITE(OGW0, 0);
-	I915_WRITE(OGW1, 0);
-	I915_WRITE(EG0, 0x00007f00);
-	I915_WRITE(EG1, 0x0000000e);
-	I915_WRITE(EG2, 0x000e0000);
-	I915_WRITE(EG3, 0x68000300);
-	I915_WRITE(EG4, 0x42000000);
-	I915_WRITE(EG5, 0x00140031);
-	I915_WRITE(EG6, 0);
-	I915_WRITE(EG7, 0);
-
-	for (i = 0; i < 8; i++)
-		I915_WRITE(PXWL + (i * 4), 0);
-
-	/* Enable PMON + select events */
-	I915_WRITE(ECR, 0x80000019);
-
-	lcfuse = I915_READ(LCFUSE02);
-
-	dev_priv->corr = (lcfuse & LCFUSE_HIV_MASK);
-}
-
-static int intel_enable_rc6(struct drm_device *dev)
-{
-	/*
-	 * Respect the kernel parameter if it is set
-	 */
-	if (i915_enable_rc6 >= 0)
-		return i915_enable_rc6;
-
-	/*
-	 * Disable RC6 on Ironlake
-	 */
-	if (INTEL_INFO(dev)->gen == 5)
-		return 0;
-
-	/*
-	 * Enable rc6 on Sandybridge if DMA remapping is disabled
-	 */
-	if (INTEL_INFO(dev)->gen == 6) {
-		DRM_DEBUG_DRIVER(
-		    "Sandybridge: intel_iommu_enabled %s -- RC6 %sabled\n",
-		     intel_iommu_enabled ? "true" : "false",
-		     !intel_iommu_enabled ? "en" : "dis");
-		return (intel_iommu_enabled ? 0 : INTEL_RC6_ENABLE);
-	}
-	DRM_DEBUG_DRIVER("RC6 and deep RC6 enabled\n");
-	return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
-}
-
-void gen6_enable_rps(struct drm_i915_private *dev_priv)
-{
-	struct drm_device *dev = dev_priv->dev;
-	u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
-	u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
-	u32 pcu_mbox, rc6_mask = 0;
-	u32 gtfifodbg;
-	int cur_freq, min_freq, max_freq;
-	int rc6_mode;
-	int i;
-
-	/* Here begins a magic sequence of register writes to enable
-	 * auto-downclocking.
-	 *
-	 * Perhaps there might be some value in exposing these to
-	 * userspace...
-	 */
-	I915_WRITE(GEN6_RC_STATE, 0);
-	DRM_LOCK(dev);
-
-	/* Clear the DBG now so we don't confuse earlier errors */
-	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
-		DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
-		I915_WRITE(GTFIFODBG, gtfifodbg);
-	}
-
-	gen6_gt_force_wake_get(dev_priv);
-
-	/* disable the counters and set deterministic thresholds */
-	I915_WRITE(GEN6_RC_CONTROL, 0);
-
-	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
-	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
-	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
-	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
-	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
-
-	for (i = 0; i < I915_NUM_RINGS; i++)
-		I915_WRITE(RING_MAX_IDLE(dev_priv->rings[i].mmio_base), 10);
-
-	I915_WRITE(GEN6_RC_SLEEP, 0);
-	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
-	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
-	I915_WRITE(GEN6_RC6p_THRESHOLD, 100000);
-	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
-
-	rc6_mode = intel_enable_rc6(dev_priv->dev);
-	if (rc6_mode & INTEL_RC6_ENABLE)
-		rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
-
-	if (rc6_mode & INTEL_RC6p_ENABLE)
-		rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
-
-	if (rc6_mode & INTEL_RC6pp_ENABLE)
-		rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
-
-	DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
-			(rc6_mode & INTEL_RC6_ENABLE) ? "on" : "off",
-			(rc6_mode & INTEL_RC6p_ENABLE) ? "on" : "off",
-			(rc6_mode & INTEL_RC6pp_ENABLE) ? "on" : "off");
-
-	I915_WRITE(GEN6_RC_CONTROL,
-		   rc6_mask |
-		   GEN6_RC_CTL_EI_MODE(1) |
-		   GEN6_RC_CTL_HW_ENABLE);
-
-	I915_WRITE(GEN6_RPNSWREQ,
-		   GEN6_FREQUENCY(10) |
-		   GEN6_OFFSET(0) |
-		   GEN6_AGGRESSIVE_TURBO);
-	I915_WRITE(GEN6_RC_VIDEO_FREQ,
-		   GEN6_FREQUENCY(12));
-
-	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
-	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
-		   18 << 24 |
-		   6 << 16);
-	I915_WRITE(GEN6_RP_UP_THRESHOLD, 10000);
-	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 1000000);
-	I915_WRITE(GEN6_RP_UP_EI, 100000);
-	I915_WRITE(GEN6_RP_DOWN_EI, 5000000);
-	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
-	I915_WRITE(GEN6_RP_CONTROL,
-		   GEN6_RP_MEDIA_TURBO |
-		   GEN6_RP_MEDIA_HW_MODE |
-		   GEN6_RP_MEDIA_IS_GFX |
-		   GEN6_RP_ENABLE |
-		   GEN6_RP_UP_BUSY_AVG |
-		   GEN6_RP_DOWN_IDLE_CONT);
-
-	if (_intel_wait_for(dev,
-	    (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, 500,
-	    1, "915pr1"))
-		DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
-
-	I915_WRITE(GEN6_PCODE_DATA, 0);
-	I915_WRITE(GEN6_PCODE_MAILBOX,
-		   GEN6_PCODE_READY |
-		   GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
-	if (_intel_wait_for(dev,
-	    (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, 500,
-	    1, "915pr2"))
-		DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
-
-	min_freq = (rp_state_cap & 0xff0000) >> 16;
-	max_freq = rp_state_cap & 0xff;
-	cur_freq = (gt_perf_status & 0xff00) >> 8;
-
-	/* Check for overclock support */
-	if (_intel_wait_for(dev,
-	    (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, 500,
-	    1, "915pr3"))
-		DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
-	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_READ_OC_PARAMS);
-	pcu_mbox = I915_READ(GEN6_PCODE_DATA);
-	if (_intel_wait_for(dev,
-	    (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0, 500,
-	    1, "915pr4"))
-		DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
-	if (pcu_mbox & (1<<31)) { /* OC supported */
-		max_freq = pcu_mbox & 0xff;
-		DRM_DEBUG("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 50);
-	}
-
-	/* In units of 100MHz */
-	dev_priv->max_delay = max_freq;
-	dev_priv->min_delay = min_freq;
-	dev_priv->cur_delay = cur_freq;
-
-	/* requires MSI enabled */
-	I915_WRITE(GEN6_PMIER,
-		   GEN6_PM_MBOX_EVENT |
-		   GEN6_PM_THERMAL_EVENT |
-		   GEN6_PM_RP_DOWN_TIMEOUT |
-		   GEN6_PM_RP_UP_THRESHOLD |
-		   GEN6_PM_RP_DOWN_THRESHOLD |
-		   GEN6_PM_RP_UP_EI_EXPIRED |
-		   GEN6_PM_RP_DOWN_EI_EXPIRED);
-	mtx_lock(&dev_priv->rps_lock);
-	if (dev_priv->pm_iir != 0)
-		printf("pm_iir %x\n", dev_priv->pm_iir);
-	I915_WRITE(GEN6_PMIMR, 0);
-	mtx_unlock(&dev_priv->rps_lock);
-	/* enable all PM interrupts */
-	I915_WRITE(GEN6_PMINTRMSK, 0);
-
-	gen6_gt_force_wake_put(dev_priv);
-	DRM_UNLOCK(dev);
-}
-
-void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
-{
-	struct drm_device *dev;
-	int min_freq = 15;
-	int gpu_freq, ia_freq, max_ia_freq;
-	int scaling_factor = 180;
-	uint64_t tsc_freq;
-
-	dev = dev_priv->dev;
-#if 0
-	max_ia_freq = cpufreq_quick_get_max(0);
-	/*
-	 * Default to measured freq if none found, PCU will ensure we don't go
-	 * over
-	 */
-	if (!max_ia_freq)
-		max_ia_freq = tsc_freq;
-
-	/* Convert from Hz to MHz */
-	max_ia_freq /= 1000;
-#else
-	tsc_freq = atomic_load_acq_64(&tsc_freq);
-	max_ia_freq = tsc_freq / 1000 / 1000;
-#endif
-
-	DRM_LOCK(dev);
-
-	/*
-	 * For each potential GPU frequency, load a ring frequency we'd like
-	 * to use for memory access.  We do this by specifying the IA frequency
-	 * the PCU should use as a reference to determine the ring frequency.
-	 */
-	for (gpu_freq = dev_priv->max_delay; gpu_freq >= dev_priv->min_delay;
-	     gpu_freq--) {
-		int diff = dev_priv->max_delay - gpu_freq;
-		int d;
-
-		/*
-		 * For GPU frequencies less than 750MHz, just use the lowest
-		 * ring freq.
-		 */
-		if (gpu_freq < min_freq)
-			ia_freq = 800;
-		else
-			ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
-		d = 100;
-		ia_freq = (ia_freq + d / 2) / d;
-
-		I915_WRITE(GEN6_PCODE_DATA,
-			   (ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT) |
-			   gpu_freq);
-		I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY |
-			   GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
-		if (_intel_wait_for(dev,
-		    (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
-		    10, 1, "915frq")) {
-			DRM_ERROR("pcode write of freq table timed out\n");
-			continue;
-		}
-	}
-
-	DRM_UNLOCK(dev);
-}
-
-static void ironlake_init_clock_gating(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
-
-	/* Required for FBC */
-	dspclk_gate |= DPFCUNIT_CLOCK_GATE_DISABLE |
-		DPFCRUNIT_CLOCK_GATE_DISABLE |
-		DPFDUNIT_CLOCK_GATE_DISABLE;
-	/* Required for CxSR */
-	dspclk_gate |= DPARBUNIT_CLOCK_GATE_DISABLE;
-
-	I915_WRITE(PCH_3DCGDIS0,
-		   MARIUNIT_CLOCK_GATE_DISABLE |
-		   SVSMUNIT_CLOCK_GATE_DISABLE);
-	I915_WRITE(PCH_3DCGDIS1,
-		   VFMUNIT_CLOCK_GATE_DISABLE);
-
-	I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
-
-	/*
-	 * According to the spec the following bits should be set in
-	 * order to enable memory self-refresh
-	 * The bit 22/21 of 0x42004
-	 * The bit 5 of 0x42020
-	 * The bit 15 of 0x45000
-	 */
-	I915_WRITE(ILK_DISPLAY_CHICKEN2,
-		   (I915_READ(ILK_DISPLAY_CHICKEN2) |
-		    ILK_DPARB_GATE | ILK_VSDPFD_FULL));
-	I915_WRITE(ILK_DSPCLK_GATE,
-		   (I915_READ(ILK_DSPCLK_GATE) |
-		    ILK_DPARB_CLK_GATE));
-	I915_WRITE(DISP_ARB_CTL,
-		   (I915_READ(DISP_ARB_CTL) |
-		    DISP_FBC_WM_DIS));
-	I915_WRITE(WM3_LP_ILK, 0);
-	I915_WRITE(WM2_LP_ILK, 0);
-	I915_WRITE(WM1_LP_ILK, 0);
-
-	/*
-	 * Based on the document from hardware guys the following bits
-	 * should be set unconditionally in order to enable FBC.
-	 * The bit 22 of 0x42000
-	 * The bit 22 of 0x42004
-	 * The bit 7,8,9 of 0x42020.
-	 */
-	if (IS_IRONLAKE_M(dev)) {
-		I915_WRITE(ILK_DISPLAY_CHICKEN1,
-			   I915_READ(ILK_DISPLAY_CHICKEN1) |
-			   ILK_FBCQ_DIS);
-		I915_WRITE(ILK_DISPLAY_CHICKEN2,
-			   I915_READ(ILK_DISPLAY_CHICKEN2) |
-			   ILK_DPARB_GATE);
-		I915_WRITE(ILK_DSPCLK_GATE,
-			   I915_READ(ILK_DSPCLK_GATE) |
-			   ILK_DPFC_DIS1 |
-			   ILK_DPFC_DIS2 |
-			   ILK_CLK_FBC);
-	}
-
-	I915_WRITE(ILK_DISPLAY_CHICKEN2,
-		   I915_READ(ILK_DISPLAY_CHICKEN2) |
-		   ILK_ELPIN_409_SELECT);
-	I915_WRITE(_3D_CHICKEN2,
-		   _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
-		   _3D_CHICKEN2_WM_READ_PIPELINED);
-}
-
-static void gen6_init_clock_gating(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	int pipe;
-	uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
-
-	I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
-
-	I915_WRITE(ILK_DISPLAY_CHICKEN2,
-		   I915_READ(ILK_DISPLAY_CHICKEN2) |
-		   ILK_ELPIN_409_SELECT);
-
-	I915_WRITE(WM3_LP_ILK, 0);
-	I915_WRITE(WM2_LP_ILK, 0);
-	I915_WRITE(WM1_LP_ILK, 0);
-
-	I915_WRITE(GEN6_UCGCTL1,
-		   I915_READ(GEN6_UCGCTL1) |
-		   GEN6_BLBUNIT_CLOCK_GATE_DISABLE);
-
-	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
-	 * gating disable must be set.  Failure to set it results in
-	 * flickering pixels due to Z write ordering failures after
-	 * some amount of runtime in the Mesa "fire" demo, and Unigine
-	 * Sanctuary and Tropics, and apparently anything else with
-	 * alpha test or pixel discard.
-	 *
-	 * According to the spec, bit 11 (RCCUNIT) must also be set,
-	 * but we didn't debug actual testcases to find it out.
-	 */
-	I915_WRITE(GEN6_UCGCTL2,
-		   GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
-		   GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
-
-	/*
-	 * According to the spec the following bits should be
-	 * set in order to enable memory self-refresh and fbc:
-	 * The bit21 and bit22 of 0x42000
-	 * The bit21 and bit22 of 0x42004
-	 * The bit5 and bit7 of 0x42020
-	 * The bit14 of 0x70180
-	 * The bit14 of 0x71180
-	 */
-	I915_WRITE(ILK_DISPLAY_CHICKEN1,
-		   I915_READ(ILK_DISPLAY_CHICKEN1) |
-		   ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
-	I915_WRITE(ILK_DISPLAY_CHICKEN2,
-		   I915_READ(ILK_DISPLAY_CHICKEN2) |
-		   ILK_DPARB_GATE | ILK_VSDPFD_FULL);
-	I915_WRITE(ILK_DSPCLK_GATE,
-		   I915_READ(ILK_DSPCLK_GATE) |
-		   ILK_DPARB_CLK_GATE  |
-		   ILK_DPFD_CLK_GATE);
-
-	for_each_pipe(pipe) {
-		I915_WRITE(DSPCNTR(pipe),
-			   I915_READ(DSPCNTR(pipe)) |
-			   DISPPLANE_TRICKLE_FEED_DISABLE);
-		intel_flush_display_plane(dev_priv, pipe);
-	}
-}
-
-static void ivybridge_init_clock_gating(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	int pipe;
-	uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
-
-	I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
-
-	I915_WRITE(WM3_LP_ILK, 0);
-	I915_WRITE(WM2_LP_ILK, 0);
-	I915_WRITE(WM1_LP_ILK, 0);
-
-	/* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
-	 * This implements the WaDisableRCZUnitClockGating workaround.
-	 */
-	I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
-
-	I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
-
-	I915_WRITE(IVB_CHICKEN3,
-		   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
-		   CHICKEN3_DGMG_DONE_FIX_DISABLE);
-
-	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
-	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
-		   GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
-
-	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
-	I915_WRITE(GEN7_L3CNTLREG1,
-			GEN7_WA_FOR_GEN7_L3_CONTROL);
-	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
-			GEN7_WA_L3_CHICKEN_MODE);
-
-	/* This is required by WaCatErrorRejectionIssue */
-	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
-			I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
-			GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
-
-	for_each_pipe(pipe) {
-		I915_WRITE(DSPCNTR(pipe),
-			   I915_READ(DSPCNTR(pipe)) |
-			   DISPPLANE_TRICKLE_FEED_DISABLE);
-		intel_flush_display_plane(dev_priv, pipe);
-	}
-}
-
-static void g4x_init_clock_gating(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	uint32_t dspclk_gate;
-
-	I915_WRITE(RENCLK_GATE_D1, 0);
-	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
-		   GS_UNIT_CLOCK_GATE_DISABLE |
-		   CL_UNIT_CLOCK_GATE_DISABLE);
-	I915_WRITE(RAMCLK_GATE_D, 0);
-	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
-		OVRUNIT_CLOCK_GATE_DISABLE |
-		OVCUNIT_CLOCK_GATE_DISABLE;
-	if (IS_GM45(dev))
-		dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
-	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
-}
-
-static void crestline_init_clock_gating(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
-	I915_WRITE(RENCLK_GATE_D2, 0);
-	I915_WRITE(DSPCLK_GATE_D, 0);
-	I915_WRITE(RAMCLK_GATE_D, 0);
-	I915_WRITE16(DEUC, 0);
-}
-
-static void broadwater_init_clock_gating(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
-		   I965_RCC_CLOCK_GATE_DISABLE |
-		   I965_RCPB_CLOCK_GATE_DISABLE |
-		   I965_ISC_CLOCK_GATE_DISABLE |
-		   I965_FBC_CLOCK_GATE_DISABLE);
-	I915_WRITE(RENCLK_GATE_D2, 0);
-}
-
-static void gen3_init_clock_gating(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	u32 dstate = I915_READ(D_STATE);
-
-	dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
-		DSTATE_DOT_CLOCK_GATING;
-	I915_WRITE(D_STATE, dstate);
-}
-
-static void i85x_init_clock_gating(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
-}
-
-static void i830_init_clock_gating(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
-}
-
-static void ibx_init_clock_gating(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	/*
-	 * On Ibex Peak and Cougar Point, we need to disable clock
-	 * gating for the panel power sequencer or it will fail to
-	 * start up when no ports are active.
-	 */
-	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
-}
-
-static void cpt_init_clock_gating(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	int pipe;
-
-	/*
-	 * On Ibex Peak and Cougar Point, we need to disable clock
-	 * gating for the panel power sequencer or it will fail to
-	 * start up when no ports are active.
-	 */
-	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
-	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
-		   DPLS_EDP_PPS_FIX_DIS);
-	/* Without this, mode sets may fail silently on FDI */
-	for_each_pipe(pipe)
-		I915_WRITE(TRANS_CHICKEN2(pipe), TRANS_AUTOTRAIN_GEN_STALL_DIS);
-}
-
-static void ironlake_teardown_rc6(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	if (dev_priv->renderctx) {
-		i915_gem_object_unpin(dev_priv->renderctx);
-		drm_gem_object_unreference(&dev_priv->renderctx->base);
-		dev_priv->renderctx = NULL;
-	}
-
-	if (dev_priv->pwrctx) {
-		i915_gem_object_unpin(dev_priv->pwrctx);
-		drm_gem_object_unreference(&dev_priv->pwrctx->base);
-		dev_priv->pwrctx = NULL;
-	}
-}
-
-static void ironlake_disable_rc6(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	if (I915_READ(PWRCTXA)) {
-		/* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
-		I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) | RCX_SW_EXIT);
-		(void)_intel_wait_for(dev,
-		    ((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
-		    50, 1, "915pro");
-
-		I915_WRITE(PWRCTXA, 0);
-		POSTING_READ(PWRCTXA);
-
-		I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
-		POSTING_READ(RSTDBYCTL);
-	}
-
-	ironlake_teardown_rc6(dev);
-}
-
-static int ironlake_setup_rc6(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	if (dev_priv->renderctx == NULL)
-		dev_priv->renderctx = intel_alloc_context_page(dev);
-	if (!dev_priv->renderctx)
-		return -ENOMEM;
-
-	if (dev_priv->pwrctx == NULL)
-		dev_priv->pwrctx = intel_alloc_context_page(dev);
-	if (!dev_priv->pwrctx) {
-		ironlake_teardown_rc6(dev);
-		return -ENOMEM;
-	}
-
-	return 0;
-}
-
-void ironlake_enable_rc6(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-	int ret;
-
-	/* rc6 disabled by default due to repeated reports of hanging during
-	 * boot and resume.
-	 */
-	if (!intel_enable_rc6(dev))
-		return;
-
-	DRM_LOCK(dev);
-	ret = ironlake_setup_rc6(dev);
-	if (ret) {
-		DRM_UNLOCK(dev);
-		return;
-	}
-
-	/*
-	 * GPU can automatically power down the render unit if given a page
-	 * to save state.
-	 */
-	ret = BEGIN_LP_RING(6);
-	if (ret) {
-		ironlake_teardown_rc6(dev);
-		DRM_UNLOCK(dev);
-		return;
-	}
-
-	OUT_RING(MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN);
-	OUT_RING(MI_SET_CONTEXT);
-	OUT_RING(dev_priv->renderctx->gtt_offset |
-		 MI_MM_SPACE_GTT |
-		 MI_SAVE_EXT_STATE_EN |
-		 MI_RESTORE_EXT_STATE_EN |
-		 MI_RESTORE_INHIBIT);
-	OUT_RING(MI_SUSPEND_FLUSH);
-	OUT_RING(MI_NOOP);
-	OUT_RING(MI_FLUSH);
-	ADVANCE_LP_RING();
-
-	/*
-	 * Wait for the command parser to advance past MI_SET_CONTEXT. The HW
-	 * does an implicit flush, combined with MI_FLUSH above, it should be
-	 * safe to assume that renderctx is valid
-	 */
-	ret = intel_wait_ring_idle(LP_RING(dev_priv));
-	if (ret) {
-		DRM_ERROR("failed to enable ironlake power power savings\n");
-		ironlake_teardown_rc6(dev);
-		DRM_UNLOCK(dev);
-		return;
-	}
-
-	I915_WRITE(PWRCTXA, dev_priv->pwrctx->gtt_offset | PWRCTX_EN);
-	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
-	DRM_UNLOCK(dev);
-}
-
-void intel_init_clock_gating(struct drm_device *dev)
-{
-	struct drm_i915_private *dev_priv = dev->dev_private;
-
-	dev_priv->display.init_clock_gating(dev);
-
-	if (dev_priv->display.init_pch_clock_gating)
-		dev_priv->display.init_pch_clock_gating(dev);
-}
-
 /* Set up chip specific display functions */
 static void intel_init_display(struct drm_device *dev)
 {
@@ -8969,32 +6783,20 @@ static void intel_init_display(struct drm_device *dev)
 	if (HAS_PCH_SPLIT(dev)) {
 		dev_priv->display.dpms = ironlake_crtc_dpms;
 		dev_priv->display.crtc_mode_set = ironlake_crtc_mode_set;
+		dev_priv->display.off = ironlake_crtc_off;
 		dev_priv->display.update_plane = ironlake_update_plane;
 	} else {
 		dev_priv->display.dpms = i9xx_crtc_dpms;
 		dev_priv->display.crtc_mode_set = i9xx_crtc_mode_set;
+		dev_priv->display.off = i9xx_crtc_off;
 		dev_priv->display.update_plane = i9xx_update_plane;
 	}
 
-	if (I915_HAS_FBC(dev)) {
-		if (HAS_PCH_SPLIT(dev)) {
-			dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
-			dev_priv->display.enable_fbc = ironlake_enable_fbc;
-			dev_priv->display.disable_fbc = ironlake_disable_fbc;
-		} else if (IS_GM45(dev)) {
-			dev_priv->display.fbc_enabled = g4x_fbc_enabled;
-			dev_priv->display.enable_fbc = g4x_enable_fbc;
-			dev_priv->display.disable_fbc = g4x_disable_fbc;
-		} else if (IS_CRESTLINE(dev)) {
-			dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
-			dev_priv->display.enable_fbc = i8xx_enable_fbc;
-			dev_priv->display.disable_fbc = i8xx_disable_fbc;
-		}
-		/* 855GM needs testing */
-	}
-
 	/* Returns the core display clock speed */
-	if (IS_I945G(dev) || (IS_G33(dev) && !IS_PINEVIEW_M(dev)))
+	if (IS_VALLEYVIEW(dev))
+		dev_priv->display.get_display_clock_speed =
+			valleyview_get_display_clock_speed;
+	else if (IS_I945G(dev) || (IS_G33(dev) && !IS_PINEVIEW_M(dev)))
 		dev_priv->display.get_display_clock_speed =
 			i945_get_display_clock_speed;
 	else if (IS_I915G(dev))
@@ -9016,124 +6818,27 @@ static void intel_init_display(struct drm_device *dev)
 		dev_priv->display.get_display_clock_speed =
 			i830_get_display_clock_speed;
 
-	/* For FIFO watermark updates */
 	if (HAS_PCH_SPLIT(dev)) {
-		dev_priv->display.force_wake_get = __gen6_gt_force_wake_get;
-		dev_priv->display.force_wake_put = __gen6_gt_force_wake_put;
-
-		/* IVB configs may use multi-threaded forcewake */
-		if (IS_IVYBRIDGE(dev)) {
-			u32	ecobus;
-
-			/* A small trick here - if the bios hasn't configured MT forcewake,
-			 * and if the device is in RC6, then force_wake_mt_get will not wake
-			 * the device and the ECOBUS read will return zero. Which will be
-			 * (correctly) interpreted by the test below as MT forcewake being
-			 * disabled.
-			 */
-			DRM_LOCK(dev);
-			__gen6_gt_force_wake_mt_get(dev_priv);
-			ecobus = I915_READ_NOTRACE(ECOBUS);
-			__gen6_gt_force_wake_mt_put(dev_priv);
-			DRM_UNLOCK(dev);
-
-			if (ecobus & FORCEWAKE_MT_ENABLE) {
-				DRM_DEBUG_KMS("Using MT version of forcewake\n");
-				dev_priv->display.force_wake_get =
-					__gen6_gt_force_wake_mt_get;
-				dev_priv->display.force_wake_put =
-					__gen6_gt_force_wake_mt_put;
-			}
-		}
-
-		if (HAS_PCH_IBX(dev))
-			dev_priv->display.init_pch_clock_gating = ibx_init_clock_gating;
-		else if (HAS_PCH_CPT(dev))
-			dev_priv->display.init_pch_clock_gating = cpt_init_clock_gating;
-
 		if (IS_GEN5(dev)) {
-			if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK)
-				dev_priv->display.update_wm = ironlake_update_wm;
-			else {
-				DRM_DEBUG_KMS("Failed to get proper latency. "
-					      "Disable CxSR\n");
-				dev_priv->display.update_wm = NULL;
-			}
 			dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
-			dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
 			dev_priv->display.write_eld = ironlake_write_eld;
 		} else if (IS_GEN6(dev)) {
-			if (SNB_READ_WM0_LATENCY()) {
-				dev_priv->display.update_wm = sandybridge_update_wm;
-				dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
-			} else {
-				DRM_DEBUG_KMS("Failed to read display plane latency. "
-					      "Disable CxSR\n");
-				dev_priv->display.update_wm = NULL;
-			}
 			dev_priv->display.fdi_link_train = gen6_fdi_link_train;
-			dev_priv->display.init_clock_gating = gen6_init_clock_gating;
 			dev_priv->display.write_eld = ironlake_write_eld;
 		} else if (IS_IVYBRIDGE(dev)) {
 			/* FIXME: detect B0+ stepping and use auto training */
 			dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
-			if (SNB_READ_WM0_LATENCY()) {
-				dev_priv->display.update_wm = sandybridge_update_wm;
-				dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
-			} else {
-				DRM_DEBUG_KMS("Failed to read display plane latency. "
-					      "Disable CxSR\n");
-				dev_priv->display.update_wm = NULL;
-			}
-			dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
+			dev_priv->display.write_eld = ironlake_write_eld;
+		} else if (IS_HASWELL(dev)) {
+			dev_priv->display.fdi_link_train = hsw_fdi_link_train;
 			dev_priv->display.write_eld = ironlake_write_eld;
 		} else
 			dev_priv->display.update_wm = NULL;
-	} else if (IS_PINEVIEW(dev)) {
-		if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
-					    dev_priv->is_ddr3,
-					    dev_priv->fsb_freq,
-					    dev_priv->mem_freq)) {
-			DRM_INFO("failed to find known CxSR latency "
-				 "(found ddr%s fsb freq %d, mem freq %d), "
-				 "disabling CxSR\n",
-				 (dev_priv->is_ddr3 == 1) ? "3" : "2",
-				 dev_priv->fsb_freq, dev_priv->mem_freq);
-			/* Disable CxSR and never update its watermark again */
-			pineview_disable_cxsr(dev);
-			dev_priv->display.update_wm = NULL;
-		} else
-			dev_priv->display.update_wm = pineview_update_wm;
-		dev_priv->display.init_clock_gating = gen3_init_clock_gating;
+	} else if (IS_VALLEYVIEW(dev)) {
+		dev_priv->display.force_wake_get = vlv_force_wake_get;
+		dev_priv->display.force_wake_put = vlv_force_wake_put;
 	} else if (IS_G4X(dev)) {
 		dev_priv->display.write_eld = g4x_write_eld;
-		dev_priv->display.update_wm = g4x_update_wm;
-		dev_priv->display.init_clock_gating = g4x_init_clock_gating;
-	} else if (IS_GEN4(dev)) {
-		dev_priv->display.update_wm = i965_update_wm;
-		if (IS_CRESTLINE(dev))
-			dev_priv->display.init_clock_gating = crestline_init_clock_gating;
-		else if (IS_BROADWATER(dev))
-			dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
-	} else if (IS_GEN3(dev)) {
-		dev_priv->display.update_wm = i9xx_update_wm;
-		dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
-		dev_priv->display.init_clock_gating = gen3_init_clock_gating;
-	} else if (IS_I865G(dev)) {
-		dev_priv->display.update_wm = i830_update_wm;
-		dev_priv->display.init_clock_gating = i85x_init_clock_gating;
-		dev_priv->display.get_fifo_size = i830_get_fifo_size;
-	} else if (IS_I85X(dev)) {
-		dev_priv->display.update_wm = i9xx_update_wm;
-		dev_priv->display.get_fifo_size = i85x_get_fifo_size;
-		dev_priv->display.init_clock_gating = i85x_init_clock_gating;
-	} else {
-		dev_priv->display.update_wm = i830_update_wm;
-		dev_priv->display.init_clock_gating = i830_init_clock_gating;
-		if (IS_845G(dev))
-			dev_priv->display.get_fifo_size = i845_get_fifo_size;
-		else
-			dev_priv->display.get_fifo_size = i830_get_fifo_size;
 	}
 
 	/* Default just returns -ENODEV to indicate unsupported */
@@ -9172,7 +6877,7 @@ static void quirk_pipea_force(struct drm_device *dev)
 	struct drm_i915_private *dev_priv = dev->dev_private;
 
 	dev_priv->quirks |= QUIRK_PIPEA_FORCE;
-	DRM_DEBUG("applying pipe a force quirk\n");
+	DRM_INFO("applying pipe a force quirk\n");
 }
 
 /*
@@ -9182,6 +6887,18 @@ static void quirk_ssc_force_disable(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE;
+	DRM_INFO("applying lvds SSC disable quirk\n");
+}
+
+/*
+ * A machine (e.g. Acer Aspire 5734Z) may need to invert the panel backlight
+ * brightness value
+ */
+static void quirk_invert_brightness(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	dev_priv->quirks |= QUIRK_INVERT_BRIGHTNESS;
+	DRM_INFO("applying inverted panel brightness quirk\n");
 }
 
 struct intel_quirk {
@@ -9193,7 +6910,7 @@ struct intel_quirk {
 
 #define	PCI_ANY_ID	(~0u)
 
-struct intel_quirk intel_quirks[] = {
+static struct intel_quirk intel_quirks[] = {
 	/* HP Mini needs pipe A force quirk (LP: #322104) */
 	{ 0x27ae, 0x103c, 0x361a, quirk_pipea_force },
 
@@ -9218,6 +6935,9 @@ struct intel_quirk intel_quirks[] = {
 
 	/* Sony Vaio Y cannot use SSC on LVDS */
 	{ 0x0046, 0x104d, 0x9076, quirk_ssc_force_disable },
+
+	/* Acer Aspire 5734Z must invert backlight brightness */
+	{ 0x2a42, 0x1025, 0x0459, quirk_invert_brightness },
 };
 
 static void intel_init_quirks(struct drm_device *dev)
@@ -9253,7 +6973,7 @@ static void i915_disable_vga(struct drm_device *dev)
 #if 0
 	vga_get_uninterruptible(dev->pdev, VGA_RSRC_LEGACY_IO);
 #endif
-	outb(VGA_SR_INDEX, 1);
+	outb(VGA_SR_INDEX, SR01);
 	sr1 = inb(VGA_SR_DATA);
 	outb(VGA_SR_DATA, sr1 | 1 << 5);
 #if 0
@@ -9265,6 +6985,40 @@ static void i915_disable_vga(struct drm_device *dev)
 	POSTING_READ(vga_reg);
 }
 
+static void ivb_pch_pwm_override(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	/*
+	 * IVB has CPU eDP backlight regs too, set things up to let the
+	 * PCH regs control the backlight
+	 */
+	I915_WRITE(BLC_PWM_CPU_CTL2, PWM_ENABLE);
+	I915_WRITE(BLC_PWM_CPU_CTL, 0);
+	I915_WRITE(BLC_PWM_PCH_CTL1, PWM_ENABLE | (1<<30));
+}
+
+void intel_modeset_init_hw(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	intel_init_clock_gating(dev);
+
+	if (IS_IRONLAKE_M(dev)) {
+		ironlake_enable_drps(dev);
+		ironlake_enable_rc6(dev);
+		intel_init_emon(dev);
+	}
+
+	if ((IS_GEN6(dev) || IS_GEN7(dev)) && !IS_VALLEYVIEW(dev)) {
+		gen6_enable_rps(dev_priv);
+		gen6_update_ring_freq(dev_priv);
+	}
+
+	if (IS_IVYBRIDGE(dev))
+		ivb_pch_pwm_override(dev);
+}
+
 void intel_modeset_init(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
@@ -9278,11 +7032,14 @@ void intel_modeset_init(struct drm_device *dev)
 	dev->mode_config.preferred_depth = 24;
 	dev->mode_config.prefer_shadow = 1;
 
-	dev->mode_config.funcs = __DECONST(struct drm_mode_config_funcs *,
-	    &intel_mode_funcs);
+	dev->mode_config.funcs = &intel_mode_funcs;
 
 	intel_init_quirks(dev);
 
+	intel_init_pm(dev);
+
+	intel_prepare_ddi(dev);
+
 	intel_init_display(dev);
 
 	if (IS_GEN2(dev)) {
@@ -9307,30 +7064,19 @@ void intel_modeset_init(struct drm_device *dev)
 			DRM_DEBUG_KMS("plane %d init failed: %d\n", i, ret);
 	}
 
+	intel_pch_pll_init(dev);
+
 	/* Just disable it once at startup */
 	i915_disable_vga(dev);
 	intel_setup_outputs(dev);
 
-	intel_init_clock_gating(dev);
-
-	if (IS_IRONLAKE_M(dev)) {
-		ironlake_enable_drps(dev);
-		intel_init_emon(dev);
-	}
-
-	if (IS_GEN6(dev)) {
-		gen6_enable_rps(dev_priv);
-		gen6_update_ring_freq(dev_priv);
-	}
-
 	TASK_INIT(&dev_priv->idle_task, 0, intel_idle_update, dev_priv);
 	callout_init(&dev_priv->idle_callout, CALLOUT_MPSAFE);
 }
 
 void intel_modeset_gem_init(struct drm_device *dev)
 {
-	if (IS_IRONLAKE_M(dev))
-		ironlake_enable_rc6(dev);
+	intel_modeset_init_hw(dev);
 
 	intel_setup_overlay(dev);
 }
@@ -9361,12 +7107,15 @@ void intel_modeset_cleanup(struct drm_device *dev)
 
 	if (IS_IRONLAKE_M(dev))
 		ironlake_disable_drps(dev);
-	if (IS_GEN6(dev))
+	if ((IS_GEN6(dev) || IS_GEN7(dev)) && !IS_VALLEYVIEW(dev))
 		gen6_disable_rps(dev);
 
 	if (IS_IRONLAKE_M(dev))
 		ironlake_disable_rc6(dev);
 
+	if (IS_VALLEYVIEW(dev))
+		vlv_init_dpio(dev);
+
 	/* Disable the irq before mode object teardown, for the irq might
 	 * enqueue unpin/hotplug work. */
 	drm_irq_uninstall(dev);
diff --git a/sys/dev/drm2/i915/intel_dp.c b/sys/dev/drm2/i915/intel_dp.c
index e0df0bc2749..6382387e535 100644
--- a/sys/dev/drm2/i915/intel_dp.c
+++ b/sys/dev/drm2/i915/intel_dp.c
@@ -673,7 +673,7 @@ intel_dp_i2c_init(struct intel_dp *intel_dp,
 }
 
 static bool
-intel_dp_mode_fixup(struct drm_encoder *encoder, const struct drm_display_mode *mode,
+intel_dp_mode_fixup(struct drm_encoder *encoder, struct drm_display_mode *mode,
 		    struct drm_display_mode *adjusted_mode)
 {
 	struct drm_device *dev = encoder->dev;
@@ -681,33 +681,44 @@ intel_dp_mode_fixup(struct drm_encoder *encoder, const struct drm_display_mode *
 	int lane_count, clock;
 	int max_lane_count = intel_dp_max_lane_count(intel_dp);
 	int max_clock = intel_dp_max_link_bw(intel_dp) == DP_LINK_BW_2_7 ? 1 : 0;
-	int bpp;
+	int bpp, mode_rate;
 	static int bws[2] = { DP_LINK_BW_1_62, DP_LINK_BW_2_7 };
 
 	if (is_edp(intel_dp) && intel_dp->panel_fixed_mode) {
 		intel_fixed_panel_mode(intel_dp->panel_fixed_mode, adjusted_mode);
 		intel_pch_panel_fitting(dev, DRM_MODE_SCALE_FULLSCREEN,
 					mode, adjusted_mode);
+		/*
+		 * the mode->clock is used to calculate the Data&Link M/N
+		 * of the pipe. For the eDP the fixed clock should be used.
+		 */
+		mode->clock = intel_dp->panel_fixed_mode->clock;
 	}
 
+	DRM_DEBUG_KMS("DP link computation with max lane count %i "
+		      "max bw %02x pixel clock %iKHz\n",
+		      max_lane_count, bws[max_clock], mode->clock);
+
 	if (!intel_dp_adjust_dithering(intel_dp, adjusted_mode, adjusted_mode))
 		return false;
 
 	bpp = adjusted_mode->private_flags & INTEL_MODE_DP_FORCE_6BPC ? 18 : 24;
+	mode_rate = intel_dp_link_required(mode->clock, bpp);
 
 	for (lane_count = 1; lane_count <= max_lane_count; lane_count <<= 1) {
 		for (clock = 0; clock <= max_clock; clock++) {
 			int link_avail = intel_dp_max_data_rate(intel_dp_link_clock(bws[clock]), lane_count);
 
-			if (intel_dp_link_required(adjusted_mode->clock, bpp)
-					<= link_avail) {
+			if (mode_rate <= link_avail) {
 				intel_dp->link_bw = bws[clock];
 				intel_dp->lane_count = lane_count;
 				adjusted_mode->clock = intel_dp_link_clock(intel_dp->link_bw);
-				DRM_DEBUG_KMS("Display port link bw %02x lane "
-						"count %d clock %d\n",
+				DRM_DEBUG_KMS("DP link bw %02x lane "
+						"count %d clock %d bpp %d\n",
 				       intel_dp->link_bw, intel_dp->lane_count,
-				       adjusted_mode->clock);
+				       adjusted_mode->clock, bpp);
+				DRM_DEBUG_KMS("DP link bw required %i available %i\n",
+					      mode_rate, link_avail);
 				return true;
 			}
 		}
@@ -1140,6 +1151,7 @@ static void ironlake_edp_panel_off(struct intel_dp *intel_dp)
 
 	if (intel_dp->want_panel_vdd)
 		printf("Cannot turn power off while VDD is on\n");
+	ironlake_panel_vdd_off_sync(intel_dp); /* finish any pending work */
 
 	pp = ironlake_get_pp_control(dev_priv);
 	pp &= ~(POWER_TARGET_ON | EDP_FORCE_VDD | PANEL_POWER_RESET | EDP_BLC_ENABLE);
@@ -1937,6 +1949,23 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
 	return false;
 }
 
+static void
+intel_dp_probe_oui(struct intel_dp *intel_dp)
+{
+	u8 buf[3];
+
+	if (!(intel_dp->dpcd[DP_DOWN_STREAM_PORT_COUNT] & DP_OUI_SUPPORT))
+		return;
+
+	if (intel_dp_aux_native_read_retry(intel_dp, DP_SINK_OUI, buf, 3))
+		DRM_DEBUG_KMS("Sink OUI: %02x%02x%02x\n",
+			      buf[0], buf[1], buf[2]);
+
+	if (intel_dp_aux_native_read_retry(intel_dp, DP_BRANCH_OUI, buf, 3))
+		DRM_DEBUG_KMS("Branch OUI: %02x%02x%02x\n",
+			      buf[0], buf[1], buf[2]);
+}
+
 static bool
 intel_dp_get_sink_irq(struct intel_dp *intel_dp, u8 *sink_irq_vector)
 {
@@ -2114,6 +2143,8 @@ intel_dp_detect(struct drm_connector *connector, bool force)
 	if (status != connector_status_connected)
 		return status;
 
+	intel_dp_probe_oui(intel_dp);
+
 	if (intel_dp->force_audio != HDMI_AUDIO_AUTO) {
 		intel_dp->has_audio = (intel_dp->force_audio == HDMI_AUDIO_ON);
 	} else {
@@ -2428,6 +2459,7 @@ intel_dp_init(struct drm_device *dev, int output_reg)
 	}
 
 	intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
+
 	connector->interlace_allowed = true;
 	connector->doublescan_allowed = 0;
 
@@ -2475,6 +2507,13 @@ intel_dp_init(struct drm_device *dev, int output_reg)
 		pp_off = I915_READ(PCH_PP_OFF_DELAYS);
 		pp_div = I915_READ(PCH_PP_DIVISOR);
 
+		if (!pp_on || !pp_off || !pp_div) {
+			DRM_INFO("bad panel power sequencing delays, disabling panel\n");
+			intel_dp_encoder_destroy(&intel_dp->base.base);
+			intel_dp_destroy(&intel_connector->base);
+			return;
+		}
+
 		/* Pull timing values out of registers */
 		cur.t1_t3 = (pp_on & PANEL_POWER_UP_DELAY_MASK) >>
 			PANEL_POWER_UP_DELAY_SHIFT;
diff --git a/sys/dev/drm2/i915/intel_drv.h b/sys/dev/drm2/i915/intel_drv.h
index bc2c55c6b91..f2b55225a2e 100644
--- a/sys/dev/drm2/i915/intel_drv.h
+++ b/sys/dev/drm2/i915/intel_drv.h
@@ -55,6 +55,21 @@
 	ret;								\
 })
 
+#define wait_for_atomic_us(COND, US) ({ \
+	int i, ret__ = -ETIMEDOUT;	\
+	for (i = 0; i < (US); i++) {	\
+		if ((COND)) {		\
+			ret__ = 0;	\
+			break;		\
+		}			\
+		DELAY(1);		\
+	}				\
+	ret__;				\
+})
+
+#define	wait_for(COND, MS) _intel_wait_for(NULL, COND, MS, 1, "915wfi")
+#define	wait_for_atomic(COND, MS) _intel_wait_for(NULL, COND, MS, 0, "915wfa")
+
 #define KHz(x) (1000*x)
 #define MHz(x) KHz(1000*x)
 
@@ -174,8 +189,8 @@ struct intel_crtc {
 	bool cursor_visible;
 	unsigned int bpp;
 
-	bool no_pll; /* tertiary pipe for IVB */
-	bool use_pll_a;
+	/* We can share PLLs across outputs if the timings match */
+	struct intel_pch_pll *pch_pll;
 };
 
 struct intel_plane {
@@ -199,6 +214,25 @@ struct intel_plane {
 			     struct drm_intel_sprite_colorkey *key);
 };
 
+struct intel_watermark_params {
+	unsigned long fifo_size;
+	unsigned long max_wm;
+	unsigned long default_wm;
+	unsigned long guard_size;
+	unsigned long cacheline_size;
+};
+
+struct cxsr_latency {
+	int is_desktop;
+	int is_ddr3;
+	unsigned long fsb_freq;
+	unsigned long mem_freq;
+	unsigned long display_sr;
+	unsigned long display_hpll_disable;
+	unsigned long cursor_sr;
+	unsigned long cursor_hpll_disable;
+};
+
 #define to_intel_crtc(x) container_of(x, struct intel_crtc, base)
 #define to_intel_connector(x) container_of(x, struct intel_connector, base)
 #define to_intel_encoder(x) container_of(x, struct intel_encoder, base)
@@ -210,6 +244,8 @@ struct intel_plane {
 #define DIP_TYPE_AVI    0x82
 #define DIP_VERSION_AVI 0x2
 #define DIP_LEN_AVI     13
+#define DIP_AVI_PR_1    0
+#define DIP_AVI_PR_2    1
 
 #define DIP_TYPE_SPD	0x83
 #define DIP_VERSION_SPD	0x1
@@ -243,23 +279,36 @@ struct dip_infoframe {
 			uint8_t ITC_EC_Q_SC;
 			/* PB4 - VIC 6:0 */
 			uint8_t VIC;
-			/* PB5 - PR 3:0 */
-			uint8_t PR;
+			/* PB5 - YQ 7:6, CN 5:4, PR 3:0 */
+			uint8_t YQ_CN_PR;
 			/* PB6 to PB13 */
 			uint16_t top_bar_end;
 			uint16_t bottom_bar_start;
 			uint16_t left_bar_end;
 			uint16_t right_bar_start;
-		} avi;
+		} __attribute__ ((packed)) avi;
 		struct {
 			uint8_t vn[8];
 			uint8_t pd[16];
 			uint8_t sdi;
-		} spd;
+		} __attribute__ ((packed)) spd;
 		uint8_t payload[27];
 	} __attribute__ ((packed)) body;
 } __attribute__((packed));
 
+struct intel_hdmi {
+	struct intel_encoder base;
+	u32 sdvox_reg;
+	int ddc_bus;
+	int ddi_port;
+	uint32_t color_range;
+	bool has_hdmi_sink;
+	bool has_audio;
+	enum hdmi_force_audio force_audio;
+	void (*write_infoframe)(struct drm_encoder *encoder,
+				struct dip_infoframe *frame);
+};
+
 static inline struct drm_crtc *
 intel_get_crtc_for_pipe(struct drm_device *dev, int pipe)
 {
@@ -299,8 +348,13 @@ extern void intel_attach_broadcast_rgb_property(struct drm_connector *connector)
 
 extern void intel_crt_init(struct drm_device *dev);
 extern void intel_hdmi_init(struct drm_device *dev, int sdvox_reg);
-void intel_dip_infoframe_csum(struct dip_infoframe *avi_if);
-extern bool intel_sdvo_init(struct drm_device *dev, int output_device);
+extern struct intel_hdmi *enc_to_intel_hdmi(struct drm_encoder *encoder);
+extern void intel_hdmi_set_avi_infoframe(struct drm_encoder *encoder,
+			    struct drm_display_mode *adjusted_mode);
+extern void intel_hdmi_set_spd_infoframe(struct drm_encoder *encoder);
+extern void intel_dip_infoframe_csum(struct dip_infoframe *avi_if);
+extern bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg,
+			    bool is_sdvob);
 extern void intel_dvo_init(struct drm_device *dev);
 extern void intel_tv_init(struct drm_device *dev);
 extern void intel_mark_busy(struct drm_device *dev,
@@ -314,6 +368,10 @@ extern bool intel_dpd_is_edp(struct drm_device *dev);
 extern void intel_edp_link_config(struct intel_encoder *, int *, int *);
 extern bool intel_encoder_is_pch_edp(struct drm_encoder *encoder);
 extern int intel_plane_init(struct drm_device *dev, enum pipe pipe);
+extern void intel_flush_display_plane(struct drm_i915_private *dev_priv,
+				      enum plane plane);
+
+void intel_sanitize_pm(struct drm_device *dev);
 
 /* intel_panel.c */
 extern void intel_fixed_panel_mode(struct drm_display_mode *fixed_mode,
@@ -371,12 +429,19 @@ extern void intel_crtc_fb_gamma_set(struct drm_crtc *crtc, u16 red, u16 green,
 extern void intel_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
 				    u16 *blue, int regno);
 extern void intel_enable_clock_gating(struct drm_device *dev);
+extern void ironlake_disable_rc6(struct drm_device *dev);
 extern void ironlake_enable_drps(struct drm_device *dev);
 extern void ironlake_disable_drps(struct drm_device *dev);
 extern void gen6_enable_rps(struct drm_i915_private *dev_priv);
 extern void gen6_update_ring_freq(struct drm_i915_private *dev_priv);
 extern void gen6_disable_rps(struct drm_device *dev);
 extern void intel_init_emon(struct drm_device *dev);
+extern int intel_enable_rc6(const struct drm_device *dev);
+
+extern void intel_ddi_dpms(struct drm_encoder *encoder, int mode);
+extern void intel_ddi_mode_set(struct drm_encoder *encoder,
+			       struct drm_display_mode *mode,
+			       struct drm_display_mode *adjusted_mode);
 
 extern int intel_pin_and_fence_fb_obj(struct drm_device *dev,
 				      struct drm_i915_gem_object *obj,
@@ -414,15 +479,30 @@ extern void intel_init_clock_gating(struct drm_device *dev);
 extern void intel_write_eld(struct drm_encoder *encoder,
 			    struct drm_display_mode *mode);
 extern void intel_cpt_verify_modeset(struct drm_device *dev, int pipe);
+extern void intel_prepare_ddi(struct drm_device *dev);
+extern void hsw_fdi_link_train(struct drm_crtc *crtc);
+extern void intel_ddi_init(struct drm_device *dev, enum port port);
 
 /* For use by IVB LP watermark workaround in intel_sprite.c */
-extern void sandybridge_update_wm(struct drm_device *dev);
+extern void intel_update_watermarks(struct drm_device *dev);
 extern void intel_update_sprite_watermarks(struct drm_device *dev, int pipe,
 					   uint32_t sprite_width,
 					   int pixel_size);
+extern void intel_update_linetime_watermarks(struct drm_device *dev, int pipe,
+			 struct drm_display_mode *mode);
+
 extern int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
 				     struct drm_file *file_priv);
 extern int intel_sprite_get_colorkey(struct drm_device *dev, void *data,
 				     struct drm_file *file_priv);
 
-#endif
+extern u32 intel_dpio_read(struct drm_i915_private *dev_priv, int reg);
+
+/* Power-related functions, located in intel_pm.c */
+extern void intel_init_pm(struct drm_device *dev);
+/* FBC */
+extern bool intel_fbc_enabled(struct drm_device *dev);
+extern void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval);
+extern void intel_update_fbc(struct drm_device *dev);
+
+#endif /* __INTEL_DRV_H__ */
diff --git a/sys/dev/drm2/i915/intel_fb.c b/sys/dev/drm2/i915/intel_fb.c
index f8a68ec92b3..124b7693e1b 100644
--- a/sys/dev/drm2/i915/intel_fb.c
+++ b/sys/dev/drm2/i915/intel_fb.c
@@ -73,7 +73,7 @@ static int intelfb_create(struct intel_fbdev *ifbdev,
 	DRM_LOCK(dev);
 
 	/* Flush everything out, we'll be doing GTT only from now on */
-	ret = intel_pin_and_fence_fb_obj(dev, obj, false);
+	ret = intel_pin_and_fence_fb_obj(dev, obj, NULL);
 	if (ret) {
 		DRM_ERROR("failed to pin fb: %d\n", ret);
 		goto out_unref;
diff --git a/sys/dev/drm2/i915/intel_hdmi.c b/sys/dev/drm2/i915/intel_hdmi.c
index f076608d30d..3f0572f260c 100644
--- a/sys/dev/drm2/i915/intel_hdmi.c
+++ b/sys/dev/drm2/i915/intel_hdmi.c
@@ -37,19 +37,7 @@ __FBSDID("$FreeBSD$");
 #include <dev/drm2/i915/i915_drv.h>
 #include <dev/drm2/i915/intel_drv.h>
 
-struct intel_hdmi {
-	struct intel_encoder base;
-	u32 sdvox_reg;
-	int ddc_bus;
-	uint32_t color_range;
-	bool has_hdmi_sink;
-	bool has_audio;
-	enum hdmi_force_audio force_audio;
-	void (*write_infoframe)(struct drm_encoder *encoder,
-				struct dip_infoframe *frame);
-};
-
-static struct intel_hdmi *enc_to_intel_hdmi(struct drm_encoder *encoder)
+struct intel_hdmi *enc_to_intel_hdmi(struct drm_encoder *encoder)
 {
 	return container_of(encoder, struct intel_hdmi, base.base);
 }
@@ -75,107 +63,244 @@ void intel_dip_infoframe_csum(struct dip_infoframe *frame)
 	frame->checksum = 0x100 - sum;
 }
 
-static u32 intel_infoframe_index(struct dip_infoframe *frame)
+static u32 g4x_infoframe_index(struct dip_infoframe *frame)
 {
-	u32 flags = 0;
-
 	switch (frame->type) {
 	case DIP_TYPE_AVI:
-		flags |= VIDEO_DIP_SELECT_AVI;
-		break;
+		return VIDEO_DIP_SELECT_AVI;
 	case DIP_TYPE_SPD:
-		flags |= VIDEO_DIP_SELECT_SPD;
-		break;
+		return VIDEO_DIP_SELECT_SPD;
 	default:
-		DRM_DEBUG("unknown info frame type %d\n", frame->type);
-		break;
+		DRM_DEBUG_DRIVER("unknown info frame type %d\n", frame->type);
+		return 0;
 	}
-
-	return flags;
 }
 
-static u32 intel_infoframe_flags(struct dip_infoframe *frame)
+static u32 g4x_infoframe_enable(struct dip_infoframe *frame)
 {
-	u32 flags = 0;
-
 	switch (frame->type) {
 	case DIP_TYPE_AVI:
-		flags |= VIDEO_DIP_ENABLE_AVI | VIDEO_DIP_FREQ_VSYNC;
-		break;
+		return VIDEO_DIP_ENABLE_AVI;
 	case DIP_TYPE_SPD:
-		flags |= VIDEO_DIP_ENABLE_SPD | VIDEO_DIP_FREQ_VSYNC;
-		break;
+		return VIDEO_DIP_ENABLE_SPD;
 	default:
-		DRM_DEBUG("unknown info frame type %d\n", frame->type);
-		break;
+		DRM_DEBUG_DRIVER("unknown info frame type %d\n", frame->type);
+		return 0;
 	}
-
-	return flags;
 }
 
-static void i9xx_write_infoframe(struct drm_encoder *encoder,
-				 struct dip_infoframe *frame)
+static u32 hsw_infoframe_enable(struct dip_infoframe *frame)
+{
+	switch (frame->type) {
+	case DIP_TYPE_AVI:
+		return VIDEO_DIP_ENABLE_AVI_HSW;
+	case DIP_TYPE_SPD:
+		return VIDEO_DIP_ENABLE_SPD_HSW;
+	default:
+		DRM_DEBUG_DRIVER("unknown info frame type %d\n", frame->type);
+		return 0;
+	}
+}
+
+static u32 hsw_infoframe_data_reg(struct dip_infoframe *frame, enum pipe pipe)
+{
+	switch (frame->type) {
+	case DIP_TYPE_AVI:
+		return HSW_TVIDEO_DIP_AVI_DATA(pipe);
+	case DIP_TYPE_SPD:
+		return HSW_TVIDEO_DIP_SPD_DATA(pipe);
+	default:
+		DRM_DEBUG_DRIVER("unknown info frame type %d\n", frame->type);
+		return 0;
+	}
+}
+
+static void g4x_write_infoframe(struct drm_encoder *encoder,
+				struct dip_infoframe *frame)
 {
 	uint32_t *data = (uint32_t *)frame;
 	struct drm_device *dev = encoder->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
-	u32 port, flags, val = I915_READ(VIDEO_DIP_CTL);
+	u32 val = I915_READ(VIDEO_DIP_CTL);
 	unsigned i, len = DIP_HEADER_SIZE + frame->len;
 
-
-	/* XXX first guess at handling video port, is this corrent? */
+	val &= ~VIDEO_DIP_PORT_MASK;
 	if (intel_hdmi->sdvox_reg == SDVOB)
-		port = VIDEO_DIP_PORT_B;
+		val |= VIDEO_DIP_PORT_B;
 	else if (intel_hdmi->sdvox_reg == SDVOC)
-		port = VIDEO_DIP_PORT_C;
+		val |= VIDEO_DIP_PORT_C;
 	else
 		return;
 
-	flags = intel_infoframe_index(frame);
+	val &= ~(VIDEO_DIP_SELECT_MASK | 0xf); /* clear DIP data offset */
+	val |= g4x_infoframe_index(frame);
 
-	val &= ~VIDEO_DIP_SELECT_MASK;
+	val &= ~g4x_infoframe_enable(frame);
+	val |= VIDEO_DIP_ENABLE;
 
-	I915_WRITE(VIDEO_DIP_CTL, VIDEO_DIP_ENABLE | val | port | flags);
+	I915_WRITE(VIDEO_DIP_CTL, val);
 
 	for (i = 0; i < len; i += 4) {
 		I915_WRITE(VIDEO_DIP_DATA, *data);
 		data++;
 	}
 
-	flags |= intel_infoframe_flags(frame);
+	val |= g4x_infoframe_enable(frame);
+	val &= ~VIDEO_DIP_FREQ_MASK;
+	val |= VIDEO_DIP_FREQ_VSYNC;
 
-	I915_WRITE(VIDEO_DIP_CTL, VIDEO_DIP_ENABLE | val | port | flags);
+	I915_WRITE(VIDEO_DIP_CTL, val);
 }
 
-static void ironlake_write_infoframe(struct drm_encoder *encoder,
-				     struct dip_infoframe *frame)
+static void ibx_write_infoframe(struct drm_encoder *encoder,
+				struct dip_infoframe *frame)
 {
 	uint32_t *data = (uint32_t *)frame;
 	struct drm_device *dev = encoder->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_crtc *crtc = encoder->crtc;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
+	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
 	int reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
 	unsigned i, len = DIP_HEADER_SIZE + frame->len;
-	u32 flags, val = I915_READ(reg);
+	u32 val = I915_READ(reg);
+
+	val &= ~VIDEO_DIP_PORT_MASK;
+	switch (intel_hdmi->sdvox_reg) {
+	case HDMIB:
+		val |= VIDEO_DIP_PORT_B;
+		break;
+	case HDMIC:
+		val |= VIDEO_DIP_PORT_C;
+		break;
+	case HDMID:
+		val |= VIDEO_DIP_PORT_D;
+		break;
+	default:
+		return;
+	}
 
 	intel_wait_for_vblank(dev, intel_crtc->pipe);
 
-	flags = intel_infoframe_index(frame);
-
 	val &= ~(VIDEO_DIP_SELECT_MASK | 0xf); /* clear DIP data offset */
+	val |= g4x_infoframe_index(frame);
 
-	I915_WRITE(reg, VIDEO_DIP_ENABLE | val | flags);
+	val &= ~g4x_infoframe_enable(frame);
+	val |= VIDEO_DIP_ENABLE;
+
+	I915_WRITE(reg, val);
 
 	for (i = 0; i < len; i += 4) {
 		I915_WRITE(TVIDEO_DIP_DATA(intel_crtc->pipe), *data);
 		data++;
 	}
 
-	flags |= intel_infoframe_flags(frame);
+	val |= g4x_infoframe_enable(frame);
+	val &= ~VIDEO_DIP_FREQ_MASK;
+	val |= VIDEO_DIP_FREQ_VSYNC;
 
-	I915_WRITE(reg, VIDEO_DIP_ENABLE | val | flags);
+	I915_WRITE(reg, val);
+}
+
+static void cpt_write_infoframe(struct drm_encoder *encoder,
+				struct dip_infoframe *frame)
+{
+	uint32_t *data = (uint32_t *)frame;
+	struct drm_device *dev = encoder->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
+	int reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
+	unsigned i, len = DIP_HEADER_SIZE + frame->len;
+	u32 val = I915_READ(reg);
+
+	intel_wait_for_vblank(dev, intel_crtc->pipe);
+
+	val &= ~(VIDEO_DIP_SELECT_MASK | 0xf); /* clear DIP data offset */
+	val |= g4x_infoframe_index(frame);
+
+	/* The DIP control register spec says that we need to update the AVI
+	 * infoframe without clearing its enable bit */
+	if (frame->type == DIP_TYPE_AVI)
+		val |= VIDEO_DIP_ENABLE_AVI;
+	else
+		val &= ~g4x_infoframe_enable(frame);
+
+	val |= VIDEO_DIP_ENABLE;
+
+	I915_WRITE(reg, val);
+
+	for (i = 0; i < len; i += 4) {
+		I915_WRITE(TVIDEO_DIP_DATA(intel_crtc->pipe), *data);
+		data++;
+	}
+
+	val |= g4x_infoframe_enable(frame);
+	val &= ~VIDEO_DIP_FREQ_MASK;
+	val |= VIDEO_DIP_FREQ_VSYNC;
+
+	I915_WRITE(reg, val);
+}
+
+static void vlv_write_infoframe(struct drm_encoder *encoder,
+				     struct dip_infoframe *frame)
+{
+	uint32_t *data = (uint32_t *)frame;
+	struct drm_device *dev = encoder->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
+	int reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe);
+	unsigned i, len = DIP_HEADER_SIZE + frame->len;
+	u32 val = I915_READ(reg);
+
+	intel_wait_for_vblank(dev, intel_crtc->pipe);
+
+	val &= ~(VIDEO_DIP_SELECT_MASK | 0xf); /* clear DIP data offset */
+	val |= g4x_infoframe_index(frame);
+
+	val &= ~g4x_infoframe_enable(frame);
+	val |= VIDEO_DIP_ENABLE;
+
+	I915_WRITE(reg, val);
+
+	for (i = 0; i < len; i += 4) {
+		I915_WRITE(VLV_TVIDEO_DIP_DATA(intel_crtc->pipe), *data);
+		data++;
+	}
+
+	val |= g4x_infoframe_enable(frame);
+	val &= ~VIDEO_DIP_FREQ_MASK;
+	val |= VIDEO_DIP_FREQ_VSYNC;
+
+	I915_WRITE(reg, val);
+}
+
+static void hsw_write_infoframe(struct drm_encoder *encoder,
+				struct dip_infoframe *frame)
+{
+	uint32_t *data = (uint32_t *)frame;
+	struct drm_device *dev = encoder->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
+	u32 ctl_reg = HSW_TVIDEO_DIP_CTL(intel_crtc->pipe);
+	u32 data_reg = hsw_infoframe_data_reg(frame, intel_crtc->pipe);
+	unsigned int i, len = DIP_HEADER_SIZE + frame->len;
+	u32 val = I915_READ(ctl_reg);
+
+	if (data_reg == 0)
+		return;
+
+	intel_wait_for_vblank(dev, intel_crtc->pipe);
+
+	val &= ~hsw_infoframe_enable(frame);
+	I915_WRITE(ctl_reg, val);
+
+	for (i = 0; i < len; i += 4) {
+		I915_WRITE(data_reg + i, *data);
+		data++;
+	}
+
+	val |= hsw_infoframe_enable(frame);
+	I915_WRITE(ctl_reg, val);
 }
 
 static void intel_set_infoframe(struct drm_encoder *encoder,
@@ -190,7 +315,8 @@ static void intel_set_infoframe(struct drm_encoder *encoder,
 	intel_hdmi->write_infoframe(encoder, frame);
 }
 
-static void intel_hdmi_set_avi_infoframe(struct drm_encoder *encoder)
+void intel_hdmi_set_avi_infoframe(struct drm_encoder *encoder,
+					 struct drm_display_mode *adjusted_mode)
 {
 	struct dip_infoframe avi_if = {
 		.type = DIP_TYPE_AVI,
@@ -198,10 +324,13 @@ static void intel_hdmi_set_avi_infoframe(struct drm_encoder *encoder)
 		.len = DIP_LEN_AVI,
 	};
 
+	if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK)
+		avi_if.body.avi.YQ_CN_PR |= DIP_AVI_PR_2;
+
 	intel_set_infoframe(encoder, &avi_if);
 }
 
-static void intel_hdmi_set_spd_infoframe(struct drm_encoder *encoder)
+void intel_hdmi_set_spd_infoframe(struct drm_encoder *encoder)
 {
 	struct dip_infoframe spd_if;
 
@@ -222,8 +351,7 @@ static void intel_hdmi_mode_set(struct drm_encoder *encoder,
 {
 	struct drm_device *dev = encoder->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
-	struct drm_crtc *crtc = encoder->crtc;
-	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
 	struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
 	u32 sdvox;
 
@@ -260,7 +388,7 @@ static void intel_hdmi_mode_set(struct drm_encoder *encoder,
 	I915_WRITE(intel_hdmi->sdvox_reg, sdvox);
 	POSTING_READ(intel_hdmi->sdvox_reg);
 
-	intel_hdmi_set_avi_infoframe(encoder);
+	intel_hdmi_set_avi_infoframe(encoder, adjusted_mode);
 	intel_hdmi_set_spd_infoframe(encoder);
 }
 
@@ -318,7 +446,7 @@ static int intel_hdmi_mode_valid(struct drm_connector *connector,
 }
 
 static bool intel_hdmi_mode_fixup(struct drm_encoder *encoder,
-				  const struct drm_display_mode *mode,
+				  struct drm_display_mode *mode,
 				  struct drm_display_mode *adjusted_mode)
 {
 	return true;
@@ -334,7 +462,8 @@ intel_hdmi_detect(struct drm_connector *connector, bool force)
 
 	intel_hdmi->has_hdmi_sink = false;
 	intel_hdmi->has_audio = false;
-	edid = drm_get_edid(connector, dev_priv->gmbus[intel_hdmi->ddc_bus]);
+	edid = drm_get_edid(connector, intel_gmbus_get_adapter(dev_priv,
+	    intel_hdmi->ddc_bus));
 
 	if (edid) {
 		if (edid->input & DRM_EDID_INPUT_DIGITAL) {
@@ -371,7 +500,8 @@ static int intel_hdmi_get_modes(struct drm_connector *connector)
 	 */
 
 	return intel_ddc_get_modes(connector,
-	    dev_priv->gmbus[intel_hdmi->ddc_bus]);
+				   intel_gmbus_get_adapter(dev_priv,
+							   intel_hdmi->ddc_bus));
 }
 
 static bool
@@ -382,7 +512,9 @@ intel_hdmi_detect_audio(struct drm_connector *connector)
 	struct edid *edid;
 	bool has_audio = false;
 
-	edid = drm_get_edid(connector, dev_priv->gmbus[intel_hdmi->ddc_bus]);
+	edid = drm_get_edid(connector,
+			    intel_gmbus_get_adapter(dev_priv,
+						    intel_hdmi->ddc_bus));
 	if (edid) {
 		if (edid->input & DRM_EDID_INPUT_DIGITAL)
 			has_audio = drm_detect_monitor_audio(edid);
@@ -458,6 +590,14 @@ static void intel_hdmi_destroy(struct drm_connector *connector)
 	free(connector, DRM_MEM_KMS);
 }
 
+static const struct drm_encoder_helper_funcs intel_hdmi_helper_funcs_hsw = {
+	.dpms = intel_ddi_dpms,
+	.mode_fixup = intel_hdmi_mode_fixup,
+	.prepare = intel_encoder_prepare,
+	.mode_set = intel_ddi_mode_set,
+	.commit = intel_encoder_commit,
+};
+
 static const struct drm_encoder_helper_funcs intel_hdmi_helper_funcs = {
 	.dpms = intel_hdmi_dpms,
 	.mode_fixup = intel_hdmi_mode_fixup,
@@ -542,21 +682,59 @@ void intel_hdmi_init(struct drm_device *dev, int sdvox_reg)
 		intel_encoder->clone_mask = (1 << INTEL_HDMIF_CLONE_BIT);
 		intel_hdmi->ddc_bus = GMBUS_PORT_DPD;
 		dev_priv->hotplug_supported_mask |= HDMID_HOTPLUG_INT_STATUS;
+	} else if (sdvox_reg == DDI_BUF_CTL(PORT_B)) {
+		DRM_DEBUG_DRIVER("LPT: detected output on DDI B\n");
+		intel_encoder->clone_mask = (1 << INTEL_HDMIB_CLONE_BIT);
+		intel_hdmi->ddc_bus = GMBUS_PORT_DPB;
+		intel_hdmi->ddi_port = PORT_B;
+		dev_priv->hotplug_supported_mask |= HDMIB_HOTPLUG_INT_STATUS;
+	} else if (sdvox_reg == DDI_BUF_CTL(PORT_C)) {
+		DRM_DEBUG_DRIVER("LPT: detected output on DDI C\n");
+		intel_encoder->clone_mask = (1 << INTEL_HDMIC_CLONE_BIT);
+		intel_hdmi->ddc_bus = GMBUS_PORT_DPC;
+		intel_hdmi->ddi_port = PORT_C;
+		dev_priv->hotplug_supported_mask |= HDMIC_HOTPLUG_INT_STATUS;
+	} else if (sdvox_reg == DDI_BUF_CTL(PORT_D)) {
+		DRM_DEBUG_DRIVER("LPT: detected output on DDI D\n");
+		intel_encoder->clone_mask = (1 << INTEL_HDMID_CLONE_BIT);
+		intel_hdmi->ddc_bus = GMBUS_PORT_DPD;
+		intel_hdmi->ddi_port = PORT_D;
+		dev_priv->hotplug_supported_mask |= HDMID_HOTPLUG_INT_STATUS;
+	} else {
+		/* If we got an unknown sdvox_reg, things are pretty much broken
+		 * in a way that we should let the kernel know about it */
+		DRM_DEBUG_KMS("unknown sdvox_reg %d\n", sdvox_reg);
 	}
 
-
 	intel_hdmi->sdvox_reg = sdvox_reg;
-
 	if (!HAS_PCH_SPLIT(dev)) {
-		intel_hdmi->write_infoframe = i9xx_write_infoframe;
+		intel_hdmi->write_infoframe = g4x_write_infoframe;
 		I915_WRITE(VIDEO_DIP_CTL, 0);
+	} else if (IS_VALLEYVIEW(dev)) {
+		intel_hdmi->write_infoframe = vlv_write_infoframe;
+		for_each_pipe(i)
+			I915_WRITE(VLV_TVIDEO_DIP_CTL(i), 0);
+	} else if (IS_HASWELL(dev)) {
+		/* FIXME: Haswell has a new set of DIP frame registers, but we are
+		 * just doing the minimal required for HDMI to work at this stage.
+		 */
+		intel_hdmi->write_infoframe = hsw_write_infoframe;
+		for_each_pipe(i)
+			I915_WRITE(HSW_TVIDEO_DIP_CTL(i), 0);
+	} else if (HAS_PCH_IBX(dev)) {
+		intel_hdmi->write_infoframe = ibx_write_infoframe;
+		for_each_pipe(i)
+			I915_WRITE(TVIDEO_DIP_CTL(i), 0);
 	} else {
-		intel_hdmi->write_infoframe = ironlake_write_infoframe;
+		intel_hdmi->write_infoframe = cpt_write_infoframe;
 		for_each_pipe(i)
 			I915_WRITE(TVIDEO_DIP_CTL(i), 0);
 	}
 
-	drm_encoder_helper_add(&intel_encoder->base, &intel_hdmi_helper_funcs);
+	if (IS_HASWELL(dev))
+		drm_encoder_helper_add(&intel_encoder->base, &intel_hdmi_helper_funcs_hsw);
+	else
+		drm_encoder_helper_add(&intel_encoder->base, &intel_hdmi_helper_funcs);
 
 	intel_hdmi_add_properties(intel_hdmi, connector);
 
diff --git a/sys/dev/drm2/i915/intel_iic.c b/sys/dev/drm2/i915/intel_iic.c
index 412349c8d9d..fdd59e5d1ed 100644
--- a/sys/dev/drm2/i915/intel_iic.c
+++ b/sys/dev/drm2/i915/intel_iic.c
@@ -67,9 +67,22 @@ __FBSDID("$FreeBSD$");
 #include "iicbus_if.h"
 #include "iicbb_if.h"
 
-static int intel_iic_quirk_xfer(device_t idev, struct iic_msg *msgs, int nmsgs);
 static void intel_teardown_gmbus_m(struct drm_device *dev, int m);
 
+struct gmbus_port {
+	const char *name;
+	int reg;
+};
+
+static const struct gmbus_port gmbus_ports[] = {
+	{ "ssc", GPIOB },
+	{ "vga", GPIOA },
+	{ "panel", GPIOC },
+	{ "dpc", GPIOD },
+	{ "dpb", GPIOE },
+	{ "dpd", GPIOF },
+};
+
 /* Intel GPIO access functions */
 
 #define I2C_RISEFALL_TIME 10
@@ -128,10 +141,7 @@ intel_iic_reset(struct drm_device *dev)
 	struct drm_i915_private *dev_priv;
 
 	dev_priv = dev->dev_private;
-	if (HAS_PCH_SPLIT(dev))
-		I915_WRITE(PCH_GMBUS0, 0);
-	else
-		I915_WRITE(GMBUS0, 0);
+	I915_WRITE(dev_priv->gpio_mmio_base + GMBUS0, 0);
 }
 
 static int
@@ -224,129 +234,232 @@ intel_iicbb_getscl(device_t idev)
 	return ((I915_READ_NOTRACE(sc->reg) & GPIO_CLOCK_VAL_IN) != 0);
 }
 
+static int
+gmbus_xfer_read(struct drm_i915_private *dev_priv, struct iic_msg *msg,
+    u32 gmbus1_index)
+{
+	int reg_offset = dev_priv->gpio_mmio_base;
+	u16 len = msg->len;
+	u8 *buf = msg->buf;
+
+	I915_WRITE(GMBUS1 + reg_offset,
+		   gmbus1_index |
+		   GMBUS_CYCLE_WAIT |
+		   (len << GMBUS_BYTE_COUNT_SHIFT) |
+		   (msg->slave << (GMBUS_SLAVE_ADDR_SHIFT - 1)) |
+		   GMBUS_SLAVE_READ | GMBUS_SW_RDY);
+	while (len) {
+		int ret;
+		u32 val, loop = 0;
+		u32 gmbus2;
+
+		ret = _intel_wait_for(sc->drm_dev,
+		    ((gmbus2 = I915_READ(GMBUS2 + reg_offset)) &
+		    (GMBUS_SATOER | GMBUS_HW_RDY)),
+		    50, 1, "915gbr");
+		if (ret)
+			return (ETIMEDOUT);
+		if (gmbus2 & GMBUS_SATOER)
+			return (ENXIO);
+
+		val = I915_READ(GMBUS3 + reg_offset);
+		do {
+			*buf++ = val & 0xff;
+			val >>= 8;
+		} while (--len != 0 && ++loop < 4);
+	}
+
+	return 0;
+}
+
+static int
+gmbus_xfer_write(struct drm_i915_private *dev_priv, struct iic_msg *msg)
+{
+	int reg_offset = dev_priv->gpio_mmio_base;
+	u16 len = msg->len;
+	u8 *buf = msg->buf;
+	u32 val, loop;
+
+	val = loop = 0;
+	while (len && loop < 4) {
+		val |= *buf++ << (8 * loop++);
+		len -= 1;
+	}
+
+	I915_WRITE(GMBUS3 + reg_offset, val);
+	I915_WRITE(GMBUS1 + reg_offset,
+		   GMBUS_CYCLE_WAIT |
+		   (msg->len << GMBUS_BYTE_COUNT_SHIFT) |
+		   (msg->slave << (GMBUS_SLAVE_ADDR_SHIFT - 1)) |
+		   GMBUS_SLAVE_WRITE | GMBUS_SW_RDY);
+	while (len) {
+		int ret;
+		u32 gmbus2;
+
+		val = loop = 0;
+		do {
+			val |= *buf++ << (8 * loop);
+		} while (--len != 0 && ++loop < 4);
+
+		I915_WRITE(GMBUS3 + reg_offset, val);
+
+		ret = _intel_wait_for(sc->drm_dev,
+		    ((gmbus2 = I915_READ(GMBUS2 + reg_offset)) &
+		    (GMBUS_SATOER | GMBUS_HW_RDY)),
+		    50, 1, "915gbw");
+		if (ret)
+			return (ETIMEDOUT);
+		if (gmbus2 & GMBUS_SATOER)
+			return (ENXIO);
+	}
+	return 0;
+}
+
+/*
+ * The gmbus controller can combine a 1 or 2 byte write with a read that
+ * immediately follows it by using an "INDEX" cycle.
+ */
+static bool
+gmbus_is_index_read(struct iic_msg *msgs, int i, int num)
+{
+	return (i + 1 < num &&
+		!(msgs[i].flags & IIC_M_RD) && msgs[i].len <= 2 &&
+		(msgs[i + 1].flags & IIC_M_RD));
+}
+
+static int
+gmbus_xfer_index_read(struct drm_i915_private *dev_priv, struct iic_msg *msgs)
+{
+	int reg_offset = dev_priv->gpio_mmio_base;
+	u32 gmbus1_index = 0;
+	u32 gmbus5 = 0;
+	int ret;
+
+	if (msgs[0].len == 2)
+		gmbus5 = GMBUS_2BYTE_INDEX_EN |
+			 msgs[0].buf[1] | (msgs[0].buf[0] << 8);
+	if (msgs[0].len == 1)
+		gmbus1_index = GMBUS_CYCLE_INDEX |
+			       (msgs[0].buf[0] << GMBUS_SLAVE_INDEX_SHIFT);
+
+	/* GMBUS5 holds 16-bit index */
+	if (gmbus5)
+		I915_WRITE(GMBUS5 + reg_offset, gmbus5);
+
+	ret = gmbus_xfer_read(dev_priv, &msgs[1], gmbus1_index);
+
+	/* Clear GMBUS5 after each index transfer */
+	if (gmbus5)
+		I915_WRITE(GMBUS5 + reg_offset, 0);
+
+	return ret;
+}
+
 static int
 intel_gmbus_transfer(device_t idev, struct iic_msg *msgs, uint32_t nmsgs)
 {
 	struct intel_iic_softc *sc;
 	struct drm_i915_private *dev_priv;
-	u8 *buf;
-	int error, i, reg_offset, unit;
-	u32 val, loop;
-	u16 len;
+	int error, i, ret, reg_offset, unit;
 
+	error = 0;
 	sc = device_get_softc(idev);
 	dev_priv = sc->drm_dev->dev_private;
 	unit = device_get_unit(idev);
 
 	sx_xlock(&dev_priv->gmbus_sx);
 	if (sc->force_bit_dev) {
-		error = intel_iic_quirk_xfer(dev_priv->bbbus[unit], msgs, nmsgs);
+		error = IICBUS_TRANSFER(dev_priv->bbbus[unit], msgs, nmsgs);
 		goto out;
 	}
 
-	reg_offset = HAS_PCH_SPLIT(dev_priv->dev) ? PCH_GMBUS0 - GMBUS0 : 0;
+	reg_offset = dev_priv->gpio_mmio_base;
 
 	I915_WRITE(GMBUS0 + reg_offset, sc->reg0);
 
 	for (i = 0; i < nmsgs; i++) {
-		len = msgs[i].len;
-		buf = msgs[i].buf;
+		u32 gmbus2;
 
-		if ((msgs[i].flags & IIC_M_RD) != 0) {
-			I915_WRITE(GMBUS1 + reg_offset, GMBUS_CYCLE_WAIT |
-			    (i + 1 == nmsgs ? GMBUS_CYCLE_STOP : 0) |
-			    (len << GMBUS_BYTE_COUNT_SHIFT) |
-			    (msgs[i].slave << (GMBUS_SLAVE_ADDR_SHIFT - 1)) |
-			    GMBUS_SLAVE_READ | GMBUS_SW_RDY);
-			POSTING_READ(GMBUS2 + reg_offset);
-			do {
-				loop = 0;
-
-				if (_intel_wait_for(sc->drm_dev,
-				    (I915_READ(GMBUS2 + reg_offset) &
-					(GMBUS_SATOER | GMBUS_HW_RDY)) != 0,
-				    50, 1, "915gbr"))
-					goto timeout;
-				if ((I915_READ(GMBUS2 + reg_offset) &
-				    GMBUS_SATOER) != 0)
-					goto clear_err;
-
-				val = I915_READ(GMBUS3 + reg_offset);
-				do {
-					*buf++ = val & 0xff;
-					val >>= 8;
-				} while (--len != 0 && ++loop < 4);
-			} while (len != 0);
+		if (gmbus_is_index_read(msgs, i, nmsgs)) {
+			error = gmbus_xfer_index_read(dev_priv, &msgs[i]);
+			i += 1;  /* set i to the index of the read xfer */
+		} else if (msgs[i].flags & IIC_M_RD) {
+			error = gmbus_xfer_read(dev_priv, &msgs[i], 0);
 		} else {
-			val = loop = 0;
-			do {
-				val |= *buf++ << (8 * loop);
-			} while (--len != 0 && ++loop < 4);
-
-			I915_WRITE(GMBUS3 + reg_offset, val);
-			I915_WRITE(GMBUS1 + reg_offset, GMBUS_CYCLE_WAIT |
-			    (i + 1 == nmsgs ? GMBUS_CYCLE_STOP : 0) |
-			    (msgs[i].len << GMBUS_BYTE_COUNT_SHIFT) |
-			    (msgs[i].slave << (GMBUS_SLAVE_ADDR_SHIFT - 1)) |
-			    GMBUS_SLAVE_WRITE | GMBUS_SW_RDY);
-			POSTING_READ(GMBUS2+reg_offset);
-
-			while (len != 0) {
-				if (_intel_wait_for(sc->drm_dev,
-				    (I915_READ(GMBUS2 + reg_offset) &
-					(GMBUS_SATOER | GMBUS_HW_RDY)) != 0,
-				    50, 1, "915gbw"))
-					goto timeout;
-				if (I915_READ(GMBUS2 + reg_offset) & GMBUS_SATOER)
-					goto clear_err;
-
-				val = loop = 0;
-				do {
-					val |= *buf++ << (8 * loop);
-				} while (--len != 0 && ++loop < 4);
-
-				I915_WRITE(GMBUS3 + reg_offset, val);
-				POSTING_READ(GMBUS2 + reg_offset);
-			}
+			error = gmbus_xfer_write(dev_priv, &msgs[i]);
 		}
 
-		if (i + 1 < nmsgs && _intel_wait_for(sc->drm_dev,
-		    (I915_READ(GMBUS2 + reg_offset) & (GMBUS_SATOER |
-			GMBUS_HW_WAIT_PHASE)) != 0,
-		    50, 1, "915gbh"))
+		if (error == ETIMEDOUT)
 			goto timeout;
-		if ((I915_READ(GMBUS2 + reg_offset) & GMBUS_SATOER) != 0)
+		if (error == ENXIO)
+			goto clear_err;
+
+		ret = _intel_wait_for(sc->drm_dev,
+		    ((gmbus2 = I915_READ(GMBUS2 + reg_offset)) &
+		    (GMBUS_SATOER | GMBUS_HW_WAIT_PHASE)),
+		    50, 1, "915gbh");
+		if (ret)
+			goto timeout;
+		if (gmbus2 & GMBUS_SATOER)
 			goto clear_err;
 	}
 
-	error = 0;
-done:
+	/* Generate a STOP condition on the bus. Note that gmbus can't generata
+	 * a STOP on the very first cycle. To simplify the code we
+	 * unconditionally generate the STOP condition with an additional gmbus
+	 * cycle. */
+	I915_WRITE(GMBUS1 + reg_offset, GMBUS_CYCLE_STOP | GMBUS_SW_RDY);
+
 	/* Mark the GMBUS interface as disabled after waiting for idle.
 	 * We will re-enable it at the start of the next xfer,
 	 * till then let it sleep.
  	 */
 	if (_intel_wait_for(dev,
 	    (I915_READ(GMBUS2 + reg_offset) & GMBUS_ACTIVE) == 0,
-	    10, 1, "915gbu"))
-		DRM_INFO("GMBUS timed out waiting for idle\n");
+	    10, 1, "915gbu")) {
+		DRM_DEBUG_KMS("GMBUS [%s] timed out waiting for idle\n",
+		    sc->name);
+		error = ETIMEDOUT;
+	}
 	I915_WRITE(GMBUS0 + reg_offset, 0);
-out:
-	sx_xunlock(&dev_priv->gmbus_sx);
-	return (error);
+	goto out;
 
 clear_err:
+	/*
+	 * Wait for bus to IDLE before clearing NAK.
+	 * If we clear the NAK while bus is still active, then it will stay
+	 * active and the next transaction may fail.
+	 */
+	if (_intel_wait_for(dev,
+	    (I915_READ(GMBUS2 + reg_offset) & GMBUS_ACTIVE) == 0,
+	    10, 1, "915gbu"))
+		DRM_DEBUG_KMS("GMBUS [%s] timed out after NAK\n", sc->name);
+
 	/* Toggle the Software Clear Interrupt bit. This has the effect
 	 * of resetting the GMBUS controller and so clearing the
 	 * BUS_ERROR raised by the slave's NAK.
 	 */
 	I915_WRITE(GMBUS1 + reg_offset, GMBUS_SW_CLR_INT);
 	I915_WRITE(GMBUS1 + reg_offset, 0);
-	error = EIO;
-	goto done;
+	I915_WRITE(GMBUS0 + reg_offset, 0);
+
+	DRM_DEBUG_KMS("GMBUS [%s] NAK for addr: %04x %c(%d)\n",
+			 sc->name, msgs[i].slave,
+			 (msgs[i].flags & IIC_M_RD) ? 'r' : 'w', msgs[i].len);
+
+	/*
+	 * If no ACK is received during the address phase of a transaction,
+	 * the adapter must report -ENXIO.
+	 * It is not clear what to return if no ACK is received at other times.
+	 * So, we always return -ENXIO in all NAK cases, to ensure we send
+	 * it at least during the one case that is specified.
+	 */
+	error = ENXIO;
+	goto out;
 
 timeout:
-	DRM_INFO("GMBUS timed out, falling back to bit banging on pin %d [%s]\n",
-	    sc->reg0 & 0xff, sc->name);
+	DRM_INFO("GMBUS [%s] timed out, falling back to bit banging on pin %d\n",
+	    sc->name, sc->reg0 & 0xff);
 	I915_WRITE(GMBUS0 + reg_offset, 0);
 
 	/*
@@ -354,9 +467,23 @@ timeout:
 	 * Try GPIO bitbanging instead.
 	 */
 	sc->force_bit_dev = true;
-
-	error = intel_iic_quirk_xfer(dev_priv->bbbus[unit], msgs, nmsgs);
+	error = IICBUS_TRANSFER(idev, msgs, nmsgs);
 	goto out;
+
+out:
+	sx_xunlock(&dev_priv->gmbus_sx);
+	return (error);
+}
+
+device_t 
+intel_gmbus_get_adapter(struct drm_i915_private *dev_priv,
+    unsigned port)
+{
+
+	if (!intel_gmbus_is_port_valid(port))
+		DRM_ERROR("GMBUS get adapter %d: invalid port\n", port);
+	return (intel_gmbus_is_port_valid(port) ? dev_priv->gmbus[port - 1] :
+	    NULL);
 }
 
 void
@@ -379,46 +506,35 @@ intel_gmbus_force_bit(device_t idev, bool force_bit)
 }
 
 static int
-intel_iic_quirk_xfer(device_t idev, struct iic_msg *msgs, int nmsgs)
+intel_iicbb_pre_xfer(device_t idev)
 {
-	device_t bridge_dev;
 	struct intel_iic_softc *sc;
 	struct drm_i915_private *dev_priv;
-	int ret;
-	int i;
 
-	bridge_dev = device_get_parent(device_get_parent(idev));
-	sc = device_get_softc(bridge_dev);
+	sc = device_get_softc(idev);
 	dev_priv = sc->drm_dev->dev_private;
 
 	intel_iic_reset(sc->drm_dev);
 	intel_iic_quirk_set(dev_priv, true);
-	IICBB_SETSDA(bridge_dev, 1);
-	IICBB_SETSCL(bridge_dev, 1);
+	IICBB_SETSDA(idev, 1);
+	IICBB_SETSCL(idev, 1);
 	DELAY(I2C_RISEFALL_TIME);
-
-	for (i = 0; i < nmsgs - 1; i++) {
-		/* force use of repeated start instead of default stop+start */
-		msgs[i].flags |= IIC_M_NOSTOP;
-	}
-	ret = iicbus_transfer(idev, msgs, nmsgs);
-	IICBB_SETSDA(bridge_dev, 1);
-	IICBB_SETSCL(bridge_dev, 1);
-	intel_iic_quirk_set(dev_priv, false);
-
-	return (ret);
+	return (0);
 }
 
-static const char *gpio_names[GMBUS_NUM_PORTS] = {
-	"disabled",
-	"ssc",
-	"vga",
-	"panel",
-	"dpc",
-	"dpb",
-	"reserved",
-	"dpd",
-};
+static void
+intel_iicbb_post_xfer(device_t idev)
+{
+	struct intel_iic_softc *sc;
+	struct drm_i915_private *dev_priv;
+
+	sc = device_get_softc(idev);
+	dev_priv = sc->drm_dev->dev_private;
+
+	IICBB_SETSDA(idev, 1);
+	IICBB_SETSCL(idev, 1);
+	intel_iic_quirk_set(dev_priv, false);
+}
 
 static int
 intel_gmbus_probe(device_t dev)
@@ -432,23 +548,28 @@ intel_gmbus_attach(device_t idev)
 {
 	struct drm_i915_private *dev_priv;
 	struct intel_iic_softc *sc;
-	int pin;
+	int pin, port;
 
 	sc = device_get_softc(idev);
 	sc->drm_dev = device_get_softc(device_get_parent(idev));
 	dev_priv = sc->drm_dev->dev_private;
 	pin = device_get_unit(idev);
+	port = pin + 1;
 
-	snprintf(sc->name, sizeof(sc->name), "gmbus bus %s", gpio_names[pin]);
+	snprintf(sc->name, sizeof(sc->name), "gmbus %s", gmbus_ports[pin].name);
 	device_set_desc(idev, sc->name);
 
 	/* By default use a conservative clock rate */
-	sc->reg0 = pin | GMBUS_RATE_100KHZ;
+	sc->reg0 = port | GMBUS_RATE_100KHZ;
 
-	/* XXX force bit banging until GMBUS is fully debugged */
+	/* gmbus seems to be broken on i830 */
+	if (IS_I830(sc->drm_dev))
+		sc->force_bit_dev = true;
+#if 0
 	if (IS_GEN2(sc->drm_dev)) {
 		sc->force_bit_dev = true;
 	}
+#endif
 
 	/* add bus interface device */
 	sc->iic_dev = device_add_child(idev, "iicbus", -1);
@@ -490,17 +611,6 @@ intel_iicbb_probe(device_t dev)
 static int
 intel_iicbb_attach(device_t idev)
 {
-	static const int map_pin_to_reg[] = {
-		0,
-		GPIOB,
-		GPIOA,
-		GPIOC,
-		GPIOD,
-		GPIOE,
-		0,
-		GPIOF
-	};
-
 	struct intel_iic_softc *sc;
 	struct drm_i915_private *dev_priv;
 	int pin;
@@ -510,13 +620,12 @@ intel_iicbb_attach(device_t idev)
 	dev_priv = sc->drm_dev->dev_private;
 	pin = device_get_unit(idev);
 
-	snprintf(sc->name, sizeof(sc->name), "i915 iicbb %s", gpio_names[pin]);
+	snprintf(sc->name, sizeof(sc->name), "i915 iicbb %s",
+	    gmbus_ports[pin].name);
 	device_set_desc(idev, sc->name);
 
 	sc->reg0 = pin | GMBUS_RATE_100KHZ;
-	sc->reg = map_pin_to_reg[pin];
-	if (HAS_PCH_SPLIT(dev_priv->dev))
-		sc->reg += PCH_GPIOA - GPIOA;
+	sc->reg = dev_priv->gpio_mmio_base + gmbus_ports[pin].reg;
 
 	/* add generic bit-banging code */
 	sc->iic_dev = device_add_child(idev, "iicbb", -1);
@@ -524,6 +633,7 @@ intel_iicbb_attach(device_t idev)
 		return (ENXIO);
 	device_quiet(sc->iic_dev);
 	bus_generic_attach(idev);
+	iicbus_set_nostop(idev, true);
 
 	return (0);
 }
@@ -574,6 +684,8 @@ static device_method_t intel_iicbb_methods[] =	{
 	DEVMETHOD(iicbb_setscl,		intel_iicbb_setscl),
 	DEVMETHOD(iicbb_getsda,		intel_iicbb_getsda),
 	DEVMETHOD(iicbb_getscl,		intel_iicbb_getscl),
+	DEVMETHOD(iicbb_pre_xfer,	intel_iicbb_pre_xfer),
+	DEVMETHOD(iicbb_post_xfer,	intel_iicbb_post_xfer),
 	DEVMETHOD_END
 };
 static driver_t intel_iicbb_driver = {
@@ -595,14 +707,10 @@ intel_setup_gmbus(struct drm_device *dev)
 
 	dev_priv = dev->dev_private;
 	sx_init(&dev_priv->gmbus_sx, "gmbus");
-	dev_priv->gmbus_bridge = malloc(sizeof(device_t) * GMBUS_NUM_PORTS,
-	    DRM_MEM_DRIVER, M_WAITOK | M_ZERO);
-	dev_priv->bbbus_bridge = malloc(sizeof(device_t) * GMBUS_NUM_PORTS,
-	    DRM_MEM_DRIVER, M_WAITOK | M_ZERO);
-	dev_priv->gmbus = malloc(sizeof(device_t) * GMBUS_NUM_PORTS,
-	    DRM_MEM_DRIVER, M_WAITOK | M_ZERO);
-	dev_priv->bbbus = malloc(sizeof(device_t) * GMBUS_NUM_PORTS,
-	    DRM_MEM_DRIVER, M_WAITOK | M_ZERO);
+	if (HAS_PCH_SPLIT(dev))
+		dev_priv->gpio_mmio_base = PCH_GPIOA - GPIOA;
+	else
+		dev_priv->gpio_mmio_base = 0;
 
 	/*
 	 * The Giant there is recursed, most likely.  Normally, the
@@ -610,7 +718,7 @@ intel_setup_gmbus(struct drm_device *dev)
 	 * driver.
 	 */
 	mtx_lock(&Giant);
-	for (i = 0; i < GMBUS_NUM_PORTS; i++) {
+	for (i = 0; i <= GMBUS_NUM_PORTS; i++) {
 		/*
 		 * Initialized bbbus_bridge before gmbus_bridge, since
 		 * gmbus may decide to force quirk transfer in the
@@ -689,14 +797,6 @@ intel_teardown_gmbus_m(struct drm_device *dev, int m)
 
 	dev_priv = dev->dev_private;
 
-	free(dev_priv->gmbus, DRM_MEM_DRIVER);
-	dev_priv->gmbus = NULL;
-	free(dev_priv->bbbus, DRM_MEM_DRIVER);
-	dev_priv->bbbus = NULL;
-	free(dev_priv->gmbus_bridge, DRM_MEM_DRIVER);
-	dev_priv->gmbus_bridge = NULL;
-	free(dev_priv->bbbus_bridge, DRM_MEM_DRIVER);
-	dev_priv->bbbus_bridge = NULL;
 	sx_destroy(&dev_priv->gmbus_sx);
 }
 
diff --git a/sys/dev/drm2/i915/intel_lvds.c b/sys/dev/drm2/i915/intel_lvds.c
index cbf3ea4f7c2..5f749cedcd9 100644
--- a/sys/dev/drm2/i915/intel_lvds.c
+++ b/sys/dev/drm2/i915/intel_lvds.c
@@ -230,7 +230,7 @@ static inline u32 panel_fitter_scaling(u32 source, u32 target)
 }
 
 static bool intel_lvds_mode_fixup(struct drm_encoder *encoder,
-				  const struct drm_display_mode *mode,
+				  struct drm_display_mode *mode,
 				  struct drm_display_mode *adjusted_mode)
 {
 	struct drm_device *dev = encoder->dev;
@@ -482,7 +482,7 @@ static int intel_lvds_get_modes(struct drm_connector *connector)
 
 static int intel_no_modeset_on_lid_dmi_callback(const struct dmi_system_id *id)
 {
-	DRM_DEBUG_KMS("Skipping forced modeset for %s\n", id->ident);
+	DRM_INFO("Skipping forced modeset for %s\n", id->ident);
 	return 1;
 }
 
@@ -638,7 +638,7 @@ static const struct drm_encoder_funcs intel_lvds_enc_funcs = {
 
 static int intel_no_lvds_dmi_callback(const struct dmi_system_id *id)
 {
-	DRM_DEBUG_KMS("Skipping LVDS initialization for %s\n", id->ident);
+	DRM_INFO("Skipping LVDS initialization for %s\n", id->ident);
 	return 1;
 }
 
@@ -861,8 +861,8 @@ static bool lvds_is_present_in_vbt(struct drm_device *dev,
 		    child->device_type != DEVICE_TYPE_LFP)
 			continue;
 
-		if (child->i2c_pin)
-		    *i2c_pin = child->i2c_pin;
+		if (intel_gmbus_is_port_valid(child->i2c_pin))
+			*i2c_pin = child->i2c_pin;
 
 		/* However, we cannot trust the BIOS writers to populate
 		 * the VBT correctly.  Since LVDS requires additional
@@ -996,7 +996,8 @@ bool intel_lvds_init(struct drm_device *dev)
 	 * Attempt to get the fixed panel mode from DDC.  Assume that the
 	 * preferred mode is the right one.
 	 */
-	intel_lvds->edid = drm_get_edid(connector, dev_priv->gmbus[pin]);
+	intel_lvds->edid = drm_get_edid(connector,
+					intel_gmbus_get_adapter(dev_priv, pin));
 	if (intel_lvds->edid) {
 		if (drm_add_edid_modes(connector,
 				       intel_lvds->edid)) {
diff --git a/sys/dev/drm2/i915/intel_modes.c b/sys/dev/drm2/i915/intel_modes.c
index 22969d38100..6b83bae76ea 100644
--- a/sys/dev/drm2/i915/intel_modes.c
+++ b/sys/dev/drm2/i915/intel_modes.c
@@ -58,8 +58,8 @@ bool intel_ddc_probe(struct intel_encoder *intel_encoder, int ddc_bus)
 		}
 	};
 
-	return (iicbus_transfer(dev_priv->gmbus[ddc_bus], msgs, 2)
-	    == 0/* XXXKIB  2*/);
+	return (iicbus_transfer(intel_gmbus_get_adapter(dev_priv, ddc_bus),
+	    msgs, 2) == 0/* XXXKIB  2*/);
 }
 
 /**
diff --git a/sys/dev/drm2/i915/intel_overlay.c b/sys/dev/drm2/i915/intel_overlay.c
index 34f2c39224f..715da2fe0f1 100644
--- a/sys/dev/drm2/i915/intel_overlay.c
+++ b/sys/dev/drm2/i915/intel_overlay.c
@@ -219,20 +219,21 @@ static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
 {
 	struct drm_device *dev = overlay->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
 	int ret;
 
 	KASSERT(!overlay->last_flip_req, ("Overlay already has flip req"));
-	ret = i915_add_request(LP_RING(dev_priv), NULL, request);
+	ret = i915_add_request(ring, NULL, request);
 	if (ret) {
 		free(request, DRM_I915_GEM);
 		return ret;
 	}
 	overlay->last_flip_req = request->seqno;
 	overlay->flip_tail = tail;
-	ret = i915_wait_request(LP_RING(dev_priv), overlay->last_flip_req,
-				true);
+	ret = i915_wait_request(ring, overlay->last_flip_req);
 	if (ret)
 		return ret;
+	i915_gem_retire_requests(dev);
 
 	overlay->last_flip_req = 0;
 	return 0;
@@ -266,7 +267,7 @@ i830_activate_pipe_a(struct drm_device *dev)
 	DRM_DEBUG_DRIVER("Enabling pipe A in order to enable overlay\n");
 
 	mode = drm_mode_duplicate(dev, &vesa_640x480);
-	drm_mode_set_crtcinfo(mode, 0);
+
 	if (!drm_crtc_helper_set_mode(&crtc->base, mode,
 				       crtc->base.x, crtc->base.y,
 				       crtc->base.fb))
@@ -291,6 +292,7 @@ static int intel_overlay_on(struct intel_overlay *overlay)
 {
 	struct drm_device *dev = overlay->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
 	struct drm_i915_gem_request *request;
 	int pipe_a_quirk = 0;
 	int ret;
@@ -306,17 +308,17 @@ static int intel_overlay_on(struct intel_overlay *overlay)
 
 	request = malloc(sizeof(*request), DRM_I915_GEM, M_WAITOK | M_ZERO);
 
-	ret = BEGIN_LP_RING(4);
+	ret = intel_ring_begin(ring, 4);
 	if (ret) {
 		free(request, DRM_I915_GEM);
 		goto out;
 	}
 
-	OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_ON);
-	OUT_RING(overlay->flip_addr | OFC_UPDATE);
-	OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
-	OUT_RING(MI_NOOP);
-	ADVANCE_LP_RING();
+	intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_ON);
+	intel_ring_emit(ring, overlay->flip_addr | OFC_UPDATE);
+	intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
 
 	ret = intel_overlay_do_wait_request(overlay, request, NULL);
 out:
@@ -332,6 +334,7 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
 {
 	struct drm_device *dev = overlay->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
 	struct drm_i915_gem_request *request;
 	u32 flip_addr = overlay->flip_addr;
 	u32 tmp;
@@ -349,16 +352,16 @@ static int intel_overlay_continue(struct intel_overlay *overlay,
 	if (tmp & (1 << 17))
 		DRM_DEBUG("overlay underrun, DOVSTA: %x\n", tmp);
 
-	ret = BEGIN_LP_RING(2);
+	ret = intel_ring_begin(ring, 2);
 	if (ret) {
 		free(request, DRM_I915_GEM);
 		return ret;
 	}
-	OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
-	OUT_RING(flip_addr);
-	ADVANCE_LP_RING();
+	intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
+	intel_ring_emit(ring, flip_addr);
+	intel_ring_advance(ring);
 
-	ret = i915_add_request(LP_RING(dev_priv), NULL, request);
+	ret = i915_add_request(ring, NULL, request);
 	if (ret) {
 		free(request, DRM_I915_GEM);
 		return ret;
@@ -399,6 +402,7 @@ static int intel_overlay_off(struct intel_overlay *overlay)
 {
 	struct drm_device *dev = overlay->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
 	u32 flip_addr = overlay->flip_addr;
 	struct drm_i915_gem_request *request;
 	int ret;
@@ -413,20 +417,20 @@ static int intel_overlay_off(struct intel_overlay *overlay)
 	 * of the hw. Do it in both cases */
 	flip_addr |= OFC_UPDATE;
 
-	ret = BEGIN_LP_RING(6);
+	ret = intel_ring_begin(ring, 6);
 	if (ret) {
 		free(request, DRM_I915_GEM);
 		return ret;
 	}
 	/* wait for overlay to go idle */
-	OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
-	OUT_RING(flip_addr);
-	OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
+	intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_CONTINUE);
+	intel_ring_emit(ring, flip_addr);
+	intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
 	/* turn overlay off */
-	OUT_RING(MI_OVERLAY_FLIP | MI_OVERLAY_OFF);
-	OUT_RING(flip_addr);
-	OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
-	ADVANCE_LP_RING();
+	intel_ring_emit(ring, MI_OVERLAY_FLIP | MI_OVERLAY_OFF);
+	intel_ring_emit(ring, flip_addr);
+	intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
+	intel_ring_advance(ring);
 
 	return intel_overlay_do_wait_request(overlay, request,
 					     intel_overlay_off_tail);
@@ -438,15 +442,16 @@ static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay)
 {
 	struct drm_device *dev = overlay->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
 	int ret;
 
 	if (overlay->last_flip_req == 0)
 		return 0;
 
-	ret = i915_wait_request(LP_RING(dev_priv), overlay->last_flip_req,
-				true);
+	ret = i915_wait_request(ring, overlay->last_flip_req);
 	if (ret)
 		return ret;
+	i915_gem_retire_requests(dev);
 
 	if (overlay->flip_tail)
 		overlay->flip_tail(overlay);
@@ -463,6 +468,7 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
 {
 	struct drm_device *dev = overlay->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
+	struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
 	int ret;
 
 	/* Only wait if there is actually an old frame to release to
@@ -477,15 +483,15 @@ static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
 		/* synchronous slowpath */
 		request = malloc(sizeof(*request), DRM_I915_GEM, M_WAITOK | M_ZERO);
 
-		ret = BEGIN_LP_RING(2);
+		ret = intel_ring_begin(ring, 2);
 		if (ret) {
 			free(request, DRM_I915_GEM);
 			return ret;
 		}
 
-		OUT_RING(MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
-		OUT_RING(MI_NOOP);
-		ADVANCE_LP_RING();
+		intel_ring_emit(ring, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
+		intel_ring_emit(ring, MI_NOOP);
+		intel_ring_advance(ring);
 
 		ret = intel_overlay_do_wait_request(overlay, request,
 						    intel_overlay_release_old_vid_tail);
@@ -764,6 +770,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 	int ret, tmp_width;
 	struct overlay_registers *regs;
 	bool scale_changed = false;
+	u32 swidth, swidthsw, sheight, ostride;
 
 	KASSERT(overlay != NULL, ("No overlay ?"));
 	DRM_LOCK_ASSERT(overlay->dev);
@@ -782,16 +789,18 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 		goto out_unpin;
 
 	if (!overlay->active) {
+		u32 oconfig;
 		regs = intel_overlay_map_regs(overlay);
 		if (!regs) {
 			ret = -ENOMEM;
 			goto out_unpin;
 		}
-		regs->OCONFIG = OCONF_CC_OUT_8BIT;
+		oconfig = OCONF_CC_OUT_8BIT;
 		if (IS_GEN4(overlay->dev))
-			regs->OCONFIG |= OCONF_CSC_MODE_BT709;
-		regs->OCONFIG |= overlay->crtc->pipe == 0 ?
+			oconfig |= OCONF_CSC_MODE_BT709;
+		oconfig |= overlay->crtc->pipe == 0 ?
 			OCONF_PIPE_A : OCONF_PIPE_B;
+		regs->OCONFIG = oconfig;
 		intel_overlay_unmap_regs(overlay, regs);
 
 		ret = intel_overlay_on(overlay);
@@ -813,29 +822,33 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
 	else
 		tmp_width = params->src_w;
 
-	regs->SWIDTH = params->src_w;
-	regs->SWIDTHSW = calc_swidthsw(overlay->dev,
-				       params->offset_Y, tmp_width);
-	regs->SHEIGHT = params->src_h;
+	swidth = params->src_w;
+	swidthsw = calc_swidthsw(overlay->dev, params->offset_Y, tmp_width);
+	sheight = params->src_h;
 	regs->OBUF_0Y = new_bo->gtt_offset + params->offset_Y;
-	regs->OSTRIDE = params->stride_Y;
+	ostride = params->stride_Y;
 
 	if (params->format & I915_OVERLAY_YUV_PLANAR) {
 		int uv_hscale = uv_hsubsampling(params->format);
 		int uv_vscale = uv_vsubsampling(params->format);
 		u32 tmp_U, tmp_V;
-		regs->SWIDTH |= (params->src_w/uv_hscale) << 16;
+		swidth |= (params->src_w/uv_hscale) << 16;
 		tmp_U = calc_swidthsw(overlay->dev, params->offset_U,
 				      params->src_w/uv_hscale);
 		tmp_V = calc_swidthsw(overlay->dev, params->offset_V,
 				      params->src_w/uv_hscale);
-		regs->SWIDTHSW |= max_u32(tmp_U, tmp_V) << 16;
-		regs->SHEIGHT |= (params->src_h/uv_vscale) << 16;
+		swidthsw |= max_u32(tmp_U, tmp_V) << 16;
+		sheight |= (params->src_h/uv_vscale) << 16;
 		regs->OBUF_0U = new_bo->gtt_offset + params->offset_U;
 		regs->OBUF_0V = new_bo->gtt_offset + params->offset_V;
-		regs->OSTRIDE |= params->stride_UV << 16;
+		ostride |= params->stride_UV << 16;
 	}
 
+	regs->SWIDTH = swidth;
+	regs->SWIDTHSW = swidthsw;
+	regs->SHEIGHT = sheight;
+	regs->OSTRIDE = ostride;
+
 	scale_changed = update_scaling_factors(overlay, regs, params);
 
 	update_colorkey(overlay, regs);
@@ -1108,11 +1121,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
 	struct put_image_params *params;
 	int ret;
 
-	if (!dev_priv) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
-	}
-
+	/* No need to check for DRIVER_MODESET - we don't set it up then. */
 	overlay = dev_priv->overlay;
 	if (!overlay) {
 		DRM_DEBUG("userspace bug: no overlay\n");
@@ -1307,11 +1316,7 @@ int intel_overlay_attrs(struct drm_device *dev, void *data,
 	struct overlay_registers *regs;
 	int ret;
 
-	if (!dev_priv) {
-		DRM_ERROR("called with no initialization\n");
-		return -EINVAL;
-	}
-
+	/* No need to check for DRIVER_MODESET - we don't set it up then. */
 	overlay = dev_priv->overlay;
 	if (!overlay) {
 		DRM_DEBUG("userspace bug: no overlay\n");
diff --git a/sys/dev/drm2/i915/intel_panel.c b/sys/dev/drm2/i915/intel_panel.c
index 504e2ac4de4..ef60f4945fe 100644
--- a/sys/dev/drm2/i915/intel_panel.c
+++ b/sys/dev/drm2/i915/intel_panel.c
@@ -197,6 +197,20 @@ u32 intel_panel_get_max_backlight(struct drm_device *dev)
 	return max;
 }
 
+static u32 intel_panel_compute_brightness(struct drm_device *dev, u32 val)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (i915_panel_invert_brightness < 0)
+		return val;
+
+	if (i915_panel_invert_brightness > 0 ||
+	    dev_priv->quirks & QUIRK_INVERT_BRIGHTNESS)
+		return intel_panel_get_max_backlight(dev) - val;
+
+	return val;
+}
+
 u32 intel_panel_get_backlight(struct drm_device *dev)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
@@ -217,7 +231,8 @@ u32 intel_panel_get_backlight(struct drm_device *dev)
 		}
 	}
 
-	DRM_DEBUG("get backlight PWM = %d\n", val);
+	val = intel_panel_compute_brightness(dev, val);
+	DRM_DEBUG_DRIVER("get backlight PWM = %d\n", val);
 	return val;
 }
 
@@ -233,7 +248,8 @@ static void intel_panel_actually_set_backlight(struct drm_device *dev, u32 level
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	u32 tmp;
 
-	DRM_DEBUG("set backlight PWM = %d\n", level);
+	DRM_DEBUG_DRIVER("set backlight PWM = %d\n", level);
+	level = intel_panel_compute_brightness(dev, level);
 
 	if (HAS_PCH_SPLIT(dev))
 		return intel_pch_panel_set_backlight(dev, level);
diff --git a/sys/dev/drm2/i915/intel_pm.c b/sys/dev/drm2/i915/intel_pm.c
new file mode 100644
index 00000000000..7a8adbdbd8c
--- /dev/null
+++ b/sys/dev/drm2/i915/intel_pm.c
@@ -0,0 +1,3788 @@
+/*
+ * Copyright © 2012 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Eugeni Dodonov <eugeni.dodonov@intel.com>
+ *
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <dev/drm2/drmP.h>
+#include <dev/drm2/drm.h>
+#include <dev/drm2/i915/i915_drm.h>
+#include <dev/drm2/i915/i915_drv.h>
+#include <dev/drm2/i915/intel_drv.h>
+#include <sys/kdb.h>
+
+static struct drm_i915_private *i915_mch_dev;
+/*
+ * Lock protecting IPS related data structures
+ *   - i915_mch_dev
+ *   - dev_priv->max_delay
+ *   - dev_priv->min_delay
+ *   - dev_priv->fmax
+ *   - dev_priv->gpu_busy
+ */
+static struct mtx mchdev_lock;
+MTX_SYSINIT(mchdev, &mchdev_lock, "mchdev", MTX_DEF);
+
+/* FBC, or Frame Buffer Compression, is a technique employed to compress the
+ * framebuffer contents in-memory, aiming at reducing the required bandwidth
+ * during in-memory transfers and, therefore, reduce the power packet.
+ *
+ * The benefits of FBC are mostly visible with solid backgrounds and
+ * variation-less patterns.
+ *
+ * FBC-related functionality can be enabled by the means of the
+ * i915.i915_enable_fbc parameter
+ */
+
+static void i8xx_disable_fbc(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 fbc_ctl;
+
+	/* Disable compression */
+	fbc_ctl = I915_READ(FBC_CONTROL);
+	if ((fbc_ctl & FBC_CTL_EN) == 0)
+		return;
+
+	fbc_ctl &= ~FBC_CTL_EN;
+	I915_WRITE(FBC_CONTROL, fbc_ctl);
+
+	/* Wait for compressing bit to clear */
+	if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
+		DRM_DEBUG_KMS("FBC idle timed out\n");
+		return;
+	}
+
+	DRM_DEBUG_KMS("disabled FBC\n");
+}
+
+static void i8xx_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_framebuffer *fb = crtc->fb;
+	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
+	struct drm_i915_gem_object *obj = intel_fb->obj;
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	int cfb_pitch;
+	int plane, i;
+	u32 fbc_ctl, fbc_ctl2;
+
+	cfb_pitch = dev_priv->cfb_size / FBC_LL_SIZE;
+	if (fb->pitches[0] < cfb_pitch)
+		cfb_pitch = fb->pitches[0];
+
+	/* FBC_CTL wants 64B units */
+	cfb_pitch = (cfb_pitch / 64) - 1;
+	plane = intel_crtc->plane == 0 ? FBC_CTL_PLANEA : FBC_CTL_PLANEB;
+
+	/* Clear old tags */
+	for (i = 0; i < (FBC_LL_SIZE / 32) + 1; i++)
+		I915_WRITE(FBC_TAG + (i * 4), 0);
+
+	/* Set it up... */
+	fbc_ctl2 = FBC_CTL_FENCE_DBL | FBC_CTL_IDLE_IMM | FBC_CTL_CPU_FENCE;
+	fbc_ctl2 |= plane;
+	I915_WRITE(FBC_CONTROL2, fbc_ctl2);
+	I915_WRITE(FBC_FENCE_OFF, crtc->y);
+
+	/* enable it... */
+	fbc_ctl = FBC_CTL_EN | FBC_CTL_PERIODIC;
+	if (IS_I945GM(dev))
+		fbc_ctl |= FBC_CTL_C3_IDLE; /* 945 needs special SR handling */
+	fbc_ctl |= (cfb_pitch & 0xff) << FBC_CTL_STRIDE_SHIFT;
+	fbc_ctl |= (interval & 0x2fff) << FBC_CTL_INTERVAL_SHIFT;
+	fbc_ctl |= obj->fence_reg;
+	I915_WRITE(FBC_CONTROL, fbc_ctl);
+
+	DRM_DEBUG_KMS("enabled FBC, pitch %d, yoff %d, plane %d, ",
+		      cfb_pitch, crtc->y, intel_crtc->plane);
+}
+
+static bool i8xx_fbc_enabled(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	return I915_READ(FBC_CONTROL) & FBC_CTL_EN;
+}
+
+static void g4x_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_framebuffer *fb = crtc->fb;
+	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
+	struct drm_i915_gem_object *obj = intel_fb->obj;
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
+	unsigned long stall_watermark = 200;
+	u32 dpfc_ctl;
+
+	dpfc_ctl = plane | DPFC_SR_EN | DPFC_CTL_LIMIT_1X;
+	dpfc_ctl |= DPFC_CTL_FENCE_EN | obj->fence_reg;
+	I915_WRITE(DPFC_CHICKEN, DPFC_HT_MODIFY);
+
+	I915_WRITE(DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
+		   (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
+		   (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
+	I915_WRITE(DPFC_FENCE_YOFF, crtc->y);
+
+	/* enable it... */
+	I915_WRITE(DPFC_CONTROL, I915_READ(DPFC_CONTROL) | DPFC_CTL_EN);
+
+	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
+}
+
+static void g4x_disable_fbc(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 dpfc_ctl;
+
+	/* Disable compression */
+	dpfc_ctl = I915_READ(DPFC_CONTROL);
+	if (dpfc_ctl & DPFC_CTL_EN) {
+		dpfc_ctl &= ~DPFC_CTL_EN;
+		I915_WRITE(DPFC_CONTROL, dpfc_ctl);
+
+		DRM_DEBUG_KMS("disabled FBC\n");
+	}
+}
+
+static bool g4x_fbc_enabled(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	return I915_READ(DPFC_CONTROL) & DPFC_CTL_EN;
+}
+
+static void sandybridge_blit_fbc_update(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 blt_ecoskpd;
+
+	/* Make sure blitter notifies FBC of writes */
+	gen6_gt_force_wake_get(dev_priv);
+	blt_ecoskpd = I915_READ(GEN6_BLITTER_ECOSKPD);
+	blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY <<
+		GEN6_BLITTER_LOCK_SHIFT;
+	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
+	blt_ecoskpd |= GEN6_BLITTER_FBC_NOTIFY;
+	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
+	blt_ecoskpd &= ~(GEN6_BLITTER_FBC_NOTIFY <<
+			 GEN6_BLITTER_LOCK_SHIFT);
+	I915_WRITE(GEN6_BLITTER_ECOSKPD, blt_ecoskpd);
+	POSTING_READ(GEN6_BLITTER_ECOSKPD);
+	gen6_gt_force_wake_put(dev_priv);
+}
+
+static void ironlake_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
+{
+	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_framebuffer *fb = crtc->fb;
+	struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
+	struct drm_i915_gem_object *obj = intel_fb->obj;
+	struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+	int plane = intel_crtc->plane == 0 ? DPFC_CTL_PLANEA : DPFC_CTL_PLANEB;
+	unsigned long stall_watermark = 200;
+	u32 dpfc_ctl;
+
+	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
+	dpfc_ctl &= DPFC_RESERVED;
+	dpfc_ctl |= (plane | DPFC_CTL_LIMIT_1X);
+	/* Set persistent mode for front-buffer rendering, ala X. */
+	dpfc_ctl |= DPFC_CTL_PERSISTENT_MODE;
+	dpfc_ctl |= (DPFC_CTL_FENCE_EN | obj->fence_reg);
+	I915_WRITE(ILK_DPFC_CHICKEN, DPFC_HT_MODIFY);
+
+	I915_WRITE(ILK_DPFC_RECOMP_CTL, DPFC_RECOMP_STALL_EN |
+		   (stall_watermark << DPFC_RECOMP_STALL_WM_SHIFT) |
+		   (interval << DPFC_RECOMP_TIMER_COUNT_SHIFT));
+	I915_WRITE(ILK_DPFC_FENCE_YOFF, crtc->y);
+	I915_WRITE(ILK_FBC_RT_BASE, obj->gtt_offset | ILK_FBC_RT_VALID);
+	/* enable it... */
+	I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
+
+	if (IS_GEN6(dev)) {
+		I915_WRITE(SNB_DPFC_CTL_SA,
+			   SNB_CPU_FENCE_ENABLE | obj->fence_reg);
+		I915_WRITE(DPFC_CPU_FENCE_OFFSET, crtc->y);
+		sandybridge_blit_fbc_update(dev);
+	}
+
+	DRM_DEBUG_KMS("enabled fbc on plane %d\n", intel_crtc->plane);
+}
+
+static void ironlake_disable_fbc(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 dpfc_ctl;
+
+	/* Disable compression */
+	dpfc_ctl = I915_READ(ILK_DPFC_CONTROL);
+	if (dpfc_ctl & DPFC_CTL_EN) {
+		dpfc_ctl &= ~DPFC_CTL_EN;
+		I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl);
+
+		DRM_DEBUG_KMS("disabled FBC\n");
+	}
+}
+
+static bool ironlake_fbc_enabled(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	return I915_READ(ILK_DPFC_CONTROL) & DPFC_CTL_EN;
+}
+
+bool intel_fbc_enabled(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (!dev_priv->display.fbc_enabled)
+		return false;
+
+	return dev_priv->display.fbc_enabled(dev);
+}
+
+static void intel_fbc_work_fn(void *arg, int pending)
+{
+	struct intel_fbc_work *work = arg;
+	struct drm_device *dev = work->crtc->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	DRM_LOCK(dev);
+	if (work == dev_priv->fbc_work) {
+		/* Double check that we haven't switched fb without cancelling
+		 * the prior work.
+		 */
+		if (work->crtc->fb == work->fb) {
+			dev_priv->display.enable_fbc(work->crtc,
+						     work->interval);
+
+			dev_priv->cfb_plane = to_intel_crtc(work->crtc)->plane;
+			dev_priv->cfb_fb = work->crtc->fb->base.id;
+			dev_priv->cfb_y = work->crtc->y;
+		}
+
+		dev_priv->fbc_work = NULL;
+	}
+	DRM_UNLOCK(dev);
+
+	free(work, DRM_MEM_KMS);
+}
+
+static void intel_cancel_fbc_work(struct drm_i915_private *dev_priv)
+{
+	u_int pending;
+
+	if (dev_priv->fbc_work == NULL)
+		return;
+
+	DRM_DEBUG_KMS("cancelling pending FBC enable\n");
+
+	/* Synchronisation is provided by struct_mutex and checking of
+	 * dev_priv->fbc_work, so we can perform the cancellation
+	 * entirely asynchronously.
+	 */
+	if (taskqueue_cancel_timeout(dev_priv->tq, &dev_priv->fbc_work->task,
+	    &pending) == 0)
+		/* tasklet was killed before being run, clean up */
+		free(dev_priv->fbc_work, DRM_MEM_KMS);
+
+	/* Mark the work as no longer wanted so that if it does
+	 * wake-up (because the work was already running and waiting
+	 * for our mutex), it will discover that is no longer
+	 * necessary to run.
+	 */
+	dev_priv->fbc_work = NULL;
+}
+
+void intel_enable_fbc(struct drm_crtc *crtc, unsigned long interval)
+{
+	struct intel_fbc_work *work;
+	struct drm_device *dev = crtc->dev;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (!dev_priv->display.enable_fbc)
+		return;
+
+	intel_cancel_fbc_work(dev_priv);
+
+	work = malloc(sizeof(*work), DRM_MEM_KMS, M_WAITOK | M_ZERO);
+
+	work->crtc = crtc;
+	work->fb = crtc->fb;
+	work->interval = interval;
+	TIMEOUT_TASK_INIT(dev_priv->tq, &work->task, 0, intel_fbc_work_fn,
+	    work);
+
+	dev_priv->fbc_work = work;
+
+	DRM_DEBUG_KMS("scheduling delayed FBC enable\n");
+
+	/* Delay the actual enabling to let pageflipping cease and the
+	 * display to settle before starting the compression. Note that
+	 * this delay also serves a second purpose: it allows for a
+	 * vblank to pass after disabling the FBC before we attempt
+	 * to modify the control registers.
+	 *
+	 * A more complicated solution would involve tracking vblanks
+	 * following the termination of the page-flipping sequence
+	 * and indeed performing the enable as a co-routine and not
+	 * waiting synchronously upon the vblank.
+	 */
+	taskqueue_enqueue_timeout(dev_priv->tq, &work->task,
+	    msecs_to_jiffies(50));
+}
+
+void intel_disable_fbc(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	intel_cancel_fbc_work(dev_priv);
+
+	if (!dev_priv->display.disable_fbc)
+		return;
+
+	dev_priv->display.disable_fbc(dev);
+	dev_priv->cfb_plane = -1;
+}
+
+/**
+ * intel_update_fbc - enable/disable FBC as needed
+ * @dev: the drm_device
+ *
+ * Set up the framebuffer compression hardware at mode set time.  We
+ * enable it if possible:
+ *   - plane A only (on pre-965)
+ *   - no pixel mulitply/line duplication
+ *   - no alpha buffer discard
+ *   - no dual wide
+ *   - framebuffer <= 2048 in width, 1536 in height
+ *
+ * We can't assume that any compression will take place (worst case),
+ * so the compressed buffer has to be the same size as the uncompressed
+ * one.  It also must reside (along with the line length buffer) in
+ * stolen memory.
+ *
+ * We need to enable/disable FBC on a global basis.
+ */
+void intel_update_fbc(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_crtc *crtc = NULL, *tmp_crtc;
+	struct intel_crtc *intel_crtc;
+	struct drm_framebuffer *fb;
+	struct intel_framebuffer *intel_fb;
+	struct drm_i915_gem_object *obj;
+	int enable_fbc;
+
+	DRM_DEBUG_KMS("\n");
+
+	if (!i915_powersave)
+		return;
+
+	if (!I915_HAS_FBC(dev))
+		return;
+
+	/*
+	 * If FBC is already on, we just have to verify that we can
+	 * keep it that way...
+	 * Need to disable if:
+	 *   - more than one pipe is active
+	 *   - changing FBC params (stride, fence, mode)
+	 *   - new fb is too large to fit in compressed buffer
+	 *   - going to an unsupported config (interlace, pixel multiply, etc.)
+	 */
+	list_for_each_entry(tmp_crtc, &dev->mode_config.crtc_list, head) {
+		if (tmp_crtc->enabled && tmp_crtc->fb) {
+			if (crtc) {
+				DRM_DEBUG_KMS("more than one pipe active, disabling compression\n");
+				dev_priv->no_fbc_reason = FBC_MULTIPLE_PIPES;
+				goto out_disable;
+			}
+			crtc = tmp_crtc;
+		}
+	}
+
+	if (!crtc || crtc->fb == NULL) {
+		DRM_DEBUG_KMS("no output, disabling\n");
+		dev_priv->no_fbc_reason = FBC_NO_OUTPUT;
+		goto out_disable;
+	}
+
+	intel_crtc = to_intel_crtc(crtc);
+	fb = crtc->fb;
+	intel_fb = to_intel_framebuffer(fb);
+	obj = intel_fb->obj;
+
+	enable_fbc = i915_enable_fbc;
+	if (enable_fbc < 0) {
+		DRM_DEBUG_KMS("fbc set to per-chip default\n");
+		enable_fbc = 1;
+		if (INTEL_INFO(dev)->gen <= 6)
+			enable_fbc = 0;
+	}
+	if (!enable_fbc) {
+		DRM_DEBUG_KMS("fbc disabled per module param\n");
+		dev_priv->no_fbc_reason = FBC_MODULE_PARAM;
+		goto out_disable;
+	}
+	if (intel_fb->obj->base.size > dev_priv->cfb_size) {
+		DRM_DEBUG_KMS("framebuffer too large, disabling "
+			      "compression\n");
+		dev_priv->no_fbc_reason = FBC_STOLEN_TOO_SMALL;
+		goto out_disable;
+	}
+	if ((crtc->mode.flags & DRM_MODE_FLAG_INTERLACE) ||
+	    (crtc->mode.flags & DRM_MODE_FLAG_DBLSCAN)) {
+		DRM_DEBUG_KMS("mode incompatible with compression, "
+			      "disabling\n");
+		dev_priv->no_fbc_reason = FBC_UNSUPPORTED_MODE;
+		goto out_disable;
+	}
+	if ((crtc->mode.hdisplay > 2048) ||
+	    (crtc->mode.vdisplay > 1536)) {
+		DRM_DEBUG_KMS("mode too large for compression, disabling\n");
+		dev_priv->no_fbc_reason = FBC_MODE_TOO_LARGE;
+		goto out_disable;
+	}
+	if ((IS_I915GM(dev) || IS_I945GM(dev)) && intel_crtc->plane != 0) {
+		DRM_DEBUG_KMS("plane not 0, disabling compression\n");
+		dev_priv->no_fbc_reason = FBC_BAD_PLANE;
+		goto out_disable;
+	}
+
+	/* The use of a CPU fence is mandatory in order to detect writes
+	 * by the CPU to the scanout and trigger updates to the FBC.
+	 */
+	if (obj->tiling_mode != I915_TILING_X ||
+	    obj->fence_reg == I915_FENCE_REG_NONE) {
+		DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
+		dev_priv->no_fbc_reason = FBC_NOT_TILED;
+		goto out_disable;
+	}
+
+	/* If the kernel debugger is active, always disable compression */
+	if (kdb_active)
+		goto out_disable;
+
+	/* If the scanout has not changed, don't modify the FBC settings.
+	 * Note that we make the fundamental assumption that the fb->obj
+	 * cannot be unpinned (and have its GTT offset and fence revoked)
+	 * without first being decoupled from the scanout and FBC disabled.
+	 */
+	if (dev_priv->cfb_plane == intel_crtc->plane &&
+	    dev_priv->cfb_fb == fb->base.id &&
+	    dev_priv->cfb_y == crtc->y)
+		return;
+
+	if (intel_fbc_enabled(dev)) {
+		/* We update FBC along two paths, after changing fb/crtc
+		 * configuration (modeswitching) and after page-flipping
+		 * finishes. For the latter, we know that not only did
+		 * we disable the FBC at the start of the page-flip
+		 * sequence, but also more than one vblank has passed.
+		 *
+		 * For the former case of modeswitching, it is possible
+		 * to switch between two FBC valid configurations
+		 * instantaneously so we do need to disable the FBC
+		 * before we can modify its control registers. We also
+		 * have to wait for the next vblank for that to take
+		 * effect. However, since we delay enabling FBC we can
+		 * assume that a vblank has passed since disabling and
+		 * that we can safely alter the registers in the deferred
+		 * callback.
+		 *
+		 * In the scenario that we go from a valid to invalid
+		 * and then back to valid FBC configuration we have
+		 * no strict enforcement that a vblank occurred since
+		 * disabling the FBC. However, along all current pipe
+		 * disabling paths we do need to wait for a vblank at
+		 * some point. And we wait before enabling FBC anyway.
+		 */
+		DRM_DEBUG_KMS("disabling active FBC for update\n");
+		intel_disable_fbc(dev);
+	}
+
+	intel_enable_fbc(crtc, 500);
+	return;
+
+out_disable:
+	/* Multiple disables should be harmless */
+	if (intel_fbc_enabled(dev)) {
+		DRM_DEBUG_KMS("unsupported config, disabling FBC\n");
+		intel_disable_fbc(dev);
+	}
+}
+
+static void i915_pineview_get_mem_freq(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	u32 tmp;
+
+	tmp = I915_READ(CLKCFG);
+
+	switch (tmp & CLKCFG_FSB_MASK) {
+	case CLKCFG_FSB_533:
+		dev_priv->fsb_freq = 533; /* 133*4 */
+		break;
+	case CLKCFG_FSB_800:
+		dev_priv->fsb_freq = 800; /* 200*4 */
+		break;
+	case CLKCFG_FSB_667:
+		dev_priv->fsb_freq =  667; /* 167*4 */
+		break;
+	case CLKCFG_FSB_400:
+		dev_priv->fsb_freq = 400; /* 100*4 */
+		break;
+	}
+
+	switch (tmp & CLKCFG_MEM_MASK) {
+	case CLKCFG_MEM_533:
+		dev_priv->mem_freq = 533;
+		break;
+	case CLKCFG_MEM_667:
+		dev_priv->mem_freq = 667;
+		break;
+	case CLKCFG_MEM_800:
+		dev_priv->mem_freq = 800;
+		break;
+	}
+
+	/* detect pineview DDR3 setting */
+	tmp = I915_READ(CSHRDDR3CTL);
+	dev_priv->is_ddr3 = (tmp & CSHRDDR3CTL_DDR3) ? 1 : 0;
+}
+
+static void i915_ironlake_get_mem_freq(struct drm_device *dev)
+{
+	drm_i915_private_t *dev_priv = dev->dev_private;
+	u16 ddrpll, csipll;
+
+	ddrpll = I915_READ16(DDRMPLL1);
+	csipll = I915_READ16(CSIPLL0);
+
+	switch (ddrpll & 0xff) {
+	case 0xc:
+		dev_priv->mem_freq = 800;
+		break;
+	case 0x10:
+		dev_priv->mem_freq = 1066;
+		break;
+	case 0x14:
+		dev_priv->mem_freq = 1333;
+		break;
+	case 0x18:
+		dev_priv->mem_freq = 1600;
+		break;
+	default:
+		DRM_DEBUG("unknown memory frequency 0x%02x\n",
+				 ddrpll & 0xff);
+		dev_priv->mem_freq = 0;
+		break;
+	}
+
+	dev_priv->r_t = dev_priv->mem_freq;
+
+	switch (csipll & 0x3ff) {
+	case 0x00c:
+		dev_priv->fsb_freq = 3200;
+		break;
+	case 0x00e:
+		dev_priv->fsb_freq = 3733;
+		break;
+	case 0x010:
+		dev_priv->fsb_freq = 4266;
+		break;
+	case 0x012:
+		dev_priv->fsb_freq = 4800;
+		break;
+	case 0x014:
+		dev_priv->fsb_freq = 5333;
+		break;
+	case 0x016:
+		dev_priv->fsb_freq = 5866;
+		break;
+	case 0x018:
+		dev_priv->fsb_freq = 6400;
+		break;
+	default:
+		DRM_DEBUG("unknown fsb frequency 0x%04x\n",
+				 csipll & 0x3ff);
+		dev_priv->fsb_freq = 0;
+		break;
+	}
+
+	if (dev_priv->fsb_freq == 3200) {
+		dev_priv->c_m = 0;
+	} else if (dev_priv->fsb_freq > 3200 && dev_priv->fsb_freq <= 4800) {
+		dev_priv->c_m = 1;
+	} else {
+		dev_priv->c_m = 2;
+	}
+}
+
+static const struct cxsr_latency cxsr_latency_table[] = {
+	{1, 0, 800, 400, 3382, 33382, 3983, 33983},    /* DDR2-400 SC */
+	{1, 0, 800, 667, 3354, 33354, 3807, 33807},    /* DDR2-667 SC */
+	{1, 0, 800, 800, 3347, 33347, 3763, 33763},    /* DDR2-800 SC */
+	{1, 1, 800, 667, 6420, 36420, 6873, 36873},    /* DDR3-667 SC */
+	{1, 1, 800, 800, 5902, 35902, 6318, 36318},    /* DDR3-800 SC */
+
+	{1, 0, 667, 400, 3400, 33400, 4021, 34021},    /* DDR2-400 SC */
+	{1, 0, 667, 667, 3372, 33372, 3845, 33845},    /* DDR2-667 SC */
+	{1, 0, 667, 800, 3386, 33386, 3822, 33822},    /* DDR2-800 SC */
+	{1, 1, 667, 667, 6438, 36438, 6911, 36911},    /* DDR3-667 SC */
+	{1, 1, 667, 800, 5941, 35941, 6377, 36377},    /* DDR3-800 SC */
+
+	{1, 0, 400, 400, 3472, 33472, 4173, 34173},    /* DDR2-400 SC */
+	{1, 0, 400, 667, 3443, 33443, 3996, 33996},    /* DDR2-667 SC */
+	{1, 0, 400, 800, 3430, 33430, 3946, 33946},    /* DDR2-800 SC */
+	{1, 1, 400, 667, 6509, 36509, 7062, 37062},    /* DDR3-667 SC */
+	{1, 1, 400, 800, 5985, 35985, 6501, 36501},    /* DDR3-800 SC */
+
+	{0, 0, 800, 400, 3438, 33438, 4065, 34065},    /* DDR2-400 SC */
+	{0, 0, 800, 667, 3410, 33410, 3889, 33889},    /* DDR2-667 SC */
+	{0, 0, 800, 800, 3403, 33403, 3845, 33845},    /* DDR2-800 SC */
+	{0, 1, 800, 667, 6476, 36476, 6955, 36955},    /* DDR3-667 SC */
+	{0, 1, 800, 800, 5958, 35958, 6400, 36400},    /* DDR3-800 SC */
+
+	{0, 0, 667, 400, 3456, 33456, 4103, 34106},    /* DDR2-400 SC */
+	{0, 0, 667, 667, 3428, 33428, 3927, 33927},    /* DDR2-667 SC */
+	{0, 0, 667, 800, 3443, 33443, 3905, 33905},    /* DDR2-800 SC */
+	{0, 1, 667, 667, 6494, 36494, 6993, 36993},    /* DDR3-667 SC */
+	{0, 1, 667, 800, 5998, 35998, 6460, 36460},    /* DDR3-800 SC */
+
+	{0, 0, 400, 400, 3528, 33528, 4255, 34255},    /* DDR2-400 SC */
+	{0, 0, 400, 667, 3500, 33500, 4079, 34079},    /* DDR2-667 SC */
+	{0, 0, 400, 800, 3487, 33487, 4029, 34029},    /* DDR2-800 SC */
+	{0, 1, 400, 667, 6566, 36566, 7145, 37145},    /* DDR3-667 SC */
+	{0, 1, 400, 800, 6042, 36042, 6584, 36584},    /* DDR3-800 SC */
+};
+
+static const struct cxsr_latency *intel_get_cxsr_latency(int is_desktop,
+							 int is_ddr3,
+							 int fsb,
+							 int mem)
+{
+	const struct cxsr_latency *latency;
+	int i;
+
+	if (fsb == 0 || mem == 0)
+		return NULL;
+
+	for (i = 0; i < DRM_ARRAY_SIZE(cxsr_latency_table); i++) {
+		latency = &cxsr_latency_table[i];
+		if (is_desktop == latency->is_desktop &&
+		    is_ddr3 == latency->is_ddr3 &&
+		    fsb == latency->fsb_freq && mem == latency->mem_freq)
+			return latency;
+	}
+
+	DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
+
+	return NULL;
+}
+
+static void pineview_disable_cxsr(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	/* deactivate cxsr */
+	I915_WRITE(DSPFW3, I915_READ(DSPFW3) & ~PINEVIEW_SELF_REFRESH_EN);
+}
+
+/*
+ * Latency for FIFO fetches is dependent on several factors:
+ *   - memory configuration (speed, channels)
+ *   - chipset
+ *   - current MCH state
+ * It can be fairly high in some situations, so here we assume a fairly
+ * pessimal value.  It's a tradeoff between extra memory fetches (if we
+ * set this value too high, the FIFO will fetch frequently to stay full)
+ * and power consumption (set it too low to save power and we might see
+ * FIFO underruns and display "flicker").
+ *
+ * A value of 5us seems to be a good balance; safe for very low end
+ * platforms but not overly aggressive on lower latency configs.
+ */
+static const int latency_ns = 5000;
+
+static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	uint32_t dsparb = I915_READ(DSPARB);
+	int size;
+
+	size = dsparb & 0x7f;
+	if (plane)
+		size = ((dsparb >> DSPARB_CSTART_SHIFT) & 0x7f) - size;
+
+	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
+		      plane ? "B" : "A", size);
+
+	return size;
+}
+
+static int i85x_get_fifo_size(struct drm_device *dev, int plane)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	uint32_t dsparb = I915_READ(DSPARB);
+	int size;
+
+	size = dsparb & 0x1ff;
+	if (plane)
+		size = ((dsparb >> DSPARB_BEND_SHIFT) & 0x1ff) - size;
+	size >>= 1; /* Convert to cachelines */
+
+	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
+		      plane ? "B" : "A", size);
+
+	return size;
+}
+
+static int i845_get_fifo_size(struct drm_device *dev, int plane)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	uint32_t dsparb = I915_READ(DSPARB);
+	int size;
+
+	size = dsparb & 0x7f;
+	size >>= 2; /* Convert to cachelines */
+
+	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
+		      plane ? "B" : "A",
+		      size);
+
+	return size;
+}
+
+static int i830_get_fifo_size(struct drm_device *dev, int plane)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	uint32_t dsparb = I915_READ(DSPARB);
+	int size;
+
+	size = dsparb & 0x7f;
+	size >>= 1; /* Convert to cachelines */
+
+	DRM_DEBUG_KMS("FIFO size - (0x%08x) %s: %d\n", dsparb,
+		      plane ? "B" : "A", size);
+
+	return size;
+}
+
+/* Pineview has different values for various configs */
+static const struct intel_watermark_params pineview_display_wm = {
+	PINEVIEW_DISPLAY_FIFO,
+	PINEVIEW_MAX_WM,
+	PINEVIEW_DFT_WM,
+	PINEVIEW_GUARD_WM,
+	PINEVIEW_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params pineview_display_hplloff_wm = {
+	PINEVIEW_DISPLAY_FIFO,
+	PINEVIEW_MAX_WM,
+	PINEVIEW_DFT_HPLLOFF_WM,
+	PINEVIEW_GUARD_WM,
+	PINEVIEW_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params pineview_cursor_wm = {
+	PINEVIEW_CURSOR_FIFO,
+	PINEVIEW_CURSOR_MAX_WM,
+	PINEVIEW_CURSOR_DFT_WM,
+	PINEVIEW_CURSOR_GUARD_WM,
+	PINEVIEW_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params pineview_cursor_hplloff_wm = {
+	PINEVIEW_CURSOR_FIFO,
+	PINEVIEW_CURSOR_MAX_WM,
+	PINEVIEW_CURSOR_DFT_WM,
+	PINEVIEW_CURSOR_GUARD_WM,
+	PINEVIEW_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params g4x_wm_info = {
+	G4X_FIFO_SIZE,
+	G4X_MAX_WM,
+	G4X_MAX_WM,
+	2,
+	G4X_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params g4x_cursor_wm_info = {
+	I965_CURSOR_FIFO,
+	I965_CURSOR_MAX_WM,
+	I965_CURSOR_DFT_WM,
+	2,
+	G4X_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params valleyview_wm_info = {
+	VALLEYVIEW_FIFO_SIZE,
+	VALLEYVIEW_MAX_WM,
+	VALLEYVIEW_MAX_WM,
+	2,
+	G4X_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params valleyview_cursor_wm_info = {
+	I965_CURSOR_FIFO,
+	VALLEYVIEW_CURSOR_MAX_WM,
+	I965_CURSOR_DFT_WM,
+	2,
+	G4X_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params i965_cursor_wm_info = {
+	I965_CURSOR_FIFO,
+	I965_CURSOR_MAX_WM,
+	I965_CURSOR_DFT_WM,
+	2,
+	I915_FIFO_LINE_SIZE,
+};
+static const struct intel_watermark_params i945_wm_info = {
+	I945_FIFO_SIZE,
+	I915_MAX_WM,
+	1,
+	2,
+	I915_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params i915_wm_info = {
+	I915_FIFO_SIZE,
+	I915_MAX_WM,
+	1,
+	2,
+	I915_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params i855_wm_info = {
+	I855GM_FIFO_SIZE,
+	I915_MAX_WM,
+	1,
+	2,
+	I830_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params i830_wm_info = {
+	I830_FIFO_SIZE,
+	I915_MAX_WM,
+	1,
+	2,
+	I830_FIFO_LINE_SIZE
+};
+
+static const struct intel_watermark_params ironlake_display_wm_info = {
+	ILK_DISPLAY_FIFO,
+	ILK_DISPLAY_MAXWM,
+	ILK_DISPLAY_DFTWM,
+	2,
+	ILK_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params ironlake_cursor_wm_info = {
+	ILK_CURSOR_FIFO,
+	ILK_CURSOR_MAXWM,
+	ILK_CURSOR_DFTWM,
+	2,
+	ILK_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params ironlake_display_srwm_info = {
+	ILK_DISPLAY_SR_FIFO,
+	ILK_DISPLAY_MAX_SRWM,
+	ILK_DISPLAY_DFT_SRWM,
+	2,
+	ILK_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params ironlake_cursor_srwm_info = {
+	ILK_CURSOR_SR_FIFO,
+	ILK_CURSOR_MAX_SRWM,
+	ILK_CURSOR_DFT_SRWM,
+	2,
+	ILK_FIFO_LINE_SIZE
+};
+
+static const struct intel_watermark_params sandybridge_display_wm_info = {
+	SNB_DISPLAY_FIFO,
+	SNB_DISPLAY_MAXWM,
+	SNB_DISPLAY_DFTWM,
+	2,
+	SNB_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params sandybridge_cursor_wm_info = {
+	SNB_CURSOR_FIFO,
+	SNB_CURSOR_MAXWM,
+	SNB_CURSOR_DFTWM,
+	2,
+	SNB_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params sandybridge_display_srwm_info = {
+	SNB_DISPLAY_SR_FIFO,
+	SNB_DISPLAY_MAX_SRWM,
+	SNB_DISPLAY_DFT_SRWM,
+	2,
+	SNB_FIFO_LINE_SIZE
+};
+static const struct intel_watermark_params sandybridge_cursor_srwm_info = {
+	SNB_CURSOR_SR_FIFO,
+	SNB_CURSOR_MAX_SRWM,
+	SNB_CURSOR_DFT_SRWM,
+	2,
+	SNB_FIFO_LINE_SIZE
+};
+
+
+/**
+ * intel_calculate_wm - calculate watermark level
+ * @clock_in_khz: pixel clock
+ * @wm: chip FIFO params
+ * @pixel_size: display pixel size
+ * @latency_ns: memory latency for the platform
+ *
+ * Calculate the watermark level (the level at which the display plane will
+ * start fetching from memory again).  Each chip has a different display
+ * FIFO size and allocation, so the caller needs to figure that out and pass
+ * in the correct intel_watermark_params structure.
+ *
+ * As the pixel clock runs, the FIFO will be drained at a rate that depends
+ * on the pixel size.  When it reaches the watermark level, it'll start
+ * fetching FIFO line sized based chunks from memory until the FIFO fills
+ * past the watermark point.  If the FIFO drains completely, a FIFO underrun
+ * will occur, and a display engine hang could result.
+ */
+static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
+					const struct intel_watermark_params *wm,
+					int fifo_size,
+					int pixel_size,
+					unsigned long latency_ns)
+{
+	long entries_required, wm_size;
+
+	/*
+	 * Note: we need to make sure we don't overflow for various clock &
+	 * latency values.
+	 * clocks go from a few thousand to several hundred thousand.
+	 * latency is usually a few thousand
+	 */
+	entries_required = ((clock_in_khz / 1000) * pixel_size * latency_ns) /
+		1000;
+	entries_required = DIV_ROUND_UP(entries_required, wm->cacheline_size);
+
+	DRM_DEBUG_KMS("FIFO entries required for mode: %ld\n", entries_required);
+
+	wm_size = fifo_size - (entries_required + wm->guard_size);
+
+	DRM_DEBUG_KMS("FIFO watermark level: %ld\n", wm_size);
+
+	/* Don't promote wm_size to unsigned... */
+	if (wm_size > (long)wm->max_wm)
+		wm_size = wm->max_wm;
+	if (wm_size <= 0)
+		wm_size = wm->default_wm;
+	return wm_size;
+}
+
+static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
+{
+	struct drm_crtc *crtc, *enabled = NULL;
+
+	list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
+		if (crtc->enabled && crtc->fb) {
+			if (enabled)
+				return NULL;
+			enabled = crtc;
+		}
+	}
+
+	return enabled;
+}
+
+static void pineview_update_wm(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_crtc *crtc;
+	const struct cxsr_latency *latency;
+	u32 reg;
+	unsigned long wm;
+
+	latency = intel_get_cxsr_latency(IS_PINEVIEW_G(dev), dev_priv->is_ddr3,
+					 dev_priv->fsb_freq, dev_priv->mem_freq);
+	if (!latency) {
+		DRM_DEBUG_KMS("Unknown FSB/MEM found, disable CxSR\n");
+		pineview_disable_cxsr(dev);
+		return;
+	}
+
+	crtc = single_enabled_crtc(dev);
+	if (crtc) {
+		int clock = crtc->mode.clock;
+		int pixel_size = crtc->fb->bits_per_pixel / 8;
+
+		/* Display SR */
+		wm = intel_calculate_wm(clock, &pineview_display_wm,
+					pineview_display_wm.fifo_size,
+					pixel_size, latency->display_sr);
+		reg = I915_READ(DSPFW1);
+		reg &= ~DSPFW_SR_MASK;
+		reg |= wm << DSPFW_SR_SHIFT;
+		I915_WRITE(DSPFW1, reg);
+		DRM_DEBUG_KMS("DSPFW1 register is %x\n", reg);
+
+		/* cursor SR */
+		wm = intel_calculate_wm(clock, &pineview_cursor_wm,
+					pineview_display_wm.fifo_size,
+					pixel_size, latency->cursor_sr);
+		reg = I915_READ(DSPFW3);
+		reg &= ~DSPFW_CURSOR_SR_MASK;
+		reg |= (wm & 0x3f) << DSPFW_CURSOR_SR_SHIFT;
+		I915_WRITE(DSPFW3, reg);
+
+		/* Display HPLL off SR */
+		wm = intel_calculate_wm(clock, &pineview_display_hplloff_wm,
+					pineview_display_hplloff_wm.fifo_size,
+					pixel_size, latency->display_hpll_disable);
+		reg = I915_READ(DSPFW3);
+		reg &= ~DSPFW_HPLL_SR_MASK;
+		reg |= wm & DSPFW_HPLL_SR_MASK;
+		I915_WRITE(DSPFW3, reg);
+
+		/* cursor HPLL off SR */
+		wm = intel_calculate_wm(clock, &pineview_cursor_hplloff_wm,
+					pineview_display_hplloff_wm.fifo_size,
+					pixel_size, latency->cursor_hpll_disable);
+		reg = I915_READ(DSPFW3);
+		reg &= ~DSPFW_HPLL_CURSOR_MASK;
+		reg |= (wm & 0x3f) << DSPFW_HPLL_CURSOR_SHIFT;
+		I915_WRITE(DSPFW3, reg);
+		DRM_DEBUG_KMS("DSPFW3 register is %x\n", reg);
+
+		/* activate cxsr */
+		I915_WRITE(DSPFW3,
+			   I915_READ(DSPFW3) | PINEVIEW_SELF_REFRESH_EN);
+		DRM_DEBUG_KMS("Self-refresh is enabled\n");
+	} else {
+		pineview_disable_cxsr(dev);
+		DRM_DEBUG_KMS("Self-refresh is disabled\n");
+	}
+}
+
+static bool g4x_compute_wm0(struct drm_device *dev,
+			    int plane,
+			    const struct intel_watermark_params *display,
+			    int display_latency_ns,
+			    const struct intel_watermark_params *cursor,
+			    int cursor_latency_ns,
+			    int *plane_wm,
+			    int *cursor_wm)
+{
+	struct drm_crtc *crtc;
+	int htotal, hdisplay, clock, pixel_size;
+	int line_time_us, line_count;
+	int entries, tlb_miss;
+
+	crtc = intel_get_crtc_for_plane(dev, plane);
+	if (crtc->fb == NULL || !crtc->enabled) {
+		*cursor_wm = cursor->guard_size;
+		*plane_wm = display->guard_size;
+		return false;
+	}
+
+	htotal = crtc->mode.htotal;
+	hdisplay = crtc->mode.hdisplay;
+	clock = crtc->mode.clock;
+	pixel_size = crtc->fb->bits_per_pixel / 8;
+
+	/* Use the small buffer method to calculate plane watermark */
+	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
+	tlb_miss = display->fifo_size*display->cacheline_size - hdisplay * 8;
+	if (tlb_miss > 0)
+		entries += tlb_miss;
+	entries = DIV_ROUND_UP(entries, display->cacheline_size);
+	*plane_wm = entries + display->guard_size;
+	if (*plane_wm > (int)display->max_wm)
+		*plane_wm = display->max_wm;
+
+	/* Use the large buffer method to calculate cursor watermark */
+	line_time_us = ((htotal * 1000) / clock);
+	line_count = (cursor_latency_ns / line_time_us + 1000) / 1000;
+	entries = line_count * 64 * pixel_size;
+	tlb_miss = cursor->fifo_size*cursor->cacheline_size - hdisplay * 8;
+	if (tlb_miss > 0)
+		entries += tlb_miss;
+	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
+	*cursor_wm = entries + cursor->guard_size;
+	if (*cursor_wm > (int)cursor->max_wm)
+		*cursor_wm = (int)cursor->max_wm;
+
+	return true;
+}
+
+/*
+ * Check the wm result.
+ *
+ * If any calculated watermark values is larger than the maximum value that
+ * can be programmed into the associated watermark register, that watermark
+ * must be disabled.
+ */
+static bool g4x_check_srwm(struct drm_device *dev,
+			   int display_wm, int cursor_wm,
+			   const struct intel_watermark_params *display,
+			   const struct intel_watermark_params *cursor)
+{
+	DRM_DEBUG_KMS("SR watermark: display plane %d, cursor %d\n",
+		      display_wm, cursor_wm);
+
+	if (display_wm > display->max_wm) {
+		DRM_DEBUG_KMS("display watermark is too large(%d/%ld), disabling\n",
+			      display_wm, display->max_wm);
+		return false;
+	}
+
+	if (cursor_wm > cursor->max_wm) {
+		DRM_DEBUG_KMS("cursor watermark is too large(%d/%ld), disabling\n",
+			      cursor_wm, cursor->max_wm);
+		return false;
+	}
+
+	if (!(display_wm || cursor_wm)) {
+		DRM_DEBUG_KMS("SR latency is 0, disabling\n");
+		return false;
+	}
+
+	return true;
+}
+
+static bool g4x_compute_srwm(struct drm_device *dev,
+			     int plane,
+			     int latency_ns,
+			     const struct intel_watermark_params *display,
+			     const struct intel_watermark_params *cursor,
+			     int *display_wm, int *cursor_wm)
+{
+	struct drm_crtc *crtc;
+	int hdisplay, htotal, pixel_size, clock;
+	unsigned long line_time_us;
+	int line_count, line_size;
+	int small, large;
+	int entries;
+
+	if (!latency_ns) {
+		*display_wm = *cursor_wm = 0;
+		return false;
+	}
+
+	crtc = intel_get_crtc_for_plane(dev, plane);
+	hdisplay = crtc->mode.hdisplay;
+	htotal = crtc->mode.htotal;
+	clock = crtc->mode.clock;
+	pixel_size = crtc->fb->bits_per_pixel / 8;
+
+	line_time_us = (htotal * 1000) / clock;
+	line_count = (latency_ns / line_time_us + 1000) / 1000;
+	line_size = hdisplay * pixel_size;
+
+	/* Use the minimum of the small and large buffer method for primary */
+	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
+	large = line_count * line_size;
+
+	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
+	*display_wm = entries + display->guard_size;
+
+	/* calculate the self-refresh watermark for display cursor */
+	entries = line_count * pixel_size * 64;
+	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
+	*cursor_wm = entries + cursor->guard_size;
+
+	return g4x_check_srwm(dev,
+			      *display_wm, *cursor_wm,
+			      display, cursor);
+}
+
+static bool vlv_compute_drain_latency(struct drm_device *dev,
+				     int plane,
+				     int *plane_prec_mult,
+				     int *plane_dl,
+				     int *cursor_prec_mult,
+				     int *cursor_dl)
+{
+	struct drm_crtc *crtc;
+	int clock, pixel_size;
+	int entries;
+
+	crtc = intel_get_crtc_for_plane(dev, plane);
+	if (crtc->fb == NULL || !crtc->enabled)
+		return false;
+
+	clock = crtc->mode.clock;	/* VESA DOT Clock */
+	pixel_size = crtc->fb->bits_per_pixel / 8;	/* BPP */
+
+	entries = (clock / 1000) * pixel_size;
+	*plane_prec_mult = (entries > 256) ?
+		DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
+	*plane_dl = (64 * (*plane_prec_mult) * 4) / ((clock / 1000) *
+						     pixel_size);
+
+	entries = (clock / 1000) * 4;	/* BPP is always 4 for cursor */
+	*cursor_prec_mult = (entries > 256) ?
+		DRAIN_LATENCY_PRECISION_32 : DRAIN_LATENCY_PRECISION_16;
+	*cursor_dl = (64 * (*cursor_prec_mult) * 4) / ((clock / 1000) * 4);
+
+	return true;
+}
+
+/*
+ * Update drain latency registers of memory arbiter
+ *
+ * Valleyview SoC has a new memory arbiter and needs drain latency registers
+ * to be programmed. Each plane has a drain latency multiplier and a drain
+ * latency value.
+ */
+
+static void vlv_update_drain_latency(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int planea_prec, planea_dl, planeb_prec, planeb_dl;
+	int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
+	int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
+							either 16 or 32 */
+
+	/* For plane A, Cursor A */
+	if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
+				      &cursor_prec_mult, &cursora_dl)) {
+		cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
+			DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_16;
+		planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
+			DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_16;
+
+		I915_WRITE(VLV_DDL1, cursora_prec |
+				(cursora_dl << DDL_CURSORA_SHIFT) |
+				planea_prec | planea_dl);
+	}
+
+	/* For plane B, Cursor B */
+	if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
+				      &cursor_prec_mult, &cursorb_dl)) {
+		cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
+			DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_16;
+		planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
+			DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_16;
+
+		I915_WRITE(VLV_DDL2, cursorb_prec |
+				(cursorb_dl << DDL_CURSORB_SHIFT) |
+				planeb_prec | planeb_dl);
+	}
+}
+
+#define single_plane_enabled(mask) ((mask) != 0 && powerof2(mask))
+
+static void valleyview_update_wm(struct drm_device *dev)
+{
+	static const int sr_latency_ns = 12000;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
+	int plane_sr, cursor_sr;
+	unsigned int enabled = 0;
+
+	vlv_update_drain_latency(dev);
+
+	if (g4x_compute_wm0(dev, 0,
+			    &valleyview_wm_info, latency_ns,
+			    &valleyview_cursor_wm_info, latency_ns,
+			    &planea_wm, &cursora_wm))
+		enabled |= 1;
+
+	if (g4x_compute_wm0(dev, 1,
+			    &valleyview_wm_info, latency_ns,
+			    &valleyview_cursor_wm_info, latency_ns,
+			    &planeb_wm, &cursorb_wm))
+		enabled |= 2;
+
+	plane_sr = cursor_sr = 0;
+	if (single_plane_enabled(enabled) &&
+	    g4x_compute_srwm(dev, ffs(enabled) - 1,
+			     sr_latency_ns,
+			     &valleyview_wm_info,
+			     &valleyview_cursor_wm_info,
+			     &plane_sr, &cursor_sr))
+		I915_WRITE(FW_BLC_SELF_VLV, FW_CSPWRDWNEN);
+	else
+		I915_WRITE(FW_BLC_SELF_VLV,
+			   I915_READ(FW_BLC_SELF_VLV) & ~FW_CSPWRDWNEN);
+
+	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
+		      planea_wm, cursora_wm,
+		      planeb_wm, cursorb_wm,
+		      plane_sr, cursor_sr);
+
+	I915_WRITE(DSPFW1,
+		   (plane_sr << DSPFW_SR_SHIFT) |
+		   (cursorb_wm << DSPFW_CURSORB_SHIFT) |
+		   (planeb_wm << DSPFW_PLANEB_SHIFT) |
+		   planea_wm);
+	I915_WRITE(DSPFW2,
+		   (I915_READ(DSPFW2) & DSPFW_CURSORA_MASK) |
+		   (cursora_wm << DSPFW_CURSORA_SHIFT));
+	I915_WRITE(DSPFW3,
+		   (I915_READ(DSPFW3) | (cursor_sr << DSPFW_CURSOR_SR_SHIFT)));
+}
+
+static void g4x_update_wm(struct drm_device *dev)
+{
+	static const int sr_latency_ns = 12000;
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
+	int plane_sr, cursor_sr;
+	unsigned int enabled = 0;
+
+	if (g4x_compute_wm0(dev, 0,
+			    &g4x_wm_info, latency_ns,
+			    &g4x_cursor_wm_info, latency_ns,
+			    &planea_wm, &cursora_wm))
+		enabled |= 1;
+
+	if (g4x_compute_wm0(dev, 1,
+			    &g4x_wm_info, latency_ns,
+			    &g4x_cursor_wm_info, latency_ns,
+			    &planeb_wm, &cursorb_wm))
+		enabled |= 2;
+
+	plane_sr = cursor_sr = 0;
+	if (single_plane_enabled(enabled) &&
+	    g4x_compute_srwm(dev, ffs(enabled) - 1,
+			     sr_latency_ns,
+			     &g4x_wm_info,
+			     &g4x_cursor_wm_info,
+			     &plane_sr, &cursor_sr))
+		I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
+	else
+		I915_WRITE(FW_BLC_SELF,
+			   I915_READ(FW_BLC_SELF) & ~FW_BLC_SELF_EN);
+
+	DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
+		      planea_wm, cursora_wm,
+		      planeb_wm, cursorb_wm,
+		      plane_sr, cursor_sr);
+
+	I915_WRITE(DSPFW1,
+		   (plane_sr << DSPFW_SR_SHIFT) |
+		   (cursorb_wm << DSPFW_CURSORB_SHIFT) |
+		   (planeb_wm << DSPFW_PLANEB_SHIFT) |
+		   planea_wm);
+	I915_WRITE(DSPFW2,
+		   (I915_READ(DSPFW2) & DSPFW_CURSORA_MASK) |
+		   (cursora_wm << DSPFW_CURSORA_SHIFT));
+	/* HPLL off in SR has some issues on G4x... disable it */
+	I915_WRITE(DSPFW3,
+		   (I915_READ(DSPFW3) & ~DSPFW_HPLL_SR_EN) |
+		   (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
+}
+
+static void i965_update_wm(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_crtc *crtc;
+	int srwm = 1;
+	int cursor_sr = 16;
+
+	/* Calc sr entries for one plane configs */
+	crtc = single_enabled_crtc(dev);
+	if (crtc) {
+		/* self-refresh has much higher latency */
+		static const int sr_latency_ns = 12000;
+		int clock = crtc->mode.clock;
+		int htotal = crtc->mode.htotal;
+		int hdisplay = crtc->mode.hdisplay;
+		int pixel_size = crtc->fb->bits_per_pixel / 8;
+		unsigned long line_time_us;
+		int entries;
+
+		line_time_us = ((htotal * 1000) / clock);
+
+		/* Use ns/us then divide to preserve precision */
+		entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
+			pixel_size * hdisplay;
+		entries = DIV_ROUND_UP(entries, I915_FIFO_LINE_SIZE);
+		srwm = I965_FIFO_SIZE - entries;
+		if (srwm < 0)
+			srwm = 1;
+		srwm &= 0x1ff;
+		DRM_DEBUG_KMS("self-refresh entries: %d, wm: %d\n",
+			      entries, srwm);
+
+		entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
+			pixel_size * 64;
+		entries = DIV_ROUND_UP(entries,
+					  i965_cursor_wm_info.cacheline_size);
+		cursor_sr = i965_cursor_wm_info.fifo_size -
+			(entries + i965_cursor_wm_info.guard_size);
+
+		if (cursor_sr > i965_cursor_wm_info.max_wm)
+			cursor_sr = i965_cursor_wm_info.max_wm;
+
+		DRM_DEBUG_KMS("self-refresh watermark: display plane %d "
+			      "cursor %d\n", srwm, cursor_sr);
+
+		if (IS_CRESTLINE(dev))
+			I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN);
+	} else {
+		/* Turn off self refresh if both pipes are enabled */
+		if (IS_CRESTLINE(dev))
+			I915_WRITE(FW_BLC_SELF, I915_READ(FW_BLC_SELF)
+				   & ~FW_BLC_SELF_EN);
+	}
+
+	DRM_DEBUG_KMS("Setting FIFO watermarks - A: 8, B: 8, C: 8, SR %d\n",
+		      srwm);
+
+	/* 965 has limitations... */
+	I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
+		   (8 << 16) | (8 << 8) | (8 << 0));
+	I915_WRITE(DSPFW2, (8 << 8) | (8 << 0));
+	/* update cursor SR watermark */
+	I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
+}
+
+static void i9xx_update_wm(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	const struct intel_watermark_params *wm_info;
+	uint32_t fwater_lo;
+	uint32_t fwater_hi;
+	int cwm, srwm = 1;
+	int fifo_size;
+	int planea_wm, planeb_wm;
+	struct drm_crtc *crtc, *enabled = NULL;
+
+	if (IS_I945GM(dev))
+		wm_info = &i945_wm_info;
+	else if (!IS_GEN2(dev))
+		wm_info = &i915_wm_info;
+	else
+		wm_info = &i855_wm_info;
+
+	fifo_size = dev_priv->display.get_fifo_size(dev, 0);
+	crtc = intel_get_crtc_for_plane(dev, 0);
+	if (crtc->enabled && crtc->fb) {
+		planea_wm = intel_calculate_wm(crtc->mode.clock,
+					       wm_info, fifo_size,
+					       crtc->fb->bits_per_pixel / 8,
+					       latency_ns);
+		enabled = crtc;
+	} else
+		planea_wm = fifo_size - wm_info->guard_size;
+
+	fifo_size = dev_priv->display.get_fifo_size(dev, 1);
+	crtc = intel_get_crtc_for_plane(dev, 1);
+	if (crtc->enabled && crtc->fb) {
+		planeb_wm = intel_calculate_wm(crtc->mode.clock,
+					       wm_info, fifo_size,
+					       crtc->fb->bits_per_pixel / 8,
+					       latency_ns);
+		if (enabled == NULL)
+			enabled = crtc;
+		else
+			enabled = NULL;
+	} else
+		planeb_wm = fifo_size - wm_info->guard_size;
+
+	DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
+
+	/*
+	 * Overlay gets an aggressive default since video jitter is bad.
+	 */
+	cwm = 2;
+
+	/* Play safe and disable self-refresh before adjusting watermarks. */
+	if (IS_I945G(dev) || IS_I945GM(dev))
+		I915_WRITE(FW_BLC_SELF, FW_BLC_SELF_EN_MASK | 0);
+	else if (IS_I915GM(dev))
+		I915_WRITE(INSTPM, I915_READ(INSTPM) & ~INSTPM_SELF_EN);
+
+	/* Calc sr entries for one plane configs */
+	if (HAS_FW_BLC(dev) && enabled) {
+		/* self-refresh has much higher latency */
+		static const int sr_latency_ns = 6000;
+		int clock = enabled->mode.clock;
+		int htotal = enabled->mode.htotal;
+		int hdisplay = enabled->mode.hdisplay;
+		int pixel_size = enabled->fb->bits_per_pixel / 8;
+		unsigned long line_time_us;
+		int entries;
+
+		line_time_us = (htotal * 1000) / clock;
+
+		/* Use ns/us then divide to preserve precision */
+		entries = (((sr_latency_ns / line_time_us) + 1000) / 1000) *
+			pixel_size * hdisplay;
+		entries = DIV_ROUND_UP(entries, wm_info->cacheline_size);
+		DRM_DEBUG_KMS("self-refresh entries: %d\n", entries);
+		srwm = wm_info->fifo_size - entries;
+		if (srwm < 0)
+			srwm = 1;
+
+		if (IS_I945G(dev) || IS_I945GM(dev))
+			I915_WRITE(FW_BLC_SELF,
+				   FW_BLC_SELF_FIFO_MASK | (srwm & 0xff));
+		else if (IS_I915GM(dev))
+			I915_WRITE(FW_BLC_SELF, srwm & 0x3f);
+	}
+
+	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d, B: %d, C: %d, SR %d\n",
+		      planea_wm, planeb_wm, cwm, srwm);
+
+	fwater_lo = ((planeb_wm & 0x3f) << 16) | (planea_wm & 0x3f);
+	fwater_hi = (cwm & 0x1f);
+
+	/* Set request length to 8 cachelines per fetch */
+	fwater_lo = fwater_lo | (1 << 24) | (1 << 8);
+	fwater_hi = fwater_hi | (1 << 8);
+
+	I915_WRITE(FW_BLC, fwater_lo);
+	I915_WRITE(FW_BLC2, fwater_hi);
+
+	if (HAS_FW_BLC(dev)) {
+		if (enabled) {
+			if (IS_I945G(dev) || IS_I945GM(dev))
+				I915_WRITE(FW_BLC_SELF,
+					   FW_BLC_SELF_EN_MASK | FW_BLC_SELF_EN);
+			else if (IS_I915GM(dev))
+				I915_WRITE(INSTPM, I915_READ(INSTPM) | INSTPM_SELF_EN);
+			DRM_DEBUG_KMS("memory self refresh enabled\n");
+		} else
+			DRM_DEBUG_KMS("memory self refresh disabled\n");
+	}
+}
+
+static void i830_update_wm(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct drm_crtc *crtc;
+	uint32_t fwater_lo;
+	int planea_wm;
+
+	crtc = single_enabled_crtc(dev);
+	if (crtc == NULL)
+		return;
+
+	planea_wm = intel_calculate_wm(crtc->mode.clock, &i830_wm_info,
+				       dev_priv->display.get_fifo_size(dev, 0),
+				       crtc->fb->bits_per_pixel / 8,
+				       latency_ns);
+	fwater_lo = I915_READ(FW_BLC) & ~0xfff;
+	fwater_lo |= (3<<8) | planea_wm;
+
+	DRM_DEBUG_KMS("Setting FIFO watermarks - A: %d\n", planea_wm);
+
+	I915_WRITE(FW_BLC, fwater_lo);
+}
+
+#define ILK_LP0_PLANE_LATENCY		700
+#define ILK_LP0_CURSOR_LATENCY		1300
+
+/*
+ * Check the wm result.
+ *
+ * If any calculated watermark values is larger than the maximum value that
+ * can be programmed into the associated watermark register, that watermark
+ * must be disabled.
+ */
+static bool ironlake_check_srwm(struct drm_device *dev, int level,
+				int fbc_wm, int display_wm, int cursor_wm,
+				const struct intel_watermark_params *display,
+				const struct intel_watermark_params *cursor)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	DRM_DEBUG_KMS("watermark %d: display plane %d, fbc lines %d,"
+		      " cursor %d\n", level, display_wm, fbc_wm, cursor_wm);
+
+	if (fbc_wm > SNB_FBC_MAX_SRWM) {
+		DRM_DEBUG_KMS("fbc watermark(%d) is too large(%d), disabling wm%d+\n",
+			      fbc_wm, SNB_FBC_MAX_SRWM, level);
+
+		/* fbc has it's own way to disable FBC WM */
+		I915_WRITE(DISP_ARB_CTL,
+			   I915_READ(DISP_ARB_CTL) | DISP_FBC_WM_DIS);
+		return false;
+	}
+
+	if (display_wm > display->max_wm) {
+		DRM_DEBUG_KMS("display watermark(%d) is too large(%d), disabling wm%d+\n",
+			      display_wm, SNB_DISPLAY_MAX_SRWM, level);
+		return false;
+	}
+
+	if (cursor_wm > cursor->max_wm) {
+		DRM_DEBUG_KMS("cursor watermark(%d) is too large(%d), disabling wm%d+\n",
+			      cursor_wm, SNB_CURSOR_MAX_SRWM, level);
+		return false;
+	}
+
+	if (!(fbc_wm || display_wm || cursor_wm)) {
+		DRM_DEBUG_KMS("latency %d is 0, disabling wm%d+\n", level, level);
+		return false;
+	}
+
+	return true;
+}
+
+/*
+ * Compute watermark values of WM[1-3],
+ */
+static bool ironlake_compute_srwm(struct drm_device *dev, int level, int plane,
+				  int latency_ns,
+				  const struct intel_watermark_params *display,
+				  const struct intel_watermark_params *cursor,
+				  int *fbc_wm, int *display_wm, int *cursor_wm)
+{
+	struct drm_crtc *crtc;
+	unsigned long line_time_us;
+	int hdisplay, htotal, pixel_size, clock;
+	int line_count, line_size;
+	int small, large;
+	int entries;
+
+	if (!latency_ns) {
+		*fbc_wm = *display_wm = *cursor_wm = 0;
+		return false;
+	}
+
+	crtc = intel_get_crtc_for_plane(dev, plane);
+	hdisplay = crtc->mode.hdisplay;
+	htotal = crtc->mode.htotal;
+	clock = crtc->mode.clock;
+	pixel_size = crtc->fb->bits_per_pixel / 8;
+
+	line_time_us = (htotal * 1000) / clock;
+	line_count = (latency_ns / line_time_us + 1000) / 1000;
+	line_size = hdisplay * pixel_size;
+
+	/* Use the minimum of the small and large buffer method for primary */
+	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
+	large = line_count * line_size;
+
+	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
+	*display_wm = entries + display->guard_size;
+
+	/*
+	 * Spec says:
+	 * FBC WM = ((Final Primary WM * 64) / number of bytes per line) + 2
+	 */
+	*fbc_wm = DIV_ROUND_UP(*display_wm * 64, line_size) + 2;
+
+	/* calculate the self-refresh watermark for display cursor */
+	entries = line_count * pixel_size * 64;
+	entries = DIV_ROUND_UP(entries, cursor->cacheline_size);
+	*cursor_wm = entries + cursor->guard_size;
+
+	return ironlake_check_srwm(dev, level,
+				   *fbc_wm, *display_wm, *cursor_wm,
+				   display, cursor);
+}
+
+static void ironlake_update_wm(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int fbc_wm, plane_wm, cursor_wm;
+	unsigned int enabled;
+
+	enabled = 0;
+	if (g4x_compute_wm0(dev, 0,
+			    &ironlake_display_wm_info,
+			    ILK_LP0_PLANE_LATENCY,
+			    &ironlake_cursor_wm_info,
+			    ILK_LP0_CURSOR_LATENCY,
+			    &plane_wm, &cursor_wm)) {
+		I915_WRITE(WM0_PIPEA_ILK,
+			   (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
+		DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
+			      " plane %d, " "cursor: %d\n",
+			      plane_wm, cursor_wm);
+		enabled |= 1;
+	}
+
+	if (g4x_compute_wm0(dev, 1,
+			    &ironlake_display_wm_info,
+			    ILK_LP0_PLANE_LATENCY,
+			    &ironlake_cursor_wm_info,
+			    ILK_LP0_CURSOR_LATENCY,
+			    &plane_wm, &cursor_wm)) {
+		I915_WRITE(WM0_PIPEB_ILK,
+			   (plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm);
+		DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
+			      " plane %d, cursor: %d\n",
+			      plane_wm, cursor_wm);
+		enabled |= 2;
+	}
+
+	/*
+	 * Calculate and update the self-refresh watermark only when one
+	 * display plane is used.
+	 */
+	I915_WRITE(WM3_LP_ILK, 0);
+	I915_WRITE(WM2_LP_ILK, 0);
+	I915_WRITE(WM1_LP_ILK, 0);
+
+	if (!single_plane_enabled(enabled))
+		return;
+	enabled = ffs(enabled) - 1;
+
+	/* WM1 */
+	if (!ironlake_compute_srwm(dev, 1, enabled,
+				   ILK_READ_WM1_LATENCY() * 500,
+				   &ironlake_display_srwm_info,
+				   &ironlake_cursor_srwm_info,
+				   &fbc_wm, &plane_wm, &cursor_wm))
+		return;
+
+	I915_WRITE(WM1_LP_ILK,
+		   WM1_LP_SR_EN |
+		   (ILK_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+		   (fbc_wm << WM1_LP_FBC_SHIFT) |
+		   (plane_wm << WM1_LP_SR_SHIFT) |
+		   cursor_wm);
+
+	/* WM2 */
+	if (!ironlake_compute_srwm(dev, 2, enabled,
+				   ILK_READ_WM2_LATENCY() * 500,
+				   &ironlake_display_srwm_info,
+				   &ironlake_cursor_srwm_info,
+				   &fbc_wm, &plane_wm, &cursor_wm))
+		return;
+
+	I915_WRITE(WM2_LP_ILK,
+		   WM2_LP_EN |
+		   (ILK_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+		   (fbc_wm << WM1_LP_FBC_SHIFT) |
+		   (plane_wm << WM1_LP_SR_SHIFT) |
+		   cursor_wm);
+
+	/*
+	 * WM3 is unsupported on ILK, probably because we don't have latency
+	 * data for that power state
+	 */
+}
+
+static void sandybridge_update_wm(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int latency = SNB_READ_WM0_LATENCY() * 100;	/* In unit 0.1us */
+	u32 val;
+	int fbc_wm, plane_wm, cursor_wm;
+	unsigned int enabled;
+
+	enabled = 0;
+	if (g4x_compute_wm0(dev, 0,
+			    &sandybridge_display_wm_info, latency,
+			    &sandybridge_cursor_wm_info, latency,
+			    &plane_wm, &cursor_wm)) {
+		val = I915_READ(WM0_PIPEA_ILK);
+		val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
+		I915_WRITE(WM0_PIPEA_ILK, val |
+			   ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
+		DRM_DEBUG_KMS("FIFO watermarks For pipe A -"
+			      " plane %d, " "cursor: %d\n",
+			      plane_wm, cursor_wm);
+		enabled |= 1;
+	}
+
+	if (g4x_compute_wm0(dev, 1,
+			    &sandybridge_display_wm_info, latency,
+			    &sandybridge_cursor_wm_info, latency,
+			    &plane_wm, &cursor_wm)) {
+		val = I915_READ(WM0_PIPEB_ILK);
+		val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
+		I915_WRITE(WM0_PIPEB_ILK, val |
+			   ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
+		DRM_DEBUG_KMS("FIFO watermarks For pipe B -"
+			      " plane %d, cursor: %d\n",
+			      plane_wm, cursor_wm);
+		enabled |= 2;
+	}
+
+	if ((dev_priv->num_pipe == 3) &&
+	    g4x_compute_wm0(dev, 2,
+			    &sandybridge_display_wm_info, latency,
+			    &sandybridge_cursor_wm_info, latency,
+			    &plane_wm, &cursor_wm)) {
+		val = I915_READ(WM0_PIPEC_IVB);
+		val &= ~(WM0_PIPE_PLANE_MASK | WM0_PIPE_CURSOR_MASK);
+		I915_WRITE(WM0_PIPEC_IVB, val |
+			   ((plane_wm << WM0_PIPE_PLANE_SHIFT) | cursor_wm));
+		DRM_DEBUG_KMS("FIFO watermarks For pipe C -"
+			      " plane %d, cursor: %d\n",
+			      plane_wm, cursor_wm);
+		enabled |= 3;
+	}
+
+	/*
+	 * Calculate and update the self-refresh watermark only when one
+	 * display plane is used.
+	 *
+	 * SNB support 3 levels of watermark.
+	 *
+	 * WM1/WM2/WM2 watermarks have to be enabled in the ascending order,
+	 * and disabled in the descending order
+	 *
+	 */
+	I915_WRITE(WM3_LP_ILK, 0);
+	I915_WRITE(WM2_LP_ILK, 0);
+	I915_WRITE(WM1_LP_ILK, 0);
+
+	if (!single_plane_enabled(enabled) ||
+	    dev_priv->sprite_scaling_enabled)
+		return;
+	enabled = ffs(enabled) - 1;
+
+	/* WM1 */
+	if (!ironlake_compute_srwm(dev, 1, enabled,
+				   SNB_READ_WM1_LATENCY() * 500,
+				   &sandybridge_display_srwm_info,
+				   &sandybridge_cursor_srwm_info,
+				   &fbc_wm, &plane_wm, &cursor_wm))
+		return;
+
+	I915_WRITE(WM1_LP_ILK,
+		   WM1_LP_SR_EN |
+		   (SNB_READ_WM1_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+		   (fbc_wm << WM1_LP_FBC_SHIFT) |
+		   (plane_wm << WM1_LP_SR_SHIFT) |
+		   cursor_wm);
+
+	/* WM2 */
+	if (!ironlake_compute_srwm(dev, 2, enabled,
+				   SNB_READ_WM2_LATENCY() * 500,
+				   &sandybridge_display_srwm_info,
+				   &sandybridge_cursor_srwm_info,
+				   &fbc_wm, &plane_wm, &cursor_wm))
+		return;
+
+	I915_WRITE(WM2_LP_ILK,
+		   WM2_LP_EN |
+		   (SNB_READ_WM2_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+		   (fbc_wm << WM1_LP_FBC_SHIFT) |
+		   (plane_wm << WM1_LP_SR_SHIFT) |
+		   cursor_wm);
+
+	/* WM3 */
+	if (!ironlake_compute_srwm(dev, 3, enabled,
+				   SNB_READ_WM3_LATENCY() * 500,
+				   &sandybridge_display_srwm_info,
+				   &sandybridge_cursor_srwm_info,
+				   &fbc_wm, &plane_wm, &cursor_wm))
+		return;
+
+	I915_WRITE(WM3_LP_ILK,
+		   WM3_LP_EN |
+		   (SNB_READ_WM3_LATENCY() << WM1_LP_LATENCY_SHIFT) |
+		   (fbc_wm << WM1_LP_FBC_SHIFT) |
+		   (plane_wm << WM1_LP_SR_SHIFT) |
+		   cursor_wm);
+}
+
+static void
+haswell_update_linetime_wm(struct drm_device *dev, int pipe,
+				 struct drm_display_mode *mode)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 temp;
+
+	temp = I915_READ(PIPE_WM_LINETIME(pipe));
+	temp &= ~PIPE_WM_LINETIME_MASK;
+
+	/* The WM are computed with base on how long it takes to fill a single
+	 * row at the given clock rate, multiplied by 8.
+	 * */
+	temp |= PIPE_WM_LINETIME_TIME(
+		((mode->crtc_hdisplay * 1000) / mode->clock) * 8);
+
+	/* IPS watermarks are only used by pipe A, and are ignored by
+	 * pipes B and C.  They are calculated similarly to the common
+	 * linetime values, except that we are using CD clock frequency
+	 * in MHz instead of pixel rate for the division.
+	 *
+	 * This is a placeholder for the IPS watermark calculation code.
+	 */
+
+	I915_WRITE(PIPE_WM_LINETIME(pipe), temp);
+}
+
+static bool
+sandybridge_compute_sprite_wm(struct drm_device *dev, int plane,
+			      uint32_t sprite_width, int pixel_size,
+			      const struct intel_watermark_params *display,
+			      int display_latency_ns, int *sprite_wm)
+{
+	struct drm_crtc *crtc;
+	int clock;
+	int entries, tlb_miss;
+
+	crtc = intel_get_crtc_for_plane(dev, plane);
+	if (crtc->fb == NULL || !crtc->enabled) {
+		*sprite_wm = display->guard_size;
+		return false;
+	}
+
+	clock = crtc->mode.clock;
+
+	/* Use the small buffer method to calculate the sprite watermark */
+	entries = ((clock * pixel_size / 1000) * display_latency_ns) / 1000;
+	tlb_miss = display->fifo_size*display->cacheline_size -
+		sprite_width * 8;
+	if (tlb_miss > 0)
+		entries += tlb_miss;
+	entries = DIV_ROUND_UP(entries, display->cacheline_size);
+	*sprite_wm = entries + display->guard_size;
+	if (*sprite_wm > (int)display->max_wm)
+		*sprite_wm = display->max_wm;
+
+	return true;
+}
+
+static bool
+sandybridge_compute_sprite_srwm(struct drm_device *dev, int plane,
+				uint32_t sprite_width, int pixel_size,
+				const struct intel_watermark_params *display,
+				int latency_ns, int *sprite_wm)
+{
+	struct drm_crtc *crtc;
+	unsigned long line_time_us;
+	int clock;
+	int line_count, line_size;
+	int small, large;
+	int entries;
+
+	if (!latency_ns) {
+		*sprite_wm = 0;
+		return false;
+	}
+
+	crtc = intel_get_crtc_for_plane(dev, plane);
+	clock = crtc->mode.clock;
+	if (!clock) {
+		*sprite_wm = 0;
+		return false;
+	}
+
+	line_time_us = (sprite_width * 1000) / clock;
+	if (!line_time_us) {
+		*sprite_wm = 0;
+		return false;
+	}
+
+	line_count = (latency_ns / line_time_us + 1000) / 1000;
+	line_size = sprite_width * pixel_size;
+
+	/* Use the minimum of the small and large buffer method for primary */
+	small = ((clock * pixel_size / 1000) * latency_ns) / 1000;
+	large = line_count * line_size;
+
+	entries = DIV_ROUND_UP(min(small, large), display->cacheline_size);
+	*sprite_wm = entries + display->guard_size;
+
+	return *sprite_wm > 0x3ff ? false : true;
+}
+
+static void sandybridge_update_sprite_wm(struct drm_device *dev, int pipe,
+					 uint32_t sprite_width, int pixel_size)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int latency = SNB_READ_WM0_LATENCY() * 100;	/* In unit 0.1us */
+	u32 val;
+	int sprite_wm, reg;
+	int ret;
+
+	switch (pipe) {
+	case 0:
+		reg = WM0_PIPEA_ILK;
+		break;
+	case 1:
+		reg = WM0_PIPEB_ILK;
+		break;
+	case 2:
+		reg = WM0_PIPEC_IVB;
+		break;
+	default:
+		return; /* bad pipe */
+	}
+
+	ret = sandybridge_compute_sprite_wm(dev, pipe, sprite_width, pixel_size,
+					    &sandybridge_display_wm_info,
+					    latency, &sprite_wm);
+	if (!ret) {
+		DRM_DEBUG_KMS("failed to compute sprite wm for pipe %d\n",
+			      pipe);
+		return;
+	}
+
+	val = I915_READ(reg);
+	val &= ~WM0_PIPE_SPRITE_MASK;
+	I915_WRITE(reg, val | (sprite_wm << WM0_PIPE_SPRITE_SHIFT));
+	DRM_DEBUG_KMS("sprite watermarks For pipe %d - %d\n", pipe, sprite_wm);
+
+
+	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
+					      pixel_size,
+					      &sandybridge_display_srwm_info,
+					      SNB_READ_WM1_LATENCY() * 500,
+					      &sprite_wm);
+	if (!ret) {
+		DRM_DEBUG_KMS("failed to compute sprite lp1 wm on pipe %d\n",
+			      pipe);
+		return;
+	}
+	I915_WRITE(WM1S_LP_ILK, sprite_wm);
+
+	/* Only IVB has two more LP watermarks for sprite */
+	if (!IS_IVYBRIDGE(dev))
+		return;
+
+	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
+					      pixel_size,
+					      &sandybridge_display_srwm_info,
+					      SNB_READ_WM2_LATENCY() * 500,
+					      &sprite_wm);
+	if (!ret) {
+		DRM_DEBUG_KMS("failed to compute sprite lp2 wm on pipe %d\n",
+			      pipe);
+		return;
+	}
+	I915_WRITE(WM2S_LP_IVB, sprite_wm);
+
+	ret = sandybridge_compute_sprite_srwm(dev, pipe, sprite_width,
+					      pixel_size,
+					      &sandybridge_display_srwm_info,
+					      SNB_READ_WM3_LATENCY() * 500,
+					      &sprite_wm);
+	if (!ret) {
+		DRM_DEBUG_KMS("failed to compute sprite lp3 wm on pipe %d\n",
+			      pipe);
+		return;
+	}
+	I915_WRITE(WM3S_LP_IVB, sprite_wm);
+}
+
+/**
+ * intel_update_watermarks - update FIFO watermark values based on current modes
+ *
+ * Calculate watermark values for the various WM regs based on current mode
+ * and plane configuration.
+ *
+ * There are several cases to deal with here:
+ *   - normal (i.e. non-self-refresh)
+ *   - self-refresh (SR) mode
+ *   - lines are large relative to FIFO size (buffer can hold up to 2)
+ *   - lines are small relative to FIFO size (buffer can hold more than 2
+ *     lines), so need to account for TLB latency
+ *
+ *   The normal calculation is:
+ *     watermark = dotclock * bytes per pixel * latency
+ *   where latency is platform & configuration dependent (we assume pessimal
+ *   values here).
+ *
+ *   The SR calculation is:
+ *     watermark = (trunc(latency/line time)+1) * surface width *
+ *       bytes per pixel
+ *   where
+ *     line time = htotal / dotclock
+ *     surface width = hdisplay for normal plane and 64 for cursor
+ *   and latency is assumed to be high, as above.
+ *
+ * The final value programmed to the register should always be rounded up,
+ * and include an extra 2 entries to account for clock crossings.
+ *
+ * We don't use the sprite, so we can ignore that.  And on Crestline we have
+ * to set the non-SR watermarks to 8.
+ */
+void intel_update_watermarks(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (dev_priv->display.update_wm)
+		dev_priv->display.update_wm(dev);
+}
+
+void intel_update_linetime_watermarks(struct drm_device *dev,
+		int pipe, struct drm_display_mode *mode)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (dev_priv->display.update_linetime_wm)
+		dev_priv->display.update_linetime_wm(dev, pipe, mode);
+}
+
+void intel_update_sprite_watermarks(struct drm_device *dev, int pipe,
+				    uint32_t sprite_width, int pixel_size)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (dev_priv->display.update_sprite_wm)
+		dev_priv->display.update_sprite_wm(dev, pipe, sprite_width,
+						   pixel_size);
+}
+
+static struct drm_i915_gem_object *
+intel_alloc_context_page(struct drm_device *dev)
+{
+	struct drm_i915_gem_object *ctx;
+	int ret;
+
+	DRM_LOCK_ASSERT(dev);
+
+	ctx = i915_gem_alloc_object(dev, 4096);
+	if (!ctx) {
+		DRM_DEBUG("failed to alloc power context, RC6 disabled\n");
+		return NULL;
+	}
+
+	ret = i915_gem_object_pin(ctx, 4096, true);
+	if (ret) {
+		DRM_ERROR("failed to pin power context: %d\n", ret);
+		goto err_unref;
+	}
+
+	ret = i915_gem_object_set_to_gtt_domain(ctx, 1);
+	if (ret) {
+		DRM_ERROR("failed to set-domain on power context: %d\n", ret);
+		goto err_unpin;
+	}
+
+	return ctx;
+
+err_unpin:
+	i915_gem_object_unpin(ctx);
+err_unref:
+	drm_gem_object_unreference(&ctx->base);
+	DRM_UNLOCK(dev);
+	return NULL;
+}
+
+bool ironlake_set_drps(struct drm_device *dev, u8 val)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u16 rgvswctl;
+
+	rgvswctl = I915_READ16(MEMSWCTL);
+	if (rgvswctl & MEMCTL_CMD_STS) {
+		DRM_DEBUG("gpu busy, RCS change rejected\n");
+		return false; /* still busy with another command */
+	}
+
+	rgvswctl = (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) |
+		(val << MEMCTL_FREQ_SHIFT) | MEMCTL_SFCAVM;
+	I915_WRITE16(MEMSWCTL, rgvswctl);
+	POSTING_READ16(MEMSWCTL);
+
+	rgvswctl |= MEMCTL_CMD_STS;
+	I915_WRITE16(MEMSWCTL, rgvswctl);
+
+	return true;
+}
+
+void ironlake_enable_drps(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 rgvmodectl = I915_READ(MEMMODECTL);
+	u8 fmax, fmin, fstart, vstart;
+
+	/* Enable temp reporting */
+	I915_WRITE16(PMMISC, I915_READ(PMMISC) | MCPPCE_EN);
+	I915_WRITE16(TSC1, I915_READ(TSC1) | TSE);
+
+	/* 100ms RC evaluation intervals */
+	I915_WRITE(RCUPEI, 100000);
+	I915_WRITE(RCDNEI, 100000);
+
+	/* Set max/min thresholds to 90ms and 80ms respectively */
+	I915_WRITE(RCBMAXAVG, 90000);
+	I915_WRITE(RCBMINAVG, 80000);
+
+	I915_WRITE(MEMIHYST, 1);
+
+	/* Set up min, max, and cur for interrupt handling */
+	fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT;
+	fmin = (rgvmodectl & MEMMODE_FMIN_MASK);
+	fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >>
+		MEMMODE_FSTART_SHIFT;
+
+	vstart = (I915_READ(PXVFREQ_BASE + (fstart * 4)) & PXVFREQ_PX_MASK) >>
+		PXVFREQ_PX_SHIFT;
+
+	dev_priv->fmax = fmax; /* IPS callback will increase this */
+	dev_priv->fstart = fstart;
+
+	dev_priv->max_delay = fstart;
+	dev_priv->min_delay = fmin;
+	dev_priv->cur_delay = fstart;
+
+	DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n",
+			 fmax, fmin, fstart);
+
+	I915_WRITE(MEMINTREN, MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN);
+
+	/*
+	 * Interrupts will be enabled in ironlake_irq_postinstall
+	 */
+
+	I915_WRITE(VIDSTART, vstart);
+	POSTING_READ(VIDSTART);
+
+	rgvmodectl |= MEMMODE_SWMODE_EN;
+	I915_WRITE(MEMMODECTL, rgvmodectl);
+
+	if (wait_for((I915_READ(MEMSWCTL) & MEMCTL_CMD_STS) == 0, 10))
+		DRM_ERROR("stuck trying to change perf mode\n");
+	pause("915dsp", 1);
+
+	ironlake_set_drps(dev, fstart);
+
+	dev_priv->last_count1 = I915_READ(0x112e4) + I915_READ(0x112e8) +
+		I915_READ(0x112e0);
+	dev_priv->last_time1 = jiffies_to_msecs(jiffies);
+	dev_priv->last_count2 = I915_READ(0x112f4);
+	nanotime(&dev_priv->last_time2);
+}
+
+void ironlake_disable_drps(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u16 rgvswctl = I915_READ16(MEMSWCTL);
+
+	/* Ack interrupts, disable EFC interrupt */
+	I915_WRITE(MEMINTREN, I915_READ(MEMINTREN) & ~MEMINT_EVAL_CHG_EN);
+	I915_WRITE(MEMINTRSTS, MEMINT_EVAL_CHG);
+	I915_WRITE(DEIER, I915_READ(DEIER) & ~DE_PCU_EVENT);
+	I915_WRITE(DEIIR, DE_PCU_EVENT);
+	I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
+
+	/* Go back to the starting frequency */
+	ironlake_set_drps(dev, dev_priv->fstart);
+	pause("915dsp", 1);
+	rgvswctl |= MEMCTL_CMD_STS;
+	I915_WRITE(MEMSWCTL, rgvswctl);
+	pause("915dsp", 1);
+
+}
+
+void gen6_set_rps(struct drm_device *dev, u8 val)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 swreq;
+
+	swreq = (val & 0x3ff) << 25;
+	I915_WRITE(GEN6_RPNSWREQ, swreq);
+}
+
+void gen6_disable_rps(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
+	I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
+	I915_WRITE(GEN6_PMIER, 0);
+	/* Complete PM interrupt masking here doesn't race with the rps work
+	 * item again unmasking PM interrupts because that is using a different
+	 * register (PMIMR) to mask PM interrupts. The only risk is in leaving
+	 * stale bits in PMIIR and PMIMR which gen6_enable_rps will clean up. */
+
+	mtx_lock(&dev_priv->rps_lock);
+	dev_priv->pm_iir = 0;
+	mtx_unlock(&dev_priv->rps_lock);
+
+	I915_WRITE(GEN6_PMIIR, I915_READ(GEN6_PMIIR));
+}
+
+int intel_enable_rc6(const struct drm_device *dev)
+{
+	/*
+	 * Respect the kernel parameter if it is set
+	 */
+	if (i915_enable_rc6 >= 0)
+		return i915_enable_rc6;
+
+	/*
+	 * Disable RC6 on Ironlake
+	 */
+	if (INTEL_INFO(dev)->gen == 5)
+		return 0;
+
+	/* Sorry Haswell, no RC6 for you for now. */
+	if (IS_HASWELL(dev))
+		return 0;
+
+	/*
+	 * Disable rc6 on Sandybridge
+	 */
+	if (INTEL_INFO(dev)->gen == 6) {
+		DRM_DEBUG_DRIVER("Sandybridge: deep RC6 disabled\n");
+		return INTEL_RC6_ENABLE;
+	}
+	DRM_DEBUG_DRIVER("RC6 and deep RC6 enabled\n");
+	return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
+}
+
+void gen6_enable_rps(struct drm_i915_private *dev_priv)
+{
+	struct intel_ring_buffer *ring;
+	u32 rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
+	u32 gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
+	u32 pcu_mbox, rc6_mask = 0;
+	u32 gtfifodbg;
+	int cur_freq, min_freq, max_freq;
+	int rc6_mode;
+	int i;
+
+	/* Here begins a magic sequence of register writes to enable
+	 * auto-downclocking.
+	 *
+	 * Perhaps there might be some value in exposing these to
+	 * userspace...
+	 */
+	I915_WRITE(GEN6_RC_STATE, 0);
+	DRM_LOCK(dev_priv->dev);
+
+	/* Clear the DBG now so we don't confuse earlier errors */
+	if ((gtfifodbg = I915_READ(GTFIFODBG))) {
+		DRM_ERROR("GT fifo had a previous error %x\n", gtfifodbg);
+		I915_WRITE(GTFIFODBG, gtfifodbg);
+	}
+
+	gen6_gt_force_wake_get(dev_priv);
+
+	/* disable the counters and set deterministic thresholds */
+	I915_WRITE(GEN6_RC_CONTROL, 0);
+
+	I915_WRITE(GEN6_RC1_WAKE_RATE_LIMIT, 1000 << 16);
+	I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16 | 30);
+	I915_WRITE(GEN6_RC6pp_WAKE_RATE_LIMIT, 30);
+	I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000);
+	I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25);
+
+	for_each_ring(ring, dev_priv, i)
+		I915_WRITE(RING_MAX_IDLE(ring->mmio_base), 10);
+
+	I915_WRITE(GEN6_RC_SLEEP, 0);
+	I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
+	I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
+	I915_WRITE(GEN6_RC6p_THRESHOLD, 100000);
+	I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
+
+	rc6_mode = intel_enable_rc6(dev_priv->dev);
+	if (rc6_mode & INTEL_RC6_ENABLE)
+		rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
+
+	if (rc6_mode & INTEL_RC6p_ENABLE)
+		rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
+
+	if (rc6_mode & INTEL_RC6pp_ENABLE)
+		rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
+
+	DRM_INFO("Enabling RC6 states: RC6 %s, RC6p %s, RC6pp %s\n",
+			(rc6_mode & INTEL_RC6_ENABLE) ? "on" : "off",
+			(rc6_mode & INTEL_RC6p_ENABLE) ? "on" : "off",
+			(rc6_mode & INTEL_RC6pp_ENABLE) ? "on" : "off");
+
+	I915_WRITE(GEN6_RC_CONTROL,
+		   rc6_mask |
+		   GEN6_RC_CTL_EI_MODE(1) |
+		   GEN6_RC_CTL_HW_ENABLE);
+
+	I915_WRITE(GEN6_RPNSWREQ,
+		   GEN6_FREQUENCY(10) |
+		   GEN6_OFFSET(0) |
+		   GEN6_AGGRESSIVE_TURBO);
+	I915_WRITE(GEN6_RC_VIDEO_FREQ,
+		   GEN6_FREQUENCY(12));
+
+	I915_WRITE(GEN6_RP_DOWN_TIMEOUT, 1000000);
+	I915_WRITE(GEN6_RP_INTERRUPT_LIMITS,
+		   18 << 24 |
+		   6 << 16);
+	I915_WRITE(GEN6_RP_UP_THRESHOLD, 10000);
+	I915_WRITE(GEN6_RP_DOWN_THRESHOLD, 1000000);
+	I915_WRITE(GEN6_RP_UP_EI, 100000);
+	I915_WRITE(GEN6_RP_DOWN_EI, 5000000);
+	I915_WRITE(GEN6_RP_IDLE_HYSTERSIS, 10);
+	I915_WRITE(GEN6_RP_CONTROL,
+		   GEN6_RP_MEDIA_TURBO |
+		   GEN6_RP_MEDIA_HW_MODE |
+		   GEN6_RP_MEDIA_IS_GFX |
+		   GEN6_RP_ENABLE |
+		   GEN6_RP_UP_BUSY_AVG |
+		   GEN6_RP_DOWN_IDLE_CONT);
+
+	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+		     500))
+		DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
+
+	I915_WRITE(GEN6_PCODE_DATA, 0);
+	I915_WRITE(GEN6_PCODE_MAILBOX,
+		   GEN6_PCODE_READY |
+		   GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
+	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+		     500))
+		DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
+
+	min_freq = (rp_state_cap & 0xff0000) >> 16;
+	max_freq = rp_state_cap & 0xff;
+	cur_freq = (gt_perf_status & 0xff00) >> 8;
+
+	/* Check for overclock support */
+	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+		     500))
+		DRM_ERROR("timeout waiting for pcode mailbox to become idle\n");
+	I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_READ_OC_PARAMS);
+	pcu_mbox = I915_READ(GEN6_PCODE_DATA);
+	if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
+		     500))
+		DRM_ERROR("timeout waiting for pcode mailbox to finish\n");
+	if (pcu_mbox & (1<<31)) { /* OC supported */
+		max_freq = pcu_mbox & 0xff;
+		DRM_DEBUG_DRIVER("overclocking supported, adjusting frequency max to %dMHz\n", pcu_mbox * 50);
+	}
+
+	/* In units of 100MHz */
+	dev_priv->max_delay = max_freq;
+	dev_priv->min_delay = min_freq;
+	dev_priv->cur_delay = cur_freq;
+
+	/* requires MSI enabled */
+	I915_WRITE(GEN6_PMIER,
+		   GEN6_PM_MBOX_EVENT |
+		   GEN6_PM_THERMAL_EVENT |
+		   GEN6_PM_RP_DOWN_TIMEOUT |
+		   GEN6_PM_RP_UP_THRESHOLD |
+		   GEN6_PM_RP_DOWN_THRESHOLD |
+		   GEN6_PM_RP_UP_EI_EXPIRED |
+		   GEN6_PM_RP_DOWN_EI_EXPIRED);
+	mtx_lock(&dev_priv->rps_lock);
+	if (dev_priv->pm_iir != 0)
+		printf("KMS: pm_iir %x\n", dev_priv->pm_iir);
+	I915_WRITE(GEN6_PMIMR, 0);
+	mtx_unlock(&dev_priv->rps_lock);
+	/* enable all PM interrupts */
+	I915_WRITE(GEN6_PMINTRMSK, 0);
+
+	gen6_gt_force_wake_put(dev_priv);
+	DRM_UNLOCK(dev_priv->dev);
+}
+
+void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
+{
+	int min_freq = 15;
+	int gpu_freq, ia_freq, max_ia_freq;
+	int scaling_factor = 180;
+	uint64_t tsc_freq;
+
+#if 0
+	max_ia_freq = cpufreq_quick_get_max(0);
+	/*
+	 * Default to measured freq if none found, PCU will ensure we don't go
+	 * over
+	 */
+	if (!max_ia_freq)
+		max_ia_freq = tsc_khz;
+
+	/* Convert from kHz to MHz */
+	max_ia_freq /= 1000;
+#else
+	tsc_freq = atomic_load_acq_64(&tsc_freq);
+	max_ia_freq = tsc_freq / 1000 / 1000;
+#endif
+
+	DRM_LOCK(dev_priv->dev);
+
+	/*
+	 * For each potential GPU frequency, load a ring frequency we'd like
+	 * to use for memory access.  We do this by specifying the IA frequency
+	 * the PCU should use as a reference to determine the ring frequency.
+	 */
+	for (gpu_freq = dev_priv->max_delay; gpu_freq >= dev_priv->min_delay;
+	     gpu_freq--) {
+		int diff = dev_priv->max_delay - gpu_freq;
+		int d;
+
+		/*
+		 * For GPU frequencies less than 750MHz, just use the lowest
+		 * ring freq.
+		 */
+		if (gpu_freq < min_freq)
+			ia_freq = 800;
+		else
+			ia_freq = max_ia_freq - ((diff * scaling_factor) / 2);
+		d = 100;
+		ia_freq = (ia_freq + d / 2) / d;
+
+		I915_WRITE(GEN6_PCODE_DATA,
+			   (ia_freq << GEN6_PCODE_FREQ_IA_RATIO_SHIFT) |
+			   gpu_freq);
+		I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY |
+			   GEN6_PCODE_WRITE_MIN_FREQ_TABLE);
+		if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) &
+			      GEN6_PCODE_READY) == 0, 10)) {
+			DRM_ERROR("pcode write of freq table timed out\n");
+			continue;
+		}
+	}
+
+	DRM_UNLOCK(dev_priv->dev);
+}
+
+static void ironlake_teardown_rc6(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (dev_priv->renderctx) {
+		i915_gem_object_unpin(dev_priv->renderctx);
+		drm_gem_object_unreference(&dev_priv->renderctx->base);
+		dev_priv->renderctx = NULL;
+	}
+
+	if (dev_priv->pwrctx) {
+		i915_gem_object_unpin(dev_priv->pwrctx);
+		drm_gem_object_unreference(&dev_priv->pwrctx->base);
+		dev_priv->pwrctx = NULL;
+	}
+}
+
+void ironlake_disable_rc6(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (I915_READ(PWRCTXA)) {
+		/* Wake the GPU, prevent RC6, then restore RSTDBYCTL */
+		I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) | RCX_SW_EXIT);
+		wait_for(((I915_READ(RSTDBYCTL) & RSX_STATUS_MASK) == RSX_STATUS_ON),
+			 50);
+
+		I915_WRITE(PWRCTXA, 0);
+		POSTING_READ(PWRCTXA);
+
+		I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
+		POSTING_READ(RSTDBYCTL);
+	}
+
+	ironlake_teardown_rc6(dev);
+}
+
+static int ironlake_setup_rc6(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (dev_priv->renderctx == NULL)
+		dev_priv->renderctx = intel_alloc_context_page(dev);
+	if (!dev_priv->renderctx)
+		return -ENOMEM;
+
+	if (dev_priv->pwrctx == NULL)
+		dev_priv->pwrctx = intel_alloc_context_page(dev);
+	if (!dev_priv->pwrctx) {
+		ironlake_teardown_rc6(dev);
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+void ironlake_enable_rc6(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
+	int ret;
+
+	/* rc6 disabled by default due to repeated reports of hanging during
+	 * boot and resume.
+	 */
+	if (!intel_enable_rc6(dev))
+		return;
+
+	DRM_LOCK(dev);
+	ret = ironlake_setup_rc6(dev);
+	if (ret) {
+		DRM_UNLOCK(dev);
+		return;
+	}
+
+	/*
+	 * GPU can automatically power down the render unit if given a page
+	 * to save state.
+	 */
+	ret = intel_ring_begin(ring, 6);
+	if (ret) {
+		ironlake_teardown_rc6(dev);
+		DRM_UNLOCK(dev);
+		return;
+	}
+
+	intel_ring_emit(ring, MI_SUSPEND_FLUSH | MI_SUSPEND_FLUSH_EN);
+	intel_ring_emit(ring, MI_SET_CONTEXT);
+	intel_ring_emit(ring, dev_priv->renderctx->gtt_offset |
+			MI_MM_SPACE_GTT |
+			MI_SAVE_EXT_STATE_EN |
+			MI_RESTORE_EXT_STATE_EN |
+			MI_RESTORE_INHIBIT);
+	intel_ring_emit(ring, MI_SUSPEND_FLUSH);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_emit(ring, MI_FLUSH);
+	intel_ring_advance(ring);
+
+	/*
+	 * Wait for the command parser to advance past MI_SET_CONTEXT. The HW
+	 * does an implicit flush, combined with MI_FLUSH above, it should be
+	 * safe to assume that renderctx is valid
+	 */
+	ret = intel_wait_ring_idle(ring);
+	if (ret) {
+		DRM_ERROR("failed to enable ironlake power power savings\n");
+		ironlake_teardown_rc6(dev);
+		DRM_UNLOCK(dev);
+		return;
+	}
+
+	I915_WRITE(PWRCTXA, dev_priv->pwrctx->gtt_offset | PWRCTX_EN);
+	I915_WRITE(RSTDBYCTL, I915_READ(RSTDBYCTL) & ~RCX_SW_EXIT);
+	DRM_UNLOCK(dev);
+}
+
+static unsigned long intel_pxfreq(u32 vidfreq)
+{
+	unsigned long freq;
+	int div = (vidfreq & 0x3f0000) >> 16;
+	int post = (vidfreq & 0x3000) >> 12;
+	int pre = (vidfreq & 0x7);
+
+	if (!pre)
+		return 0;
+
+	freq = ((div * 133333) / ((1<<post) * pre));
+
+	return freq;
+}
+
+static const struct cparams {
+	u16 i;
+	u16 t;
+	u16 m;
+	u16 c;
+} cparams[] = {
+	{ 1, 1333, 301, 28664 },
+	{ 1, 1066, 294, 24460 },
+	{ 1, 800, 294, 25192 },
+	{ 0, 1333, 276, 27605 },
+	{ 0, 1066, 276, 27605 },
+	{ 0, 800, 231, 23784 },
+};
+
+unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
+{
+	u64 total_count, diff, ret;
+	u32 count1, count2, count3, m = 0, c = 0;
+	unsigned long now = jiffies_to_msecs(jiffies), diff1;
+	int i;
+
+	diff1 = now - dev_priv->last_time1;
+	/*
+	 * sysctl(8) reads the value of sysctl twice in rapid
+	 * succession.  There is high chance that it happens in the
+	 * same timer tick.  Use the cached value to not divide by
+	 * zero and give the hw a chance to gather more samples.
+	 */
+	if (diff1 <= 10)
+		return (dev_priv->chipset_power);
+
+	count1 = I915_READ(DMIEC);
+	count2 = I915_READ(DDREC);
+	count3 = I915_READ(CSIEC);
+
+	total_count = count1 + count2 + count3;
+
+	/* FIXME: handle per-counter overflow */
+	if (total_count < dev_priv->last_count1) {
+		diff = ~0UL - dev_priv->last_count1;
+		diff += total_count;
+	} else {
+		diff = total_count - dev_priv->last_count1;
+	}
+
+	for (i = 0; i < DRM_ARRAY_SIZE(cparams); i++) {
+		if (cparams[i].i == dev_priv->c_m &&
+		    cparams[i].t == dev_priv->r_t) {
+			m = cparams[i].m;
+			c = cparams[i].c;
+			break;
+		}
+	}
+
+	diff = diff / diff1;
+	ret = ((m * diff) + c);
+	ret = ret / 10;
+
+	dev_priv->last_count1 = total_count;
+	dev_priv->last_time1 = now;
+
+	dev_priv->chipset_power = ret;
+	return (ret);
+}
+
+unsigned long i915_mch_val(struct drm_i915_private *dev_priv)
+{
+	unsigned long m, x, b;
+	u32 tsfs;
+
+	tsfs = I915_READ(TSFS);
+
+	m = ((tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT);
+	x = I915_READ8(I915_TR1);
+
+	b = tsfs & TSFS_INTR_MASK;
+
+	return ((m * x) / 127) - b;
+}
+
+static u16 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
+{
+	static const struct v_table {
+		u16 vd; /* in .1 mil */
+		u16 vm; /* in .1 mil */
+	} v_table[] = {
+		{ 0, 0, },
+		{ 375, 0, },
+		{ 500, 0, },
+		{ 625, 0, },
+		{ 750, 0, },
+		{ 875, 0, },
+		{ 1000, 0, },
+		{ 1125, 0, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4125, 3000, },
+		{ 4250, 3125, },
+		{ 4375, 3250, },
+		{ 4500, 3375, },
+		{ 4625, 3500, },
+		{ 4750, 3625, },
+		{ 4875, 3750, },
+		{ 5000, 3875, },
+		{ 5125, 4000, },
+		{ 5250, 4125, },
+		{ 5375, 4250, },
+		{ 5500, 4375, },
+		{ 5625, 4500, },
+		{ 5750, 4625, },
+		{ 5875, 4750, },
+		{ 6000, 4875, },
+		{ 6125, 5000, },
+		{ 6250, 5125, },
+		{ 6375, 5250, },
+		{ 6500, 5375, },
+		{ 6625, 5500, },
+		{ 6750, 5625, },
+		{ 6875, 5750, },
+		{ 7000, 5875, },
+		{ 7125, 6000, },
+		{ 7250, 6125, },
+		{ 7375, 6250, },
+		{ 7500, 6375, },
+		{ 7625, 6500, },
+		{ 7750, 6625, },
+		{ 7875, 6750, },
+		{ 8000, 6875, },
+		{ 8125, 7000, },
+		{ 8250, 7125, },
+		{ 8375, 7250, },
+		{ 8500, 7375, },
+		{ 8625, 7500, },
+		{ 8750, 7625, },
+		{ 8875, 7750, },
+		{ 9000, 7875, },
+		{ 9125, 8000, },
+		{ 9250, 8125, },
+		{ 9375, 8250, },
+		{ 9500, 8375, },
+		{ 9625, 8500, },
+		{ 9750, 8625, },
+		{ 9875, 8750, },
+		{ 10000, 8875, },
+		{ 10125, 9000, },
+		{ 10250, 9125, },
+		{ 10375, 9250, },
+		{ 10500, 9375, },
+		{ 10625, 9500, },
+		{ 10750, 9625, },
+		{ 10875, 9750, },
+		{ 11000, 9875, },
+		{ 11125, 10000, },
+		{ 11250, 10125, },
+		{ 11375, 10250, },
+		{ 11500, 10375, },
+		{ 11625, 10500, },
+		{ 11750, 10625, },
+		{ 11875, 10750, },
+		{ 12000, 10875, },
+		{ 12125, 11000, },
+		{ 12250, 11125, },
+		{ 12375, 11250, },
+		{ 12500, 11375, },
+		{ 12625, 11500, },
+		{ 12750, 11625, },
+		{ 12875, 11750, },
+		{ 13000, 11875, },
+		{ 13125, 12000, },
+		{ 13250, 12125, },
+		{ 13375, 12250, },
+		{ 13500, 12375, },
+		{ 13625, 12500, },
+		{ 13750, 12625, },
+		{ 13875, 12750, },
+		{ 14000, 12875, },
+		{ 14125, 13000, },
+		{ 14250, 13125, },
+		{ 14375, 13250, },
+		{ 14500, 13375, },
+		{ 14625, 13500, },
+		{ 14750, 13625, },
+		{ 14875, 13750, },
+		{ 15000, 13875, },
+		{ 15125, 14000, },
+		{ 15250, 14125, },
+		{ 15375, 14250, },
+		{ 15500, 14375, },
+		{ 15625, 14500, },
+		{ 15750, 14625, },
+		{ 15875, 14750, },
+		{ 16000, 14875, },
+		{ 16125, 15000, },
+	};
+	if (dev_priv->info->is_mobile)
+		return v_table[pxvid].vm;
+	else
+		return v_table[pxvid].vd;
+}
+
+void i915_update_gfx_val(struct drm_i915_private *dev_priv)
+{
+	struct timespec now, diff1;
+	u64 diff;
+	unsigned long diffms;
+	u32 count;
+
+	if (dev_priv->info->gen != 5)
+		return;
+
+	nanotime(&now);
+	diff1 = now;
+	timespecsub(&diff1, &dev_priv->last_time2);
+
+	/* Don't divide by 0 */
+	diffms = diff1.tv_sec * 1000 + diff1.tv_nsec / 1000000;
+	if (!diffms)
+		return;
+
+	count = I915_READ(GFXEC);
+
+	if (count < dev_priv->last_count2) {
+		diff = ~0UL - dev_priv->last_count2;
+		diff += count;
+	} else {
+		diff = count - dev_priv->last_count2;
+	}
+
+	dev_priv->last_count2 = count;
+	dev_priv->last_time2 = now;
+
+	/* More magic constants... */
+	diff = diff * 1181;
+	diff = diff / (diffms * 10);
+	dev_priv->gfx_power = diff;
+}
+
+unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
+{
+	unsigned long t, corr, state1, corr2, state2;
+	u32 pxvid, ext_v;
+
+	pxvid = I915_READ(PXVFREQ_BASE + (dev_priv->cur_delay * 4));
+	pxvid = (pxvid >> 24) & 0x7f;
+	ext_v = pvid_to_extvid(dev_priv, pxvid);
+
+	state1 = ext_v;
+
+	t = i915_mch_val(dev_priv);
+
+	/* Revel in the empirically derived constants */
+
+	/* Correction factor in 1/100000 units */
+	if (t > 80)
+		corr = ((t * 2349) + 135940);
+	else if (t >= 50)
+		corr = ((t * 964) + 29317);
+	else /* < 50 */
+		corr = ((t * 301) + 1004);
+
+	corr = corr * ((150142 * state1) / 10000 - 78642);
+	corr /= 100000;
+	corr2 = (corr * dev_priv->corr);
+
+	state2 = (corr2 * state1) / 10000;
+	state2 /= 100; /* convert to mW */
+
+	i915_update_gfx_val(dev_priv);
+
+	return dev_priv->gfx_power + state2;
+}
+
+/**
+ * i915_read_mch_val - return value for IPS use
+ *
+ * Calculate and return a value for the IPS driver to use when deciding whether
+ * we have thermal and power headroom to increase CPU or GPU power budget.
+ */
+unsigned long i915_read_mch_val(void)
+{
+	struct drm_i915_private *dev_priv;
+	unsigned long chipset_val, graphics_val, ret = 0;
+
+	mtx_lock(&mchdev_lock);
+	if (!i915_mch_dev)
+		goto out_unlock;
+	dev_priv = i915_mch_dev;
+
+	chipset_val = i915_chipset_val(dev_priv);
+	graphics_val = i915_gfx_val(dev_priv);
+
+	ret = chipset_val + graphics_val;
+
+out_unlock:
+	mtx_unlock(&mchdev_lock);
+
+	return ret;
+}
+
+/**
+ * i915_gpu_raise - raise GPU frequency limit
+ *
+ * Raise the limit; IPS indicates we have thermal headroom.
+ */
+bool i915_gpu_raise(void)
+{
+	struct drm_i915_private *dev_priv;
+	bool ret = true;
+
+	mtx_lock(&mchdev_lock);
+	if (!i915_mch_dev) {
+		ret = false;
+		goto out_unlock;
+	}
+	dev_priv = i915_mch_dev;
+
+	if (dev_priv->max_delay > dev_priv->fmax)
+		dev_priv->max_delay--;
+
+out_unlock:
+	mtx_unlock(&mchdev_lock);
+
+	return ret;
+}
+
+/**
+ * i915_gpu_lower - lower GPU frequency limit
+ *
+ * IPS indicates we're close to a thermal limit, so throttle back the GPU
+ * frequency maximum.
+ */
+bool i915_gpu_lower(void)
+{
+	struct drm_i915_private *dev_priv;
+	bool ret = true;
+
+	mtx_lock(&mchdev_lock);
+	if (!i915_mch_dev) {
+		ret = false;
+		goto out_unlock;
+	}
+	dev_priv = i915_mch_dev;
+
+	if (dev_priv->max_delay < dev_priv->min_delay)
+		dev_priv->max_delay++;
+
+out_unlock:
+	mtx_unlock(&mchdev_lock);
+
+	return ret;
+}
+
+/**
+ * i915_gpu_busy - indicate GPU business to IPS
+ *
+ * Tell the IPS driver whether or not the GPU is busy.
+ */
+bool i915_gpu_busy(void)
+{
+	struct drm_i915_private *dev_priv;
+	bool ret = false;
+
+	mtx_lock(&mchdev_lock);
+	if (!i915_mch_dev)
+		goto out_unlock;
+	dev_priv = i915_mch_dev;
+
+	ret = dev_priv->busy;
+
+out_unlock:
+	mtx_unlock(&mchdev_lock);
+
+	return ret;
+}
+
+/**
+ * i915_gpu_turbo_disable - disable graphics turbo
+ *
+ * Disable graphics turbo by resetting the max frequency and setting the
+ * current frequency to the default.
+ */
+bool i915_gpu_turbo_disable(void)
+{
+	struct drm_i915_private *dev_priv;
+	bool ret = true;
+
+	mtx_lock(&mchdev_lock);
+	if (!i915_mch_dev) {
+		ret = false;
+		goto out_unlock;
+	}
+	dev_priv = i915_mch_dev;
+
+	dev_priv->max_delay = dev_priv->fstart;
+
+	if (!ironlake_set_drps(dev_priv->dev, dev_priv->fstart))
+		ret = false;
+
+out_unlock:
+	mtx_unlock(&mchdev_lock);
+
+	return ret;
+}
+
+void intel_gpu_ips_init(struct drm_i915_private *dev_priv)
+{
+	mtx_lock(&mchdev_lock);
+	i915_mch_dev = dev_priv;
+	dev_priv->mchdev_lock = &mchdev_lock;
+	mtx_unlock(&mchdev_lock);
+
+#if 0
+	ips_ping_for_i915_load();
+#endif
+}
+
+void intel_gpu_ips_teardown(void)
+{
+	mtx_lock(&mchdev_lock);
+	i915_mch_dev = NULL;
+	mtx_unlock(&mchdev_lock);
+}
+
+void intel_init_emon(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 lcfuse;
+	u8 pxw[16];
+	int i;
+
+	/* Disable to program */
+	I915_WRITE(ECR, 0);
+	POSTING_READ(ECR);
+
+	/* Program energy weights for various events */
+	I915_WRITE(SDEW, 0x15040d00);
+	I915_WRITE(CSIEW0, 0x007f0000);
+	I915_WRITE(CSIEW1, 0x1e220004);
+	I915_WRITE(CSIEW2, 0x04000004);
+
+	for (i = 0; i < 5; i++)
+		I915_WRITE(PEW + (i * 4), 0);
+	for (i = 0; i < 3; i++)
+		I915_WRITE(DEW + (i * 4), 0);
+
+	/* Program P-state weights to account for frequency power adjustment */
+	for (i = 0; i < 16; i++) {
+		u32 pxvidfreq = I915_READ(PXVFREQ_BASE + (i * 4));
+		unsigned long freq = intel_pxfreq(pxvidfreq);
+		unsigned long vid = (pxvidfreq & PXVFREQ_PX_MASK) >>
+			PXVFREQ_PX_SHIFT;
+		unsigned long val;
+
+		val = vid * vid;
+		val *= (freq / 1000);
+		val *= 255;
+		val /= (127*127*900);
+		if (val > 0xff)
+			DRM_ERROR("bad pxval: %ld\n", val);
+		pxw[i] = val;
+	}
+	/* Render standby states get 0 weight */
+	pxw[14] = 0;
+	pxw[15] = 0;
+
+	for (i = 0; i < 4; i++) {
+		u32 val = (pxw[i*4] << 24) | (pxw[(i*4)+1] << 16) |
+			(pxw[(i*4)+2] << 8) | (pxw[(i*4)+3]);
+		I915_WRITE(PXW + (i * 4), val);
+	}
+
+	/* Adjust magic regs to magic values (more experimental results) */
+	I915_WRITE(OGW0, 0);
+	I915_WRITE(OGW1, 0);
+	I915_WRITE(EG0, 0x00007f00);
+	I915_WRITE(EG1, 0x0000000e);
+	I915_WRITE(EG2, 0x000e0000);
+	I915_WRITE(EG3, 0x68000300);
+	I915_WRITE(EG4, 0x42000000);
+	I915_WRITE(EG5, 0x00140031);
+	I915_WRITE(EG6, 0);
+	I915_WRITE(EG7, 0);
+
+	for (i = 0; i < 8; i++)
+		I915_WRITE(PXWL + (i * 4), 0);
+
+	/* Enable PMON + select events */
+	I915_WRITE(ECR, 0x80000019);
+
+	lcfuse = I915_READ(LCFUSE02);
+
+	dev_priv->corr = (lcfuse & LCFUSE_HIV_MASK);
+}
+
+static void ironlake_init_clock_gating(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
+
+	/* Required for FBC */
+	dspclk_gate |= DPFCUNIT_CLOCK_GATE_DISABLE |
+		DPFCRUNIT_CLOCK_GATE_DISABLE |
+		DPFDUNIT_CLOCK_GATE_DISABLE;
+	/* Required for CxSR */
+	dspclk_gate |= DPARBUNIT_CLOCK_GATE_DISABLE;
+
+	I915_WRITE(PCH_3DCGDIS0,
+		   MARIUNIT_CLOCK_GATE_DISABLE |
+		   SVSMUNIT_CLOCK_GATE_DISABLE);
+	I915_WRITE(PCH_3DCGDIS1,
+		   VFMUNIT_CLOCK_GATE_DISABLE);
+
+	I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
+
+	/*
+	 * According to the spec the following bits should be set in
+	 * order to enable memory self-refresh
+	 * The bit 22/21 of 0x42004
+	 * The bit 5 of 0x42020
+	 * The bit 15 of 0x45000
+	 */
+	I915_WRITE(ILK_DISPLAY_CHICKEN2,
+		   (I915_READ(ILK_DISPLAY_CHICKEN2) |
+		    ILK_DPARB_GATE | ILK_VSDPFD_FULL));
+	I915_WRITE(ILK_DSPCLK_GATE,
+		   (I915_READ(ILK_DSPCLK_GATE) |
+		    ILK_DPARB_CLK_GATE));
+	I915_WRITE(DISP_ARB_CTL,
+		   (I915_READ(DISP_ARB_CTL) |
+		    DISP_FBC_WM_DIS));
+	I915_WRITE(WM3_LP_ILK, 0);
+	I915_WRITE(WM2_LP_ILK, 0);
+	I915_WRITE(WM1_LP_ILK, 0);
+
+	/*
+	 * Based on the document from hardware guys the following bits
+	 * should be set unconditionally in order to enable FBC.
+	 * The bit 22 of 0x42000
+	 * The bit 22 of 0x42004
+	 * The bit 7,8,9 of 0x42020.
+	 */
+	if (IS_IRONLAKE_M(dev)) {
+		I915_WRITE(ILK_DISPLAY_CHICKEN1,
+			   I915_READ(ILK_DISPLAY_CHICKEN1) |
+			   ILK_FBCQ_DIS);
+		I915_WRITE(ILK_DISPLAY_CHICKEN2,
+			   I915_READ(ILK_DISPLAY_CHICKEN2) |
+			   ILK_DPARB_GATE);
+		I915_WRITE(ILK_DSPCLK_GATE,
+			   I915_READ(ILK_DSPCLK_GATE) |
+			   ILK_DPFC_DIS1 |
+			   ILK_DPFC_DIS2 |
+			   ILK_CLK_FBC);
+	}
+
+	I915_WRITE(ILK_DISPLAY_CHICKEN2,
+		   I915_READ(ILK_DISPLAY_CHICKEN2) |
+		   ILK_ELPIN_409_SELECT);
+	I915_WRITE(_3D_CHICKEN2,
+		   _3D_CHICKEN2_WM_READ_PIPELINED << 16 |
+		   _3D_CHICKEN2_WM_READ_PIPELINED);
+}
+
+static void gen6_init_clock_gating(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int pipe;
+	uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
+
+	I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
+
+	I915_WRITE(ILK_DISPLAY_CHICKEN2,
+		   I915_READ(ILK_DISPLAY_CHICKEN2) |
+		   ILK_ELPIN_409_SELECT);
+
+	I915_WRITE(WM3_LP_ILK, 0);
+	I915_WRITE(WM2_LP_ILK, 0);
+	I915_WRITE(WM1_LP_ILK, 0);
+
+	I915_WRITE(CACHE_MODE_0,
+		   _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
+
+	I915_WRITE(GEN6_UCGCTL1,
+		   I915_READ(GEN6_UCGCTL1) |
+		   GEN6_BLBUNIT_CLOCK_GATE_DISABLE |
+		   GEN6_CSUNIT_CLOCK_GATE_DISABLE);
+
+	/* According to the BSpec vol1g, bit 12 (RCPBUNIT) clock
+	 * gating disable must be set.  Failure to set it results in
+	 * flickering pixels due to Z write ordering failures after
+	 * some amount of runtime in the Mesa "fire" demo, and Unigine
+	 * Sanctuary and Tropics, and apparently anything else with
+	 * alpha test or pixel discard.
+	 *
+	 * According to the spec, bit 11 (RCCUNIT) must also be set,
+	 * but we didn't debug actual testcases to find it out.
+	 */
+	I915_WRITE(GEN6_UCGCTL2,
+		   GEN6_RCPBUNIT_CLOCK_GATE_DISABLE |
+		   GEN6_RCCUNIT_CLOCK_GATE_DISABLE);
+
+	/* Bspec says we need to always set all mask bits. */
+	I915_WRITE(_3D_CHICKEN, (0xFFFF << 16) |
+		   _3D_CHICKEN_SF_DISABLE_FASTCLIP_CULL);
+
+	/*
+	 * According to the spec the following bits should be
+	 * set in order to enable memory self-refresh and fbc:
+	 * The bit21 and bit22 of 0x42000
+	 * The bit21 and bit22 of 0x42004
+	 * The bit5 and bit7 of 0x42020
+	 * The bit14 of 0x70180
+	 * The bit14 of 0x71180
+	 */
+	I915_WRITE(ILK_DISPLAY_CHICKEN1,
+		   I915_READ(ILK_DISPLAY_CHICKEN1) |
+		   ILK_FBCQ_DIS | ILK_PABSTRETCH_DIS);
+	I915_WRITE(ILK_DISPLAY_CHICKEN2,
+		   I915_READ(ILK_DISPLAY_CHICKEN2) |
+		   ILK_DPARB_GATE | ILK_VSDPFD_FULL);
+	I915_WRITE(ILK_DSPCLK_GATE,
+		   I915_READ(ILK_DSPCLK_GATE) |
+		   ILK_DPARB_CLK_GATE  |
+		   ILK_DPFD_CLK_GATE);
+
+	for_each_pipe(pipe) {
+		I915_WRITE(DSPCNTR(pipe),
+			   I915_READ(DSPCNTR(pipe)) |
+			   DISPPLANE_TRICKLE_FEED_DISABLE);
+		intel_flush_display_plane(dev_priv, pipe);
+	}
+}
+
+static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
+{
+	uint32_t reg = I915_READ(GEN7_FF_THREAD_MODE);
+
+	reg &= ~GEN7_FF_SCHED_MASK;
+	reg |= GEN7_FF_TS_SCHED_HW;
+	reg |= GEN7_FF_VS_SCHED_HW;
+	reg |= GEN7_FF_DS_SCHED_HW;
+
+	I915_WRITE(GEN7_FF_THREAD_MODE, reg);
+}
+
+static void ivybridge_init_clock_gating(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int pipe;
+	uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
+
+	I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
+
+	I915_WRITE(WM3_LP_ILK, 0);
+	I915_WRITE(WM2_LP_ILK, 0);
+	I915_WRITE(WM1_LP_ILK, 0);
+
+	/* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
+	 * This implements the WaDisableRCZUnitClockGating workaround.
+	 */
+	I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
+
+	I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
+
+	I915_WRITE(IVB_CHICKEN3,
+		   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
+		   CHICKEN3_DGMG_DONE_FIX_DISABLE);
+
+	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
+	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
+		   GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
+
+	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
+	I915_WRITE(GEN7_L3CNTLREG1,
+			GEN7_WA_FOR_GEN7_L3_CONTROL);
+	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER,
+			GEN7_WA_L3_CHICKEN_MODE);
+
+	/* This is required by WaCatErrorRejectionIssue */
+	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
+			I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
+			GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
+
+	for_each_pipe(pipe) {
+		I915_WRITE(DSPCNTR(pipe),
+			   I915_READ(DSPCNTR(pipe)) |
+			   DISPPLANE_TRICKLE_FEED_DISABLE);
+		intel_flush_display_plane(dev_priv, pipe);
+	}
+
+	gen7_setup_fixed_func_scheduler(dev_priv);
+
+	/* WaDisable4x2SubspanOptimization */
+	I915_WRITE(CACHE_MODE_1,
+		   _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
+}
+
+static void valleyview_init_clock_gating(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int pipe;
+	uint32_t dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE;
+
+	I915_WRITE(PCH_DSPCLK_GATE_D, dspclk_gate);
+
+	I915_WRITE(WM3_LP_ILK, 0);
+	I915_WRITE(WM2_LP_ILK, 0);
+	I915_WRITE(WM1_LP_ILK, 0);
+
+	/* According to the spec, bit 13 (RCZUNIT) must be set on IVB.
+	 * This implements the WaDisableRCZUnitClockGating workaround.
+	 */
+	I915_WRITE(GEN6_UCGCTL2, GEN6_RCZUNIT_CLOCK_GATE_DISABLE);
+
+	I915_WRITE(ILK_DSPCLK_GATE, IVB_VRHUNIT_CLK_GATE);
+
+	I915_WRITE(IVB_CHICKEN3,
+		   CHICKEN3_DGMG_REQ_OUT_FIX_DISABLE |
+		   CHICKEN3_DGMG_DONE_FIX_DISABLE);
+
+	/* Apply the WaDisableRHWOOptimizationForRenderHang workaround. */
+	I915_WRITE(GEN7_COMMON_SLICE_CHICKEN1,
+		   GEN7_CSC1_RHWO_OPT_DISABLE_IN_RCC);
+
+	/* WaApplyL3ControlAndL3ChickenMode requires those two on Ivy Bridge */
+	I915_WRITE(GEN7_L3CNTLREG1, GEN7_WA_FOR_GEN7_L3_CONTROL);
+	I915_WRITE(GEN7_L3_CHICKEN_MODE_REGISTER, GEN7_WA_L3_CHICKEN_MODE);
+
+	/* This is required by WaCatErrorRejectionIssue */
+	I915_WRITE(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG,
+		   I915_READ(GEN7_SQ_CHICKEN_MBCUNIT_CONFIG) |
+		   GEN7_SQ_CHICKEN_MBCUNIT_SQINTMOB);
+
+	for_each_pipe(pipe) {
+		I915_WRITE(DSPCNTR(pipe),
+			   I915_READ(DSPCNTR(pipe)) |
+			   DISPPLANE_TRICKLE_FEED_DISABLE);
+		intel_flush_display_plane(dev_priv, pipe);
+	}
+
+	I915_WRITE(CACHE_MODE_1,
+		   _MASKED_BIT_ENABLE(PIXEL_SUBSPAN_COLLECT_OPT_DISABLE));
+}
+
+static void g4x_init_clock_gating(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	uint32_t dspclk_gate;
+
+	I915_WRITE(RENCLK_GATE_D1, 0);
+	I915_WRITE(RENCLK_GATE_D2, VF_UNIT_CLOCK_GATE_DISABLE |
+		   GS_UNIT_CLOCK_GATE_DISABLE |
+		   CL_UNIT_CLOCK_GATE_DISABLE);
+	I915_WRITE(RAMCLK_GATE_D, 0);
+	dspclk_gate = VRHUNIT_CLOCK_GATE_DISABLE |
+		OVRUNIT_CLOCK_GATE_DISABLE |
+		OVCUNIT_CLOCK_GATE_DISABLE;
+	if (IS_GM45(dev))
+		dspclk_gate |= DSSUNIT_CLOCK_GATE_DISABLE;
+	I915_WRITE(DSPCLK_GATE_D, dspclk_gate);
+}
+
+static void crestline_init_clock_gating(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
+	I915_WRITE(RENCLK_GATE_D2, 0);
+	I915_WRITE(DSPCLK_GATE_D, 0);
+	I915_WRITE(RAMCLK_GATE_D, 0);
+	I915_WRITE16(DEUC, 0);
+}
+
+static void broadwater_init_clock_gating(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
+		   I965_RCC_CLOCK_GATE_DISABLE |
+		   I965_RCPB_CLOCK_GATE_DISABLE |
+		   I965_ISC_CLOCK_GATE_DISABLE |
+		   I965_FBC_CLOCK_GATE_DISABLE);
+	I915_WRITE(RENCLK_GATE_D2, 0);
+}
+
+static void gen3_init_clock_gating(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 dstate = I915_READ(D_STATE);
+
+	dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
+		DSTATE_DOT_CLOCK_GATING;
+	I915_WRITE(D_STATE, dstate);
+
+	if (IS_PINEVIEW(dev))
+		I915_WRITE(ECOSKPD, _MASKED_BIT_ENABLE(ECO_GATING_CX_ONLY));
+}
+
+static void i85x_init_clock_gating(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
+}
+
+static void i830_init_clock_gating(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
+}
+
+static void ibx_init_clock_gating(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	/*
+	 * On Ibex Peak and Cougar Point, we need to disable clock
+	 * gating for the panel power sequencer or it will fail to
+	 * start up when no ports are active.
+	 */
+	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
+}
+
+static void cpt_init_clock_gating(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	int pipe;
+
+	/*
+	 * On Ibex Peak and Cougar Point, we need to disable clock
+	 * gating for the panel power sequencer or it will fail to
+	 * start up when no ports are active.
+	 */
+	I915_WRITE(SOUTH_DSPCLK_GATE_D, PCH_DPLSUNIT_CLOCK_GATE_DISABLE);
+	I915_WRITE(SOUTH_CHICKEN2, I915_READ(SOUTH_CHICKEN2) |
+		   DPLS_EDP_PPS_FIX_DIS);
+	/* Without this, mode sets may fail silently on FDI */
+	for_each_pipe(pipe)
+		I915_WRITE(TRANS_CHICKEN2(pipe), TRANS_AUTOTRAIN_GEN_STALL_DIS);
+}
+
+void intel_init_clock_gating(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	dev_priv->display.init_clock_gating(dev);
+
+	if (dev_priv->display.init_pch_clock_gating)
+		dev_priv->display.init_pch_clock_gating(dev);
+}
+
+static void gen6_sanitize_pm(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	u32 limits, delay, old;
+
+	gen6_gt_force_wake_get(dev_priv);
+
+	old = limits = I915_READ(GEN6_RP_INTERRUPT_LIMITS);
+	/* Make sure we continue to get interrupts
+	 * until we hit the minimum or maximum frequencies.
+	 */
+	limits &= ~(0x3f << 16 | 0x3f << 24);
+	delay = dev_priv->cur_delay;
+	if (delay < dev_priv->max_delay)
+		limits |= (dev_priv->max_delay & 0x3f) << 24;
+	if (delay > dev_priv->min_delay)
+		limits |= (dev_priv->min_delay & 0x3f) << 16;
+
+	if (old != limits) {
+		DRM_ERROR("Power management discrepancy: GEN6_RP_INTERRUPT_LIMITS expected %08x, was %08x\n",
+			  limits, old);
+		I915_WRITE(GEN6_RP_INTERRUPT_LIMITS, limits);
+	}
+
+	gen6_gt_force_wake_put(dev_priv);
+}
+
+void intel_sanitize_pm(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (dev_priv->display.sanitize_pm)
+		dev_priv->display.sanitize_pm(dev);
+}
+
+/* Starting with Haswell, we have different power wells for
+ * different parts of the GPU. This attempts to enable them all.
+ */
+static void intel_init_power_wells(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+	unsigned long power_wells[] = {
+		HSW_PWR_WELL_CTL1,
+		HSW_PWR_WELL_CTL2,
+		HSW_PWR_WELL_CTL4
+	};
+	int i;
+
+	if (!IS_HASWELL(dev))
+		return;
+
+	DRM_LOCK(dev);
+
+	for (i = 0; i < DRM_ARRAY_SIZE(power_wells); i++) {
+		int well = I915_READ(power_wells[i]);
+
+		if ((well & HSW_PWR_WELL_STATE) == 0) {
+			I915_WRITE(power_wells[i], well & HSW_PWR_WELL_ENABLE);
+			if (wait_for(I915_READ(power_wells[i] & HSW_PWR_WELL_STATE), 20))
+				DRM_ERROR("Error enabling power well %lx\n", power_wells[i]);
+		}
+	}
+
+printf("XXXKIB HACK: HSW RC OFF\n");
+	I915_WRITE(GEN6_RC_STATE, 0);
+	I915_WRITE(GEN6_RC_CONTROL, 0);
+	DRM_UNLOCK(dev);
+}
+
+/* Set up chip specific power management-related functions */
+void intel_init_pm(struct drm_device *dev)
+{
+	struct drm_i915_private *dev_priv = dev->dev_private;
+
+	if (I915_HAS_FBC(dev)) {
+		if (HAS_PCH_SPLIT(dev)) {
+			dev_priv->display.fbc_enabled = ironlake_fbc_enabled;
+			dev_priv->display.enable_fbc = ironlake_enable_fbc;
+			dev_priv->display.disable_fbc = ironlake_disable_fbc;
+		} else if (IS_GM45(dev)) {
+			dev_priv->display.fbc_enabled = g4x_fbc_enabled;
+			dev_priv->display.enable_fbc = g4x_enable_fbc;
+			dev_priv->display.disable_fbc = g4x_disable_fbc;
+		} else if (IS_CRESTLINE(dev)) {
+			dev_priv->display.fbc_enabled = i8xx_fbc_enabled;
+			dev_priv->display.enable_fbc = i8xx_enable_fbc;
+			dev_priv->display.disable_fbc = i8xx_disable_fbc;
+		}
+		/* 855GM needs testing */
+	}
+
+	/* For cxsr */
+	if (IS_PINEVIEW(dev))
+		i915_pineview_get_mem_freq(dev);
+	else if (IS_GEN5(dev))
+		i915_ironlake_get_mem_freq(dev);
+
+	/* For FIFO watermark updates */
+	if (HAS_PCH_SPLIT(dev)) {
+		dev_priv->display.force_wake_get = __gen6_gt_force_wake_get;
+		dev_priv->display.force_wake_put = __gen6_gt_force_wake_put;
+
+		/* IVB configs may use multi-threaded forcewake */
+		if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev)) {
+			u32	ecobus;
+
+			/* A small trick here - if the bios hasn't configured MT forcewake,
+			 * and if the device is in RC6, then force_wake_mt_get will not wake
+			 * the device and the ECOBUS read will return zero. Which will be
+			 * (correctly) interpreted by the test below as MT forcewake being
+			 * disabled.
+			 */
+			DRM_LOCK(dev);
+			__gen6_gt_force_wake_mt_get(dev_priv);
+			ecobus = I915_READ_NOTRACE(ECOBUS);
+			__gen6_gt_force_wake_mt_put(dev_priv);
+			DRM_UNLOCK(dev);
+
+			if (ecobus & FORCEWAKE_MT_ENABLE) {
+				DRM_DEBUG_KMS("Using MT version of forcewake\n");
+				dev_priv->display.force_wake_get =
+					__gen6_gt_force_wake_mt_get;
+				dev_priv->display.force_wake_put =
+					__gen6_gt_force_wake_mt_put;
+			}
+		}
+
+		if (HAS_PCH_IBX(dev))
+			dev_priv->display.init_pch_clock_gating = ibx_init_clock_gating;
+		else if (HAS_PCH_CPT(dev))
+			dev_priv->display.init_pch_clock_gating = cpt_init_clock_gating;
+
+		if (IS_GEN5(dev)) {
+			if (I915_READ(MLTR_ILK) & ILK_SRLT_MASK)
+				dev_priv->display.update_wm = ironlake_update_wm;
+			else {
+				DRM_DEBUG_KMS("Failed to get proper latency. "
+					      "Disable CxSR\n");
+				dev_priv->display.update_wm = NULL;
+			}
+			dev_priv->display.init_clock_gating = ironlake_init_clock_gating;
+		} else if (IS_GEN6(dev)) {
+			if (SNB_READ_WM0_LATENCY()) {
+				dev_priv->display.update_wm = sandybridge_update_wm;
+				dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
+			} else {
+				DRM_DEBUG_KMS("Failed to read display plane latency. "
+					      "Disable CxSR\n");
+				dev_priv->display.update_wm = NULL;
+			}
+			dev_priv->display.init_clock_gating = gen6_init_clock_gating;
+			dev_priv->display.sanitize_pm = gen6_sanitize_pm;
+		} else if (IS_IVYBRIDGE(dev)) {
+			/* FIXME: detect B0+ stepping and use auto training */
+			if (SNB_READ_WM0_LATENCY()) {
+				dev_priv->display.update_wm = sandybridge_update_wm;
+				dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
+			} else {
+				DRM_DEBUG_KMS("Failed to read display plane latency. "
+					      "Disable CxSR\n");
+				dev_priv->display.update_wm = NULL;
+			}
+			dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
+			dev_priv->display.sanitize_pm = gen6_sanitize_pm;
+		} else if (IS_HASWELL(dev)) {
+			if (SNB_READ_WM0_LATENCY()) {
+				dev_priv->display.update_wm = sandybridge_update_wm;
+				dev_priv->display.update_sprite_wm = sandybridge_update_sprite_wm;
+				dev_priv->display.update_linetime_wm = haswell_update_linetime_wm;
+			} else {
+				DRM_DEBUG_KMS("Failed to read display plane latency. "
+					      "Disable CxSR\n");
+				dev_priv->display.update_wm = NULL;
+			}
+			dev_priv->display.init_clock_gating = ivybridge_init_clock_gating;
+			dev_priv->display.sanitize_pm = gen6_sanitize_pm;
+		} else
+			dev_priv->display.update_wm = NULL;
+	} else if (IS_VALLEYVIEW(dev)) {
+		dev_priv->display.update_wm = valleyview_update_wm;
+		dev_priv->display.init_clock_gating =
+			valleyview_init_clock_gating;
+		dev_priv->display.force_wake_get = vlv_force_wake_get;
+		dev_priv->display.force_wake_put = vlv_force_wake_put;
+	} else if (IS_PINEVIEW(dev)) {
+		if (!intel_get_cxsr_latency(IS_PINEVIEW_G(dev),
+					    dev_priv->is_ddr3,
+					    dev_priv->fsb_freq,
+					    dev_priv->mem_freq)) {
+			DRM_INFO("failed to find known CxSR latency "
+				 "(found ddr%s fsb freq %d, mem freq %d), "
+				 "disabling CxSR\n",
+				 (dev_priv->is_ddr3 == 1) ? "3" : "2",
+				 dev_priv->fsb_freq, dev_priv->mem_freq);
+			/* Disable CxSR and never update its watermark again */
+			pineview_disable_cxsr(dev);
+			dev_priv->display.update_wm = NULL;
+		} else
+			dev_priv->display.update_wm = pineview_update_wm;
+		dev_priv->display.init_clock_gating = gen3_init_clock_gating;
+	} else if (IS_G4X(dev)) {
+		dev_priv->display.update_wm = g4x_update_wm;
+		dev_priv->display.init_clock_gating = g4x_init_clock_gating;
+	} else if (IS_GEN4(dev)) {
+		dev_priv->display.update_wm = i965_update_wm;
+		if (IS_CRESTLINE(dev))
+			dev_priv->display.init_clock_gating = crestline_init_clock_gating;
+		else if (IS_BROADWATER(dev))
+			dev_priv->display.init_clock_gating = broadwater_init_clock_gating;
+	} else if (IS_GEN3(dev)) {
+		dev_priv->display.update_wm = i9xx_update_wm;
+		dev_priv->display.get_fifo_size = i9xx_get_fifo_size;
+		dev_priv->display.init_clock_gating = gen3_init_clock_gating;
+	} else if (IS_I865G(dev)) {
+		dev_priv->display.update_wm = i830_update_wm;
+		dev_priv->display.init_clock_gating = i85x_init_clock_gating;
+		dev_priv->display.get_fifo_size = i830_get_fifo_size;
+	} else if (IS_I85X(dev)) {
+		dev_priv->display.update_wm = i9xx_update_wm;
+		dev_priv->display.get_fifo_size = i85x_get_fifo_size;
+		dev_priv->display.init_clock_gating = i85x_init_clock_gating;
+	} else {
+		dev_priv->display.update_wm = i830_update_wm;
+		dev_priv->display.init_clock_gating = i830_init_clock_gating;
+		if (IS_845G(dev))
+			dev_priv->display.get_fifo_size = i845_get_fifo_size;
+		else
+			dev_priv->display.get_fifo_size = i830_get_fifo_size;
+	}
+
+	/* We attempt to init the necessary power wells early in the initialization
+	 * time, so the subsystems that expect power to be enabled can work.
+	 */
+	intel_init_power_wells(dev);
+}
+
diff --git a/sys/dev/drm2/i915/intel_ringbuffer.c b/sys/dev/drm2/i915/intel_ringbuffer.c
index 89a5c94c103..d3738655587 100644
--- a/sys/dev/drm2/i915/intel_ringbuffer.c
+++ b/sys/dev/drm2/i915/intel_ringbuffer.c
@@ -52,12 +52,14 @@ struct pipe_control {
 void
 i915_trace_irq_get(struct intel_ring_buffer *ring, uint32_t seqno)
 {
+	struct drm_i915_private *dev_priv;
 
 	if (ring->trace_irq_seqno == 0) {
-		mtx_lock(&ring->irq_lock);
+		dev_priv = ring->dev->dev_private;
+		mtx_lock(&dev_priv->irq_lock);
 		if (ring->irq_get(ring))
 			ring->trace_irq_seqno = seqno;
-		mtx_unlock(&ring->irq_lock);
+		mtx_unlock(&dev_priv->irq_lock);
 	}
 }
 
@@ -70,9 +72,35 @@ static inline int ring_space(struct intel_ring_buffer *ring)
 }
 
 static int
-render_ring_flush(struct intel_ring_buffer *ring,
-		  uint32_t	invalidate_domains,
-		  uint32_t	flush_domains)
+gen2_render_ring_flush(struct intel_ring_buffer *ring,
+		       u32	invalidate_domains,
+		       u32	flush_domains)
+{
+	u32 cmd;
+	int ret;
+
+	cmd = MI_FLUSH;
+	if (((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER) == 0)
+		cmd |= MI_NO_WRITE_FLUSH;
+
+	if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
+		cmd |= MI_READ_FLUSH;
+
+	ret = intel_ring_begin(ring, 2);
+	if (ret)
+		return ret;
+
+	intel_ring_emit(ring, cmd);
+	intel_ring_emit(ring, MI_NOOP);
+	intel_ring_advance(ring);
+
+	return 0;
+}
+
+static int
+gen4_render_ring_flush(struct intel_ring_buffer *ring,
+		  u32	invalidate_domains,
+		  u32	flush_domains)
 {
 	struct drm_device *dev = ring->dev;
 	uint32_t cmd;
@@ -107,17 +135,8 @@ render_ring_flush(struct intel_ring_buffer *ring,
 	 */
 
 	cmd = MI_FLUSH | MI_NO_WRITE_FLUSH;
-	if ((invalidate_domains|flush_domains) &
-	    I915_GEM_DOMAIN_RENDER)
+	if ((invalidate_domains|flush_domains) & I915_GEM_DOMAIN_RENDER)
 		cmd &= ~MI_NO_WRITE_FLUSH;
-	if (INTEL_INFO(dev)->gen < 4) {
-		/*
-		 * On the 965, the sampler cache always gets flushed
-		 * and this bit is reserved.
-		 */
-		if (invalidate_domains & I915_GEM_DOMAIN_SAMPLER)
-			cmd |= MI_READ_FLUSH;
-	}
 	if (invalidate_domains & I915_GEM_DOMAIN_INSTRUCTION)
 		cmd |= MI_EXE_FLUSH;
 
@@ -407,12 +426,11 @@ static int init_render_ring(struct intel_ring_buffer *ring)
 	int ret = init_ring_common(ring);
 
 	if (INTEL_INFO(dev)->gen > 3) {
-		int mode = VS_TIMER_DISPATCH << 16 | VS_TIMER_DISPATCH;
-		I915_WRITE(MI_MODE, mode);
+		I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
 		if (IS_GEN7(dev))
 			I915_WRITE(GFX_MODE_GEN7,
-				   GFX_MODE_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
-				   GFX_MODE_ENABLE(GFX_REPLAY_MODE));
+				   _MASKED_BIT_DISABLE(GFX_TLB_INVALIDATE_ALWAYS) |
+				   _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
 	}
 
 	if (INTEL_INFO(dev)->gen >= 5) {
@@ -429,7 +447,7 @@ static int init_render_ring(struct intel_ring_buffer *ring)
 		 *  policy is not supported."
 		 */
 		I915_WRITE(CACHE_MODE_0,
-			   CM0_STC_EVICT_DISABLE_LRA_SNB << CM0_MASK_SHIFT);
+			   _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
 
 		/* This is not explicitly set for GEN6, so read the register.
 		 * see intel_ring_mi_set_context() for why we care.
@@ -439,10 +457,8 @@ static int init_render_ring(struct intel_ring_buffer *ring)
 			!!(I915_READ(GFX_MODE) & GFX_TLB_INVALIDATE_ALWAYS);
 	}
 
-	if (INTEL_INFO(dev)->gen >= 6) {
-		I915_WRITE(INSTPM,
-			   INSTPM_FORCE_ORDERING << 16 | INSTPM_FORCE_ORDERING);
-	}
+	if (INTEL_INFO(dev)->gen >= 6)
+		I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
 
 	return ret;
 }
@@ -513,21 +529,32 @@ gen6_add_request(struct intel_ring_buffer *ring,
  * @seqno - seqno which the waiter will block on
  */
 static int
-intel_ring_sync(struct intel_ring_buffer *waiter,
-		struct intel_ring_buffer *signaller,
-		int ring,
-		u32 seqno)
+gen6_ring_sync(struct intel_ring_buffer *waiter,
+	       struct intel_ring_buffer *signaller,
+	       u32 seqno)
 {
 	int ret;
 	u32 dw1 = MI_SEMAPHORE_MBOX |
 		  MI_SEMAPHORE_COMPARE |
 		  MI_SEMAPHORE_REGISTER;
 
+	/* Throughout all of the GEM code, seqno passed implies our current
+	 * seqno is >= the last seqno executed. However for hardware the
+	 * comparison is strictly greater than.
+	 */
+	seqno -= 1;
+
+	if (signaller->semaphore_register[waiter->id] ==
+	    MI_SEMAPHORE_SYNC_INVALID)
+		printf("gen6_ring_sync semaphore_register %d invalid\n",
+		    waiter->id);
+
 	ret = intel_ring_begin(waiter, 4);
 	if (ret)
 		return ret;
 
-	intel_ring_emit(waiter, dw1 | signaller->semaphore_register[ring]);
+	intel_ring_emit(waiter,
+			dw1 | signaller->semaphore_register[waiter->id]);
 	intel_ring_emit(waiter, seqno);
 	intel_ring_emit(waiter, 0);
 	intel_ring_emit(waiter, MI_NOOP);
@@ -543,48 +570,6 @@ int gen6_bsd_ring_sync_to(struct intel_ring_buffer *waiter,
 int gen6_blt_ring_sync_to(struct intel_ring_buffer *waiter,
     struct intel_ring_buffer *signaller, u32 seqno);
 
-/* VCS->RCS (RVSYNC) or BCS->RCS (RBSYNC) */
-int
-render_ring_sync_to(struct intel_ring_buffer *waiter,
-		    struct intel_ring_buffer *signaller,
-		    u32 seqno)
-{
-	KASSERT(signaller->semaphore_register[RCS] != MI_SEMAPHORE_SYNC_INVALID,
-	    ("valid RCS semaphore"));
-	return intel_ring_sync(waiter,
-			       signaller,
-			       RCS,
-			       seqno);
-}
-
-/* RCS->VCS (VRSYNC) or BCS->VCS (VBSYNC) */
-int
-gen6_bsd_ring_sync_to(struct intel_ring_buffer *waiter,
-		      struct intel_ring_buffer *signaller,
-		      u32 seqno)
-{
-	KASSERT(signaller->semaphore_register[VCS] != MI_SEMAPHORE_SYNC_INVALID,
-	    ("Valid VCS semaphore"));
-	return intel_ring_sync(waiter,
-			       signaller,
-			       VCS,
-			       seqno);
-}
-
-/* RCS->BCS (BRSYNC) or VCS->BCS (BVSYNC) */
-int
-gen6_blt_ring_sync_to(struct intel_ring_buffer *waiter,
-		      struct intel_ring_buffer *signaller,
-		      u32 seqno)
-{
-	KASSERT(signaller->semaphore_register[BCS] != MI_SEMAPHORE_SYNC_INVALID,
-	    ("Valid BCS semaphore"));
-	return intel_ring_sync(waiter,
-			       signaller,
-			       BCS,
-			       seqno);
-}
-
 #define PIPE_CONTROL_FLUSH(ring__, addr__)					\
 do {									\
 	intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |		\
@@ -645,28 +630,7 @@ pc_render_add_request(struct intel_ring_buffer *ring,
 	return 0;
 }
 
-static int
-render_ring_add_request(struct intel_ring_buffer *ring,
-			uint32_t *result)
-{
-	u32 seqno = i915_gem_next_request_seqno(ring);
-	int ret;
-
-	ret = intel_ring_begin(ring, 4);
-	if (ret)
-		return ret;
-
-	intel_ring_emit(ring, MI_STORE_DWORD_INDEX);
-	intel_ring_emit(ring, I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-	intel_ring_emit(ring, seqno);
-	intel_ring_emit(ring, MI_USER_INTERRUPT);
-	intel_ring_advance(ring);
-
-	*result = seqno;
-	return 0;
-}
-
- static u32
+static u32
 gen6_ring_get_seqno(struct intel_ring_buffer *ring)
 {
 	struct drm_device *dev = ring->dev;
@@ -697,40 +661,8 @@ pc_render_get_seqno(struct intel_ring_buffer *ring)
 		return (-1);
 }
 
-static void
-ironlake_enable_irq(drm_i915_private_t *dev_priv, uint32_t mask)
-{
-	dev_priv->gt_irq_mask &= ~mask;
-	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
-	POSTING_READ(GTIMR);
-}
-
-static void
-ironlake_disable_irq(drm_i915_private_t *dev_priv, uint32_t mask)
-{
-	dev_priv->gt_irq_mask |= mask;
-	I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
-	POSTING_READ(GTIMR);
-}
-
-static void
-i915_enable_irq(drm_i915_private_t *dev_priv, uint32_t mask)
-{
-	dev_priv->irq_mask &= ~mask;
-	I915_WRITE(IMR, dev_priv->irq_mask);
-	POSTING_READ(IMR);
-}
-
-static void
-i915_disable_irq(drm_i915_private_t *dev_priv, uint32_t mask)
-{
-	dev_priv->irq_mask |= mask;
-	I915_WRITE(IMR, dev_priv->irq_mask);
-	POSTING_READ(IMR);
-}
-
 static bool
-render_ring_get_irq(struct intel_ring_buffer *ring)
+gen5_ring_get_irq(struct intel_ring_buffer *ring)
 {
 	struct drm_device *dev = ring->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
@@ -738,32 +670,93 @@ render_ring_get_irq(struct intel_ring_buffer *ring)
 	if (!dev->irq_enabled)
 		return false;
 
-	mtx_assert(&ring->irq_lock, MA_OWNED);
+	mtx_assert(&dev_priv->irq_lock, MA_OWNED);
 	if (ring->irq_refcount++ == 0) {
-		if (HAS_PCH_SPLIT(dev))
-			ironlake_enable_irq(dev_priv,
-					    GT_PIPE_NOTIFY | GT_USER_INTERRUPT);
-		else
-			i915_enable_irq(dev_priv, I915_USER_INTERRUPT);
+		dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
+		I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
+		POSTING_READ(GTIMR);
 	}
 
 	return true;
 }
 
 static void
-render_ring_put_irq(struct intel_ring_buffer *ring)
+gen5_ring_put_irq(struct intel_ring_buffer *ring)
 {
 	struct drm_device *dev = ring->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 
-	mtx_assert(&ring->irq_lock, MA_OWNED);
+	mtx_assert(&dev_priv->irq_lock, MA_OWNED);
 	if (--ring->irq_refcount == 0) {
-		if (HAS_PCH_SPLIT(dev))
-			ironlake_disable_irq(dev_priv,
-					     GT_USER_INTERRUPT |
-					     GT_PIPE_NOTIFY);
-		else
-			i915_disable_irq(dev_priv, I915_USER_INTERRUPT);
+		dev_priv->gt_irq_mask |= ring->irq_enable_mask;
+		I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
+		POSTING_READ(GTIMR);
+	}
+}
+
+static bool
+i9xx_ring_get_irq(struct intel_ring_buffer *ring)
+{
+	struct drm_device *dev = ring->dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+
+	if (!dev->irq_enabled)
+		return false;
+
+	mtx_assert(&dev_priv->irq_lock, MA_OWNED);
+	if (ring->irq_refcount++ == 0) {
+		dev_priv->irq_mask &= ~ring->irq_enable_mask;
+		I915_WRITE(IMR, dev_priv->irq_mask);
+		POSTING_READ(IMR);
+	}
+
+	return true;
+}
+
+static void
+i9xx_ring_put_irq(struct intel_ring_buffer *ring)
+{
+	struct drm_device *dev = ring->dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+
+	mtx_assert(&dev_priv->irq_lock, MA_OWNED);
+	if (--ring->irq_refcount == 0) {
+		dev_priv->irq_mask |= ring->irq_enable_mask;
+		I915_WRITE(IMR, dev_priv->irq_mask);
+		POSTING_READ(IMR);
+	}
+}
+
+static bool
+i8xx_ring_get_irq(struct intel_ring_buffer *ring)
+{
+	struct drm_device *dev = ring->dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+
+	if (!dev->irq_enabled)
+		return false;
+
+	mtx_assert(&dev_priv->irq_lock, MA_OWNED);
+	if (ring->irq_refcount++ == 0) {
+		dev_priv->irq_mask &= ~ring->irq_enable_mask;
+		I915_WRITE16(IMR, dev_priv->irq_mask);
+		POSTING_READ16(IMR);
+	}
+
+	return true;
+}
+
+static void
+i8xx_ring_put_irq(struct intel_ring_buffer *ring)
+{
+	struct drm_device *dev = ring->dev;
+	drm_i915_private_t *dev_priv = dev->dev_private;
+
+	mtx_assert(&dev_priv->irq_lock, MA_OWNED);
+	if (--ring->irq_refcount == 0) {
+		dev_priv->irq_mask |= ring->irq_enable_mask;
+		I915_WRITE16(IMR, dev_priv->irq_mask);
+		POSTING_READ16(IMR);
 	}
 }
 
@@ -816,10 +809,10 @@ bsd_ring_flush(struct intel_ring_buffer *ring,
 }
 
 static int
-ring_add_request(struct intel_ring_buffer *ring,
-		 uint32_t *result)
+i9xx_add_request(struct intel_ring_buffer *ring,
+		 u32 *result)
 {
-	uint32_t seqno;
+	u32 seqno;
 	int ret;
 
 	ret = intel_ring_begin(ring, 4);
@@ -839,7 +832,7 @@ ring_add_request(struct intel_ring_buffer *ring,
 }
 
 static bool
-gen6_ring_get_irq(struct intel_ring_buffer *ring, uint32_t gflag, uint32_t rflag)
+gen6_ring_get_irq(struct intel_ring_buffer *ring)
 {
 	struct drm_device *dev = ring->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
@@ -849,69 +842,36 @@ gen6_ring_get_irq(struct intel_ring_buffer *ring, uint32_t gflag, uint32_t rflag
 
 	gen6_gt_force_wake_get(dev_priv);
 
-	mtx_assert(&ring->irq_lock, MA_OWNED);
+	mtx_assert(&dev_priv->irq_lock, MA_OWNED);
 	if (ring->irq_refcount++ == 0) {
-		ring->irq_mask &= ~rflag;
-		I915_WRITE_IMR(ring, ring->irq_mask);
-		ironlake_enable_irq(dev_priv, gflag);
+		I915_WRITE_IMR(ring, ~ring->irq_enable_mask);
+		dev_priv->gt_irq_mask &= ~ring->irq_enable_mask;
+		I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
+		POSTING_READ(GTIMR);
 	}
 
 	return true;
 }
 
 static void
-gen6_ring_put_irq(struct intel_ring_buffer *ring, uint32_t gflag, uint32_t rflag)
+gen6_ring_put_irq(struct intel_ring_buffer *ring)
 {
 	struct drm_device *dev = ring->dev;
 	drm_i915_private_t *dev_priv = dev->dev_private;
 
-	mtx_assert(&ring->irq_lock, MA_OWNED);
+	mtx_assert(&dev_priv->irq_lock, MA_OWNED);
 	if (--ring->irq_refcount == 0) {
-		ring->irq_mask |= rflag;
-		I915_WRITE_IMR(ring, ring->irq_mask);
-		ironlake_disable_irq(dev_priv, gflag);
+		I915_WRITE_IMR(ring, ~0);
+		dev_priv->gt_irq_mask |= ring->irq_enable_mask;
+		I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
+		POSTING_READ(GTIMR);
 	}
 
 	gen6_gt_force_wake_put(dev_priv);
 }
 
-static bool
-bsd_ring_get_irq(struct intel_ring_buffer *ring)
-{
-	struct drm_device *dev = ring->dev;
-	drm_i915_private_t *dev_priv = dev->dev_private;
-
-	if (!dev->irq_enabled)
-		return false;
-
-	mtx_assert(&ring->irq_lock, MA_OWNED);
-	if (ring->irq_refcount++ == 0) {
-		if (IS_G4X(dev))
-			i915_enable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
-		else
-			ironlake_enable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
-	}
-
-	return true;
-}
-static void
-bsd_ring_put_irq(struct intel_ring_buffer *ring)
-{
-	struct drm_device *dev = ring->dev;
-	drm_i915_private_t *dev_priv = dev->dev_private;
-
-	mtx_assert(&ring->irq_lock, MA_OWNED);
-	if (--ring->irq_refcount == 0) {
-		if (IS_G4X(dev))
-			i915_disable_irq(dev_priv, I915_BSD_USER_INTERRUPT);
-		else
-			ironlake_disable_irq(dev_priv, GT_BSD_USER_INTERRUPT);
-	}
-}
-
 static int
-ring_dispatch_execbuffer(struct intel_ring_buffer *ring, uint32_t offset,
-    uint32_t length)
+i965_dispatch_execbuffer(struct intel_ring_buffer *ring, u32 offset, u32 length)
 {
 	int ret;
 
@@ -920,7 +880,8 @@ ring_dispatch_execbuffer(struct intel_ring_buffer *ring, uint32_t offset,
 		return ret;
 
 	intel_ring_emit(ring,
-			MI_BATCH_BUFFER_START | (2 << 6) |
+			MI_BATCH_BUFFER_START |
+			MI_BATCH_GTT |
 			MI_BATCH_NON_SECURE_I965);
 	intel_ring_emit(ring, offset);
 	intel_ring_advance(ring);
@@ -929,37 +890,36 @@ ring_dispatch_execbuffer(struct intel_ring_buffer *ring, uint32_t offset,
 }
 
 static int
-render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
-				uint32_t offset, uint32_t len)
+i830_dispatch_execbuffer(struct intel_ring_buffer *ring,
+				u32 offset, u32 len)
 {
-	struct drm_device *dev = ring->dev;
 	int ret;
 
-	if (IS_I830(dev) || IS_845G(dev)) {
-		ret = intel_ring_begin(ring, 4);
-		if (ret)
-			return ret;
+	ret = intel_ring_begin(ring, 4);
+	if (ret)
+		return ret;
 
-		intel_ring_emit(ring, MI_BATCH_BUFFER);
-		intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
-		intel_ring_emit(ring, offset + len - 8);
-		intel_ring_emit(ring, 0);
-	} else {
-		ret = intel_ring_begin(ring, 2);
-		if (ret)
-			return ret;
+	intel_ring_emit(ring, MI_BATCH_BUFFER);
+	intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
+	intel_ring_emit(ring, offset + len - 8);
+	intel_ring_emit(ring, 0);
+	intel_ring_advance(ring);
 
-		if (INTEL_INFO(dev)->gen >= 4) {
-			intel_ring_emit(ring,
-					MI_BATCH_BUFFER_START | (2 << 6) |
-					MI_BATCH_NON_SECURE_I965);
-			intel_ring_emit(ring, offset);
-		} else {
-			intel_ring_emit(ring,
-					MI_BATCH_BUFFER_START | (2 << 6));
-			intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
-		}
-	}
+	return 0;
+}
+
+static int
+i915_dispatch_execbuffer(struct intel_ring_buffer *ring,
+				u32 offset, u32 len)
+{
+	int ret;
+
+	ret = intel_ring_begin(ring, 2);
+	if (ret)
+		return ret;
+
+	intel_ring_emit(ring, MI_BATCH_BUFFER_START | MI_BATCH_GTT);
+	intel_ring_emit(ring, offset | MI_BATCH_NON_SECURE);
 	intel_ring_advance(ring);
 
 	return 0;
@@ -967,7 +927,6 @@ render_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
 
 static void cleanup_status_page(struct intel_ring_buffer *ring)
 {
-	drm_i915_private_t *dev_priv = ring->dev->dev_private;
 	struct drm_i915_gem_object *obj;
 
 	obj = ring->status_page.obj;
@@ -980,14 +939,11 @@ static void cleanup_status_page(struct intel_ring_buffer *ring)
 	i915_gem_object_unpin(obj);
 	drm_gem_object_unreference(&obj->base);
 	ring->status_page.obj = NULL;
-
-	memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 }
 
 static int init_status_page(struct intel_ring_buffer *ring)
 {
 	struct drm_device *dev = ring->dev;
-	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct drm_i915_gem_object *obj;
 	int ret;
 
@@ -1008,7 +964,6 @@ static int init_status_page(struct intel_ring_buffer *ring)
 	ring->status_page.gfx_addr = obj->gtt_offset;
 	ring->status_page.page_addr = (void *)kva_alloc(PAGE_SIZE);
 	if (ring->status_page.page_addr == NULL) {
-		memset(&dev_priv->hws_map, 0, sizeof(dev_priv->hws_map));
 		goto err_unpin;
 	}
 	pmap_qenter((vm_offset_t)ring->status_page.page_addr, &obj->pages[0],
@@ -1032,8 +987,7 @@ err:
 	return ret;
 }
 
-static
-int intel_init_ring_buffer(struct drm_device *dev,
+static int intel_init_ring_buffer(struct drm_device *dev,
 			   struct intel_ring_buffer *ring)
 {
 	struct drm_i915_gem_object *obj;
@@ -1043,9 +997,7 @@ int intel_init_ring_buffer(struct drm_device *dev,
 	INIT_LIST_HEAD(&ring->active_list);
 	INIT_LIST_HEAD(&ring->request_list);
 	INIT_LIST_HEAD(&ring->gpu_write_list);
-
-	mtx_init(&ring->irq_lock, "ringb", NULL, MTX_DEF);
-	ring->irq_mask = ~0;
+	ring->size = 32 * PAGE_SIZE;
 
 	if (I915_NEED_GFX_HWS(dev)) {
 		ret = init_status_page(ring);
@@ -1066,20 +1018,15 @@ int intel_init_ring_buffer(struct drm_device *dev,
 	if (ret)
 		goto err_unref;
 
-	ring->map.size = ring->size;
-	ring->map.offset = dev->agp->base + obj->gtt_offset;
-	ring->map.type = 0;
-	ring->map.flags = 0;
-	ring->map.mtrr = 0;
-
-	drm_core_ioremap_wc(&ring->map, dev);
-	if (ring->map.virtual == NULL) {
+	ring->virtual_start = pmap_mapdev_attr(
+	    dev->agp->base + obj->gtt_offset, ring->size,
+	    VM_MEMATTR_WRITE_COMBINING);
+	if (ring->virtual_start == NULL) {
 		DRM_ERROR("Failed to map ringbuffer.\n");
 		ret = -EINVAL;
 		goto err_unpin;
 	}
 
-	ring->virtual_start = ring->map.virtual;
 	ret = ring->init(ring);
 	if (ret)
 		goto err_unmap;
@@ -1095,7 +1042,7 @@ int intel_init_ring_buffer(struct drm_device *dev,
 	return 0;
 
 err_unmap:
-	drm_core_ioremapfree(&ring->map, dev);
+	pmap_unmapdev((vm_offset_t)ring->virtual_start, ring->size);
 err_unpin:
 	i915_gem_object_unpin(obj);
 err_unref:
@@ -1119,7 +1066,7 @@ void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
 	ret = intel_wait_ring_idle(ring);
 	I915_WRITE_CTL(ring, 0);
 
-	drm_core_ioremapfree(&ring->map, ring->dev);
+	pmap_unmapdev((vm_offset_t)ring->virtual_start, ring->size);
 
 	i915_gem_object_unpin(ring->obj);
 	drm_gem_object_unreference(&ring->obj->base);
@@ -1133,7 +1080,7 @@ void intel_cleanup_ring_buffer(struct intel_ring_buffer *ring)
 
 static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
 {
-	unsigned int *virt;
+	uint32_t *virt;
 	int rem = ring->size - ring->tail;
 
 	if (ring->space < rem) {
@@ -1142,12 +1089,10 @@ static int intel_wrap_ring_buffer(struct intel_ring_buffer *ring)
 			return ret;
 	}
 
-	virt = (unsigned int *)((char *)ring->virtual_start + ring->tail);
-	rem /= 8;
-	while (rem--) {
+	virt = (uint32_t *)((char *)ring->virtual_start + ring->tail);
+	rem /= 4;
+	while (rem--)
 		*virt++ = MI_NOOP;
-		*virt++ = MI_NOOP;
-	}
 
 	ring->tail = 0;
 	ring->space = ring_space(ring);
@@ -1168,9 +1113,11 @@ static int intel_ring_wait_seqno(struct intel_ring_buffer *ring, u32 seqno)
 	was_interruptible = dev_priv->mm.interruptible;
 	dev_priv->mm.interruptible = false;
 
-	ret = i915_wait_request(ring, seqno, true);
+	ret = i915_wait_request(ring, seqno);
 
 	dev_priv->mm.interruptible = was_interruptible;
+	if (!ret)
+		i915_gem_retire_requests_ring(ring);
 
 	return ret;
 }
@@ -1244,15 +1191,13 @@ int intel_wait_ring_buffer(struct intel_ring_buffer *ring, int n)
 		return ret;
 
 	CTR1(KTR_DRM, "ring_wait_begin %s", ring->name);
-	if (drm_core_check_feature(dev, DRIVER_GEM))
-		/* With GEM the hangcheck timer should kick us out of the loop,
-		 * leaving it early runs the risk of corrupting GEM state (due
-		 * to running on almost untested codepaths). But on resume
-		 * timers don't work yet, so prevent a complete hang in that
-		 * case by choosing an insanely large timeout. */
-		end = ticks + hz * 60;
-	else
-		end = ticks + hz * 3;
+	/* With GEM the hangcheck timer should kick us out of the loop,
+	 * leaving it early runs the risk of corrupting GEM state (due
+	 * to running on almost untested codepaths). But on resume
+	 * timers don't work yet, so prevent a complete hang in that
+	 * case by choosing an insanely large timeout. */
+	end = ticks + hz * 60;
+
 	do {
 		ring->head = I915_READ_HEAD(ring);
 		ring->space = ring_space(ring);
@@ -1310,51 +1255,17 @@ int intel_ring_begin(struct intel_ring_buffer *ring,
 
 void intel_ring_advance(struct intel_ring_buffer *ring)
 {
+	struct drm_i915_private *dev_priv = ring->dev->dev_private;
+
 	ring->tail &= ring->size - 1;
+	if (dev_priv->stop_rings & intel_ring_flag(ring))
+		return;
 	ring->write_tail(ring, ring->tail);
 }
 
-static const struct intel_ring_buffer render_ring = {
-	.name			= "render ring",
-	.id			= RCS,
-	.mmio_base		= RENDER_RING_BASE,
-	.size			= 32 * PAGE_SIZE,
-	.init			= init_render_ring,
-	.write_tail		= ring_write_tail,
-	.flush			= render_ring_flush,
-	.add_request		= render_ring_add_request,
-	.get_seqno		= ring_get_seqno,
-	.irq_get		= render_ring_get_irq,
-	.irq_put		= render_ring_put_irq,
-	.dispatch_execbuffer	= render_ring_dispatch_execbuffer,
-	.cleanup		= render_ring_cleanup,
-	.sync_to		= render_ring_sync_to,
-	.semaphore_register	= {MI_SEMAPHORE_SYNC_INVALID,
-				   MI_SEMAPHORE_SYNC_RV,
-				   MI_SEMAPHORE_SYNC_RB},
-	.signal_mbox		= {GEN6_VRSYNC, GEN6_BRSYNC},
-};
-
-/* ring buffer for bit-stream decoder */
-
-static const struct intel_ring_buffer bsd_ring = {
-	.name                   = "bsd ring",
-	.id			= VCS,
-	.mmio_base		= BSD_RING_BASE,
-	.size			= 32 * PAGE_SIZE,
-	.init			= init_ring_common,
-	.write_tail		= ring_write_tail,
-	.flush			= bsd_ring_flush,
-	.add_request		= ring_add_request,
-	.get_seqno		= ring_get_seqno,
-	.irq_get		= bsd_ring_get_irq,
-	.irq_put		= bsd_ring_put_irq,
-	.dispatch_execbuffer	= ring_dispatch_execbuffer,
-};
-
 
 static void gen6_bsd_ring_write_tail(struct intel_ring_buffer *ring,
-				     uint32_t value)
+				     u32 value)
 {
 	drm_i915_private_t *dev_priv = ring->dev->dev_private;
 
@@ -1415,81 +1326,12 @@ gen6_ring_dispatch_execbuffer(struct intel_ring_buffer *ring,
 	return 0;
 }
 
-static bool
-gen6_render_ring_get_irq(struct intel_ring_buffer *ring)
-{
-	return gen6_ring_get_irq(ring,
-				 GT_USER_INTERRUPT,
-				 GEN6_RENDER_USER_INTERRUPT);
-}
-
-static void
-gen6_render_ring_put_irq(struct intel_ring_buffer *ring)
-{
-	return gen6_ring_put_irq(ring,
-				 GT_USER_INTERRUPT,
-				 GEN6_RENDER_USER_INTERRUPT);
-}
-
-static bool
-gen6_bsd_ring_get_irq(struct intel_ring_buffer *ring)
-{
-	return gen6_ring_get_irq(ring,
-				 GT_GEN6_BSD_USER_INTERRUPT,
-				 GEN6_BSD_USER_INTERRUPT);
-}
-
-static void
-gen6_bsd_ring_put_irq(struct intel_ring_buffer *ring)
-{
-	return gen6_ring_put_irq(ring,
-				 GT_GEN6_BSD_USER_INTERRUPT,
-				 GEN6_BSD_USER_INTERRUPT);
-}
-
-/* ring buffer for Video Codec for Gen6+ */
-static const struct intel_ring_buffer gen6_bsd_ring = {
-	.name			= "gen6 bsd ring",
-	.id			= VCS,
-	.mmio_base		= GEN6_BSD_RING_BASE,
-	.size			= 32 * PAGE_SIZE,
-	.init			= init_ring_common,
-	.write_tail		= gen6_bsd_ring_write_tail,
-	.flush			= gen6_ring_flush,
-	.add_request		= gen6_add_request,
-	.get_seqno		= gen6_ring_get_seqno,
-	.irq_get		= gen6_bsd_ring_get_irq,
-	.irq_put		= gen6_bsd_ring_put_irq,
-	.dispatch_execbuffer	= gen6_ring_dispatch_execbuffer,
-	.sync_to		= gen6_bsd_ring_sync_to,
-	.semaphore_register	= {MI_SEMAPHORE_SYNC_VR,
-				   MI_SEMAPHORE_SYNC_INVALID,
-				   MI_SEMAPHORE_SYNC_VB},
-	.signal_mbox		= {GEN6_RVSYNC, GEN6_BVSYNC},
-};
-
 /* Blitter support (SandyBridge+) */
 
-static bool
-blt_ring_get_irq(struct intel_ring_buffer *ring)
-{
-	return gen6_ring_get_irq(ring,
-				 GT_BLT_USER_INTERRUPT,
-				 GEN6_BLITTER_USER_INTERRUPT);
-}
-
-static void
-blt_ring_put_irq(struct intel_ring_buffer *ring)
-{
-	gen6_ring_put_irq(ring,
-			  GT_BLT_USER_INTERRUPT,
-			  GEN6_BLITTER_USER_INTERRUPT);
-}
-
 static int blt_ring_flush(struct intel_ring_buffer *ring,
-			  uint32_t invalidate, uint32_t flush)
+			  u32 invalidate, u32 flush)
 {
-	uint32_t cmd;
+	u32 cmd;
 	int ret;
 
 	ret = intel_ring_begin(ring, 4);
@@ -1507,42 +1349,63 @@ static int blt_ring_flush(struct intel_ring_buffer *ring,
 	return 0;
 }
 
-static const struct intel_ring_buffer gen6_blt_ring = {
-	.name			= "blt ring",
-	.id			= BCS,
-	.mmio_base		= BLT_RING_BASE,
-	.size			= 32 * PAGE_SIZE,
-	.init			= init_ring_common,
-	.write_tail		= ring_write_tail,
-	.flush			= blt_ring_flush,
-	.add_request		= gen6_add_request,
-	.get_seqno		= gen6_ring_get_seqno,
-	.irq_get		= blt_ring_get_irq,
-	.irq_put		= blt_ring_put_irq,
-	.dispatch_execbuffer	= gen6_ring_dispatch_execbuffer,
-	.sync_to		= gen6_blt_ring_sync_to,
-	.semaphore_register	= {MI_SEMAPHORE_SYNC_BR,
-				   MI_SEMAPHORE_SYNC_BV,
-				   MI_SEMAPHORE_SYNC_INVALID},
-	.signal_mbox		= {GEN6_RBSYNC, GEN6_VBSYNC},
-};
-
 int intel_init_render_ring_buffer(struct drm_device *dev)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
 
-	*ring = render_ring;
+	ring->name = "render ring";
+	ring->id = RCS;
+	ring->mmio_base = RENDER_RING_BASE;
+
 	if (INTEL_INFO(dev)->gen >= 6) {
 		ring->add_request = gen6_add_request;
 		ring->flush = gen6_render_ring_flush;
-		ring->irq_get = gen6_render_ring_get_irq;
-		ring->irq_put = gen6_render_ring_put_irq;
+		ring->irq_get = gen6_ring_get_irq;
+		ring->irq_put = gen6_ring_put_irq;
+		ring->irq_enable_mask = GT_USER_INTERRUPT;
 		ring->get_seqno = gen6_ring_get_seqno;
+		ring->sync_to = gen6_ring_sync;
+		ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_INVALID;
+		ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_RV;
+		ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_RB;
+		ring->signal_mbox[0] = GEN6_VRSYNC;
+		ring->signal_mbox[1] = GEN6_BRSYNC;
 	} else if (IS_GEN5(dev)) {
 		ring->add_request = pc_render_add_request;
+		ring->flush = gen4_render_ring_flush;
 		ring->get_seqno = pc_render_get_seqno;
+		ring->irq_get = gen5_ring_get_irq;
+		ring->irq_put = gen5_ring_put_irq;
+		ring->irq_enable_mask = GT_USER_INTERRUPT | GT_PIPE_NOTIFY;
+	} else {
+		ring->add_request = i9xx_add_request;
+		if (INTEL_INFO(dev)->gen < 4)
+			ring->flush = gen2_render_ring_flush;
+		else
+			ring->flush = gen4_render_ring_flush;
+		ring->get_seqno = ring_get_seqno;
+		if (IS_GEN2(dev)) {
+			ring->irq_get = i8xx_ring_get_irq;
+			ring->irq_put = i8xx_ring_put_irq;
+		} else {
+			ring->irq_get = i9xx_ring_get_irq;
+			ring->irq_put = i9xx_ring_put_irq;
+		}
+		ring->irq_enable_mask = I915_USER_INTERRUPT;
 	}
+	ring->write_tail = ring_write_tail;
+	if (INTEL_INFO(dev)->gen >= 6)
+		ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
+	else if (INTEL_INFO(dev)->gen >= 4)
+		ring->dispatch_execbuffer = i965_dispatch_execbuffer;
+	else if (IS_I830(dev) || IS_845G(dev))
+		ring->dispatch_execbuffer = i830_dispatch_execbuffer;
+	else
+		ring->dispatch_execbuffer = i915_dispatch_execbuffer;
+	ring->init = init_render_ring;
+	ring->cleanup = render_ring_cleanup;
+
 
 	if (!I915_NEED_GFX_HWS(dev)) {
 		ring->status_page.page_addr = dev_priv->status_page_dmah->vaddr;
@@ -1552,22 +1415,47 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
 	return intel_init_ring_buffer(dev, ring);
 }
 
-int intel_render_ring_init_dri(struct drm_device *dev, uint64_t start,
-    uint32_t size)
+int intel_render_ring_init_dri(struct drm_device *dev, u64 start, u32 size)
 {
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct intel_ring_buffer *ring = &dev_priv->rings[RCS];
 
-	*ring = render_ring;
+	ring->name = "render ring";
+	ring->id = RCS;
+	ring->mmio_base = RENDER_RING_BASE;
+
 	if (INTEL_INFO(dev)->gen >= 6) {
-		ring->add_request = gen6_add_request;
-		ring->irq_get = gen6_render_ring_get_irq;
-		ring->irq_put = gen6_render_ring_put_irq;
-	} else if (IS_GEN5(dev)) {
-		ring->add_request = pc_render_add_request;
-		ring->get_seqno = pc_render_get_seqno;
+		/* non-kms not supported on gen6+ */
+		return -ENODEV;
 	}
 
+	/* Note: gem is not supported on gen5/ilk without kms (the corresponding
+	 * gem_init ioctl returns with -ENODEV). Hence we do not need to set up
+	 * the special gen5 functions. */
+	ring->add_request = i9xx_add_request;
+	if (INTEL_INFO(dev)->gen < 4)
+		ring->flush = gen2_render_ring_flush;
+	else
+		ring->flush = gen4_render_ring_flush;
+	ring->get_seqno = ring_get_seqno;
+	if (IS_GEN2(dev)) {
+		ring->irq_get = i8xx_ring_get_irq;
+		ring->irq_put = i8xx_ring_put_irq;
+	} else {
+		ring->irq_get = i9xx_ring_get_irq;
+		ring->irq_put = i9xx_ring_put_irq;
+	}
+	ring->irq_enable_mask = I915_USER_INTERRUPT;
+	ring->write_tail = ring_write_tail;
+	if (INTEL_INFO(dev)->gen >= 4)
+		ring->dispatch_execbuffer = i965_dispatch_execbuffer;
+	else if (IS_I830(dev) || IS_845G(dev))
+		ring->dispatch_execbuffer = i830_dispatch_execbuffer;
+	else
+		ring->dispatch_execbuffer = i915_dispatch_execbuffer;
+	ring->init = init_render_ring;
+	ring->cleanup = render_ring_cleanup;
+
 	ring->dev = dev;
 	INIT_LIST_HEAD(&ring->active_list);
 	INIT_LIST_HEAD(&ring->request_list);
@@ -1578,20 +1466,14 @@ int intel_render_ring_init_dri(struct drm_device *dev, uint64_t start,
 	if (IS_I830(ring->dev))
 		ring->effective_size -= 128;
 
-	ring->map.offset = start;
-	ring->map.size = size;
-	ring->map.type = 0;
-	ring->map.flags = 0;
-	ring->map.mtrr = 0;
-
-	drm_core_ioremap_wc(&ring->map, dev);
-	if (ring->map.virtual == NULL) {
+	ring->virtual_start = pmap_mapdev_attr(start, size,
+	    VM_MEMATTR_WRITE_COMBINING);
+	if (ring->virtual_start == NULL) {
 		DRM_ERROR("can not ioremap virtual address for"
 			  " ring buffer\n");
 		return -ENOMEM;
 	}
 
-	ring->virtual_start = (void *)ring->map.virtual;
 	return 0;
 }
 
@@ -1600,10 +1482,46 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev)
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct intel_ring_buffer *ring = &dev_priv->rings[VCS];
 
-	if (IS_GEN6(dev) || IS_GEN7(dev))
-		*ring = gen6_bsd_ring;
-	else
-		*ring = bsd_ring;
+	ring->name = "bsd ring";
+	ring->id = VCS;
+
+	ring->write_tail = ring_write_tail;
+	if (IS_GEN6(dev) || IS_GEN7(dev)) {
+		ring->mmio_base = GEN6_BSD_RING_BASE;
+		/* gen6 bsd needs a special wa for tail updates */
+		if (IS_GEN6(dev))
+			ring->write_tail = gen6_bsd_ring_write_tail;
+		ring->flush = gen6_ring_flush;
+		ring->add_request = gen6_add_request;
+		ring->get_seqno = gen6_ring_get_seqno;
+		ring->irq_enable_mask = GEN6_BSD_USER_INTERRUPT;
+		ring->irq_get = gen6_ring_get_irq;
+		ring->irq_put = gen6_ring_put_irq;
+		ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
+		ring->sync_to = gen6_ring_sync;
+		ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_VR;
+		ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_INVALID;
+		ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_VB;
+		ring->signal_mbox[0] = GEN6_RVSYNC;
+		ring->signal_mbox[1] = GEN6_BVSYNC;
+	} else {
+		ring->mmio_base = BSD_RING_BASE;
+		ring->flush = bsd_ring_flush;
+		ring->add_request = i9xx_add_request;
+		ring->get_seqno = ring_get_seqno;
+		if (IS_GEN5(dev)) {
+			ring->irq_enable_mask = GT_BSD_USER_INTERRUPT;
+			ring->irq_get = gen5_ring_get_irq;
+			ring->irq_put = gen5_ring_put_irq;
+		} else {
+			ring->irq_enable_mask = I915_BSD_USER_INTERRUPT;
+			ring->irq_get = i9xx_ring_get_irq;
+			ring->irq_put = i9xx_ring_put_irq;
+		}
+		ring->dispatch_execbuffer = i965_dispatch_execbuffer;
+	}
+	ring->init = init_ring_common;
+
 
 	return intel_init_ring_buffer(dev, ring);
 }
@@ -1613,7 +1531,25 @@ int intel_init_blt_ring_buffer(struct drm_device *dev)
 	drm_i915_private_t *dev_priv = dev->dev_private;
 	struct intel_ring_buffer *ring = &dev_priv->rings[BCS];
 
-	*ring = gen6_blt_ring;
+	ring->name = "blitter ring";
+	ring->id = BCS;
+
+	ring->mmio_base = BLT_RING_BASE;
+	ring->write_tail = ring_write_tail;
+	ring->flush = blt_ring_flush;
+	ring->add_request = gen6_add_request;
+	ring->get_seqno = gen6_ring_get_seqno;
+	ring->irq_enable_mask = GEN6_BLITTER_USER_INTERRUPT;
+	ring->irq_get = gen6_ring_get_irq;
+	ring->irq_put = gen6_ring_put_irq;
+	ring->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
+	ring->sync_to = gen6_ring_sync;
+	ring->semaphore_register[0] = MI_SEMAPHORE_SYNC_BR;
+	ring->semaphore_register[1] = MI_SEMAPHORE_SYNC_BV;
+	ring->semaphore_register[2] = MI_SEMAPHORE_SYNC_INVALID;
+	ring->signal_mbox[0] = GEN6_RBSYNC;
+	ring->signal_mbox[1] = GEN6_VBSYNC;
+	ring->init = init_ring_common;
 
 	return intel_init_ring_buffer(dev, ring);
 }
diff --git a/sys/dev/drm2/i915/intel_ringbuffer.h b/sys/dev/drm2/i915/intel_ringbuffer.h
index c391b12c9f9..efbc39bf967 100644
--- a/sys/dev/drm2/i915/intel_ringbuffer.h
+++ b/sys/dev/drm2/i915/intel_ringbuffer.h
@@ -6,7 +6,7 @@
 #define _INTEL_RINGBUFFER_H_
 
 struct  intel_hw_status_page {
-	uint32_t	*page_addr;
+	u32		*page_addr;
 	unsigned int	gfx_addr;
 	struct		drm_i915_gem_object *obj;
 };
@@ -38,13 +38,13 @@ struct  intel_ring_buffer {
 		BCS,
 	} id;
 #define I915_NUM_RINGS 3
-	uint32_t	mmio_base;
+	u32		mmio_base;
 	void		*virtual_start;
 	struct		drm_device *dev;
 	struct		drm_i915_gem_object *obj;
 
-	uint32_t	head;
-	uint32_t	tail;
+	u32		head;
+	u32		tail;
 	int		space;
 	int		size;
 	int		effective_size;
@@ -60,13 +60,10 @@ struct  intel_ring_buffer {
 	 */
 	u32		last_retired_head;
 
-	struct mtx	irq_lock;
-	uint32_t	irq_refcount;
-	uint32_t	irq_mask;
-	uint32_t	irq_seqno;		/* last seq seem at irq time */
-	uint32_t	trace_irq_seqno;
-	uint32_t	waiting_seqno;
-	uint32_t	sync_seqno[I915_NUM_RINGS-1];
+	u32		irq_refcount;
+	u32		irq_enable_mask;	/* bitmask to enable ring interrupt */
+	u32		trace_irq_seqno;
+	u32		sync_seqno[I915_NUM_RINGS-1];
 	bool		(*irq_get)(struct intel_ring_buffer *ring);
 	void		(*irq_put)(struct intel_ring_buffer *ring);
 
@@ -120,7 +117,7 @@ struct  intel_ring_buffer {
 	/**
 	 * Do we have some not yet emitted requests outstanding?
 	 */
-	uint32_t outstanding_lazy_request;
+	u32 outstanding_lazy_request;
 
 	/**
 	 * Do an explicit TLB flush before MI_SET_CONTEXT
@@ -169,6 +166,8 @@ static inline uint32_t
 intel_read_status_page(struct intel_ring_buffer *ring, int reg)
 {
 
+	/* Ensure that the compiler doesn't optimize away the load. */
+	__compiler_membar();
 	return (atomic_load_acq_32(ring->status_page.page_addr + reg));
 }
 
diff --git a/sys/dev/drm2/i915/intel_sdvo.c b/sys/dev/drm2/i915/intel_sdvo.c
index 8cae7662d5d..74e479a60f9 100644
--- a/sys/dev/drm2/i915/intel_sdvo.c
+++ b/sys/dev/drm2/i915/intel_sdvo.c
@@ -44,7 +44,7 @@ __FBSDID("$FreeBSD$");
 #define SDVO_TMDS_MASK (SDVO_OUTPUT_TMDS0 | SDVO_OUTPUT_TMDS1)
 #define SDVO_RGB_MASK  (SDVO_OUTPUT_RGB0 | SDVO_OUTPUT_RGB1)
 #define SDVO_LVDS_MASK (SDVO_OUTPUT_LVDS0 | SDVO_OUTPUT_LVDS1)
-#define SDVO_TV_MASK   (SDVO_OUTPUT_CVBS0 | SDVO_OUTPUT_SVID0)
+#define SDVO_TV_MASK   (SDVO_OUTPUT_CVBS0 | SDVO_OUTPUT_SVID0 | SDVO_OUTPUT_YPRPB0)
 
 #define SDVO_OUTPUT_MASK (SDVO_TMDS_MASK | SDVO_RGB_MASK | SDVO_LVDS_MASK |\
 			SDVO_TV_MASK)
@@ -77,7 +77,7 @@ struct intel_sdvo {
 	device_t ddc_iic_bus, ddc;
 
 	/* Register for the SDVO device: SDVOB or SDVOC */
-	int sdvo_reg;
+	uint32_t sdvo_reg;
 
 	/* Active outputs controlled by this SDVO output */
 	uint16_t controlled_output;
@@ -117,6 +117,9 @@ struct intel_sdvo {
 	 */
 	bool is_tv;
 
+	/* On different gens SDVOB is at different places. */
+	bool is_sdvob;
+
 	/* This is for current tv format name */
 	int tv_format_index;
 
@@ -406,12 +409,10 @@ static const struct _sdvo_cmd_name {
 	SDVO_CMD_NAME_ENTRY(SDVO_CMD_GET_HBUF_DATA),
 };
 
-#define IS_SDVOB(reg)	(reg == SDVOB || reg == PCH_SDVOB)
-#define SDVO_NAME(svdo) (IS_SDVOB((svdo)->sdvo_reg) ? "SDVOB" : "SDVOC")
+#define SDVO_NAME(svdo) ((svdo)->is_sdvob ? "SDVOB" : "SDVOC")
 
-static void
-intel_sdvo_debug_write(struct intel_sdvo *intel_sdvo, u8 cmd,
-    const void *args, int args_len)
+static void intel_sdvo_debug_write(struct intel_sdvo *intel_sdvo, u8 cmd,
+				   const void *args, int args_len)
 {
 	int i;
 
@@ -744,18 +745,18 @@ static void intel_sdvo_get_dtd_from_mode(struct intel_sdvo_dtd *dtd,
 	uint16_t h_sync_offset, v_sync_offset;
 	int mode_clock;
 
-	width = mode->crtc_hdisplay;
-	height = mode->crtc_vdisplay;
+	width = mode->hdisplay;
+	height = mode->vdisplay;
 
-	/* do some mode translations */
-	h_blank_len = mode->crtc_hblank_end - mode->crtc_hblank_start;
-	h_sync_len = mode->crtc_hsync_end - mode->crtc_hsync_start;
+ 	/* do some mode translations */
+	h_blank_len = mode->htotal - mode->hdisplay;
+	h_sync_len = mode->hsync_end - mode->hsync_start;
 
-	v_blank_len = mode->crtc_vblank_end - mode->crtc_vblank_start;
-	v_sync_len = mode->crtc_vsync_end - mode->crtc_vsync_start;
+	v_blank_len = mode->vtotal - mode->vdisplay;
+	v_sync_len = mode->vsync_end - mode->vsync_start;
 
-	h_sync_offset = mode->crtc_hsync_start - mode->crtc_hblank_start;
-	v_sync_offset = mode->crtc_vsync_start - mode->crtc_vblank_start;
+	h_sync_offset = mode->hsync_start - mode->hdisplay;
+	v_sync_offset = mode->vsync_start - mode->vdisplay;
 
 	mode_clock = mode->clock;
 	mode_clock /= intel_mode_get_pixel_multiplier(mode) ?: 1;
@@ -884,17 +885,24 @@ static bool intel_sdvo_set_avi_infoframe(struct intel_sdvo *intel_sdvo)
 	};
 	uint8_t tx_rate = SDVO_HBUF_TX_VSYNC;
 	uint8_t set_buf_index[2] = { 1, 0 };
-	uint64_t *data = (uint64_t *)&avi_if;
+	uint8_t sdvo_data[4 + sizeof(avi_if.body.avi)];
+	uint64_t *data = (uint64_t *)sdvo_data;
 	unsigned i;
 
 	intel_dip_infoframe_csum(&avi_if);
 
+	/* sdvo spec says that the ecc is handled by the hw, and it looks like
+	 * we must not send the ecc field, either. */
+	memcpy(sdvo_data, &avi_if, 3);
+	sdvo_data[3] = avi_if.checksum;
+	memcpy(&sdvo_data[4], &avi_if.body, sizeof(avi_if.body.avi));
+
 	if (!intel_sdvo_set_value(intel_sdvo,
 				  SDVO_CMD_SET_HBUF_INDEX,
 				  set_buf_index, 2))
 		return false;
 
-	for (i = 0; i < sizeof(avi_if); i += 8) {
+	for (i = 0; i < sizeof(sdvo_data); i += 8) {
 		if (!intel_sdvo_set_value(intel_sdvo,
 					  SDVO_CMD_SET_HBUF_DATA,
 					  data, 8))
@@ -964,7 +972,7 @@ intel_sdvo_set_input_timings_for_mode(struct intel_sdvo *intel_sdvo,
 }
 
 static bool intel_sdvo_mode_fixup(struct drm_encoder *encoder,
-				  const struct drm_display_mode *mode,
+				  struct drm_display_mode *mode,
 				  struct drm_display_mode *adjusted_mode)
 {
 	struct intel_sdvo *intel_sdvo = to_intel_sdvo(encoder);
@@ -1272,7 +1280,8 @@ intel_sdvo_get_analog_edid(struct drm_connector *connector)
 	struct drm_i915_private *dev_priv = connector->dev->dev_private;
 
 	return drm_get_edid(connector,
-			    dev_priv->gmbus[dev_priv->crt_ddc_pin]);
+			    intel_gmbus_get_adapter(dev_priv,
+						    dev_priv->crt_ddc_pin));
 }
 
 static enum drm_connector_status
@@ -1361,8 +1370,7 @@ intel_sdvo_detect(struct drm_connector *connector, bool force)
 		return connector_status_unknown;
 
 	/* add 30ms delay when the output type might be TV */
-	if (intel_sdvo->caps.output_flags &
-	    (SDVO_OUTPUT_SVID0 | SDVO_OUTPUT_CVBS0))
+	if (intel_sdvo->caps.output_flags & SDVO_TV_MASK)
 		drm_msleep(30, "915svo");
 
 	if (!intel_sdvo_read_response(intel_sdvo, &response, 2))
@@ -1582,9 +1590,6 @@ end:
 			intel_sdvo->sdvo_lvds_fixed_mode =
 				drm_mode_duplicate(connector->dev, newmode);
 
-			drm_mode_set_crtcinfo(intel_sdvo->sdvo_lvds_fixed_mode,
-					      0);
-
 			intel_sdvo->is_lvds = true;
 			break;
 		}
@@ -1916,7 +1921,7 @@ intel_sdvo_select_ddc_bus(struct drm_i915_private *dev_priv,
 {
 	struct sdvo_device_mapping *mapping;
 
-	if (IS_SDVOB(reg))
+	if (sdvo->is_sdvob)
 		mapping = &(dev_priv->sdvo_mappings[0]);
 	else
 		mapping = &(dev_priv->sdvo_mappings[1]);
@@ -1934,7 +1939,7 @@ intel_sdvo_select_i2c_bus(struct drm_i915_private *dev_priv,
 	struct sdvo_device_mapping *mapping;
 	u8 pin;
 
-	if (IS_SDVOB(reg))
+	if (sdvo->is_sdvob)
 		mapping = &dev_priv->sdvo_mappings[0];
 	else
 		mapping = &dev_priv->sdvo_mappings[1];
@@ -1943,12 +1948,12 @@ intel_sdvo_select_i2c_bus(struct drm_i915_private *dev_priv,
 	if (mapping->initialized)
 		pin = mapping->i2c_pin;
 
-	if (pin < GMBUS_NUM_PORTS) {
-		sdvo->i2c = dev_priv->gmbus[pin];
+	if (intel_gmbus_is_port_valid(pin)) {
+		sdvo->i2c = intel_gmbus_get_adapter(dev_priv, pin);
 		intel_gmbus_set_speed(sdvo->i2c, GMBUS_RATE_1MHZ);
 		intel_gmbus_force_bit(sdvo->i2c, true);
 	} else {
-		sdvo->i2c = dev_priv->gmbus[GMBUS_PORT_DPB];
+		sdvo->i2c = intel_gmbus_get_adapter(dev_priv, GMBUS_PORT_DPB);
 	}
 }
 
@@ -1959,12 +1964,12 @@ intel_sdvo_is_hdmi_connector(struct intel_sdvo *intel_sdvo, int device)
 }
 
 static u8
-intel_sdvo_get_slave_addr(struct drm_device *dev, int sdvo_reg)
+intel_sdvo_get_slave_addr(struct drm_device *dev, struct intel_sdvo *sdvo)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct sdvo_device_mapping *my_mapping, *other_mapping;
 
-	if (IS_SDVOB(sdvo_reg)) {
+	if (sdvo->is_sdvob) {
 		my_mapping = &dev_priv->sdvo_mappings[0];
 		other_mapping = &dev_priv->sdvo_mappings[1];
 	} else {
@@ -1989,7 +1994,7 @@ intel_sdvo_get_slave_addr(struct drm_device *dev, int sdvo_reg)
 	/* No SDVO device info is found for another DVO port,
 	 * so use mapping assumption we had before BIOS parsing.
 	 */
-	if (IS_SDVOB(sdvo_reg))
+	if (sdvo->is_sdvob)
 		return 0x70;
 	else
 		return 0x72;
@@ -2214,6 +2219,10 @@ intel_sdvo_output_setup(struct intel_sdvo *intel_sdvo, uint16_t flags)
 		if (!intel_sdvo_tv_init(intel_sdvo, SDVO_OUTPUT_CVBS0))
 			return false;
 
+	if (flags & SDVO_OUTPUT_YPRPB0)
+		if (!intel_sdvo_tv_init(intel_sdvo, SDVO_OUTPUT_YPRPB0))
+			return false;
+
 	if (flags & SDVO_OUTPUT_RGB0)
 		if (!intel_sdvo_analog_init(intel_sdvo, 0))
 			return false;
@@ -2583,7 +2592,7 @@ DRIVER_MODULE_ORDERED(intel_sdvo_ddc_proxy, drmn, intel_sdvo_ddc_proxy_driver,
     intel_sdvo_devclass, 0, 0, SI_ORDER_FIRST);
 
 
-bool intel_sdvo_init(struct drm_device *dev, int sdvo_reg)
+bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob)
 {
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_encoder *intel_encoder;
@@ -2594,7 +2603,8 @@ bool intel_sdvo_init(struct drm_device *dev, int sdvo_reg)
 	    M_WAITOK | M_ZERO);
 
 	intel_sdvo->sdvo_reg = sdvo_reg;
-	intel_sdvo->slave_addr = intel_sdvo_get_slave_addr(dev, sdvo_reg) >> 1;
+	intel_sdvo->is_sdvob = is_sdvob;
+	intel_sdvo->slave_addr = intel_sdvo_get_slave_addr(dev, intel_sdvo) >> 1;
 	intel_sdvo_select_i2c_bus(dev_priv, intel_sdvo, sdvo_reg);
 	if (!intel_sdvo_init_ddc_proxy(intel_sdvo, dev, sdvo_reg)) {
 		free(intel_sdvo, DRM_MEM_KMS);
@@ -2611,13 +2621,13 @@ bool intel_sdvo_init(struct drm_device *dev, int sdvo_reg)
 		u8 byte;
 
 		if (!intel_sdvo_read_byte(intel_sdvo, i, &byte)) {
-			DRM_DEBUG_KMS("No SDVO device found on SDVO%c\n",
-				      IS_SDVOB(sdvo_reg) ? 'B' : 'C');
+			DRM_DEBUG_KMS("No SDVO device found on %s\n",
+				      SDVO_NAME(intel_sdvo));
 			goto err;
 		}
 	}
 
-	if (IS_SDVOB(sdvo_reg))
+	if (intel_sdvo->is_sdvob)
 		dev_priv->hotplug_supported_mask |= SDVOB_HOTPLUG_INT_STATUS;
 	else
 		dev_priv->hotplug_supported_mask |= SDVOC_HOTPLUG_INT_STATUS;
@@ -2636,12 +2646,12 @@ bool intel_sdvo_init(struct drm_device *dev, int sdvo_reg)
 			     &intel_sdvo->hotplug_active, 2);
 	intel_sdvo->hotplug_active[0] &= ~0x3;
 
-	if (!intel_sdvo_output_setup(intel_sdvo,
-	    intel_sdvo->caps.output_flags)) {
-		DRM_DEBUG_KMS("SDVO output failed to setup on SDVO%c\n",
-			      IS_SDVOB(sdvo_reg) ? 'B' : 'C');
-		goto err;
-	}
+ 	if (intel_sdvo_output_setup(intel_sdvo,
+ 				    intel_sdvo->caps.output_flags) != true) {
+		DRM_DEBUG_KMS("SDVO output failed to setup on %s\n",
+			      SDVO_NAME(intel_sdvo));
+ 		goto err;
+ 	}
 
 	intel_sdvo_select_ddc_bus(dev_priv, intel_sdvo, sdvo_reg);
 
diff --git a/sys/dev/drm2/i915/intel_sprite.c b/sys/dev/drm2/i915/intel_sprite.c
index 0737a5c6cf1..35f2baab57f 100644
--- a/sys/dev/drm2/i915/intel_sprite.c
+++ b/sys/dev/drm2/i915/intel_sprite.c
@@ -114,14 +114,18 @@ ivb_update_plane(struct drm_plane *plane, struct drm_framebuffer *fb,
 	 * when scaling is disabled.
 	 */
 	if (crtc_w != src_w || crtc_h != src_h) {
-		dev_priv->sprite_scaling_enabled = true;
-		sandybridge_update_wm(dev);
-		intel_wait_for_vblank(dev, pipe);
+		if (!dev_priv->sprite_scaling_enabled) {
+			dev_priv->sprite_scaling_enabled = true;
+			intel_update_watermarks(dev);
+			intel_wait_for_vblank(dev, pipe);
+		}
 		sprscale = SPRITE_SCALE_ENABLE | (src_w << 16) | src_h;
 	} else {
-		dev_priv->sprite_scaling_enabled = false;
-		/* potentially re-enable LP watermarks */
-		sandybridge_update_wm(dev);
+		if (dev_priv->sprite_scaling_enabled) {
+			dev_priv->sprite_scaling_enabled = false;
+			/* potentially re-enable LP watermarks */
+			intel_update_watermarks(dev);
+		}
 	}
 
 	I915_WRITE(SPRSTRIDE(pipe), fb->pitches[0]);
@@ -137,7 +141,7 @@ ivb_update_plane(struct drm_plane *plane, struct drm_framebuffer *fb,
 	I915_WRITE(SPRSIZE(pipe), (crtc_h << 16) | crtc_w);
 	I915_WRITE(SPRSCALE(pipe), sprscale);
 	I915_WRITE(SPRCTL(pipe), sprctl);
-	I915_WRITE(SPRSURF(pipe), obj->gtt_offset);
+	I915_MODIFY_DISPBASE(SPRSURF(pipe), obj->gtt_offset);
 	POSTING_READ(SPRSURF(pipe));
 }
 
@@ -153,8 +157,11 @@ ivb_disable_plane(struct drm_plane *plane)
 	/* Can't leave the scaler enabled... */
 	I915_WRITE(SPRSCALE(pipe), 0);
 	/* Activate double buffered register update */
-	I915_WRITE(SPRSURF(pipe), 0);
+	I915_MODIFY_DISPBASE(SPRSURF(pipe), 0);
 	POSTING_READ(SPRSURF(pipe));
+
+	dev_priv->sprite_scaling_enabled = false;
+	intel_update_watermarks(dev);
 }
 
 static int
@@ -212,7 +219,7 @@ ivb_get_colorkey(struct drm_plane *plane, struct drm_intel_sprite_colorkey *key)
 }
 
 static void
-snb_update_plane(struct drm_plane *plane, struct drm_framebuffer *fb,
+ilk_update_plane(struct drm_plane *plane, struct drm_framebuffer *fb,
 		 struct drm_i915_gem_object *obj, int crtc_x, int crtc_y,
 		 unsigned int crtc_w, unsigned int crtc_h,
 		 uint32_t x, uint32_t y,
@@ -222,7 +229,7 @@ snb_update_plane(struct drm_plane *plane, struct drm_framebuffer *fb,
 	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct intel_plane *intel_plane = to_intel_plane(plane);
 	int pipe = intel_plane->pipe, pixel_size;
-	u32 dvscntr, dvsscale = 0;
+	u32 dvscntr, dvsscale;
 
 	dvscntr = I915_READ(DVSCNTR(pipe));
 
@@ -266,8 +273,8 @@ snb_update_plane(struct drm_plane *plane, struct drm_framebuffer *fb,
 	if (obj->tiling_mode != I915_TILING_NONE)
 		dvscntr |= DVS_TILED;
 
-	/* must disable */
-	dvscntr |= DVS_TRICKLE_FEED_DISABLE;
+	if (IS_GEN6(dev))
+		dvscntr |= DVS_TRICKLE_FEED_DISABLE; /* must disable */
 	dvscntr |= DVS_ENABLE;
 
 	/* Sizes are 0 based */
@@ -278,7 +285,8 @@ snb_update_plane(struct drm_plane *plane, struct drm_framebuffer *fb,
 
 	intel_update_sprite_watermarks(dev, pipe, crtc_w, pixel_size);
 
-	if (crtc_w != src_w || crtc_h != src_h)
+	dvsscale = 0;
+	if (IS_GEN5(dev) || crtc_w != src_w || crtc_h != src_h)
 		dvsscale = DVS_SCALE_ENABLE | (src_w << 16) | src_h;
 
 	I915_WRITE(DVSSTRIDE(pipe), fb->pitches[0]);
@@ -294,12 +302,12 @@ snb_update_plane(struct drm_plane *plane, struct drm_framebuffer *fb,
 	I915_WRITE(DVSSIZE(pipe), (crtc_h << 16) | crtc_w);
 	I915_WRITE(DVSSCALE(pipe), dvsscale);
 	I915_WRITE(DVSCNTR(pipe), dvscntr);
-	I915_WRITE(DVSSURF(pipe), obj->gtt_offset);
+	I915_MODIFY_DISPBASE(DVSSURF(pipe), obj->gtt_offset);
 	POSTING_READ(DVSSURF(pipe));
 }
 
 static void
-snb_disable_plane(struct drm_plane *plane)
+ilk_disable_plane(struct drm_plane *plane)
 {
 	struct drm_device *dev = plane->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
@@ -310,7 +318,7 @@ snb_disable_plane(struct drm_plane *plane)
 	/* Disable the scaler */
 	I915_WRITE(DVSSCALE(pipe), 0);
 	/* Flush double buffered register updates */
-	I915_WRITE(DVSSURF(pipe), 0);
+	I915_MODIFY_DISPBASE(DVSSURF(pipe), 0);
 	POSTING_READ(DVSSURF(pipe));
 }
 
@@ -337,7 +345,7 @@ intel_disable_primary(struct drm_crtc *crtc)
 }
 
 static int
-snb_update_colorkey(struct drm_plane *plane,
+ilk_update_colorkey(struct drm_plane *plane,
 		    struct drm_intel_sprite_colorkey *key)
 {
 	struct drm_device *dev = plane->dev;
@@ -366,7 +374,7 @@ snb_update_colorkey(struct drm_plane *plane,
 }
 
 static void
-snb_get_colorkey(struct drm_plane *plane, struct drm_intel_sprite_colorkey *key)
+ilk_get_colorkey(struct drm_plane *plane, struct drm_intel_sprite_colorkey *key)
 {
 	struct drm_device *dev = plane->dev;
 	struct drm_i915_private *dev_priv = dev->dev_private;
@@ -554,14 +562,13 @@ int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv)
 {
 	struct drm_intel_sprite_colorkey *set = data;
-	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_mode_object *obj;
 	struct drm_plane *plane;
 	struct intel_plane *intel_plane;
 	int ret = 0;
 
-	if (!dev_priv)
-		return -EINVAL;
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
 
 	/* Make sure we don't try to enable both src & dest simultaneously */
 	if ((set->flags & (I915_SET_COLORKEY_DESTINATION | I915_SET_COLORKEY_SOURCE)) == (I915_SET_COLORKEY_DESTINATION | I915_SET_COLORKEY_SOURCE))
@@ -588,14 +595,13 @@ int intel_sprite_get_colorkey(struct drm_device *dev, void *data,
 			      struct drm_file *file_priv)
 {
 	struct drm_intel_sprite_colorkey *get = data;
-	struct drm_i915_private *dev_priv = dev->dev_private;
 	struct drm_mode_object *obj;
 	struct drm_plane *plane;
 	struct intel_plane *intel_plane;
 	int ret = 0;
 
-	if (!dev_priv)
-		return -EINVAL;
+	if (!drm_core_check_feature(dev, DRIVER_MODESET))
+		return -ENODEV;
 
 	sx_xlock(&dev->mode_config.mutex);
 
@@ -620,6 +626,14 @@ static const struct drm_plane_funcs intel_plane_funcs = {
 	.destroy = intel_destroy_plane,
 };
 
+static uint32_t ilk_plane_formats[] = {
+	DRM_FORMAT_XRGB8888,
+	DRM_FORMAT_YUYV,
+	DRM_FORMAT_YVYU,
+	DRM_FORMAT_UYVY,
+	DRM_FORMAT_VYUY,
+};
+
 static uint32_t snb_plane_formats[] = {
 	DRM_FORMAT_XBGR8888,
 	DRM_FORMAT_XRGB8888,
@@ -634,33 +648,55 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe)
 {
 	struct intel_plane *intel_plane;
 	unsigned long possible_crtcs;
+	const uint32_t *plane_formats;
+	int num_plane_formats;
 	int ret;
 
-	if (!(IS_GEN6(dev) || IS_GEN7(dev)))
+	if (INTEL_INFO(dev)->gen < 5)
 		return -ENODEV;
 
 	intel_plane = malloc(sizeof(struct intel_plane), DRM_MEM_KMS,
 	    M_WAITOK | M_ZERO);
 
-	if (IS_GEN6(dev)) {
+	switch (INTEL_INFO(dev)->gen) {
+	case 5:
+	case 6:
 		intel_plane->max_downscale = 16;
-		intel_plane->update_plane = snb_update_plane;
-		intel_plane->disable_plane = snb_disable_plane;
-		intel_plane->update_colorkey = snb_update_colorkey;
-		intel_plane->get_colorkey = snb_get_colorkey;
-	} else if (IS_GEN7(dev)) {
+		intel_plane->update_plane = ilk_update_plane;
+		intel_plane->disable_plane = ilk_disable_plane;
+		intel_plane->update_colorkey = ilk_update_colorkey;
+		intel_plane->get_colorkey = ilk_get_colorkey;
+
+		if (IS_GEN6(dev)) {
+			plane_formats = snb_plane_formats;
+			num_plane_formats = DRM_ARRAY_SIZE(snb_plane_formats);
+		} else {
+			plane_formats = ilk_plane_formats;
+			num_plane_formats = DRM_ARRAY_SIZE(ilk_plane_formats);
+		}
+		break;
+
+	case 7:
 		intel_plane->max_downscale = 2;
 		intel_plane->update_plane = ivb_update_plane;
 		intel_plane->disable_plane = ivb_disable_plane;
 		intel_plane->update_colorkey = ivb_update_colorkey;
 		intel_plane->get_colorkey = ivb_get_colorkey;
+
+		plane_formats = snb_plane_formats;
+		num_plane_formats = DRM_ARRAY_SIZE(snb_plane_formats);
+		break;
+
+	default:
+		return -ENODEV;
 	}
 
 	intel_plane->pipe = pipe;
 	possible_crtcs = (1 << pipe);
 	ret = drm_plane_init(dev, &intel_plane->base, possible_crtcs,
-			     &intel_plane_funcs, snb_plane_formats,
-			     DRM_ARRAY_SIZE(snb_plane_formats), false);
+			     &intel_plane_funcs,
+			     plane_formats, num_plane_formats,
+			     false);
 	if (ret)
 		free(intel_plane, DRM_MEM_KMS);
 
diff --git a/sys/dev/drm2/i915/intel_tv.c b/sys/dev/drm2/i915/intel_tv.c
index 57a2181f1aa..ddc6c97ee12 100644
--- a/sys/dev/drm2/i915/intel_tv.c
+++ b/sys/dev/drm2/i915/intel_tv.c
@@ -814,7 +814,7 @@ intel_tv_mode_lookup(const char *tv_format)
 {
 	int i;
 
-	for (i = 0; i < sizeof(tv_modes) / sizeof(tv_modes[0]); i++) {
+	for (i = 0; i < DRM_ARRAY_SIZE(tv_modes); i++) {
 		const struct tv_mode *tv_mode = &tv_modes[i];
 
 		if (!strcmp(tv_format, tv_mode->name))
@@ -846,7 +846,7 @@ intel_tv_mode_valid(struct drm_connector *connector,
 
 
 static bool
-intel_tv_mode_fixup(struct drm_encoder *encoder, const struct drm_display_mode *mode,
+intel_tv_mode_fixup(struct drm_encoder *encoder, struct drm_display_mode *mode,
 		    struct drm_display_mode *adjusted_mode)
 {
 	struct drm_device *dev = encoder->dev;
@@ -1155,6 +1155,15 @@ intel_tv_detect_type(struct intel_tv *intel_tv,
 		   DAC_B_0_7_V |
 		   DAC_C_0_7_V);
 
+
+	/*
+	 * The TV sense state should be cleared to zero on cantiga platform. Otherwise
+	 * the TV is misdetected. This is hardware requirement.
+	 */
+	if (IS_GM45(dev))
+		tv_dac &= ~(TVDAC_STATE_CHG_EN | TVDAC_A_SENSE_CTL |
+			    TVDAC_B_SENSE_CTL | TVDAC_C_SENSE_CTL);
+
 	I915_WRITE(TV_CTL, tv_ctl);
 	I915_WRITE(TV_DAC, tv_dac);
 	POSTING_READ(TV_DAC);
@@ -1244,9 +1253,7 @@ intel_tv_detect(struct drm_connector *connector, bool force)
 	mode = reported_modes[0];
 	drm_mode_set_crtcinfo(&mode, 0);
 
-	if (intel_tv->base.base.crtc && intel_tv->base.base.crtc->enabled) {
-		type = intel_tv_detect_type(intel_tv, connector);
-	} else if (force) {
+	if (force) {
 		struct intel_load_detect_pipe tmp;
 
 		if (intel_get_load_detect_pipe(&intel_tv->base, connector,
diff --git a/sys/dev/drm2/radeon/atombios_encoders.c b/sys/dev/drm2/radeon/atombios_encoders.c
index db23f137104..9128aacf77f 100644
--- a/sys/dev/drm2/radeon/atombios_encoders.c
+++ b/sys/dev/drm2/radeon/atombios_encoders.c
@@ -302,7 +302,7 @@ static inline bool radeon_encoder_is_digital(struct drm_encoder *encoder)
 }
 
 static bool radeon_atom_mode_fixup(struct drm_encoder *encoder,
-				   const struct drm_display_mode *mode,
+				   struct drm_display_mode *mode,
 				   struct drm_display_mode *adjusted_mode)
 {
 	struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
@@ -2451,7 +2451,7 @@ radeon_atom_ext_dpms(struct drm_encoder *encoder, int mode)
 }
 
 static bool radeon_atom_ext_mode_fixup(struct drm_encoder *encoder,
-				       const struct drm_display_mode *mode,
+				       struct drm_display_mode *mode,
 				       struct drm_display_mode *adjusted_mode)
 {
 	return true;
diff --git a/sys/dev/drm2/radeon/radeon_legacy_encoders.c b/sys/dev/drm2/radeon/radeon_legacy_encoders.c
index 4a1ae163c9d..a4e6e284c3b 100644
--- a/sys/dev/drm2/radeon/radeon_legacy_encoders.c
+++ b/sys/dev/drm2/radeon/radeon_legacy_encoders.c
@@ -245,7 +245,7 @@ static void radeon_legacy_lvds_mode_set(struct drm_encoder *encoder,
 }
 
 static bool radeon_legacy_mode_fixup(struct drm_encoder *encoder,
-				     const struct drm_display_mode *mode,
+				     struct drm_display_mode *mode,
 				     struct drm_display_mode *adjusted_mode)
 {
 	struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
diff --git a/sys/modules/drm2/i915kms/Makefile b/sys/modules/drm2/i915kms/Makefile
index 4b4a595d02b..8886be88545 100644
--- a/sys/modules/drm2/i915kms/Makefile
+++ b/sys/modules/drm2/i915kms/Makefile
@@ -11,11 +11,13 @@ SRCS	= \
 	i915_gem_execbuffer.c \
 	i915_gem_evict.c \
 	i915_gem_gtt.c \
+	i915_gem_stolen.c \
 	i915_gem_tiling.c \
 	i915_irq.c \
 	i915_suspend.c \
 	intel_bios.c \
 	intel_crt.c \
+	intel_ddi.c \
 	intel_display.c \
 	intel_dp.c \
 	intel_fb.c \
@@ -26,6 +28,7 @@ SRCS	= \
 	intel_opregion.c \
 	intel_overlay.c \
 	intel_panel.c \
+	intel_pm.c \
 	intel_ringbuffer.c \
 	intel_sdvo.c \
 	intel_sprite.c \

From 9e8bff64cb5e57e56cc42f17ef619e8b2db526c1 Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Wed, 21 Jan 2015 16:13:37 +0000
Subject: [PATCH 151/279] Fix bug in r276630.  Do not allow pthread_sigmask()
 to block SIGCANCEL.

Reported and tested by:	royger
Sponsored by:	The FreeBSD Foundation
MFC after:	3 days
---
 lib/libthr/thread/thr_sig.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/libthr/thread/thr_sig.c b/lib/libthr/thread/thr_sig.c
index 7cd0f75412d..11a131c1c5a 100644
--- a/lib/libthr/thread/thr_sig.c
+++ b/lib/libthr/thread/thr_sig.c
@@ -604,7 +604,8 @@ __weak_reference(_pthread_sigmask, pthread_sigmask);
 int
 _pthread_sigmask(int how, const sigset_t *set, sigset_t *oset)
 {
-	if (_sigprocmask(how, set, oset))
+
+	if (__thr_sigprocmask(how, set, oset))
 		return (errno);
 	return (0);
 }

From fe6317011557e3c0f8330a80c8cd0c13fdabea19 Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Wed, 21 Jan 2015 16:32:54 +0000
Subject: [PATCH 152/279] Do not assert that the new pipepair mutex is not
 initialized.  The backing memory contains garbage and might trigger the
 assertion.

Reported and tested by:	pho
Sponsored by:	The FreeBSD Foundation
MFC after:	2 weeks
---
 sys/kern/sys_pipe.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/kern/sys_pipe.c b/sys/kern/sys_pipe.c
index f21b19c886c..9d7d44bc308 100644
--- a/sys/kern/sys_pipe.c
+++ b/sys/kern/sys_pipe.c
@@ -318,7 +318,7 @@ pipe_zone_init(void *mem, int size, int flags)
 
 	pp = (struct pipepair *)mem;
 
-	mtx_init(&pp->pp_mtx, "pipe mutex", NULL, MTX_DEF);
+	mtx_init(&pp->pp_mtx, "pipe mutex", NULL, MTX_DEF | MTX_NEW);
 	return (0);
 }
 

From 20fe2c9465839f53ae56affdb1b12962a009c1f9 Mon Sep 17 00:00:00 2001
From: Andrew Turner <andrew@FreeBSD.org>
Date: Wed, 21 Jan 2015 16:41:05 +0000
Subject: [PATCH 153/279] Merge all the copies of _tcb_ctor and _tcb_dtor.

The amd64, i386, and sparc64 versions were identical, with the one
difference where the former two used inline asm instead of _tcb_get. I
have compared the function before and after replacing the asm with _tcb_get
and found the object files to be identical.

The arm, mips, and powerpc versions were almost identical. The only
difference was the powerpc version used an alignment of 1 where arm and
mips used 16. As this is an increase in alignment is will be safe.

Along with this arm, mips, and powerpc all passed, when initial was true,
the value returned from _tcb_get as the first argument to
_rtld_allocate_tls. This would then return this pointer back to the caller.
We can remove these extra calls by checking if initial is set and setting
the thread control block directly. As this is what the sparc64 code does
we can use it directly.

As after these observations all the architectures can now have identical
code we can merge them into a common file.

Differential Revision:	https://reviews.freebsd.org/D1556
Reviewed by:	kib
Sponsored by:	The FreeBSD Foundation
---
 lib/libthr/Makefile                           |  2 +
 lib/libthr/arch/amd64/Makefile.inc            |  2 +-
 lib/libthr/arch/amd64/amd64/pthread_md.c      | 55 -----------------
 lib/libthr/arch/amd64/include/pthread_md.h    |  3 -
 lib/libthr/arch/arm/Makefile.inc              |  3 -
 lib/libthr/arch/arm/arm/pthread_md.c          | 53 -----------------
 lib/libthr/arch/arm/include/pthread_md.h      |  6 --
 lib/libthr/arch/i386/Makefile.inc             |  2 +-
 lib/libthr/arch/i386/i386/pthread_md.c        | 57 ------------------
 lib/libthr/arch/i386/include/pthread_md.h     |  6 --
 lib/libthr/arch/mips/Makefile.inc             |  3 -
 lib/libthr/arch/mips/include/pthread_md.h     |  6 --
 lib/libthr/arch/mips/mips/pthread_md.c        | 59 -------------------
 lib/libthr/arch/powerpc/Makefile.inc          |  3 -
 lib/libthr/arch/powerpc/include/pthread_md.h  |  3 -
 lib/libthr/arch/powerpc/powerpc/pthread_md.c  | 54 -----------------
 lib/libthr/arch/sparc64/Makefile.inc          |  2 +-
 lib/libthr/arch/sparc64/include/pthread_md.h  |  6 --
 lib/libthr/thread/Makefile.inc                |  1 +
 .../pthread_md.c => thread/thr_ctrdtr.c}      |  2 +-
 lib/libthr/thread/thr_private.h               |  3 +
 21 files changed, 10 insertions(+), 321 deletions(-)
 delete mode 100644 lib/libthr/arch/amd64/amd64/pthread_md.c
 delete mode 100644 lib/libthr/arch/arm/Makefile.inc
 delete mode 100644 lib/libthr/arch/arm/arm/pthread_md.c
 delete mode 100644 lib/libthr/arch/i386/i386/pthread_md.c
 delete mode 100644 lib/libthr/arch/mips/Makefile.inc
 delete mode 100644 lib/libthr/arch/mips/mips/pthread_md.c
 delete mode 100644 lib/libthr/arch/powerpc/Makefile.inc
 delete mode 100644 lib/libthr/arch/powerpc/powerpc/pthread_md.c
 rename lib/libthr/{arch/sparc64/sparc64/pthread_md.c => thread/thr_ctrdtr.c} (98%)

diff --git a/lib/libthr/Makefile b/lib/libthr/Makefile
index 08d50525489..1acb17f04d3 100644
--- a/lib/libthr/Makefile
+++ b/lib/libthr/Makefile
@@ -45,7 +45,9 @@ PRECIOUSLIB=
 
 .PATH: ${.CURDIR}/arch/${MACHINE_CPUARCH}/${MACHINE_CPUARCH}
 
+.if exists(${.CURDIR}/arch/${MACHINE_CPUARCH}/Makefile.inc)
 .include "${.CURDIR}/arch/${MACHINE_CPUARCH}/Makefile.inc"
+.endif
 .include "${.CURDIR}/sys/Makefile.inc"
 .include "${.CURDIR}/thread/Makefile.inc"
 
diff --git a/lib/libthr/arch/amd64/Makefile.inc b/lib/libthr/arch/amd64/Makefile.inc
index e6d99ec0f57..797618d4183 100644
--- a/lib/libthr/arch/amd64/Makefile.inc
+++ b/lib/libthr/arch/amd64/Makefile.inc
@@ -1,3 +1,3 @@
 #$FreeBSD$
 
-SRCS+=	pthread_md.c _umtx_op_err.S
+SRCS+=	_umtx_op_err.S
diff --git a/lib/libthr/arch/amd64/amd64/pthread_md.c b/lib/libthr/arch/amd64/amd64/pthread_md.c
deleted file mode 100644
index b661657b44c..00000000000
--- a/lib/libthr/arch/amd64/amd64/pthread_md.c
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Copyright (c) 2003 Daniel Eischen <deischen@freebsd.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Neither the name of the author nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <sys/types.h>
-#include <rtld_tls.h>
-
-#include "pthread_md.h"
-
-/*
- * The constructors.
- */
-struct tcb *
-_tcb_ctor(struct pthread *thread, int initial)
-{
-	struct tcb *tcb;
-
-	if (initial)
-		__asm __volatile("movq %%fs:0, %0" : "=r" (tcb));
-	else
-		tcb = _rtld_allocate_tls(NULL, sizeof(struct tcb), 16);
-	if (tcb)
-		tcb->tcb_thread = thread;
-	return (tcb);
-}
-
-void
-_tcb_dtor(struct tcb *tcb)
-{
-	_rtld_free_tls(tcb, sizeof(struct tcb), 16);
-}
diff --git a/lib/libthr/arch/amd64/include/pthread_md.h b/lib/libthr/arch/amd64/include/pthread_md.h
index 0ebea2e3bd2..e4a0dd2fe44 100644
--- a/lib/libthr/arch/amd64/include/pthread_md.h
+++ b/lib/libthr/arch/amd64/include/pthread_md.h
@@ -77,9 +77,6 @@ struct tcb {
 	__result;						\
 })
 
-struct tcb	*_tcb_ctor(struct pthread *, int);
-void		_tcb_dtor(struct tcb *tcb);
-
 static __inline void
 _tcb_set(struct tcb *tcb)
 {
diff --git a/lib/libthr/arch/arm/Makefile.inc b/lib/libthr/arch/arm/Makefile.inc
deleted file mode 100644
index 2ee224772e2..00000000000
--- a/lib/libthr/arch/arm/Makefile.inc
+++ /dev/null
@@ -1,3 +0,0 @@
-# $FreeBSD$
-
-SRCS+=	pthread_md.c
diff --git a/lib/libthr/arch/arm/arm/pthread_md.c b/lib/libthr/arch/arm/arm/pthread_md.c
deleted file mode 100644
index 028f06c1793..00000000000
--- a/lib/libthr/arch/arm/arm/pthread_md.c
+++ /dev/null
@@ -1,53 +0,0 @@
-/*-
- * Copyright (C) 2005 David Xu <davidxu@freebsd.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Neither the name of the author nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <stdlib.h>
-#include <sys/types.h>
-#include <rtld_tls.h>
-
-#include "pthread_md.h"
-
-struct tcb *
-_tcb_ctor(struct pthread *thread, int initial)
-{
-	struct tcb *tcb;
-
-	tcb = _rtld_allocate_tls((initial) ? _tcb_get() :  NULL,
-	    sizeof(struct tcb), 16);
-	if (tcb)
-		tcb->tcb_thread = thread;
-
-	return (tcb);
-}
-
-void
-_tcb_dtor(struct tcb *tcb)
-{
-
-	_rtld_free_tls(tcb, sizeof(struct tcb), 16);
-}
diff --git a/lib/libthr/arch/arm/include/pthread_md.h b/lib/libthr/arch/arm/include/pthread_md.h
index 3c3dd6dbea9..9a6b5233e51 100644
--- a/lib/libthr/arch/arm/include/pthread_md.h
+++ b/lib/libthr/arch/arm/include/pthread_md.h
@@ -47,12 +47,6 @@ struct tcb {
 	struct pthread		*tcb_thread;	/* our hook */
 };
 
-/*
- * The tcb constructors.
- */
-struct tcb	*_tcb_ctor(struct pthread *, int);
-void		_tcb_dtor(struct tcb *);
-
 /* Called from the thread to set its private data. */
 static __inline void
 _tcb_set(struct tcb *tcb)
diff --git a/lib/libthr/arch/i386/Makefile.inc b/lib/libthr/arch/i386/Makefile.inc
index 01290d5ddac..bdab0bc90fe 100644
--- a/lib/libthr/arch/i386/Makefile.inc
+++ b/lib/libthr/arch/i386/Makefile.inc
@@ -1,3 +1,3 @@
 # $FreeBSD$
 
-SRCS+=	pthread_md.c _umtx_op_err.S
+SRCS+=	_umtx_op_err.S
diff --git a/lib/libthr/arch/i386/i386/pthread_md.c b/lib/libthr/arch/i386/i386/pthread_md.c
deleted file mode 100644
index a8b31d503ea..00000000000
--- a/lib/libthr/arch/i386/i386/pthread_md.c
+++ /dev/null
@@ -1,57 +0,0 @@
-/*-
- * Copyright (C) 2003 David Xu <davidxu@freebsd.org>
- * Copyright (c) 2001,2003 Daniel Eischen <deischen@freebsd.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Neither the name of the author nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <sys/types.h>
-#include <machine/segments.h>
-#include <machine/sysarch.h>
-#include <string.h>
-#include <rtld_tls.h>
-
-#include "pthread_md.h"
-
-struct tcb *
-_tcb_ctor(struct pthread *thread, int initial)
-{
-	struct tcb *tcb;
-
-	if (initial)
-		__asm __volatile("movl %%gs:0, %0" : "=r" (tcb));
-	else
-		tcb = _rtld_allocate_tls(NULL, sizeof(struct tcb), 16);
-	if (tcb)
-		tcb->tcb_thread = thread;
-	return (tcb);
-}
-
-void
-_tcb_dtor(struct tcb *tcb)
-{
-
-	_rtld_free_tls(tcb, sizeof(struct tcb), 16);
-}
diff --git a/lib/libthr/arch/i386/include/pthread_md.h b/lib/libthr/arch/i386/include/pthread_md.h
index c160aa4fb4a..ba7b75ceb2d 100644
--- a/lib/libthr/arch/i386/include/pthread_md.h
+++ b/lib/libthr/arch/i386/include/pthread_md.h
@@ -76,12 +76,6 @@ struct tcb {
 	__result;						\
 })
 
-/*
- * The constructors.
- */
-struct tcb	*_tcb_ctor(struct pthread *, int);
-void		_tcb_dtor(struct tcb *tcb);
-
 /* Called from the thread to set its private data. */
 static __inline void
 _tcb_set(struct tcb *tcb)
diff --git a/lib/libthr/arch/mips/Makefile.inc b/lib/libthr/arch/mips/Makefile.inc
deleted file mode 100644
index 2ee224772e2..00000000000
--- a/lib/libthr/arch/mips/Makefile.inc
+++ /dev/null
@@ -1,3 +0,0 @@
-# $FreeBSD$
-
-SRCS+=	pthread_md.c
diff --git a/lib/libthr/arch/mips/include/pthread_md.h b/lib/libthr/arch/mips/include/pthread_md.h
index 19c9f38193d..32af964c50a 100644
--- a/lib/libthr/arch/mips/include/pthread_md.h
+++ b/lib/libthr/arch/mips/include/pthread_md.h
@@ -50,12 +50,6 @@ struct tcb {
 	struct pthread		*tcb_thread;
 };
 
-/*
- * The tcb constructors.
- */
-struct tcb	*_tcb_ctor(struct pthread *, int);
-void		_tcb_dtor(struct tcb *);
-
 /* Called from the thread to set its private data. */
 static __inline void
 _tcb_set(struct tcb *tcb)
diff --git a/lib/libthr/arch/mips/mips/pthread_md.c b/lib/libthr/arch/mips/mips/pthread_md.c
deleted file mode 100644
index e596edc0e2e..00000000000
--- a/lib/libthr/arch/mips/mips/pthread_md.c
+++ /dev/null
@@ -1,59 +0,0 @@
-/*-
- * Copyright (C) 2005 David Xu <davidxu@freebsd.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Neither the name of the author nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * from: src/lib/libthr/arch/arm/arm/pthread_md.c,v 1.2 2005/10/29 13:40:31 davidxu
- */
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <stdlib.h>
-#include <sys/types.h>
-#include <rtld_tls.h>
-#include <strings.h>
-
-#include <machine/sysarch.h>
-
-#include "pthread_md.h"
-
-struct tcb *
-_tcb_ctor(struct pthread *thread, int initial)
-{
-	struct tcb *tcb;
-
-	tcb = _rtld_allocate_tls((initial) ? _tcb_get() :  NULL,
-	    sizeof(struct tcb), 16);
-	if (tcb)
-		tcb->tcb_thread = thread;
-
-	return (tcb);
-}
-
-void
-_tcb_dtor(struct tcb *tcb)
-{
-
-	_rtld_free_tls(tcb, sizeof(struct tcb), 16);
-}
diff --git a/lib/libthr/arch/powerpc/Makefile.inc b/lib/libthr/arch/powerpc/Makefile.inc
deleted file mode 100644
index 2ee224772e2..00000000000
--- a/lib/libthr/arch/powerpc/Makefile.inc
+++ /dev/null
@@ -1,3 +0,0 @@
-# $FreeBSD$
-
-SRCS+=	pthread_md.c
diff --git a/lib/libthr/arch/powerpc/include/pthread_md.h b/lib/libthr/arch/powerpc/include/pthread_md.h
index 35632976a1b..544ee1c5d5f 100644
--- a/lib/libthr/arch/powerpc/include/pthread_md.h
+++ b/lib/libthr/arch/powerpc/include/pthread_md.h
@@ -55,9 +55,6 @@ struct tcb {
 	struct pthread		*tcb_thread;
 };
 
-struct tcb	*_tcb_ctor(struct pthread *, int);
-void		_tcb_dtor(struct tcb *);
-
 static __inline void
 _tcb_set(struct tcb *tcb)
 {
diff --git a/lib/libthr/arch/powerpc/powerpc/pthread_md.c b/lib/libthr/arch/powerpc/powerpc/pthread_md.c
deleted file mode 100644
index 66f043e294a..00000000000
--- a/lib/libthr/arch/powerpc/powerpc/pthread_md.c
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Copyright (c) 2003 Daniel Eischen <deischen@freebsd.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Neither the name of the author nor the names of its contributors
- *    may be used to endorse or promote products derived from this software
- *    without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <sys/types.h>
-#include <rtld_tls.h>
-
-#include "pthread_md.h"
-
-/*
- * The constructors.
- */
-struct tcb *
-_tcb_ctor(struct pthread *thread, int initial)
-{
-	struct tcb *tcb;
-
-	tcb = _rtld_allocate_tls((initial) ? _tcb_get() :  NULL,
-	    sizeof(struct tcb), 1);
-	if (tcb)
-		tcb->tcb_thread = thread;
-	return (tcb);
-
-}
-
-void
-_tcb_dtor(struct tcb *tcb)
-{
-	_rtld_free_tls(tcb, sizeof(struct tcb), 1);
-}
diff --git a/lib/libthr/arch/sparc64/Makefile.inc b/lib/libthr/arch/sparc64/Makefile.inc
index 88586b4840d..bdab0bc90fe 100644
--- a/lib/libthr/arch/sparc64/Makefile.inc
+++ b/lib/libthr/arch/sparc64/Makefile.inc
@@ -1,3 +1,3 @@
 # $FreeBSD$
 
-SRCS+=	_umtx_op_err.S pthread_md.c
+SRCS+=	_umtx_op_err.S
diff --git a/lib/libthr/arch/sparc64/include/pthread_md.h b/lib/libthr/arch/sparc64/include/pthread_md.h
index 7ee9654fd74..35d8405b9d5 100644
--- a/lib/libthr/arch/sparc64/include/pthread_md.h
+++ b/lib/libthr/arch/sparc64/include/pthread_md.h
@@ -50,12 +50,6 @@ struct tcb {
 	void			*tcb_spare[1];
 };
 
-/*
- * The tcb constructors.
- */
-struct tcb	*_tcb_ctor(struct pthread *, int);
-void		_tcb_dtor(struct tcb *);
-
 /* Called from the thread to set its private data. */
 static __inline void
 _tcb_set(struct tcb *tcb)
diff --git a/lib/libthr/thread/Makefile.inc b/lib/libthr/thread/Makefile.inc
index ddde6e9180c..1a7c63d14af 100644
--- a/lib/libthr/thread/Makefile.inc
+++ b/lib/libthr/thread/Makefile.inc
@@ -14,6 +14,7 @@ SRCS+= \
 	thr_cond.c \
 	thr_condattr.c \
 	thr_create.c \
+	thr_ctrdtr.c \
 	thr_detach.c \
 	thr_equal.c \
 	thr_event.c \
diff --git a/lib/libthr/arch/sparc64/sparc64/pthread_md.c b/lib/libthr/thread/thr_ctrdtr.c
similarity index 98%
rename from lib/libthr/arch/sparc64/sparc64/pthread_md.c
rename to lib/libthr/thread/thr_ctrdtr.c
index e1d439a8bad..9d4301ef719 100644
--- a/lib/libthr/arch/sparc64/sparc64/pthread_md.c
+++ b/lib/libthr/thread/thr_ctrdtr.c
@@ -32,7 +32,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/types.h>
 #include <rtld_tls.h>
 
-#include "pthread_md.h"
+#include "thr_private.h"
 
 struct tcb *
 _tcb_ctor(struct pthread *thread, int initial)
diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h
index d62de98516d..50be0596d13 100644
--- a/lib/libthr/thread/thr_private.h
+++ b/lib/libthr/thread/thr_private.h
@@ -928,6 +928,9 @@ int __thr_sigwait(const sigset_t *set, int *sig);
 int __thr_sigwaitinfo(const sigset_t *set, siginfo_t *info);
 int __thr_swapcontext(ucontext_t *oucp, const ucontext_t *ucp);
 
+struct tcb *_tcb_ctor(struct pthread *, int);
+void	_tcb_dtor(struct tcb *);
+
 __END_DECLS
 
 #endif  /* !_THR_PRIVATE_H */

From 5b86b0eabac8645621a7cc56aabb718b0d02845e Mon Sep 17 00:00:00 2001
From: Andrew Turner <andrew@FreeBSD.org>
Date: Wed, 21 Jan 2015 16:52:24 +0000
Subject: [PATCH 154/279] Update the parsing of the cpu node. We are unable to
 use the reg property as the cpu id on arm64 as it may use two cells. In it's
 place we can use the device id.

It is expected we will use the reg data on arm64 to enable cores so we
still need to read and store it even if it is not yet used.

Differential Revision:	https://reviews.freebsd.org/D1555
Reviewed by:	nwhitehorn
Sponsored by:	The FreeBSD Foundation
---
 sys/dev/ofw/ofw_cpu.c | 48 ++++++++++++++++++++++++++++++++++++-------
 1 file changed, 41 insertions(+), 7 deletions(-)

diff --git a/sys/dev/ofw/ofw_cpu.c b/sys/dev/ofw/ofw_cpu.c
index 02dc2b50055..0432560281a 100644
--- a/sys/dev/ofw/ofw_cpu.c
+++ b/sys/dev/ofw/ofw_cpu.c
@@ -51,6 +51,10 @@ static const struct ofw_bus_devinfo *ofw_cpulist_get_devinfo(device_t dev,
 
 static MALLOC_DEFINE(M_OFWCPU, "ofwcpu", "OFW CPU device information");
 
+struct ofw_cpulist_softc {
+	pcell_t	 sc_addr_cells;
+};
+
 static device_method_t ofw_cpulist_methods[] = {
 	/* Device interface */
 	DEVMETHOD(device_probe,		ofw_cpulist_probe),
@@ -74,7 +78,7 @@ static device_method_t ofw_cpulist_methods[] = {
 static driver_t ofw_cpulist_driver = {
 	"cpulist",
 	ofw_cpulist_methods,
-	0
+	sizeof(struct ofw_cpulist_softc)
 };
 
 static devclass_t ofw_cpulist_devclass;
@@ -100,12 +104,18 @@ ofw_cpulist_probe(device_t dev)
 static int 
 ofw_cpulist_attach(device_t dev) 
 {
+	struct ofw_cpulist_softc *sc;
 	phandle_t root, child;
 	device_t cdev;
 	struct ofw_bus_devinfo *dinfo;
 
+	sc = device_get_softc(dev);
 	root = ofw_bus_get_node(dev);
 
+	sc->sc_addr_cells = 1;
+	OF_getencprop(root, "#address-cells", &sc->sc_addr_cells,
+	    sizeof(sc->sc_addr_cells));
+
 	for (child = OF_child(root); child != 0; child = OF_peer(child)) {
 		dinfo = malloc(sizeof(*dinfo), M_OFWCPU, M_WAITOK | M_ZERO);
 
@@ -141,6 +151,8 @@ static int	ofw_cpu_read_ivar(device_t dev, device_t child, int index,
 struct ofw_cpu_softc {
 	struct pcpu	*sc_cpu_pcpu;
 	uint32_t	 sc_nominal_mhz;
+	boolean_t	 sc_reg_valid;
+	pcell_t		 sc_reg[2];
 };
 
 static device_method_t ofw_cpu_methods[] = {
@@ -185,17 +197,39 @@ ofw_cpu_probe(device_t dev)
 static int
 ofw_cpu_attach(device_t dev)
 {
+	struct ofw_cpulist_softc *psc;
 	struct ofw_cpu_softc *sc;
 	phandle_t node;
-	uint32_t cell;
+	pcell_t cell;
+	int rv;
 
 	sc = device_get_softc(dev);
-	node = ofw_bus_get_node(dev);
-	if (OF_getencprop(node, "reg", &cell, sizeof(cell)) < 0) {
-		cell = device_get_unit(dev);
-		device_printf(dev, "missing 'reg' property, using %u\n", cell);
+	psc = device_get_softc(device_get_parent(dev));
+
+	if (nitems(sc->sc_reg) < psc->sc_addr_cells) {
+		if (bootverbose)
+			device_printf(dev, "Too many address cells\n");
+		return (EINVAL);
 	}
-	sc->sc_cpu_pcpu = pcpu_find(cell);
+
+	node = ofw_bus_get_node(dev);
+
+	/* Read and validate the reg property for use later */
+	sc->sc_reg_valid = false;
+	rv = OF_getencprop(node, "reg", sc->sc_reg, sizeof(sc->sc_reg));
+	if (rv < 0)
+		device_printf(dev, "missing 'reg' property\n");
+	else if ((rv % 4) != 0) {
+		if (bootverbose)
+			device_printf(dev, "Malformed reg property\n");
+	} else if ((rv / 4) != psc->sc_addr_cells) {
+		if (bootverbose)
+			device_printf(dev, "Invalid reg size %u\n", rv);
+	} else
+		sc->sc_reg_valid = true;
+
+	sc->sc_cpu_pcpu = pcpu_find(device_get_unit(dev));
+
 	if (OF_getencprop(node, "clock-frequency", &cell, sizeof(cell)) < 0) {
 		if (bootverbose)
 			device_printf(dev,

From 55ddf051d8a8c43d168ce32c3b1831f9593a3241 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Wed, 21 Jan 2015 17:03:11 +0000
Subject: [PATCH 155/279] Add vfs.zfs.reference_tracking_enable sysctl/tunable.

This is primarily for developer/debugging use; it enables built-in tagged
tracking of refcounts inside ZFS.  It can only be enabled from the loader,
since it modifies how in-core state is managed.  Default remains disabled.

MFC after:	1 week
Sponsored by:	Spectra Logic
---
 sys/cddl/contrib/opensolaris/uts/common/fs/zfs/refcount.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/refcount.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/refcount.c
index 7e96e558ce5..ade681c1de8 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/refcount.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/refcount.c
@@ -30,6 +30,10 @@
 
 #ifdef _KERNEL
 int reference_tracking_enable = FALSE; /* runs out of memory too easily */
+SYSCTL_DECL(_vfs_zfs);
+SYSCTL_INT(_vfs_zfs, OID_AUTO, reference_tracking_enable, CTLFLAG_RDTUN,
+    &reference_tracking_enable, 0,
+    "Track reference holders to refcount_t objects, used mostly by ZFS");
 #else
 int reference_tracking_enable = TRUE;
 #endif

From 5751146497abe07ff6a18cd2087efaa025c98e95 Mon Sep 17 00:00:00 2001
From: Mateusz Guzik <mjg@FreeBSD.org>
Date: Wed, 21 Jan 2015 18:02:28 +0000
Subject: [PATCH 156/279] filedesc: fix whitespace nits in fget and fget_read

No functional changes.
---
 sys/kern/kern_descrip.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index 4b4af296769..2b2a53908ea 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -2499,7 +2499,7 @@ int
 fget(struct thread *td, int fd, cap_rights_t *rightsp, struct file **fpp)
 {
 
-	return(_fget(td, fd, fpp, 0, rightsp, NULL));
+	return (_fget(td, fd, fpp, 0, rightsp, NULL));
 }
 
 int
@@ -2514,7 +2514,7 @@ int
 fget_read(struct thread *td, int fd, cap_rights_t *rightsp, struct file **fpp)
 {
 
-	return(_fget(td, fd, fpp, FREAD, rightsp, NULL));
+	return (_fget(td, fd, fpp, FREAD, rightsp, NULL));
 }
 
 int

From f9051b0e02cf6f5a29b263e6af4a0f4d50d011af Mon Sep 17 00:00:00 2001
From: Mateusz Guzik <mjg@FreeBSD.org>
Date: Wed, 21 Jan 2015 18:05:42 +0000
Subject: [PATCH 157/279] filedesc: return 0 from badfo_close

The only potential in-tree consumer (_fdrop) special-cased it and returns 0
0 on its own instead of calling badfo_close.

Remove the special case since it is not needed and very unlikely to encounter
anyway.

No objections from:	kib
---
 sys/kern/kern_descrip.c | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index 2b2a53908ea..c88e55c2c38 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -2680,11 +2680,9 @@ _fdrop(struct file *fp, struct thread *td)
 {
 	int error;
 
-	error = 0;
 	if (fp->f_count != 0)
 		panic("fdrop: count %d", fp->f_count);
-	if (fp->f_ops != &badfileops)
-		error = fo_close(fp, td);
+	error = fo_close(fp, td);
 	atomic_subtract_int(&openfiles, 1);
 	crfree(fp->f_cred);
 	free(fp->f_advice, M_FADVISE);
@@ -3664,7 +3662,7 @@ static int
 badfo_close(struct file *fp, struct thread *td)
 {
 
-	return (EBADF);
+	return (0);
 }
 
 static int

From 5e7cd3ec228e73c779b1a63013aa3769d9fd1d4d Mon Sep 17 00:00:00 2001
From: Mateusz Guzik <mjg@FreeBSD.org>
Date: Wed, 21 Jan 2015 18:32:53 +0000
Subject: [PATCH 158/279] filedesc: avoid spurious copying of capabilities in
 fget_unlocked

We obtain a stable copy and store it in local 'fde' variable. Storing another
copy (based on aforementioned variable) does not serve any purpose.

No functional changes.
---
 sys/kern/kern_descrip.c | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/sys/kern/kern_descrip.c b/sys/kern/kern_descrip.c
index c88e55c2c38..31352e9737e 100644
--- a/sys/kern/kern_descrip.c
+++ b/sys/kern/kern_descrip.c
@@ -2337,7 +2337,7 @@ fget_unlocked(struct filedesc *fdp, int fd, cap_rights_t *needrightsp,
 	u_int count;
 #ifdef CAPABILITIES
 	seq_t seq;
-	cap_rights_t haverights;
+	cap_rights_t *haverights;
 	int error;
 #endif
 
@@ -2367,9 +2367,9 @@ fget_unlocked(struct filedesc *fdp, int fd, cap_rights_t *needrightsp,
 		if (fp == NULL)
 			return (EBADF);
 #ifdef CAPABILITIES
-		haverights = *cap_rights_fde(&fde);
+		haverights = cap_rights_fde(&fde);
 		if (needrightsp != NULL) {
-			error = cap_check(&haverights, needrightsp);
+			error = cap_check(haverights, needrightsp);
 			if (error != 0)
 				return (error);
 			if (cap_rights_is_set(needrightsp, CAP_FCNTL)) {
@@ -2408,7 +2408,7 @@ fget_unlocked(struct filedesc *fdp, int fd, cap_rights_t *needrightsp,
 	*fpp = fp;
 	if (haverightsp != NULL) {
 #ifdef CAPABILITIES
-		*haverightsp = haverights;
+		*haverightsp = *haverights;
 #else
 		CAP_ALL(haverightsp);
 #endif

From da010b56065a939176792952344bb6721a0ade49 Mon Sep 17 00:00:00 2001
From: Ed Maste <emaste@FreeBSD.org>
Date: Wed, 21 Jan 2015 19:04:55 +0000
Subject: [PATCH 159/279] Remove addr2line from cross elftoolchain tools list

It is not required, and there is no reason to install it just because it
came with the binutils cross tools.

Sponsored by:	The FreeBSD Foundation
---
 Makefile.inc1 | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 3d898c42be8..984d240c6ed 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -1428,7 +1428,6 @@ _binutils=	gnu/usr.bin/binutils
 .endif
 .if ${MK_ELFTOOLCHAIN_TOOLS} != "no"
 _elftctools=	lib/libelftc \
-		usr.bin/addr2line \
 		usr.bin/elfcopy \
 		usr.bin/nm \
 		usr.bin/size \

From 554dab448ecc9a0b015790ff31e1caa729c06791 Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Wed, 21 Jan 2015 19:07:45 +0000
Subject: [PATCH 160/279] Make 64-bit AIM trap handlers relocatable by changing
 all absolute branch instructions to call through pointers instead. In
 general, these are set implicitly through relocation processing. One has to
 be set explicitly in machdep.c, however, to fit one handler in the tiny (8
 instruction) space available.

Reviewed by:	andreast
Differential revision:	D1554
Tested on:	UP and SMP G5, Cell, POWER5+
---
 sys/powerpc/aim/machdep.c     |  3 +-
 sys/powerpc/aim/trap_subr64.S | 84 ++++++++++++++++++++++++++++-------
 sys/powerpc/include/trap.h    |  3 +-
 3 files changed, 72 insertions(+), 18 deletions(-)

diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c
index d0aac4c7f46..b463c48bee2 100644
--- a/sys/powerpc/aim/machdep.c
+++ b/sys/powerpc/aim/machdep.c
@@ -238,7 +238,7 @@ extern void	*trapcode64;
 #endif
 
 extern void	*rstcode, *rstsize;
-extern void	*trapcode, *trapsize;
+extern void	*trapcode, *trapsize, *trapcode2;
 extern void	*slbtrap, *slbtrapsize;
 extern void	*alitrap, *alisize;
 extern void	*dsitrap, *dsisize;
@@ -506,6 +506,7 @@ powerpc_init(vm_offset_t fdt, vm_offset_t toc, vm_offset_t ofentry, void *mdp)
 	generictrap = &trapcode;
 
 	/* Set TOC base so that the interrupt code can get at it */
+	*((void **)TRAP_GENTRAP) = &trapcode2;
 	*((register_t *)TRAP_TOCBASE) = toc;
 	#endif
 
diff --git a/sys/powerpc/aim/trap_subr64.S b/sys/powerpc/aim/trap_subr64.S
index 63ca50a4bfe..a2d5bc5b1b4 100644
--- a/sys/powerpc/aim/trap_subr64.S
+++ b/sys/powerpc/aim/trap_subr64.S
@@ -302,8 +302,13 @@ CNAME(rstcode):
 	insrdi	%r9,%r8,1,0
 	mtmsrd	%r9
 	isync
+	bl	1f
+	.llong	cpu_reset
+1:	mflr	%r9
+	ld	%r9,0(%r9)
+	mtlr	%r9
 
-	ba	cpu_reset
+	blr
 CNAME(rstsize) = . - CNAME(rstcode)
 
 cpu_reset:
@@ -342,16 +347,20 @@ cpu_reset:
 
 /*
  * This code gets copied to all the trap vectors
- * (except ISI/DSI, ALI, and the interrupts)
+ * (except ISI/DSI, ALI, and the interrupts). Has to fit in 8 instructions!
  */
 
 	.globl	CNAME(trapcode),CNAME(trapsize)
+	.p2align 3
 CNAME(trapcode):
 	mtsprg1	%r1			/* save SP */
 	mflr	%r1			/* Save the old LR in r1 */
 	mtsprg2 %r1			/* And then in SPRG2 */
+	li	%r1,TRAP_GENTRAP
+	ld	%r1,0(%r1)
+	mtlr	%r1
 	li	%r1, 0xA0		/* How to get the vector from LR */
-	bla	generictrap		/* LR & SPRG3 is exception # */
+	blrl				/* Branch to generictrap */
 CNAME(trapsize) = .-CNAME(trapcode)
 
 /*
@@ -361,6 +370,7 @@ CNAME(trapsize) = .-CNAME(trapcode)
  * the only time this can be called.
  */
 	.globl	CNAME(slbtrap),CNAME(slbtrapsize)
+	.p2align 3
 CNAME(slbtrap):
 	mtsprg1	%r1			/* save SP */
 	GET_CPUINFO(%r1)
@@ -369,17 +379,31 @@ CNAME(slbtrap):
 	std	%r2,(PC_SLBSAVE+104)(%r1)
 	mfsrr1	%r2			/* test kernel mode */
 	mtcr	%r2
-	bf	17,1f			/* branch if PSL_PR is false */
+	bf	17,2f			/* branch if PSL_PR is false */
 	/* User mode */
 	ld	%r2,(PC_SLBSAVE+104)(%r1) /* Restore CR */
 	mtcr	%r2
 	ld	%r2,(PC_SLBSAVE+16)(%r1) /* Restore R2 */
 	mflr	%r1			/* Save the old LR in r1 */
 	mtsprg2 %r1			/* And then in SPRG2 */
+					/* 52 bytes so far */
+	bl	1f
+	.llong	generictrap
+1:	mflr	%r1
+	ld	%r1,0(%r1)
+	mtlr	%r1
 	li	%r1, 0x80		/* How to get the vector from LR */
-	bla	generictrap		/* LR & SPRG3 is exception # */
-1:	mflr	%r2			/* Save the old LR in r2 */
-	bla	kern_slbtrap
+	blrl				/* Branch to generictrap */
+					/* 84 bytes */
+2:	mflr	%r2			/* Save the old LR in r2 */
+	nop
+	bl	3f			/* Begin dance to jump to kern_slbtrap*/
+	.llong	kern_slbtrap
+3:	mflr	%r1
+	ld	%r1,0(%r1)
+	mtlr	%r1
+	GET_CPUINFO(%r1)
+	blrl				/* 124 bytes -- 4 to spare */
 CNAME(slbtrapsize) = .-CNAME(slbtrap)
 
 kern_slbtrap:
@@ -518,6 +542,16 @@ CNAME(alitrap):
 	mflr	%r28			/* save LR */
 	mfcr	%r29			/* save CR */
 
+	/* Begin dance to branch to s_trap in a bit */
+	b	1f
+	.p2align 3
+1:	nop
+	bl	1f
+	.llong	s_trap
+1:	mflr	%r31
+	ld	%r31,0(%r31)
+	mtlr	%r31
+
 	/* Put our exception vector in SPRG3 */
 	li	%r31, EXC_ALI
 	mtsprg3	%r31
@@ -525,13 +559,12 @@ CNAME(alitrap):
 	/* Test whether we already had PR set */
 	mfsrr1	%r31
 	mtcr	%r31
-	bla	s_trap
+	blrl
 CNAME(alisize) = .-CNAME(alitrap)
 
 /*
  * Similar to the above for DSI
- * Has to handle BAT spills
- * and standard pagetable spills
+ * Has to handle standard pagetable spills
  */
 	.globl	CNAME(dsitrap),CNAME(dsisize)
 CNAME(dsitrap):
@@ -542,14 +575,18 @@ CNAME(dsitrap):
 	std	%r29,(PC_DISISAVE+CPUSAVE_R29)(%r1)
 	std	%r30,(PC_DISISAVE+CPUSAVE_R30)(%r1)
 	std	%r31,(PC_DISISAVE+CPUSAVE_R31)(%r1)
-	mfsprg1	%r1			/* restore SP */
 	mfcr	%r29			/* save CR */
 	mfxer	%r30			/* save XER */
 	mtsprg2	%r30			/* in SPRG2 */
 	mfsrr1	%r31			/* test kernel mode */
 	mtcr	%r31
 	mflr	%r28			/* save LR (SP already saved) */
-	bla	disitrap
+	bl	1f			/* Begin branching to disitrap */
+	.llong	disitrap
+1:	mflr	%r1
+	ld	%r1,0(%r1)
+	mtlr	%r1
+	blrl				/* Branch to generictrap */
 CNAME(dsisize) = .-CNAME(dsitrap)
 
 /*
@@ -624,7 +661,7 @@ realtrap:
 	bl	restore_kernsrs		/* enable kernel mapping */
 	mfsprg2	%r29
 	mr	%r28,%r27
-	ba s_trap
+	b	s_trap
 
 /*
  * generictrap does some standard setup for trap handling to minimize
@@ -636,6 +673,8 @@ realtrap:
  * SPRG2 - Original LR
  */
 
+	.globl	CNAME(trapcode2)
+trapcode2:
 generictrap:
 	/* Save R1 for computing the exception vector */
 	mtsprg3 %r1
@@ -657,6 +696,7 @@ generictrap:
 	mfsprg3 %r31
 	ori	%r31,%r31,0xff00
 	mflr	%r30
+	addi	%r30,%r30,-4 /* The branch instruction, not the next */
 	and	%r30,%r30,%r31
 	mtsprg3	%r30
 
@@ -804,9 +844,16 @@ CNAME(dblow):
         mfsprg2	%r29			/* ... and r29 */
         mflr	%r1			/* save LR */
 	mtsprg2 %r1			/* And then in SPRG2 */
-	li	%r1, 0	 		/* How to get the vector from LR */
 
-        bla     generictrap		/* and we look like a generic trap */
+	nop				/* Begin branching to generictrap */
+	bl	9f
+	.llong	generictrap
+9:	mflr	%r1
+	ld	%r1,0(%r1)
+	mtlr	%r1
+	li	%r1, 0	 		/* How to get the vector from LR */
+	blrl				/* Branch to generictrap */
+
 1:
 	GET_CPUINFO(%r1)
 	std	%r27,(PC_DBSAVE+CPUSAVE_R27)(%r1)	/* free r27 */
@@ -816,6 +863,11 @@ CNAME(dblow):
         std	%r30,(PC_DBSAVE+CPUSAVE_R30)(%r1)	/* free r30 */
         std	%r31,(PC_DBSAVE+CPUSAVE_R31)(%r1)	/* free r31 */
         mflr	%r28					/* save LR */
-	bla	dbtrap
+	bl	9f					/* Begin branch */
+	.llong	dbtrap
+9:	mflr	%r1
+	ld	%r1,0(%r1)
+	mtlr	%r1
+	blrl				/* Branch to generictrap */
 CNAME(dbsize) = .-CNAME(dblow)
 #endif /* KDB */
diff --git a/sys/powerpc/include/trap.h b/sys/powerpc/include/trap.h
index 3b1ade582e1..cd5ac188fd5 100644
--- a/sys/powerpc/include/trap.h
+++ b/sys/powerpc/include/trap.h
@@ -123,7 +123,8 @@
 /* DTrace trap opcode. */
 #define EXC_DTRACE	0x7c810808
 
-/* Magic pointer to store TOC base for trap handlers on ppc64 */
+/* Magic pointer to store TOC base and other info for trap handlers on ppc64 */
+#define TRAP_GENTRAP	0x1f0
 #define TRAP_TOCBASE	0x1f8
 
 #ifndef LOCORE

From 7a28efd9eee50ee4314aa101663a5d52e6367c59 Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Wed, 21 Jan 2015 19:09:15 +0000
Subject: [PATCH 161/279] Make sure to relocate tmpstk with everything else and
 avoid processing non-relative relocations that the UART code makes for absent
 modules.

---
 sys/powerpc/aim/locore64.S          | 8 ++++++--
 sys/powerpc/powerpc/elf64_machdep.c | 2 ++
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/sys/powerpc/aim/locore64.S b/sys/powerpc/aim/locore64.S
index abce9d83e5b..e1fcd3ab050 100644
--- a/sys/powerpc/aim/locore64.S
+++ b/sys/powerpc/aim/locore64.S
@@ -126,9 +126,14 @@ ASENTRY_NOPROF(__start)
 	ld	%r1,0(%r2)
 	add	%r2,%r1,%r2
 
+	/* Get load offset */
+	ld	%r31,-0x8000(%r2) /* First TOC entry is TOC base */
+	subf    %r31,%r31,%r2	/* Subtract from real TOC base to get base */
+
 	/* Set up the stack pointer */
 	ld	%r1,TOC_REF(tmpstk)(%r2)
 	addi	%r1,%r1,TMPSTKSZ-96
+	add	%r1,%r1,%r31
 
 	/* Relocate kernel */
 	std	%r3,48(%r1)
@@ -140,8 +145,7 @@ ASENTRY_NOPROF(__start)
 1:	mflr	%r3
 	ld	%r4,0(%r3)
 	add	%r3,%r4,%r3
-	ld	%r4,-0x8000(%r2) /* First TOC entry is TOC base */
-	subf    %r4,%r4,%r2	/* Subtract from real TOC base to get base */
+	mr	%r4,%r31
 	bl	elf_reloc_self
 	nop
 	ld	%r3,48(%r1)
diff --git a/sys/powerpc/powerpc/elf64_machdep.c b/sys/powerpc/powerpc/elf64_machdep.c
index d23f7ffaa2f..0b15ca3dea8 100644
--- a/sys/powerpc/powerpc/elf64_machdep.c
+++ b/sys/powerpc/powerpc/elf64_machdep.c
@@ -226,6 +226,8 @@ elf_reloc_self(Elf_Dyn *dynp, Elf_Addr relocbase)
 	 */
 	relalim = (Elf_Rela *)((caddr_t)rela + relasz);
 	for (; rela < relalim; rela++) {
+		if (ELF_R_TYPE(rela->r_info) != R_PPC_RELATIVE)
+			continue;
 		where = (Elf_Addr *)(relocbase + rela->r_offset);
 		*where = (Elf_Addr)(relocbase + rela->r_addend);
 	}

From 88a6aee146f324d2849f5cc8b11c19e9178c478c Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Wed, 21 Jan 2015 19:11:15 +0000
Subject: [PATCH 162/279] Add POWER7+ and POWER8 to the list of CPUs with 32
 SLB slots. This is mostly a no-op since all currently-supported instances of
 these CPUs give the number of SLB slots in the device tree, but keep it here
 as well just in case.

---
 sys/powerpc/aim/machdep.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c
index b463c48bee2..82a72328d63 100644
--- a/sys/powerpc/aim/machdep.c
+++ b/sys/powerpc/aim/machdep.c
@@ -394,6 +394,9 @@ powerpc_init(vm_offset_t fdt, vm_offset_t toc, vm_offset_t ofentry, void *mdp)
 			break;
 	#ifdef __powerpc64__
 		case IBMPOWER7:
+		case IBMPOWER7PLUS:
+		case IBMPOWER8:
+		case IBMPOWER8E:
 			/* XXX: get from ibm,slb-size in device tree */
 			n_slbs = 32;
 			break;

From 2a2c1d424a7799ec9832bff7a5dd7c9701f1b35f Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Wed, 21 Jan 2015 19:20:36 +0000
Subject: [PATCH 163/279] Eliminate an #ifdef illumos for zfs_ioc_rename().

Since allow_mounted is a FreeBSD-specific change, default to B_TRUE, then
locally check for the magic bit.  Unconditionally check allow_mounted below.
Convert the setting of allow_mounted to an explicit boolean.

MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	672578 (in part) on 2013/07/19
---
 .../opensolaris/uts/common/fs/zfs/zfs_ioctl.c        | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
index 8801aa5abee..a829b069834 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_ioctl.c
@@ -3751,10 +3751,12 @@ static int
 zfs_ioc_rename(zfs_cmd_t *zc)
 {
 	boolean_t recursive = zc->zc_cookie & 1;
-#ifdef __FreeBSD__
-	boolean_t allow_mounted = zc->zc_cookie & 2;
-#endif
 	char *at;
+	boolean_t allow_mounted = B_TRUE;
+
+#ifdef __FreeBSD__
+	allow_mounted = (zc->zc_cookie & 2) != 0;
+#endif
 
 	zc->zc_value[sizeof (zc->zc_value) - 1] = '\0';
 	if (dataset_namecheck(zc->zc_value, NULL, NULL) != 0 ||
@@ -3769,11 +3771,7 @@ zfs_ioc_rename(zfs_cmd_t *zc)
 		if (strncmp(zc->zc_name, zc->zc_value, at - zc->zc_name + 1))
 			return (SET_ERROR(EXDEV));
 		*at = '\0';
-#ifdef illumos
-		if (zc->zc_objset_type == DMU_OST_ZFS) {
-#else
 		if (zc->zc_objset_type == DMU_OST_ZFS && allow_mounted) {
-#endif
 			error = dmu_objset_find(zc->zc_name,
 			    recursive_unmount, at + 1,
 			    recursive ? DS_FIND_CHILDREN : 0);

From 7d948458b1708de64a783a51965f5060939ae904 Mon Sep 17 00:00:00 2001
From: Oleksandr Tymoshenko <gonzo@FreeBSD.org>
Date: Wed, 21 Jan 2015 19:23:46 +0000
Subject: [PATCH 164/279] Remove "#define DEBUG" that conflicts with "option
 DEBUG" in kernel config

---
 sys/arm/ti/ti_mbox.c  | 1 -
 sys/arm/ti/ti_pruss.c | 1 -
 2 files changed, 2 deletions(-)

diff --git a/sys/arm/ti/ti_mbox.c b/sys/arm/ti/ti_mbox.c
index 994b0a08783..3fb5c33e224 100644
--- a/sys/arm/ti/ti_mbox.c
+++ b/sys/arm/ti/ti_mbox.c
@@ -54,7 +54,6 @@ __FBSDID("$FreeBSD$");
 
 #include "mbox_if.h"
 
-#define DEBUG
 #ifdef DEBUG
 #define	DPRINTF(fmt, ...)	do {	\
 	printf("%s: ", __func__);	\
diff --git a/sys/arm/ti/ti_pruss.c b/sys/arm/ti/ti_pruss.c
index 55838d2d908..03a76761b2b 100644
--- a/sys/arm/ti/ti_pruss.c
+++ b/sys/arm/ti/ti_pruss.c
@@ -52,7 +52,6 @@ __FBSDID("$FreeBSD$");
 #include <arm/ti/ti_prcm.h>
 #include <arm/ti/ti_pruss.h>
 
-#define DEBUG
 #ifdef DEBUG
 #define	DPRINTF(fmt, ...)	do {	\
 	printf("%s: ", __func__);	\

From 35b540bfb2b7d3c22987d97261c489d62f186d71 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Wed, 21 Jan 2015 19:25:57 +0000
Subject: [PATCH 165/279] Ignore sync requests from the system syncher, i.e.
 VFS_SYNC(waitfor=MNT_LAZY).

ZFS already commits outstanding data every zfs_txg_timeout seconds, so these
syncs are unnecessarily intrusive.

Submitted by:	gibbs
Sponsored by:	Spectra Logic
MFSpectraBSD:	1105759 on 2014/12/11
---
 .../contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c     | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
index 7b74145393b..415db9edbe5 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vfsops.c
@@ -132,6 +132,13 @@ zfs_sync(vfs_t *vfsp, int waitfor)
 	if (panicstr)
 		return (0);
 
+	/*
+	 * Ignore the system syncher.  ZFS already commits async data
+	 * at zfs_txg_timeout intervals.
+	 */
+	if (waitfor == MNT_LAZY)
+		return (0);
+
 	if (vfsp != NULL) {
 		/*
 		 * Sync a specific filesystem.

From bd3a7c08c4251e6658a5b569e1da54523a70d841 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Wed, 21 Jan 2015 19:30:01 +0000
Subject: [PATCH 166/279] Remove commented log messages.

---
 sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zil.c | 2 --
 1 file changed, 2 deletions(-)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zil.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zil.c
index 2dcfe06dbff..c558708b6dc 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zil.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zil.c
@@ -2090,7 +2090,6 @@ zil_replay(objset_t *os, void *arg, zil_replay_func_t *replay_func[TX_MAX_TYPE])
 		zil_destroy(zilog, B_TRUE);
 		return;
 	}
-	//printf("ZFS: Replaying ZIL on %s...\n", os->os->os_spa->spa_name);
 
 	zr.zr_replay = replay_func;
 	zr.zr_arg = arg;
@@ -2112,7 +2111,6 @@ zil_replay(objset_t *os, void *arg, zil_replay_func_t *replay_func[TX_MAX_TYPE])
 	zil_destroy(zilog, B_FALSE);
 	txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg);
 	zilog->zl_replay = B_FALSE;
-	//printf("ZFS: Replay of ZIL on %s finished.\n", os->os->os_spa->spa_name);
 }
 
 boolean_t

From 49f159beefdb052e182816178db93c83abbfb649 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Wed, 21 Jan 2015 19:53:52 +0000
Subject: [PATCH 167/279] Garbage collect dragonfly and legacy FreeBSD system
 support from dcons(4).

Submitted by:	gibbs
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1110990 on 2015/01/06
---
 sys/dev/dcons/dcons.c      |  4 +---
 sys/dev/dcons/dcons_crom.c | 39 +++++---------------------------------
 sys/dev/dcons/dcons_os.h   |  1 -
 3 files changed, 6 insertions(+), 38 deletions(-)

diff --git a/sys/dev/dcons/dcons.c b/sys/dev/dcons/dcons.c
index ace8f7e62c6..da5183e15cd 100644
--- a/sys/dev/dcons/dcons.c
+++ b/sys/dev/dcons/dcons.c
@@ -37,11 +37,9 @@
 
 #include <sys/param.h>
 
-#if defined(__DragonFly__) || defined(_BOOT)
-#include "dcons.h"
 #if defined(_BOOT)
+#include "dcons.h"
 #include "stand.h"
-#endif
 #else
 #include <dev/dcons/dcons.h>
 #endif
diff --git a/sys/dev/dcons/dcons_crom.c b/sys/dev/dcons/dcons_crom.c
index ee263e95ac7..cb206752be9 100644
--- a/sys/dev/dcons/dcons_crom.c
+++ b/sys/dev/dcons/dcons_crom.c
@@ -46,40 +46,25 @@
 #include <sys/bus.h>
 #include <machine/bus.h>
 
-#ifdef __DragonFly__
-#include <bus/firewire/firewire.h>
-#include <bus/firewire/firewirereg.h>
-#include <bus/firewire/iec13213.h>
-#include "dcons.h"
-#include "dcons_os.h"
-#else
 #include <dev/firewire/firewire.h>
 #include <dev/firewire/firewirereg.h>
 #include <dev/firewire/iec13213.h>
 #include <dev/dcons/dcons.h>
 #include <dev/dcons/dcons_os.h>
-#endif
 
 #include <sys/cons.h>
 
-#define EXPOSE_IDT_ADDR 1
-
-#if (defined(__i386__) || defined(__amd64__)) && defined(EXPOSE_IDT_ADDR)
+#if (defined(__i386__) || defined(__amd64__))
 #include <vm/vm.h>
 #include <vm/vm_param.h>
 #include <vm/pmap.h>
 #include <machine/segments.h> /* for idt */
 #endif
+
 static bus_addr_t dcons_paddr;
 
-#if __FreeBSD_version >= 500000
 static int force_console = 0;
 TUNABLE_INT("hw.firewire.dcons_crom.force_console", &force_console);
-#endif
-
-#ifndef CSRVAL_VENDOR_PRIVATE
-#define NEED_NEW_DRIVER
-#endif
 
 #define ADDR_HI(x)	(((x) >> 24) & 0xffffff)
 #define ADDR_LO(x)	((x) & 0xffffff)
@@ -115,8 +100,7 @@ dcons_crom_probe(device_t dev)
 	return (0);
 }
 
-#ifndef NEED_NEW_DRIVER
-#if (defined(__i386__) || defined(__amd64__)) && defined(EXPOSE_IDT_ADDR)
+#if (defined(__i386__) || defined(__amd64__))
 static void
 dcons_crom_expose_idt(struct dcons_crom_softc *sc)
 {
@@ -129,6 +113,7 @@ dcons_crom_expose_idt(struct dcons_crom_softc *sc)
 	crom_add_entry(&sc->unit, DCONS_CSR_KEY_RESET_LO, ADDR_LO(idt_paddr));
 }
 #endif
+
 static void
 dcons_crom_post_busreset(void *arg)
 {
@@ -149,11 +134,10 @@ dcons_crom_post_busreset(void *arg)
 	crom_add_simple_text(src, &sc->unit, &sc->ver, "dcons");
 	crom_add_entry(&sc->unit, DCONS_CSR_KEY_HI, ADDR_HI(dcons_paddr));
 	crom_add_entry(&sc->unit, DCONS_CSR_KEY_LO, ADDR_LO(dcons_paddr));
-#if (defined(__i386__) || defined(__amd64__)) && defined(EXPOSE_IDT_ADDR)
+#if (defined(__i386__) || defined(__amd64__))
 	dcons_crom_expose_idt(sc);
 #endif
 }
-#endif
 
 static void
 dmamap_cb(void *arg, bus_dma_segment_t *segments, int seg, int error)
@@ -168,11 +152,7 @@ dmamap_cb(void *arg, bus_dma_segment_t *segments, int seg, int error)
 
 	bus_dmamap_sync(sc->dma_tag, sc->dma_map, BUS_DMASYNC_PREWRITE);
 	device_printf(sc->fd.dev,
-#if __FreeBSD_version < 500000
-	    "bus_addr 0x%x\n", sc->bus_addr);
-#else
 	    "bus_addr 0x%jx\n", (uintmax_t)sc->bus_addr);
-#endif
 	if (dcons_paddr != 0) {
 		/* XXX */
 		device_printf(sc->fd.dev, "dcons_paddr is already set\n");
@@ -182,11 +162,9 @@ dmamap_cb(void *arg, bus_dma_segment_t *segments, int seg, int error)
 	dcons_conf->dma_map = sc->dma_map;
 	dcons_paddr = sc->bus_addr;
 
-#if __FreeBSD_version >= 500000
 	/* Force to be the high-level console */
 	if (force_console)
 		cnselect(dcons_conf->cdev);
-#endif
 }
 
 static void
@@ -200,10 +178,6 @@ dcons_crom_poll(void *p, int arg)
 static int
 dcons_crom_attach(device_t dev)
 {
-#ifdef NEED_NEW_DRIVER
-	printf("dcons_crom: you need newer firewire driver\n");
-	return (-1);
-#else
 	struct dcons_crom_softc *sc;
 	int error;
 
@@ -227,10 +201,8 @@ dcons_crom_attach(device_t dev)
 		/*nsegments*/ 1,
 		/*maxsegsz*/ BUS_SPACE_MAXSIZE_32BIT,
 		/*flags*/ BUS_DMA_ALLOCNOW,
-#if __FreeBSD_version >= 501102
 		/*lockfunc*/busdma_lock_mutex,
 		/*lockarg*/&Giant,
-#endif
 		&sc->dma_tag);
 	if (error != 0)
 		return (error);
@@ -245,7 +217,6 @@ dcons_crom_attach(device_t dev)
 	sc->ehand = EVENTHANDLER_REGISTER(dcons_poll, dcons_crom_poll,
 			 (void *)sc, 0);
 	return (0);
-#endif
 }
 
 static int
diff --git a/sys/dev/dcons/dcons_os.h b/sys/dev/dcons/dcons_os.h
index 56a32c3e3ea..bd6dfeff800 100644
--- a/sys/dev/dcons/dcons_os.h
+++ b/sys/dev/dcons/dcons_os.h
@@ -34,7 +34,6 @@
  * $FreeBSD$
  */
 
-
 typedef void (*dcons_poll_fn)(void *, int);
 EVENTHANDLER_DECLARE(dcons_poll, dcons_poll_fn);
 

From 26f1289e4995a203e4b533bfb81d18020a09dc4f Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Wed, 21 Jan 2015 19:59:09 +0000
Subject: [PATCH 168/279] Fix one cause of firewire panics.

sys/dev/firewire/firewire.c:
	In fw_xfer_unload(), clear the FWXF_INQ flag on the
	xfer under protection of the FW_GMTX, after the
	xfer is removeed from the tx/rx queue.  Otherwise
	it is possible for the xfer to be removed again
	(corrupting the list or immediately panicing) from
	another thread that has found this xfer in the
	transaction label table.

Submitted by:	gibbs
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1110200 on 2015/01/02
---
 sys/dev/firewire/firewire.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sys/dev/firewire/firewire.c b/sys/dev/firewire/firewire.c
index c5f5dbac1a7..f06ac30cb18 100644
--- a/sys/dev/firewire/firewire.c
+++ b/sys/dev/firewire/firewire.c
@@ -1166,6 +1166,7 @@ fw_xfer_unload(struct fw_xfer *xfer)
 		s = splfw();
 		FW_GLOCK(xfer->fc);
 		STAILQ_REMOVE(&xfer->q->q, xfer, fw_xfer, link);
+		xfer->flag &= ~FWXF_INQ;
 #if 0
 		xfer->q->queued--;
 #endif

From ff9ae2210b2a3b36f13caf1df2a13025e9084d18 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Wed, 21 Jan 2015 20:02:16 +0000
Subject: [PATCH 169/279] Fix a FWXF_INQ race in the firewire driver.

sys/dev/firewire/firewire.c:
	In fw_xfer_unload() expand lock coverage so that
	the test for FWXF_INQ doesn't race with it being
	cleared in another thread.

Submitted by:	gibbs
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1110207 on 2015/01/02
---
 sys/dev/firewire/firewire.c | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/sys/dev/firewire/firewire.c b/sys/dev/firewire/firewire.c
index f06ac30cb18..6fff13299a1 100644
--- a/sys/dev/firewire/firewire.c
+++ b/sys/dev/firewire/firewire.c
@@ -1022,9 +1022,7 @@ static void
 fw_tl_free(struct firewire_comm *fc, struct fw_xfer *xfer)
 {
 	struct fw_xfer *txfer;
-	int s;
 
-	s = splfw();
 	mtx_lock(&fc->tlabel_lock);
 	if (xfer->tl < 0) {
 		mtx_unlock(&fc->tlabel_lock);
@@ -1042,14 +1040,12 @@ fw_tl_free(struct firewire_comm *fc, struct fw_xfer *xfer)
 		fw_dump_hdr(&xfer->recv.hdr, "recv");
 		kdb_backtrace();
 		mtx_unlock(&fc->tlabel_lock);
-		splx(s);
 		return;
 	}
 
 	STAILQ_REMOVE(&fc->tlabels[xfer->tl], xfer, fw_xfer, tlabel);
 	xfer->tl = -1;
 	mtx_unlock(&fc->tlabel_lock);
-	splx(s);
 	return;
 }
 
@@ -1157,22 +1153,18 @@ fw_xfer_done(struct fw_xfer *xfer)
 void
 fw_xfer_unload(struct fw_xfer *xfer)
 {
-	int s;
 
 	if (xfer == NULL)
 		return;
+	FW_GLOCK(xfer->fc);
 	if (xfer->flag & FWXF_INQ) {
-		printf("fw_xfer_free FWXF_INQ\n");
-		s = splfw();
-		FW_GLOCK(xfer->fc);
 		STAILQ_REMOVE(&xfer->q->q, xfer, fw_xfer, link);
 		xfer->flag &= ~FWXF_INQ;
 #if 0
 		xfer->q->queued--;
 #endif
-		FW_GUNLOCK(xfer->fc);
-		splx(s);
 	}
+	FW_GUNLOCK(xfer->fc);
 	if (xfer->fc != NULL) {
 		/*
 		 * Ensure that any tlabel owner can't access this

From 7eaab60d36eb0ea0c3e05b16db9fc2aa8a37ff63 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Wed, 21 Jan 2015 20:03:46 +0000
Subject: [PATCH 170/279] Fix panic in firewire and creation of invalid config
 ROM.

sys/boot/i386/libfirewire/firewire.c:
sys/dev/firewire/firewire.c:
	Fix configuration ROM generation count wrapping logic
	so that the generation count is never outside of
	allowed limits (0x2 -> 0xF).

sys/dev/firewire/firewire.c:
	In fw_xfer_unload(), xfer->fc may be NULL.  Protect
	against this before taking the fc lock.

Submitted by:	gibbs
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1110685 on 2015/01/05
---
 sys/dev/firewire/firewire.c | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/sys/dev/firewire/firewire.c b/sys/dev/firewire/firewire.c
index 6fff13299a1..026bd056d40 100644
--- a/sys/dev/firewire/firewire.c
+++ b/sys/dev/firewire/firewire.c
@@ -761,8 +761,15 @@ fw_busreset(struct firewire_comm *fc, uint32_t new_status)
 	src = &fc->crom_src_buf->src;
 	crom_load(src, newrom, CROMSIZE);
 	if (bcmp(newrom, fc->config_rom, CROMSIZE) != 0) {
-		if (src->businfo.generation++ > FW_MAX_GENERATION)
+		/* Bump generation and reload. */
+		src->businfo.generation++;
+
+		/* Handle generation count wraps. */
+		if (src->businfo.generation < FW_GENERATION_CHANGEABLE)
 			src->businfo.generation = FW_GENERATION_CHANGEABLE;
+
+		/* Recalculate CRC to account for generation change. */
+		crom_load(src, newrom, CROMSIZE);
 		bcopy(newrom, fc->config_rom, CROMSIZE);
 	}
 	free(newrom, M_FW);
@@ -1156,16 +1163,18 @@ fw_xfer_unload(struct fw_xfer *xfer)
 
 	if (xfer == NULL)
 		return;
-	FW_GLOCK(xfer->fc);
-	if (xfer->flag & FWXF_INQ) {
-		STAILQ_REMOVE(&xfer->q->q, xfer, fw_xfer, link);
-		xfer->flag &= ~FWXF_INQ;
-#if 0
-		xfer->q->queued--;
-#endif
-	}
-	FW_GUNLOCK(xfer->fc);
+
 	if (xfer->fc != NULL) {
+		FW_GLOCK(xfer->fc);
+		if (xfer->flag & FWXF_INQ) {
+			STAILQ_REMOVE(&xfer->q->q, xfer, fw_xfer, link);
+			xfer->flag &= ~FWXF_INQ;
+	#if 0
+			xfer->q->queued--;
+	#endif
+		}
+		FW_GUNLOCK(xfer->fc);
+
 		/*
 		 * Ensure that any tlabel owner can't access this
 		 * xfer after it's freed.

From ed80123329840074d632ef16f077c84ec0dae7ac Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Wed, 21 Jan 2015 20:05:10 +0000
Subject: [PATCH 171/279] Properly lock accesss to the firewire_comm->devices
 list.

sys/dev/firewire/firewire.c:
	Add missing FW_GLOCK/UNLOCK() usage to fw_noderesolve_nodeid().

sys/dev/firewire/firewire.c:
sys/dev/firewire/fwmem.c:
	Remove no-op splfw() calls from functions that have been
	audited for proper lock usage.

Submitted by:	gibbs
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1110992 on 2015/01/06
---
 sys/dev/firewire/firewire.c | 18 +++++-------------
 sys/dev/firewire/fwmem.c    |  4 +---
 2 files changed, 6 insertions(+), 16 deletions(-)

diff --git a/sys/dev/firewire/firewire.c b/sys/dev/firewire/firewire.c
index 026bd056d40..976be69e721 100644
--- a/sys/dev/firewire/firewire.c
+++ b/sys/dev/firewire/firewire.c
@@ -146,13 +146,12 @@ struct fw_device *
 fw_noderesolve_nodeid(struct firewire_comm *fc, int dst)
 {
 	struct fw_device *fwdev;
-	int s;
 
-	s = splfw();
+	FW_GLOCK(fc);
 	STAILQ_FOREACH(fwdev, &fc->devices, link)
 		if (fwdev->dst == dst && fwdev->status != FWDEVINVAL)
 			break;
-	splx(s);
+	FW_GUNLOCK(fc);
 
 	return fwdev;
 }
@@ -164,15 +163,12 @@ struct fw_device *
 fw_noderesolve_eui64(struct firewire_comm *fc, struct fw_eui64 *eui)
 {
 	struct fw_device *fwdev;
-	int s;
 
-	s = splfw();
 	FW_GLOCK(fc);
 	STAILQ_FOREACH(fwdev, &fc->devices, link)
 		if (FW_EUI64_EQUAL(fwdev->eui, *eui))
 			break;
 	FW_GUNLOCK(fc);
-	splx(s);
 
 	if (fwdev == NULL)
 		return NULL;
@@ -301,9 +297,7 @@ static void
 fw_asystart(struct fw_xfer *xfer)
 {
 	struct firewire_comm *fc = xfer->fc;
-	int s;
 
-	s = splfw();
 	/* Protect from interrupt/timeout */
 	FW_GLOCK(fc);
 	xfer->flag = FWXF_INQ;
@@ -312,7 +306,6 @@ fw_asystart(struct fw_xfer *xfer)
 	xfer->q->queued++;
 #endif
 	FW_GUNLOCK(fc);
-	splx(s);
 	/* XXX just queue for mbuf */
 	if (xfer->mbuf == NULL)
 		xfer->q->start(fc);
@@ -332,6 +325,7 @@ firewire_probe(device_t dev)
 	return (0);
 }
 
+/* Just use a per-packet callout? */
 static void
 firewire_xfer_timeout(void *arg, int pending)
 {
@@ -340,7 +334,7 @@ firewire_xfer_timeout(void *arg, int pending)
 	struct timeval tv;
 	struct timeval split_timeout;
 	STAILQ_HEAD(, fw_xfer) xfer_timeout;
-	int i, s;
+	int i;
 
 	split_timeout.tv_sec = 0;
 	split_timeout.tv_usec = 200 * 1000;	 /* 200 msec */
@@ -349,9 +343,8 @@ firewire_xfer_timeout(void *arg, int pending)
 	timevalsub(&tv, &split_timeout);
 	STAILQ_INIT(&xfer_timeout);
 
-	s = splfw();
 	mtx_lock(&fc->tlabel_lock);
-	for (i = 0; i < 0x40; i++) {
+	for (i = 0; i < nitems(fc->tlabels); i++) {
 		while ((xfer = STAILQ_FIRST(&fc->tlabels[i])) != NULL) {
 			if ((xfer->flag & FWXF_SENT) == 0)
 				/* not sent yet */
@@ -370,7 +363,6 @@ firewire_xfer_timeout(void *arg, int pending)
 		}
 	}
 	mtx_unlock(&fc->tlabel_lock);
-	splx(s);
 	fc->timeout(fc);
 
 	STAILQ_FOREACH_SAFE(xfer, &xfer_timeout, tlabel, txfer)
diff --git a/sys/dev/firewire/fwmem.c b/sys/dev/firewire/fwmem.c
index 182fc8aba51..6498daa480a 100644
--- a/sys/dev/firewire/fwmem.c
+++ b/sys/dev/firewire/fwmem.c
@@ -348,12 +348,11 @@ fwmem_strategy(struct bio *bp)
 	struct fw_device *fwdev;
 	struct fw_xfer *xfer;
 	struct cdev *dev;
-	int err = 0, s, iolen;
+	int err = 0, iolen;
 
 	dev = bp->bio_dev;
 	/* XXX check request length */
 
-	s = splfw();
 	fms = dev->si_drv1;
 	fwdev = fw_noderesolve_eui64(fms->sc->fc, &fms->eui);
 	if (fwdev == NULL) {
@@ -395,7 +394,6 @@ fwmem_strategy(struct bio *bp)
 	/* XXX */
 	bp->bio_resid = bp->bio_bcount - iolen;
 error:
-	splx(s);
 	if (err != 0) {
 		if (fwmem_debug)
 			printf("%s: err=%d\n", __func__, err);

From 4dd896995c50676c65eccf8a3b87241507dd0726 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Wed, 21 Jan 2015 20:06:25 +0000
Subject: [PATCH 172/279] Fix firewire panic when issuing a reply to an
 unhandled asynchronous remote dma request (DMA request that the hardware
 cannot automatically handle).

sys/dev/firewire/firewire.c
	In fw_rcv(), add missing early return in the error
	path for DMA requests to unregistered regions.

Submitted by:	gibbs
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1110993 on 2015/01/06
---
 sys/dev/firewire/firewire.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sys/dev/firewire/firewire.c b/sys/dev/firewire/firewire.c
index 976be69e721..72341887f46 100644
--- a/sys/dev/firewire/firewire.c
+++ b/sys/dev/firewire/firewire.c
@@ -2030,6 +2030,7 @@ fw_rcv(struct fw_rcv_buf *rb)
 			rb->xfer->hand = fw_xfer_free;
 			if (fw_asyreq(rb->fc, -1, rb->xfer))
 				fw_xfer_free(rb->xfer);
+			return;
 		}
 		len = 0;
 		for (i = 0; i < rb->nvec; i++)

From 91291042274de728fce6f7ffad79152d68fccd2f Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Wed, 21 Jan 2015 20:08:24 +0000
Subject: [PATCH 173/279] Fix remote DMA based firewire debugging when
 targeting systems with more than 4GB of physical memory.

To remotely debug the system 'stealthy' which has a kernel
with this change installed and firewire properly configured:

	% fwcontrol -m stealthy (or stealthy's firewire EUI64)
	% kgdb kernel /dev/fwmem0.0

sys/dev/firewire/fwohci.c:
	Rather than hard code the upper limit for hw based
	automatic responses to remote DMA requests at 4GB,
	program the hardware using Maxmem, the page number
	one higher than the highest physical page detected
	in the system.

	While here, garbage collect more useless splfw()
	calls.

Submitted by:	gibbs
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1110994 on 2015/01/06
---
 sys/dev/firewire/fwohci.c | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/sys/dev/firewire/fwohci.c b/sys/dev/firewire/fwohci.c
index 00a54e28da6..70134af686e 100644
--- a/sys/dev/firewire/fwohci.c
+++ b/sys/dev/firewire/fwohci.c
@@ -48,6 +48,7 @@
 #include <sys/kdb.h>
 
 #include <machine/bus.h>
+#include <machine/md_var.h>
 
 #include <dev/firewire/firewire.h>
 #include <dev/firewire/firewirereg.h>
@@ -188,6 +189,7 @@ static void fwohci_task_dma(void *, int);
 #define	OHCI_PREQLO		0x118
 #define	OHCI_PREQLOCLR		0x11c
 #define	OHCI_PREQUPPER		0x120
+#define OHCI_PREQUPPER_MAX	0xffff0000
 
 #define	OHCI_SID_BUF		0x64
 #define	OHCI_SID_CNT		0x68
@@ -854,7 +856,7 @@ fwohci_execute_db2(void *arg, bus_dma_segment_t *segs, int nseg,
 static void
 fwohci_start(struct fwohci_softc *sc, struct fwohci_dbch *dbch)
 {
-	int i, s;
+	int i;
 	int tcode, hdr_len, pl_off;
 	int fsegment = -1;
 	uint32_t off;
@@ -880,7 +882,6 @@ fwohci_start(struct fwohci_softc *sc, struct fwohci_dbch *dbch)
 	if (dbch->flags & FWOHCI_DBCH_FULL)
 		return;
 
-	s = splfw();
 	db_tr = dbch->top;
 txloop:
 	xfer = STAILQ_FIRST(&dbch->xferq.q);
@@ -1030,7 +1031,6 @@ kick:
 	}
 
 	dbch->top = db_tr;
-	splx(s);
 	return;
 }
 
@@ -1821,6 +1821,7 @@ static void
 fwohci_intr_core(struct fwohci_softc *sc, uint32_t stat, int count)
 {
 	struct firewire_comm *fc = (struct firewire_comm *)sc;
+	uintmax_t prequpper;
 	uint32_t node_id, plen;
 
 	FW_GLOCK_ASSERT(fc);
@@ -1852,8 +1853,17 @@ fwohci_intr_core(struct fwohci_softc *sc, uint32_t stat, int count)
 			/* allow from all nodes */
 			OWRITE(sc, OHCI_PREQHI, 0x7fffffff);
 			OWRITE(sc, OHCI_PREQLO, 0xffffffff);
-			/* 0 to 4GB region */
-			OWRITE(sc, OHCI_PREQUPPER, 0x10000);
+			prequpper = ((uintmax_t)Maxmem << PAGE_SHIFT) >> 16;
+			if (prequpper > OHCI_PREQUPPER_MAX) {
+				device_printf(fc->dev,
+				    "Physical memory size of 0x%jx exceeds "
+				    "fire wire address space.  Limiting dma "
+				    "to memory below 0x%jx\n",
+				    (uintmax_t)Maxmem << PAGE_SHIFT,
+				    (uintmax_t)OHCI_PREQUPPER_MAX << 16);
+				prequpper = OHCI_PREQUPPER_MAX;
+			}
+			OWRITE(sc, OHCI_PREQUPPER, prequpper & 0xffffffff);
 		}
 		/* Set ATRetries register */
 		OWRITE(sc, OHCI_ATRETRY, 1<<(13 + 16) | 0xfff);
@@ -2171,7 +2181,7 @@ fwohci_rbuf_update(struct fwohci_softc *sc, int dmach)
 	struct fw_bulkxfer *chunk;
 	struct fw_xferq *ir;
 	uint32_t stat;
-	int s, w = 0, ldesc;
+	int w = 0, ldesc;
 
 	ir = fc->ir[dmach];
 	ldesc = sc->ir[dmach].ndesc - 1;
@@ -2179,7 +2189,6 @@ fwohci_rbuf_update(struct fwohci_softc *sc, int dmach)
 #if 0
 	dump_db(sc, dmach);
 #endif
-	s = splfw();
 	if ((ir->flag & FWXFERQ_HANDLER) == 0)
 		FW_GLOCK(fc);
 	fwdma_sync_multiseg_all(sc->ir[dmach].am, BUS_DMASYNC_POSTREAD);
@@ -2218,7 +2227,6 @@ fwohci_rbuf_update(struct fwohci_softc *sc, int dmach)
 	}
 	if ((ir->flag & FWXFERQ_HANDLER) == 0)
 		FW_GUNLOCK(fc);
-	splx(s);
 	if (w == 0)
 		return;
 

From ede35faed245c193f3cc8b990d370032f3672672 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 20:12:35 +0000
Subject: [PATCH 174/279] Micro-optimize the new arm inline bus_space
 implementation by grouping all the data the inline functions access together
 at the start of the bus_space struct.  The start-of part isn't so important,
 it's the grouping-together that's the point: now all the most-accessed data
 should be in one cache line.

Suggested by:	  cognet
---
 sys/arm/arm/bus_space_base.c |  2 +-
 sys/arm/include/bus.h        | 37 +++++++++++++++++++++---------------
 2 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/sys/arm/arm/bus_space_base.c b/sys/arm/arm/bus_space_base.c
index a80b06466cc..ba138d59e00 100644
--- a/sys/arm/arm/bus_space_base.c
+++ b/sys/arm/arm/bus_space_base.c
@@ -150,7 +150,7 @@ static struct bus_space arm_base_bus_space = {
 	.bs_wr_2_s	= generic_bs_wr_2,
 	.bs_wr_4_s	= generic_bs_wr_4,
 	.bs_wr_8_s	= BS_UNIMPLEMENTED,
-};
+} __aligned(CACHE_LINE_SIZE);
 
 #ifdef FDT
 bus_space_tag_t fdtbus_bs_tag = &arm_base_bus_space;
diff --git a/sys/arm/include/bus.h b/sys/arm/include/bus.h
index 757919974a1..ed9d317cd0a 100644
--- a/sys/arm/include/bus.h
+++ b/sys/arm/include/bus.h
@@ -79,9 +79,27 @@
 #define	BUS_SPACE_MAP_LINEAR		0x02
 #define	BUS_SPACE_MAP_PREFETCHABLE     	0x04
 
+/*
+ * Bus space for ARM.
+ *
+ * The functions used most often are grouped together at the beginning to ensure
+ * that all the data fits into a single cache line.  The inline implementations
+ * of single read/write access these values a lot.
+ */
 struct bus_space {
-	/* cookie */
-	void		*bs_privdata;
+	/* Read/write single and barrier: the most commonly used functions. */
+	uint8_t	 (*bs_r_1)(bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	uint32_t (*bs_r_4)(bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	void	 (*bs_w_1)(bus_space_tag_t, bus_space_handle_t,
+			   bus_size_t, uint8_t);
+	void	 (*bs_w_4)(bus_space_tag_t, bus_space_handle_t,
+			   bus_size_t, uint32_t);
+	void	 (*bs_barrier)(bus_space_tag_t, bus_space_handle_t,
+			       bus_size_t, bus_size_t, int);
+
+	/* Backlink to parent (if copied), and implementation private data. */
+	struct bus_space *bs_parent;
+	void		 *bs_privdata;
 
 	/* mapping/unmapping */
 	int		(*bs_map) (bus_space_tag_t, bus_addr_t, bus_size_t,
@@ -97,15 +115,8 @@ struct bus_space {
 	void		(*bs_free) (bus_space_tag_t, bus_space_handle_t,
 			    bus_size_t);
 
-	/* get kernel virtual address */
-	/* barrier */
-	void		(*bs_barrier) (bus_space_tag_t, bus_space_handle_t,
-			    bus_size_t, bus_size_t, int);
-
-	/* read (single) */
-	uint8_t		(*bs_r_1) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	/* Read single, the less commonly used functions. */
 	uint16_t	(*bs_r_2) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
-	uint32_t	(*bs_r_4) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
 	uint64_t	(*bs_r_8) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
 
 	/* read multiple */
@@ -128,13 +139,9 @@ struct bus_space {
 	void		(*bs_rr_8) (bus_space_tag_t, bus_space_handle_t,
 			    bus_size_t, uint64_t *, bus_size_t);
 					
-	/* write (single) */
-	void		(*bs_w_1) (bus_space_tag_t, bus_space_handle_t,
-			    bus_size_t, uint8_t);
+	/* Write single, the less commonly used functions. */
 	void		(*bs_w_2) (bus_space_tag_t, bus_space_handle_t,
 			    bus_size_t, uint16_t);
-	void		(*bs_w_4) (bus_space_tag_t, bus_space_handle_t,
-			    bus_size_t, uint32_t);
 	void		(*bs_w_8) (bus_space_tag_t, bus_space_handle_t,
 			    bus_size_t, uint64_t);
 

From b44e442e5043e04d57f700a60327608cfd979700 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Wed, 21 Jan 2015 20:22:53 +0000
Subject: [PATCH 175/279] Change 1112791 by kenm@ken.spectrabsd8 on 2015/01/15
 16:45:13

Fix SCSI status byte reporting on 4Gb and 8Gb Qlogic boards.

The newer boards don't have the response field that indicates
whether the SCSI status byte is present.  You have to just look to
see whether it is non-zero.

The code was looking to see whether the sense length was valid
before propagating the SCSI status byte (and sense information) up
the stack.  With a status like Reservation Conflict, there is no
sense information, only the SCSI status byte.  So it wasn't getting
correctly returned.

isp.c:
	In isp_intr(), if we are on a 2400 or 2500 type board and
	get a response, look at the actual contents of the
	SCSI status value and set the RQSF_GOT_STATUS flag
	accordingly so that return any SCSI status value we get.  The
	RQSF_GOT_SENSE flag will get set later on if there is
	actual sense information returned.

Submitted by:	ken
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1112791 on 2015/01/15
---
 sys/dev/isp/isp_freebsd.c | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/sys/dev/isp/isp_freebsd.c b/sys/dev/isp/isp_freebsd.c
index f9f3ed9cb69..11658157ea9 100644
--- a/sys/dev/isp/isp_freebsd.c
+++ b/sys/dev/isp/isp_freebsd.c
@@ -5123,19 +5123,34 @@ isp_action(struct cam_sim *sim, union ccb *ccb)
 		break;
 #endif
 	case XPT_RESET_DEV:		/* BDR the specified SCSI device */
+	{
+		struct isp_fc *fc;
 
 		bus = cam_sim_bus(xpt_path_sim(ccb->ccb_h.path));
 		tgt = ccb->ccb_h.target_id;
 		tgt |= (bus << 16);
+		if (IS_FC(isp))
+			fc = ISP_FC_PC(isp, bus);
+		else
+			fc = NULL;
 
 		error = isp_control(isp, ISPCTL_RESET_DEV, bus, tgt);
 		if (error) {
 			ccb->ccb_h.status = CAM_REQ_CMP_ERR;
 		} else {
+			/*
+			 * If we have a FC device, reset the Command
+			 * Reference Number, because the target will expect
+			 * that we re-start the CRN at 1 after a reset.
+			 */
+			if (fc != NULL)
+				isp_fcp_reset_crn(fc, tgt, /*tgt_set*/ 1);
+
 			ccb->ccb_h.status = CAM_REQ_CMP;
 		}
 		xpt_done(ccb);
 		break;
+	}
 	case XPT_ABORT:			/* Abort the specified CCB */
 	{
 		union ccb *accb = ccb->cab.abort_ccb;

From 5b14cb4136509a17c4d67778293456ab1faa88e9 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Wed, 21 Jan 2015 20:27:11 +0000
Subject: [PATCH 176/279] Force commit to record the correct log for r277513.

If the user sends an XPT_RESET_DEV CCB, make sure to reset the
Fibre Channel Command Reference Number if we're running on a FC
controller.

We send a SCSI Target Reset when we get this CCB, and as a result
need to reset the CRN to 1 on the next command.

isp_freebsd.c:
	In the XPT_RESET_DEV implementation in isp_action(), reset
	the CRN if we're on a FC controller.

Submitted by:	ken
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1112787 on 2015/01/15
---
 sys/dev/isp/isp_freebsd.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sys/dev/isp/isp_freebsd.c b/sys/dev/isp/isp_freebsd.c
index 11658157ea9..881f58aa0d9 100644
--- a/sys/dev/isp/isp_freebsd.c
+++ b/sys/dev/isp/isp_freebsd.c
@@ -29,6 +29,7 @@
  */
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
+
 #include <dev/isp/isp_freebsd.h>
 #include <sys/unistd.h>
 #include <sys/kthread.h>

From 35d002dc8fa47753de577d46ac63231bf8f09dde Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Wed, 21 Jan 2015 20:32:36 +0000
Subject: [PATCH 177/279] Fix SCSI status byte reporting on 4Gb and 8Gb Qlogic
 boards.

The newer boards don't have the response field that indicates
whether the SCSI status byte is present.  You have to just look to
see whether it is non-zero.

The code was looking to see whether the sense length was valid
before propagating the SCSI status byte (and sense information) up
the stack.  With a status like Reservation Conflict, there is no
sense information, only the SCSI status byte.  So it wasn't getting
correctly returned.

isp.c:
	In isp_intr(), if we are on a 2400 or 2500 type board and
	get a response, look at the actual contents of the
	SCSI status value and set the RQSF_GOT_STATUS flag
	accordingly so that return any SCSI status value we get.  The
	RQSF_GOT_SENSE flag will get set later on if there is
	actual sense information returned.

Submitted by:	ken
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1112791 on 2015/01/15
---
 sys/dev/isp/isp.c | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/sys/dev/isp/isp.c b/sys/dev/isp/isp.c
index 46361b95e48..dce666b23b3 100644
--- a/sys/dev/isp/isp.c
+++ b/sys/dev/isp/isp.c
@@ -5224,7 +5224,10 @@ again:
 			}
 			scsi_status = sp2->req_scsi_status;
 			completion_status = sp2->req_completion_status;
-			req_state_flags = 0;
+			if ((scsi_status & 0xff) != 0)
+				req_state_flags = RQSF_GOT_STATUS;
+			else
+				req_state_flags = 0;
 			resid = sp2->req_resid;
 		} else if (etype == RQSTYPE_RESPONSE) {
 			isp_get_response(isp, (ispstatusreq_t *) hp, sp);

From b9a128519124fd193c47b81e7f828ea5e87a82c7 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Wed, 21 Jan 2015 21:31:26 +0000
Subject: [PATCH 178/279] Move the __aligned() declaration to where it will
 actually do something.

---
 sys/arm/arm/bus_space_base.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sys/arm/arm/bus_space_base.c b/sys/arm/arm/bus_space_base.c
index ba138d59e00..4e4f33ce35a 100644
--- a/sys/arm/arm/bus_space_base.c
+++ b/sys/arm/arm/bus_space_base.c
@@ -45,7 +45,7 @@ bs_protos(generic);
  * The bus space tag.  This is constant for all instances, so
  * we never have to explicitly "create" it.
  */
-static struct bus_space arm_base_bus_space = {
+static struct bus_space arm_base_bus_space __aligned(CACHE_LINE_SIZE) = {
 	/* privdata is whatever the implementer wants; unused in base tag */
 	.bs_privdata	= NULL,
 
@@ -150,7 +150,7 @@ static struct bus_space arm_base_bus_space = {
 	.bs_wr_2_s	= generic_bs_wr_2,
 	.bs_wr_4_s	= generic_bs_wr_4,
 	.bs_wr_8_s	= BS_UNIMPLEMENTED,
-} __aligned(CACHE_LINE_SIZE);
+};
 
 #ifdef FDT
 bus_space_tag_t fdtbus_bs_tag = &arm_base_bus_space;

From 8b817932f716a73867ee4064d2b1b4a4ef6b383a Mon Sep 17 00:00:00 2001
From: Ed Maste <emaste@FreeBSD.org>
Date: Wed, 21 Jan 2015 21:49:03 +0000
Subject: [PATCH 179/279] Fix bootstrap on systems with old libdwarf and
 WITHOUT_CDDL

ELF Tool Chain tools need libelf and libdwarf.

Submitted by:	jmallett (earlier version)
Reviewed by:	jmallett
Sponsored by:	The FreeBSD Foundation
---
 Makefile.inc1 | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/Makefile.inc1 b/Makefile.inc1
index 984d240c6ed..afa49408e68 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -1292,12 +1292,16 @@ _clang_tblgen= \
 	usr.bin/clang/clang-tblgen
 .endif
 
+# ELF Tool Chain libraries are needed for ELF tools and dtrace tools.
 # dtrace tools are required for older bootstrap env and cross-build
 # pre libdwarf
-.if ${MK_CDDL} != "no" && (${BOOTSTRAPPING} < 1100006 \
-      || (${MACHINE} != ${TARGET} || ${MACHINE_ARCH} != ${TARGET_ARCH}))
-_dtrace_tools= cddl/usr.bin/sgsmsg cddl/lib/libctf lib/libelf \
-    lib/libdwarf cddl/usr.bin/ctfconvert cddl/usr.bin/ctfmerge
+.if ${BOOTSTRAPPING} < 1100006 || (${MACHINE} != ${TARGET} || \
+    ${MACHINE_ARCH} != ${TARGET_ARCH})
+_elftoolchain_libs= lib/libelf lib/libdwarf 
+.if ${MK_CDDL} != "no"
+_dtrace_tools= cddl/usr.bin/sgsmsg cddl/lib/libctf cddl/usr.bin/ctfconvert \
+    cddl/usr.bin/ctfmerge
+.endif
 .endif
 
 # Default to building the GPL DTC, but build the BSDL one if users explicitly
@@ -1324,6 +1328,7 @@ bootstrap-tools: .MAKE
 .for _tool in \
     ${_clang_tblgen} \
     ${_kerberos5_bootstrap_tools} \
+    ${_elftoolchain_libs} \
     ${_dtrace_tools} \
     ${_strfile} \
     ${_gperf} \

From 36fe5321f2cfec8cbc98478c31d6ecb7fbdd96ce Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Thu, 22 Jan 2015 00:52:34 +0000
Subject: [PATCH 180/279] Enable nanobsd.sh to be executed when pwd !=
 NANO_SRC.

While here, fix a bug in which NANO_PMAKE would not be appended at the
appropriate time.

Simply move both checks to after the call to set_defaults_and_export().

Tested by:	lstewart
Sponsored by:	Spectra Logic
---
 tools/tools/nanobsd/nanobsd.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)
 mode change 100644 => 100755 tools/tools/nanobsd/nanobsd.sh

diff --git a/tools/tools/nanobsd/nanobsd.sh b/tools/tools/nanobsd/nanobsd.sh
old mode 100644
new mode 100755
index a4464e0caf5..776b246599f
--- a/tools/tools/nanobsd/nanobsd.sh
+++ b/tools/tools/nanobsd/nanobsd.sh
@@ -121,6 +121,13 @@ if [ $# -gt 0 ] ; then
 	usage
 fi
 
+#######################################################################
+# And then it is as simple as that...
+
+# File descriptor 3 is used for logging output, see pprint
+exec 3>&1
+set_defaults_and_export
+
 if [ ! -d "${NANO_TOOLS}" ]; then
 	echo "NANO_TOOLS directory does not exist" 1>&2
 	exit 1
@@ -130,13 +137,6 @@ if ! $do_clean; then
 	NANO_PMAKE="${NANO_PMAKE} -DNO_CLEAN"
 fi
 
-#######################################################################
-# And then it is as simple as that...
-
-# File descriptor 3 is used for logging output, see pprint
-exec 3>&1
-set_defaults_and_export
-
 pprint 1 "NanoBSD image ${NANO_NAME} build starting"
 
 if $do_world ; then

From efc6c51ffae26a42f712389cb931e2c459c17243 Mon Sep 17 00:00:00 2001
From: Gleb Smirnoff <glebius@FreeBSD.org>
Date: Thu, 22 Jan 2015 01:23:16 +0000
Subject: [PATCH 181/279] Back out r276841, r276756, r276747, r276746. The
 change in r276747 is very very questionable, since it makes vimages more
 dependent on each other. But the reason for the backout is that it screwed up
 shutting down the pf purge threads, and now kernel immedially panics on pf
 module unload. Although module unloading isn't an advertised feature of pf,
 it is very important for development process.

I'd like to not backout r276746, since in general it is good. But since it
has introduced numerous build breakages, that later were addressed in
r276841, r276756, r276747, I need to back it out as well. Better replay it
in clean fashion from scratch.
---
 sys/net/pfvar.h           |  7 +--
 sys/netpfil/pf/pf.c       | 97 +++++++++++++++++++++++++++------------
 sys/netpfil/pf/pf_if.c    | 36 +++++++--------
 sys/netpfil/pf/pf_ioctl.c | 48 ++++++++++++-------
 sys/netpfil/pf/pf_norm.c  |  9 ++--
 sys/netpfil/pf/pf_table.c | 46 +++++++++----------
 6 files changed, 148 insertions(+), 95 deletions(-)

diff --git a/sys/net/pfvar.h b/sys/net/pfvar.h
index 47febfdf721..78a4e8a53d2 100644
--- a/sys/net/pfvar.h
+++ b/sys/net/pfvar.h
@@ -829,6 +829,7 @@ typedef int pflog_packet_t(struct pfi_kif *, struct mbuf *, sa_family_t,
     struct pf_ruleset *, struct pf_pdesc *, int);
 extern pflog_packet_t		*pflog_packet_ptr;
 
+#define	V_pf_end_threads	VNET(pf_end_threads)
 #endif /* _KERNEL */
 
 #define	PFSYNC_FLAG_SRCNODE	0x04
@@ -1494,7 +1495,7 @@ VNET_DECLARE(struct pf_altqqueue *,	 pf_altqs_inactive);
 VNET_DECLARE(struct pf_rulequeue, pf_unlinked_rules);
 #define	V_pf_unlinked_rules	VNET(pf_unlinked_rules)
 
-void				 pf_vnet_initialize(void);
+void				 pf_initialize(void);
 void				 pf_mtag_initialize(void);
 void				 pf_mtag_cleanup(void);
 void				 pf_cleanup(void);
@@ -1586,7 +1587,7 @@ int	pf_match_addr_range(struct pf_addr *, struct pf_addr *,
 	    struct pf_addr *, sa_family_t);
 int	pf_match_port(u_int8_t, u_int16_t, u_int16_t, u_int16_t);
 
-void	pf_vnet_normalize_init(void);
+void	pf_normalize_init(void);
 void	pf_normalize_cleanup(void);
 int	pf_normalize_ip(struct mbuf **, int, struct pfi_kif *, u_short *,
 	    struct pf_pdesc *);
@@ -1648,7 +1649,7 @@ MALLOC_DECLARE(PFI_MTYPE);
 VNET_DECLARE(struct pfi_kif *,		 pfi_all);
 #define	V_pfi_all	 		 VNET(pfi_all)
 
-void		 pfi_vnet_initialize(void);
+void		 pfi_initialize(void);
 void		 pfi_cleanup(void);
 void		 pfi_kif_ref(struct pfi_kif *);
 void		 pfi_kif_unref(struct pfi_kif *);
diff --git a/sys/netpfil/pf/pf.c b/sys/netpfil/pf/pf.c
index bf47c185aec..15667a6031e 100644
--- a/sys/netpfil/pf/pf.c
+++ b/sys/netpfil/pf/pf.c
@@ -151,7 +151,6 @@ static VNET_DEFINE(struct pf_send_head, pf_sendqueue);
 #define	V_pf_sendqueue	VNET(pf_sendqueue)
 
 static struct mtx pf_sendqueue_mtx;
-MTX_SYSINIT(pf_sendqueue_mtx, &pf_sendqueue_mtx, "pf send queue", MTX_DEF);
 #define	PF_SENDQ_LOCK()		mtx_lock(&pf_sendqueue_mtx)
 #define	PF_SENDQ_UNLOCK()	mtx_unlock(&pf_sendqueue_mtx)
 
@@ -173,15 +172,11 @@ static VNET_DEFINE(struct task, pf_overloadtask);
 #define	V_pf_overloadtask	VNET(pf_overloadtask)
 
 static struct mtx pf_overloadqueue_mtx;
-MTX_SYSINIT(pf_overloadqueue_mtx, &pf_overloadqueue_mtx,
-    "pf overload/flush queue", MTX_DEF);
 #define	PF_OVERLOADQ_LOCK()	mtx_lock(&pf_overloadqueue_mtx)
 #define	PF_OVERLOADQ_UNLOCK()	mtx_unlock(&pf_overloadqueue_mtx)
 
 VNET_DEFINE(struct pf_rulequeue, pf_unlinked_rules);
 struct mtx pf_unlnkdrules_mtx;
-MTX_SYSINIT(pf_unlnkdrules_mtx, &pf_unlnkdrules_mtx, "pf unlinked rules",
-    MTX_DEF);
 
 static VNET_DEFINE(uma_zone_t,	pf_sources_z);
 #define	V_pf_sources_z	VNET(pf_sources_z)
@@ -295,6 +290,8 @@ static void		 pf_route6(struct mbuf **, struct pf_rule *, int,
 
 int in4_cksum(struct mbuf *m, u_int8_t nxt, int off, int len);
 
+VNET_DECLARE(int, pf_end_threads);
+
 VNET_DEFINE(struct pf_limit, pf_limits[PF_LIMIT_MAX]);
 
 #define	PACKET_LOOPED(pd)	((pd)->pf_mtag &&			\
@@ -731,7 +728,7 @@ pf_mtag_initialize()
 
 /* Per-vnet data storage structures initialization. */
 void
-pf_vnet_initialize()
+pf_initialize()
 {
 	struct pf_keyhash	*kh;
 	struct pf_idhash	*ih;
@@ -791,9 +788,13 @@ pf_vnet_initialize()
 	STAILQ_INIT(&V_pf_sendqueue);
 	SLIST_INIT(&V_pf_overloadqueue);
 	TASK_INIT(&V_pf_overloadtask, 0, pf_overload_task, curvnet);
+	mtx_init(&pf_sendqueue_mtx, "pf send queue", NULL, MTX_DEF);
+	mtx_init(&pf_overloadqueue_mtx, "pf overload/flush queue", NULL,
+	    MTX_DEF);
 
 	/* Unlinked, but may be referenced rules. */
 	TAILQ_INIT(&V_pf_unlinked_rules);
+	mtx_init(&pf_unlnkdrules_mtx, "pf unlinked rules", NULL, MTX_DEF);
 }
 
 void
@@ -836,6 +837,10 @@ pf_cleanup()
 		free(pfse, M_PFTEMP);
 	}
 
+	mtx_destroy(&pf_sendqueue_mtx);
+	mtx_destroy(&pf_overloadqueue_mtx);
+	mtx_destroy(&pf_unlnkdrules_mtx);
+
 	uma_zdestroy(V_pf_sources_z);
 	uma_zdestroy(V_pf_state_z);
 	uma_zdestroy(V_pf_state_key_z);
@@ -1381,37 +1386,71 @@ pf_intr(void *v)
 }
 
 void
-pf_purge_thread(void *v __unused)
+pf_purge_thread(void *v)
 {
 	u_int idx = 0;
-	VNET_ITERATOR_DECL(vnet_iter);
+
+	CURVNET_SET((struct vnet *)v);
 
 	for (;;) {
-		tsleep(pf_purge_thread, PWAIT, "pftm", hz / 10);
-		VNET_LIST_RLOCK();
-		VNET_FOREACH(vnet_iter) {
-			CURVNET_SET(vnet_iter);
-			/* Process 1/interval fraction of the state table every run. */
-			idx = pf_purge_expired_states(idx, pf_hashmask /
-				    (V_pf_default_rule.timeout[PFTM_INTERVAL] * 10));
+		PF_RULES_RLOCK();
+		rw_sleep(pf_purge_thread, &pf_rules_lock, 0, "pftm", hz / 10);
 
-			/* Purge other expired types every PFTM_INTERVAL seconds. */
-			if (idx == 0) {
-				/*
-				 * Order is important:
-				 * - states and src nodes reference rules
-				 * - states and rules reference kifs
-				 */
-				pf_purge_expired_fragments();
-				pf_purge_expired_src_nodes();
-				pf_purge_unlinked_rules();
-				pfi_kif_purge();
-			}
-			CURVNET_RESTORE();
+		if (V_pf_end_threads) {
+			/*
+			 * To cleanse up all kifs and rules we need
+			 * two runs: first one clears reference flags,
+			 * then pf_purge_expired_states() doesn't
+			 * raise them, and then second run frees.
+			 */
+			PF_RULES_RUNLOCK();
+			pf_purge_unlinked_rules();
+			pfi_kif_purge();
+
+			/*
+			 * Now purge everything.
+			 */
+			pf_purge_expired_states(0, pf_hashmask);
+			pf_purge_expired_fragments();
+			pf_purge_expired_src_nodes();
+
+			/*
+			 * Now all kifs & rules should be unreferenced,
+			 * thus should be successfully freed.
+			 */
+			pf_purge_unlinked_rules();
+			pfi_kif_purge();
+
+			/*
+			 * Announce success and exit.
+			 */
+			PF_RULES_RLOCK();
+			V_pf_end_threads++;
+			PF_RULES_RUNLOCK();
+			wakeup(pf_purge_thread);
+			kproc_exit(0);
+		}
+		PF_RULES_RUNLOCK();
+
+		/* Process 1/interval fraction of the state table every run. */
+		idx = pf_purge_expired_states(idx, pf_hashmask /
+			    (V_pf_default_rule.timeout[PFTM_INTERVAL] * 10));
+
+		/* Purge other expired types every PFTM_INTERVAL seconds. */
+		if (idx == 0) {
+			/*
+			 * Order is important:
+			 * - states and src nodes reference rules
+			 * - states and rules reference kifs
+			 */
+			pf_purge_expired_fragments();
+			pf_purge_expired_src_nodes();
+			pf_purge_unlinked_rules();
+			pfi_kif_purge();
 		}
-		VNET_LIST_RUNLOCK();
 	}
 	/* not reached */
+	CURVNET_RESTORE();
 }
 
 u_int32_t
diff --git a/sys/netpfil/pf/pf_if.c b/sys/netpfil/pf/pf_if.c
index da79ed6f276..41acc7dd477 100644
--- a/sys/netpfil/pf/pf_if.c
+++ b/sys/netpfil/pf/pf_if.c
@@ -102,13 +102,10 @@ MALLOC_DEFINE(PFI_MTYPE, "pf_ifnet", "pf(4) interface database");
 LIST_HEAD(pfi_list, pfi_kif);
 static VNET_DEFINE(struct pfi_list, pfi_unlinked_kifs);
 #define	V_pfi_unlinked_kifs	VNET(pfi_unlinked_kifs)
-
 static struct mtx pfi_unlnkdkifs_mtx;
-MTX_SYSINIT(pfi_unlnkdkifs_mtx, &pfi_unlnkdkifs_mtx, "pf unlinked interfaces",
-    MTX_DEF);
 
 void
-pfi_vnet_initialize(void)
+pfi_initialize(void)
 {
 	struct ifg_group *ifg;
 	struct ifnet *ifp;
@@ -117,6 +114,9 @@ pfi_vnet_initialize(void)
 	V_pfi_buffer_max = 64;
 	V_pfi_buffer = malloc(V_pfi_buffer_max * sizeof(*V_pfi_buffer),
 	    PFI_MTYPE, M_WAITOK);
+
+	mtx_init(&pfi_unlnkdkifs_mtx, "pf unlinked interfaces", NULL, MTX_DEF);
+
 	kif = malloc(sizeof(*kif), PFI_MTYPE, M_WAITOK);
 	PF_RULES_WLOCK();
 	V_pfi_all = pfi_kif_attach(kif, IFG_ALL);
@@ -129,20 +129,18 @@ pfi_vnet_initialize(void)
 		pfi_attach_ifnet(ifp);
 	IFNET_RUNLOCK();
 
-	if (IS_DEFAULT_VNET(curvnet)) {
-	    pfi_attach_cookie = EVENTHANDLER_REGISTER(ifnet_arrival_event,
-		pfi_attach_ifnet_event, NULL, EVENTHANDLER_PRI_ANY);
-	    pfi_detach_cookie = EVENTHANDLER_REGISTER(ifnet_departure_event,
-		pfi_detach_ifnet_event, NULL, EVENTHANDLER_PRI_ANY);
-	    pfi_attach_group_cookie = EVENTHANDLER_REGISTER(group_attach_event,
-		pfi_attach_group_event, curvnet, EVENTHANDLER_PRI_ANY);
-	    pfi_change_group_cookie = EVENTHANDLER_REGISTER(group_change_event,
-		pfi_change_group_event, curvnet, EVENTHANDLER_PRI_ANY);
-	    pfi_detach_group_cookie = EVENTHANDLER_REGISTER(group_detach_event,
-		pfi_detach_group_event, curvnet, EVENTHANDLER_PRI_ANY);
-	    pfi_ifaddr_event_cookie = EVENTHANDLER_REGISTER(ifaddr_event,
-		pfi_ifaddr_event, NULL, EVENTHANDLER_PRI_ANY);
-	}
+	pfi_attach_cookie = EVENTHANDLER_REGISTER(ifnet_arrival_event,
+	    pfi_attach_ifnet_event, NULL, EVENTHANDLER_PRI_ANY);
+	pfi_detach_cookie = EVENTHANDLER_REGISTER(ifnet_departure_event,
+	    pfi_detach_ifnet_event, NULL, EVENTHANDLER_PRI_ANY);
+	pfi_attach_group_cookie = EVENTHANDLER_REGISTER(group_attach_event,
+	    pfi_attach_group_event, curvnet, EVENTHANDLER_PRI_ANY);
+	pfi_change_group_cookie = EVENTHANDLER_REGISTER(group_change_event,
+	    pfi_change_group_event, curvnet, EVENTHANDLER_PRI_ANY);
+	pfi_detach_group_cookie = EVENTHANDLER_REGISTER(group_detach_event,
+	    pfi_detach_group_event, curvnet, EVENTHANDLER_PRI_ANY);
+	pfi_ifaddr_event_cookie = EVENTHANDLER_REGISTER(ifaddr_event,
+	    pfi_ifaddr_event, NULL, EVENTHANDLER_PRI_ANY);
 }
 
 void
@@ -168,6 +166,8 @@ pfi_cleanup(void)
 		free(p, PFI_MTYPE);
 	}
 
+	mtx_destroy(&pfi_unlnkdkifs_mtx);
+
 	free(V_pfi_buffer, PFI_MTYPE);
 }
 
diff --git a/sys/netpfil/pf/pf_ioctl.c b/sys/netpfil/pf/pf_ioctl.c
index ef479bc4976..213e49cb52f 100644
--- a/sys/netpfil/pf/pf_ioctl.c
+++ b/sys/netpfil/pf/pf_ioctl.c
@@ -87,7 +87,7 @@ __FBSDID("$FreeBSD$");
 #include <altq/altq.h>
 #endif
 
-static int		 pf_vnet_init(void);
+static int		 pfattach(void);
 static struct pf_pool	*pf_get_pool(char *, u_int32_t, u_int8_t, u_int32_t,
 			    u_int8_t, u_int8_t, u_int8_t);
 
@@ -189,6 +189,7 @@ static struct cdevsw pf_cdevsw = {
 
 static volatile VNET_DEFINE(int, pf_pfil_hooked);
 #define V_pf_pfil_hooked	VNET(pf_pfil_hooked)
+VNET_DEFINE(int,		pf_end_threads);
 
 struct rwlock			pf_rules_lock;
 struct sx			pf_ioctl_lock;
@@ -204,20 +205,17 @@ pfsync_defer_t			*pfsync_defer_ptr = NULL;
 pflog_packet_t			*pflog_packet_ptr = NULL;
 
 static int
-pf_vnet_init(void)
+pfattach(void)
 {
 	u_int32_t *my_timeout = V_pf_default_rule.timeout;
 	int error;
 
 	if (IS_DEFAULT_VNET(curvnet))
 		pf_mtag_initialize();
-	TAILQ_INIT(&V_pf_tags);
-	TAILQ_INIT(&V_pf_qids);
-
-	pf_vnet_initialize();
+	pf_initialize();
 	pfr_initialize();
-	pfi_vnet_initialize();
-	pf_vnet_normalize_init();
+	pfi_initialize();
+	pf_normalize_init();
 
 	V_pf_limits[PF_LIMIT_STATES].limit = PFSTATE_HIWAT;
 	V_pf_limits[PF_LIMIT_SRC_NODES].limit = PFSNODE_HIWAT;
@@ -278,13 +276,10 @@ pf_vnet_init(void)
 	for (int i = 0; i < SCNT_MAX; i++)
 		V_pf_status.scounters[i] = counter_u64_alloc(M_WAITOK);
 
-	if (IS_DEFAULT_VNET(curvnet)) {
-	    if ((error = kproc_create(pf_purge_thread, curvnet, NULL, 0, 0,
-		"pf purge")) != 0) {
-		    /* XXXGL: leaked all above. */
-		    return (error);
-	    }
-	}
+	if ((error = kproc_create(pf_purge_thread, curvnet, NULL, 0, 0,
+	    "pf purge")) != 0)
+		/* XXXGL: leaked all above. */
+		return (error);
 	if ((error = swi_add(NULL, "pf send", pf_intr, curvnet, SWI_NET,
 	    INTR_MPSAFE, &V_pf_swi_cookie)) != 0)
 		/* XXXGL: leaked all above. */
@@ -3720,11 +3715,27 @@ dehook_pf(void)
 static int
 pf_load(void)
 {
+	int error;
+
+	VNET_ITERATOR_DECL(vnet_iter);
+
+	VNET_LIST_RLOCK();
+	VNET_FOREACH(vnet_iter) {
+		CURVNET_SET(vnet_iter);
+		V_pf_pfil_hooked = 0;
+		V_pf_end_threads = 0;
+		TAILQ_INIT(&V_pf_tags);
+		TAILQ_INIT(&V_pf_qids);
+		CURVNET_RESTORE();
+	}
+	VNET_LIST_RUNLOCK();
 
 	rw_init(&pf_rules_lock, "pf rulesets");
 	sx_init(&pf_ioctl_lock, "pf ioctl");
 
 	pf_dev = make_dev(&pf_cdevsw, 0, 0, 0, 0600, PF_NAME);
+	if ((error = pfattach()) != 0)
+		return (error);
 
 	return (0);
 }
@@ -3748,6 +3759,11 @@ pf_unload(void)
 	}
 	PF_RULES_WLOCK();
 	shutdown_pf();
+	V_pf_end_threads = 1;
+	while (V_pf_end_threads < 2) {
+		wakeup_one(pf_purge_thread);
+		rw_sleep(pf_purge_thread, &pf_rules_lock, 0, "pftmo", 0);
+	}
 	pf_normalize_cleanup();
 	pfi_cleanup();
 	pfr_cleanup();
@@ -3797,5 +3813,3 @@ static moduledata_t pf_mod = {
 
 DECLARE_MODULE(pf, pf_mod, SI_SUB_PSEUDO, SI_ORDER_FIRST);
 MODULE_VERSION(pf, PF_MODVER);
-VNET_SYSINIT(pf_vnet_init, SI_SUB_PROTO_IFATTACHDOMAIN, SI_ORDER_ANY - 255,
-	    pf_vnet_init, NULL);
diff --git a/sys/netpfil/pf/pf_norm.c b/sys/netpfil/pf/pf_norm.c
index fa9acd66b28..fb30331a80c 100644
--- a/sys/netpfil/pf/pf_norm.c
+++ b/sys/netpfil/pf/pf_norm.c
@@ -33,7 +33,6 @@ __FBSDID("$FreeBSD$");
 #include "opt_pf.h"
 
 #include <sys/param.h>
-#include <sys/kernel.h>
 #include <sys/lock.h>
 #include <sys/mbuf.h>
 #include <sys/mutex.h>
@@ -93,7 +92,6 @@ struct pf_fragment {
 };
 
 static struct mtx pf_frag_mtx;
-MTX_SYSINIT(pf_frag_mtx, &pf_frag_mtx, "pf fragments", MTX_DEF);
 #define PF_FRAG_LOCK()		mtx_lock(&pf_frag_mtx)
 #define PF_FRAG_UNLOCK()	mtx_unlock(&pf_frag_mtx)
 #define PF_FRAG_ASSERT()	mtx_assert(&pf_frag_mtx, MA_OWNED)
@@ -148,7 +146,7 @@ static void		 pf_scrub_ip6(struct mbuf **, u_int8_t);
 } while(0)
 
 void
-pf_vnet_normalize_init(void)
+pf_normalize_init(void)
 {
 
 	V_pf_frag_z = uma_zcreate("pf frags", sizeof(struct pf_fragment),
@@ -163,6 +161,9 @@ pf_vnet_normalize_init(void)
 	V_pf_limits[PF_LIMIT_FRAGS].limit = PFFRAG_FRENT_HIWAT;
 	uma_zone_set_max(V_pf_frent_z, PFFRAG_FRENT_HIWAT);
 	uma_zone_set_warning(V_pf_frent_z, "PF frag entries limit reached");
+
+	mtx_init(&pf_frag_mtx, "pf fragments", NULL, MTX_DEF);
+
 	TAILQ_INIT(&V_pf_fragqueue);
 	TAILQ_INIT(&V_pf_cachequeue);
 }
@@ -174,6 +175,8 @@ pf_normalize_cleanup(void)
 	uma_zdestroy(V_pf_state_scrub_z);
 	uma_zdestroy(V_pf_frent_z);
 	uma_zdestroy(V_pf_frag_z);
+
+	mtx_destroy(&pf_frag_mtx);
 }
 
 static int
diff --git a/sys/netpfil/pf/pf_table.c b/sys/netpfil/pf/pf_table.c
index 7cdca4039f8..b9f13b93de0 100644
--- a/sys/netpfil/pf/pf_table.c
+++ b/sys/netpfil/pf/pf_table.c
@@ -184,13 +184,9 @@ static struct pfr_kentry
 static RB_PROTOTYPE(pfr_ktablehead, pfr_ktable, pfrkt_tree, pfr_ktable_compare);
 static RB_GENERATE(pfr_ktablehead, pfr_ktable, pfrkt_tree, pfr_ktable_compare);
 
-VNET_DEFINE(struct pfr_ktablehead, pfr_ktables);
-#define V_pfr_ktables		VNET(pfr_ktables)
-
+struct pfr_ktablehead	 pfr_ktables;
 struct pfr_table	 pfr_nulltable;
-
-VNET_DEFINE(int, pfr_ktable_cnt);
-#define V_pfr_ktable_cnt	VNET(pfr_ktable_cnt)
+int			 pfr_ktable_cnt;
 
 void
 pfr_initialize(void)
@@ -1087,7 +1083,7 @@ pfr_clr_tables(struct pfr_table *filter, int *ndel, int flags)
 		return (ENOENT);
 
 	SLIST_INIT(&workq);
-	RB_FOREACH(p, pfr_ktablehead, &V_pfr_ktables) {
+	RB_FOREACH(p, pfr_ktablehead, &pfr_ktables) {
 		if (pfr_skip_table(filter, p, flags))
 			continue;
 		if (!strcmp(p->pfrkt_anchor, PF_RESERVED_ANCHOR))
@@ -1122,7 +1118,7 @@ pfr_add_tables(struct pfr_table *tbl, int size, int *nadd, int flags)
 		    flags & PFR_FLAG_USERIOCTL))
 			senderr(EINVAL);
 		key.pfrkt_flags |= PFR_TFLAG_ACTIVE;
-		p = RB_FIND(pfr_ktablehead, &V_pfr_ktables, &key);
+		p = RB_FIND(pfr_ktablehead, &pfr_ktables, &key);
 		if (p == NULL) {
 			p = pfr_create_ktable(&key.pfrkt_t, tzero, 1);
 			if (p == NULL)
@@ -1138,7 +1134,7 @@ pfr_add_tables(struct pfr_table *tbl, int size, int *nadd, int flags)
 
 			/* find or create root table */
 			bzero(key.pfrkt_anchor, sizeof(key.pfrkt_anchor));
-			r = RB_FIND(pfr_ktablehead, &V_pfr_ktables, &key);
+			r = RB_FIND(pfr_ktablehead, &pfr_ktables, &key);
 			if (r != NULL) {
 				p->pfrkt_root = r;
 				goto _skip;
@@ -1194,7 +1190,7 @@ pfr_del_tables(struct pfr_table *tbl, int size, int *ndel, int flags)
 		if (pfr_validate_table(&key.pfrkt_t, 0,
 		    flags & PFR_FLAG_USERIOCTL))
 			return (EINVAL);
-		p = RB_FIND(pfr_ktablehead, &V_pfr_ktables, &key);
+		p = RB_FIND(pfr_ktablehead, &pfr_ktables, &key);
 		if (p != NULL && (p->pfrkt_flags & PFR_TFLAG_ACTIVE)) {
 			SLIST_FOREACH(q, &workq, pfrkt_workq)
 				if (!pfr_ktable_compare(p, q))
@@ -1233,7 +1229,7 @@ pfr_get_tables(struct pfr_table *filter, struct pfr_table *tbl, int *size,
 		*size = n;
 		return (0);
 	}
-	RB_FOREACH(p, pfr_ktablehead, &V_pfr_ktables) {
+	RB_FOREACH(p, pfr_ktablehead, &pfr_ktables) {
 		if (pfr_skip_table(filter, p, flags))
 			continue;
 		if (n-- <= 0)
@@ -1268,7 +1264,7 @@ pfr_get_tstats(struct pfr_table *filter, struct pfr_tstats *tbl, int *size,
 		return (0);
 	}
 	SLIST_INIT(&workq);
-	RB_FOREACH(p, pfr_ktablehead, &V_pfr_ktables) {
+	RB_FOREACH(p, pfr_ktablehead, &pfr_ktables) {
 		if (pfr_skip_table(filter, p, flags))
 			continue;
 		if (n-- <= 0)
@@ -1300,7 +1296,7 @@ pfr_clr_tstats(struct pfr_table *tbl, int size, int *nzero, int flags)
 		bcopy(tbl + i, &key.pfrkt_t, sizeof(key.pfrkt_t));
 		if (pfr_validate_table(&key.pfrkt_t, 0, 0))
 			return (EINVAL);
-		p = RB_FIND(pfr_ktablehead, &V_pfr_ktables, &key);
+		p = RB_FIND(pfr_ktablehead, &pfr_ktables, &key);
 		if (p != NULL) {
 			SLIST_INSERT_HEAD(&workq, p, pfrkt_workq);
 			xzero++;
@@ -1332,7 +1328,7 @@ pfr_set_tflags(struct pfr_table *tbl, int size, int setflag, int clrflag,
 		if (pfr_validate_table(&key.pfrkt_t, 0,
 		    flags & PFR_FLAG_USERIOCTL))
 			return (EINVAL);
-		p = RB_FIND(pfr_ktablehead, &V_pfr_ktables, &key);
+		p = RB_FIND(pfr_ktablehead, &pfr_ktables, &key);
 		if (p != NULL && (p->pfrkt_flags & PFR_TFLAG_ACTIVE)) {
 			p->pfrkt_nflags = (p->pfrkt_flags | setflag) &
 			    ~clrflag;
@@ -1374,7 +1370,7 @@ pfr_ina_begin(struct pfr_table *trs, u_int32_t *ticket, int *ndel, int flags)
 	if (rs == NULL)
 		return (ENOMEM);
 	SLIST_INIT(&workq);
-	RB_FOREACH(p, pfr_ktablehead, &V_pfr_ktables) {
+	RB_FOREACH(p, pfr_ktablehead, &pfr_ktables) {
 		if (!(p->pfrkt_flags & PFR_TFLAG_INACTIVE) ||
 		    pfr_skip_table(trs, p, 0))
 			continue;
@@ -1419,7 +1415,7 @@ pfr_ina_define(struct pfr_table *tbl, struct pfr_addr *addr, int size,
 		return (EBUSY);
 	tbl->pfrt_flags |= PFR_TFLAG_INACTIVE;
 	SLIST_INIT(&tableq);
-	kt = RB_FIND(pfr_ktablehead, &V_pfr_ktables, (struct pfr_ktable *)tbl);
+	kt = RB_FIND(pfr_ktablehead, &pfr_ktables, (struct pfr_ktable *)tbl);
 	if (kt == NULL) {
 		kt = pfr_create_ktable(tbl, 0, 1);
 		if (kt == NULL)
@@ -1432,7 +1428,7 @@ pfr_ina_define(struct pfr_table *tbl, struct pfr_addr *addr, int size,
 		/* find or create root table */
 		bzero(&key, sizeof(key));
 		strlcpy(key.pfrkt_name, tbl->pfrt_name, sizeof(key.pfrkt_name));
-		rt = RB_FIND(pfr_ktablehead, &V_pfr_ktables, &key);
+		rt = RB_FIND(pfr_ktablehead, &pfr_ktables, &key);
 		if (rt != NULL) {
 			kt->pfrkt_root = rt;
 			goto _skip;
@@ -1509,7 +1505,7 @@ pfr_ina_rollback(struct pfr_table *trs, u_int32_t ticket, int *ndel, int flags)
 	if (rs == NULL || !rs->topen || ticket != rs->tticket)
 		return (0);
 	SLIST_INIT(&workq);
-	RB_FOREACH(p, pfr_ktablehead, &V_pfr_ktables) {
+	RB_FOREACH(p, pfr_ktablehead, &pfr_ktables) {
 		if (!(p->pfrkt_flags & PFR_TFLAG_INACTIVE) ||
 		    pfr_skip_table(trs, p, 0))
 			continue;
@@ -1545,7 +1541,7 @@ pfr_ina_commit(struct pfr_table *trs, u_int32_t ticket, int *nadd,
 		return (EBUSY);
 
 	SLIST_INIT(&workq);
-	RB_FOREACH(p, pfr_ktablehead, &V_pfr_ktables) {
+	RB_FOREACH(p, pfr_ktablehead, &pfr_ktables) {
 		if (!(p->pfrkt_flags & PFR_TFLAG_INACTIVE) ||
 		    pfr_skip_table(trs, p, 0))
 			continue;
@@ -1691,7 +1687,7 @@ pfr_table_count(struct pfr_table *filter, int flags)
 	PF_RULES_ASSERT();
 
 	if (flags & PFR_FLAG_ALLRSETS)
-		return (V_pfr_ktable_cnt);
+		return (pfr_ktable_cnt);
 	if (filter->pfrt_anchor[0]) {
 		rs = pf_find_ruleset(filter->pfrt_anchor);
 		return ((rs != NULL) ? rs->tables : -1);
@@ -1724,8 +1720,8 @@ pfr_insert_ktable(struct pfr_ktable *kt)
 
 	PF_RULES_WASSERT();
 
-	RB_INSERT(pfr_ktablehead, &V_pfr_ktables, kt);
-	V_pfr_ktable_cnt++;
+	RB_INSERT(pfr_ktablehead, &pfr_ktables, kt);
+	pfr_ktable_cnt++;
 	if (kt->pfrkt_root != NULL)
 		if (!kt->pfrkt_root->pfrkt_refcnt[PFR_REFCNT_ANCHOR]++)
 			pfr_setflags_ktable(kt->pfrkt_root,
@@ -1756,14 +1752,14 @@ pfr_setflags_ktable(struct pfr_ktable *kt, int newf)
 	if (!(newf & PFR_TFLAG_ACTIVE))
 		newf &= ~PFR_TFLAG_USRMASK;
 	if (!(newf & PFR_TFLAG_SETMASK)) {
-		RB_REMOVE(pfr_ktablehead, &V_pfr_ktables, kt);
+		RB_REMOVE(pfr_ktablehead, &pfr_ktables, kt);
 		if (kt->pfrkt_root != NULL)
 			if (!--kt->pfrkt_root->pfrkt_refcnt[PFR_REFCNT_ANCHOR])
 				pfr_setflags_ktable(kt->pfrkt_root,
 				    kt->pfrkt_root->pfrkt_flags &
 					~PFR_TFLAG_REFDANCHOR);
 		pfr_destroy_ktable(kt, 1);
-		V_pfr_ktable_cnt--;
+		pfr_ktable_cnt--;
 		return;
 	}
 	if (!(newf & PFR_TFLAG_ACTIVE) && kt->pfrkt_cnt) {
@@ -1884,7 +1880,7 @@ static struct pfr_ktable *
 pfr_lookup_table(struct pfr_table *tbl)
 {
 	/* struct pfr_ktable start like a struct pfr_table */
-	return (RB_FIND(pfr_ktablehead, &V_pfr_ktables,
+	return (RB_FIND(pfr_ktablehead, &pfr_ktables,
 	    (struct pfr_ktable *)tbl));
 }
 

From fbae9a50c4c88eecf51715c3672e3362925db99d Mon Sep 17 00:00:00 2001
From: Oleksandr Tymoshenko <gonzo@FreeBSD.org>
Date: Thu, 22 Jan 2015 03:32:04 +0000
Subject: [PATCH 182/279] Write ACK for all kinds of LCDC interrupts

---
 sys/arm/ti/am335x/am335x_lcd.c | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/sys/arm/ti/am335x/am335x_lcd.c b/sys/arm/ti/am335x/am335x_lcd.c
index a4c56b91492..ada08011b64 100644
--- a/sys/arm/ti/am335x/am335x_lcd.c
+++ b/sys/arm/ti/am335x/am335x_lcd.c
@@ -365,7 +365,7 @@ am335x_lcd_intr(void *arg)
 		reg = LCD_READ4(sc, LCD_RASTER_CTRL);
 		reg |= RASTER_CTRL_LCDEN;
 		LCD_WRITE4(sc, LCD_RASTER_CTRL, reg); 
-		return;
+		goto done;
 	}
 
 	if (reg & IRQ_PL) {
@@ -376,7 +376,7 @@ am335x_lcd_intr(void *arg)
 		reg = LCD_READ4(sc, LCD_RASTER_CTRL);
 		reg |= RASTER_CTRL_LCDEN;
 		LCD_WRITE4(sc, LCD_RASTER_CTRL, reg); 
-		return;
+		goto done;
 	}
 
 	if (reg & IRQ_EOF0) {
@@ -399,6 +399,7 @@ am335x_lcd_intr(void *arg)
 		/* TODO: Handle ACB */
 	}
 
+done:
 	LCD_WRITE4(sc, LCD_END_OF_INT_IND, 0);
 }
 

From 5408fdf48d24b4e65bd874d4636be65e53a001ed Mon Sep 17 00:00:00 2001
From: Oleksandr Tymoshenko <gonzo@FreeBSD.org>
Date: Thu, 22 Jan 2015 03:33:51 +0000
Subject: [PATCH 183/279] Add last_fault_code used in pmap-v6.c if kernel is
 compiled with "option DEBUG"

---
 sys/arm/arm/trap-v6.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/sys/arm/arm/trap-v6.c b/sys/arm/arm/trap-v6.c
index 463b1fa6dd3..b9cad50bb46 100644
--- a/sys/arm/arm/trap-v6.c
+++ b/sys/arm/arm/trap-v6.c
@@ -67,6 +67,10 @@ __FBSDID("$FreeBSD$");
 
 extern char fusubailout[];
 
+#ifdef DEBUG
+int last_fault_code;	/* For the benefit of pmap_fault_fixup() */
+#endif
+
 struct ksig {
 	int sig;
 	u_long code;
@@ -457,6 +461,10 @@ abort_handler(struct trapframe *tf, int prefetch)
 	if (prefetch)
 		ftype |= VM_PROT_EXECUTE;
 
+#ifdef DEBUG
+	last_fault_code = fsr;
+#endif
+
 #ifndef ARM_NEW_PMAP
 	if (pmap_fault_fixup(vmspace_pmap(td->td_proc->p_vmspace), va, ftype,
 	    usermode)) {

From 6a429fa5d7d7733866a49f776c72b5730326837d Mon Sep 17 00:00:00 2001
From: Ryan Stone <rstone@FreeBSD.org>
Date: Thu, 22 Jan 2015 03:56:23 +0000
Subject: [PATCH 184/279] style(9) cleanup

---
 sys/dev/hwpmc/hwpmc_core.c | 37 ++++++++++++++++++++++++-------------
 sys/dev/hwpmc/pmc_events.h |  4 ++--
 sys/sys/pmc.h              |  2 +-
 3 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/sys/dev/hwpmc/hwpmc_core.c b/sys/dev/hwpmc/hwpmc_core.c
index 7d27817de77..6c241422d9c 100644
--- a/sys/dev/hwpmc/hwpmc_core.c
+++ b/sys/dev/hwpmc/hwpmc_core.c
@@ -680,7 +680,8 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(08H_0EH, 0x08, 0x0E, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
     IAPDESCR(08H_10H, 0x08, 0x10, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
 	IAP_F_SBX | IAP_F_HW | IAP_F_HWX),
-    IAPDESCR(08H_20H, 0x08, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_HW | IAP_F_HWX),
+    IAPDESCR(08H_20H, 0x08, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_HW |
+        IAP_F_HWX),
     IAPDESCR(08H_40H, 0x08, 0x40, IAP_F_FM | IAP_F_I7O | IAP_F_HW | IAP_F_HWX),
     IAPDESCR(08H_60H, 0x08, 0x60, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
     IAPDESCR(08H_80H, 0x08, 0x80, IAP_F_FM | IAP_F_I7 | IAP_F_HW | IAP_F_HWX),
@@ -710,9 +711,12 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(0EH_01H, 0x0E, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_SB |
 	IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
     IAPDESCR(0EH_02H, 0x0E, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
-    IAPDESCR(0EH_10H, 0x0E, 0x10, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
-    IAPDESCR(0EH_20H, 0x0E, 0x20, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
-    IAPDESCR(0EH_40H, 0x0E, 0x40, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
+    IAPDESCR(0EH_10H, 0x0E, 0x10, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
+        IAP_F_HWX),
+    IAPDESCR(0EH_20H, 0x0E, 0x20, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
+        IAP_F_HWX),
+    IAPDESCR(0EH_40H, 0x0E, 0x40, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
+        IAP_F_HWX),
 
     IAPDESCR(0FH_01H, 0x0F, 0x01, IAP_F_FM | IAP_F_I7),
     IAPDESCR(0FH_02H, 0x0F, 0x02, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
@@ -826,7 +830,8 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(24H_AAH, 0x24, 0xAA, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
     IAPDESCR(24H_F8H, 0x24, 0xF8, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
     IAPDESCR(24H_3FH, 0x24, 0x3F, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
-    IAPDESCR(24H_FFH, 0x24, 0xFF, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_HW | IAP_F_HWX),
+    IAPDESCR(24H_FFH, 0x24, 0xFF, IAP_F_FM | IAP_F_I7 | IAP_F_WM | IAP_F_HW |
+        IAP_F_HWX),
 
     IAPDESCR(25H, 0x25, IAP_M_CORE, IAP_F_ALLCPUSCORE2),
 
@@ -967,7 +972,8 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(49H_20H, 0x49, 0x20, IAP_F_FM | IAP_F_I7 | IAP_F_HW | IAP_F_HWX),
     IAPDESCR(49H_40H, 0x49, 0x40, IAP_F_FM | IAP_F_I7O | IAP_F_HW | IAP_F_HWX),
     IAPDESCR(49H_60H, 0x49, 0x60, IAP_F_FM | IAP_F_HW | IAP_F_HWX),
-    IAPDESCR(49H_80H, 0x49, 0x80, IAP_F_FM | IAP_F_WM | IAP_F_I7 | IAP_F_HW | IAP_F_HWX),
+    IAPDESCR(49H_80H, 0x49, 0x80, IAP_F_FM | IAP_F_WM | IAP_F_I7 | IAP_F_HW |
+        IAP_F_HWX),
 
     IAPDESCR(4BH_00H, 0x4B, 0x00, IAP_F_FM | IAP_F_ALLCPUSCORE2),
     IAPDESCR(4BH_01H, 0x4B, 0x01, IAP_F_FM | IAP_F_ALLCPUSCORE2 | IAP_F_I7O),
@@ -1008,10 +1014,14 @@ static struct iap_event_descr iap_events[] = {
 
     IAPDESCR(53H_01H, 0x53, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
 
-    IAPDESCR(58H_01H, 0x58, 0x01, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
-    IAPDESCR(58H_02H, 0x58, 0x02, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
-    IAPDESCR(58H_04H, 0x58, 0x04, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
-    IAPDESCR(58H_08H, 0x58, 0x08, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
+    IAPDESCR(58H_01H, 0x58, 0x01, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
+        IAP_F_HWX),
+    IAPDESCR(58H_02H, 0x58, 0x02, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
+        IAP_F_HWX),
+    IAPDESCR(58H_04H, 0x58, 0x04, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
+        IAP_F_HWX),
+    IAPDESCR(58H_08H, 0x58, 0x08, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
+        IAP_F_HWX),
 
     IAPDESCR(59H_20H, 0x59, 0x20, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
     IAPDESCR(59H_40H, 0x59, 0x40, IAP_F_FM | IAP_F_SB | IAP_F_SBX),
@@ -1114,8 +1124,8 @@ static struct iap_event_descr iap_events[] = {
 
     IAPDESCR(79H_30H, 0x79, 0x30, IAP_F_FM | IAP_F_SB | IAP_F_IB |
 	IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
-
-    IAPDESCR(79H_3CH, 0x79, 0x3C, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW | IAP_F_HWX),
+    IAPDESCR(79H_18H, 0x79, 0x18, IAP_F_FM | IAP_F_IB | IAP_F_IBX | IAP_F_HW |
+        IAP_F_HWX),
 
     IAPDESCR(7AH, 0x7A, IAP_M_AGENT, IAP_F_CA | IAP_F_CC2),
 
@@ -1367,7 +1377,8 @@ static struct iap_event_descr iap_events[] = {
     IAPDESCR(B6H_04H, 0xB6, 0x04, IAP_F_CAS),
 
     IAPDESCR(B7H_01H, 0xB7, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM |
-	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_HWX | IAP_F_CAS),
+	IAP_F_SB | IAP_F_IB | IAP_F_SBX | IAP_F_IBX | IAP_F_HW | IAP_F_CAS |
+	IAP_F_HWX),
     IAPDESCR(B7H_02H, 0xB7, 0x02, IAP_F_CAS),
 
     IAPDESCR(B8H_01H, 0xB8, 0x01, IAP_F_FM | IAP_F_I7 | IAP_F_WM),
diff --git a/sys/dev/hwpmc/pmc_events.h b/sys/dev/hwpmc/pmc_events.h
index 567d6f44823..4953f19dbc1 100644
--- a/sys/dev/hwpmc/pmc_events.h
+++ b/sys/dev/hwpmc/pmc_events.h
@@ -2734,7 +2734,7 @@ __PMC_EV_ALIAS("PAGE_WALKER_LOADS.ITLB_MEMORY", IAP_EVENT_BCH_28H)	\
 __PMC_EV_ALIAS("TLB_FLUSH.DTLB_THREAD", IAP_EVENT_BDH_01H)		\
 __PMC_EV_ALIAS("TLB_FLUSH.STLB_ANY", IAP_EVENT_BDH_20H)			\
 __PMC_EV_ALIAS("INST_RETIRED.ANY_P", IAP_EVENT_C0H_00H)			\
-__PMC_EV_ALIAS("INST_RETIRED.PREC_DIST", IAP_EVENT_C0H_01H)			\
+__PMC_EV_ALIAS("INST_RETIRED.PREC_DIST", IAP_EVENT_C0H_01H)		\
 __PMC_EV_ALIAS("OTHER_ASSISTS.AVX_TO_SSE", IAP_EVENT_C1H_08H)		\
 __PMC_EV_ALIAS("OTHER_ASSISTS.SSE_TO_AVX", IAP_EVENT_C1H_10H)		\
 __PMC_EV_ALIAS("OTHER_ASSISTS.ANY_WB_ASSIST", IAP_EVENT_C1H_40H)	\
@@ -2963,7 +2963,7 @@ __PMC_EV_ALIAS("PAGE_WALKER_LOADS.ITLB_MEMORY", IAP_EVENT_BCH_28H)	\
 __PMC_EV_ALIAS("TLB_FLUSH.DTLB_THREAD", IAP_EVENT_BDH_01H)		\
 __PMC_EV_ALIAS("TLB_FLUSH.STLB_ANY", IAP_EVENT_BDH_20H)			\
 __PMC_EV_ALIAS("INST_RETIRED.ANY_P", IAP_EVENT_C0H_00H)			\
-__PMC_EV_ALIAS("INST_RETIRED.PREC_DIST", IAP_EVENT_C0H_01H)			\
+__PMC_EV_ALIAS("INST_RETIRED.PREC_DIST", IAP_EVENT_C0H_01H)		\
 __PMC_EV_ALIAS("OTHER_ASSISTS.AVX_TO_SSE", IAP_EVENT_C1H_08H)		\
 __PMC_EV_ALIAS("OTHER_ASSISTS.SSE_TO_AVX", IAP_EVENT_C1H_10H)		\
 __PMC_EV_ALIAS("OTHER_ASSISTS.ANY_WB_ASSIST", IAP_EVENT_C1H_40H)	\
diff --git a/sys/sys/pmc.h b/sys/sys/pmc.h
index 1812840308a..76f000a5379 100644
--- a/sys/sys/pmc.h
+++ b/sys/sys/pmc.h
@@ -94,7 +94,7 @@
 	__PMC_CPU(INTEL_ATOM_SILVERMONT, 0x92,	"Intel Atom Silvermont")    \
 	__PMC_CPU(INTEL_NEHALEM_EX, 0x93,   "Intel Nehalem Xeon 7500")	\
 	__PMC_CPU(INTEL_WESTMERE_EX, 0x94,   "Intel Westmere Xeon E7")	\
-        __PMC_CPU(INTEL_HASWELL_XEON, 0x95,   "Intel Haswell Xeon E5 v3") \
+	__PMC_CPU(INTEL_HASWELL_XEON, 0x95,   "Intel Haswell Xeon E5 v3") \
 	__PMC_CPU(INTEL_XSCALE,	0x100,	"Intel XScale")		\
 	__PMC_CPU(MIPS_24K,     0x200,  "MIPS 24K")		\
 	__PMC_CPU(MIPS_OCTEON,  0x201,  "Cavium Octeon")	\

From 2decd8fc810d0599227ed484e1510e840031fda7 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Thu, 22 Jan 2015 09:25:31 +0000
Subject: [PATCH 185/279] Make this compile with WARNS=6 and clang/gcc

MFC after: 3 days
Sponsored by: EMC / Isilon Storage Division
---
 tools/regression/file/flock/flock.c | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/tools/regression/file/flock/flock.c b/tools/regression/file/flock/flock.c
index fd426b0e625..49e47b83089 100644
--- a/tools/regression/file/flock/flock.c
+++ b/tools/regression/file/flock/flock.c
@@ -27,6 +27,7 @@
  * $FreeBSD$
  */
 
+#include <sys/param.h>
 #include <sys/file.h>
 #include <sys/time.h>
 #ifdef __FreeBSD__
@@ -52,6 +53,10 @@
 #endif
 #include <sys/cdefs.h>
 #else
+#ifndef nitems
+#define	nitems(x)	(sizeof((x)) / sizeof((x)[0]))
+#endif
+
 #ifndef __unused
 #ifdef __GNUC__
 #define	__unused	__attribute__((__unused__))
@@ -61,7 +66,7 @@
 #endif
 #endif
 
-int verbose = 0;
+static int verbose = 0;
 
 static int
 make_file(const char *pathname, off_t sz)
@@ -1519,7 +1524,7 @@ struct test {
 	int intr;		/* non-zero if the test interrupts a lock */
 };
 
-struct test tests[] = {
+static struct test tests[] = {
 	{	test1,		1,	0	},
 	{	test2,		2,	0	},
 	{	test3,		3,	1	},
@@ -1537,7 +1542,6 @@ struct test tests[] = {
 	{	test15,		15,	1	},
 	{	test16,		16,	1	},
 };
-int test_count = sizeof(tests) / sizeof(tests[0]);
 
 int
 main(int argc, const char *argv[])
@@ -1545,7 +1549,7 @@ main(int argc, const char *argv[])
 	int testnum;
 	int fd;
 	int nointr;
-	int i;
+	unsigned i;
 	struct sigaction sa;
 	int test_argc;
 	const char **test_argv;
@@ -1583,7 +1587,7 @@ main(int argc, const char *argv[])
 	}
 #endif
 
-	for (i = 0; i < test_count; i++) {
+	for (i = 0; i < nitems(tests); i++) {
 		if (tests[i].intr && nointr)
 			continue;
 		if (!testnum || tests[i].num == testnum)

From a115fb62edf8946c09ae3b4cebe097efb7ea3e38 Mon Sep 17 00:00:00 2001
From: Hans Petter Selasky <hselasky@FreeBSD.org>
Date: Thu, 22 Jan 2015 11:12:42 +0000
Subject: [PATCH 186/279] Revert for r277213:

FreeBSD developers need more time to review patches in the surrounding
areas like the TCP stack which are using MPSAFE callouts to restore
distribution of callouts on multiple CPUs.

Bump the __FreeBSD_version instead of reverting it.

Suggested by:		kmacy, adrian, glebius and kib
Differential Revision:	https://reviews.freebsd.org/D1438
---
 share/man/man9/Makefile             |    1 -
 share/man/man9/timeout.9            |  874 +++++++++++-----------
 sys/kern/init_main.c                |    3 +-
 sys/kern/kern_condvar.c             |   12 +-
 sys/kern/kern_lock.c                |    6 +-
 sys/kern/kern_switch.c              |    2 +
 sys/kern/kern_synch.c               |   13 +-
 sys/kern/kern_thread.c              |    8 +-
 sys/kern/kern_timeout.c             | 1049 ++++++++++++++-------------
 sys/kern/subr_sleepqueue.c          |  149 ++--
 sys/ofed/include/linux/completion.h |    2 -
 sys/sys/_callout.h                  |   19 +-
 sys/sys/callout.h                   |   14 +-
 sys/sys/param.h                     |    2 +-
 sys/sys/proc.h                      |    5 +-
 15 files changed, 1111 insertions(+), 1048 deletions(-)

diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile
index e59c14fa9d1..cf64df7c811 100644
--- a/share/man/man9/Makefile
+++ b/share/man/man9/Makefile
@@ -1570,7 +1570,6 @@ MLINKS+=timeout.9 callout.9 \
 	timeout.9 callout_active.9 \
 	timeout.9 callout_deactivate.9 \
 	timeout.9 callout_drain.9 \
-	timeout.9 callout_drain_async.9 \
 	timeout.9 callout_handle_init.9 \
 	timeout.9 callout_init.9 \
 	timeout.9 callout_init_mtx.9 \
diff --git a/share/man/man9/timeout.9 b/share/man/man9/timeout.9
index 4f52a19cb61..7202815a782 100644
--- a/share/man/man9/timeout.9
+++ b/share/man/man9/timeout.9
@@ -29,14 +29,13 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 14, 2015
+.Dd October 8, 2014
 .Dt TIMEOUT 9
 .Os
 .Sh NAME
 .Nm callout_active ,
 .Nm callout_deactivate ,
 .Nm callout_drain ,
-.Nm callout_drain_async ,
 .Nm callout_handle_init ,
 .Nm callout_init ,
 .Nm callout_init_mtx ,
@@ -64,232 +63,279 @@
 .In sys/systm.h
 .Bd -literal
 typedef void timeout_t (void *);
-typedef void callout_func_t (void *);
 .Ed
+.Ft int
+.Fn callout_active "struct callout *c"
+.Ft void
+.Fn callout_deactivate "struct callout *c"
+.Ft int
+.Fn callout_drain "struct callout *c"
+.Ft void
+.Fn callout_handle_init "struct callout_handle *handle"
+.Bd -literal
+struct callout_handle handle = CALLOUT_HANDLE_INITIALIZER(&handle);
+.Ed
+.Ft void
+.Fn callout_init "struct callout *c" "int mpsafe"
+.Ft void
+.Fn callout_init_mtx "struct callout *c" "struct mtx *mtx" "int flags"
+.Ft void
+.Fn callout_init_rm "struct callout *c" "struct rmlock *rm" "int flags"
+.Ft void
+.Fn callout_init_rw "struct callout *c" "struct rwlock *rw" "int flags"
+.Ft int
+.Fn callout_pending "struct callout *c"
+.Ft int
+.Fn callout_reset "struct callout *c" "int ticks" "timeout_t *func" "void *arg"
+.Ft int
+.Fn callout_reset_curcpu "struct callout *c" "int ticks" "timeout_t *func" \
+"void *arg"
+.Ft int
+.Fn callout_reset_on "struct callout *c" "int ticks" "timeout_t *func" \
+"void *arg" "int cpu"
+.Ft int
+.Fn callout_reset_sbt "struct callout *c" "sbintime_t sbt" \
+"sbintime_t pr" "timeout_t *func" "void *arg" "int flags"
+.Ft int
+.Fn callout_reset_sbt_curcpu "struct callout *c" "sbintime_t sbt" \
+"sbintime_t pr" "timeout_t *func" "void *arg" "int flags"
+.Ft int
+.Fn callout_reset_sbt_on "struct callout *c" "sbintime_t sbt" \
+"sbintime_t pr" "timeout_t *func" "void *arg" "int cpu" "int flags"
+.Ft int
+.Fn callout_schedule "struct callout *c" "int ticks"
+.Ft int
+.Fn callout_schedule_curcpu "struct callout *c" "int ticks"
+.Ft int
+.Fn callout_schedule_on "struct callout *c" "int ticks" "int cpu"
+.Ft int
+.Fn callout_schedule_sbt "struct callout *c" "sbintime_t sbt" \
+"sbintime_t pr" "int flags"
+.Ft int
+.Fn callout_schedule_sbt_curcpu "struct callout *c" "sbintime_t sbt" \
+"sbintime_t pr" "int flags"
+.Ft int
+.Fn callout_schedule_sbt_on "struct callout *c" "sbintime_t sbt" \
+"sbintime_t pr" "int cpu" "int flags"
+.Ft int
+.Fn callout_stop "struct callout *c"
+.Ft struct callout_handle
+.Fn timeout "timeout_t *func" "void *arg" "int ticks"
+.Ft void
+.Fn untimeout "timeout_t *func" "void *arg" "struct callout_handle handle"
 .Sh DESCRIPTION
 The
 .Nm callout
 API is used to schedule a call to an arbitrary function at a specific
-time in the future in a single-shot fashion.
-Consumers of this API are required to allocate a
+time in the future.
+Consumers of this API are required to allocate a callout structure
 .Pq struct callout
-structure for each pending function invocation.
-The
-.Pq struct callout
-structure stores the full state about any pending function call and
-should be drained by a call to
-.Fn callout_drain
+for each pending function invocation.
+This structure stores state about the pending function invocation including
+the function to be called and the time at which the function should be invoked.
+Pending function calls can be cancelled or rescheduled to a different time.
+In addition,
+a callout structure may be reused to schedule a new function call after a
+scheduled call is completed.
+.Pp
+Callouts only provide a single-shot mode.
+If a consumer requires a periodic timer,
+it must explicitly reschedule each function call.
+This is normally done by rescheduling the subsequent call within the called
+function.
+.Pp
+Callout functions must not sleep.
+They may not acquire sleepable locks,
+wait on condition variables,
+perform blocking allocation requests,
+or invoke any other action that might sleep.
+.Pp
+Each callout structure must be initialized by
+.Fn callout_init ,
+.Fn callout_init_mtx ,
+.Fn callout_init_rm ,
 or
-.Fn callout_drain_async
-before freeing.
-.Sh INITIALISATION
-.Ft void
-.Fn callout_handle_init "struct callout_handle *handle"
-This function is deprecated and is used to prepare a
-.Pq struct callout_handle
-structure before it can be used the first time.
-If this function is called on a pending timeout, the pending timeout
-cannot be cancelled and the
-.Fn untimeout
-function will return as if there was no timeout pending.
-.Pp
-.Fn CALLOUT_HANDLE_INITIALIZER "&handle"
-This macro is deprecated and can be used instead of
-.Fn callout_handle_init
-to assign the default state to the
-.Pq struct callout_handle
-structure when declaring static timeouts.
-.Pp
-.Ft void
-.Fn callout_init "struct callout *c" "int mpsafe"
-This function prepares a
-.Pq struct callout
-structure before it can be used.
-This function should not be used when the callout is pending a timeout.
+.Fn callout_init_rw
+before it is passed to any of the other callout functions.
+The
+.Fn callout_init
+function initializes a callout structure in
+.Fa c
+that is not associated with a specific lock.
 If the
 .Fa mpsafe
-argument is non-zero, the callback function will be running unlocked.
-Else the Giant mutex will be locked before calling the callback function.
+argument is zero,
+the callout structure is not considered to be
+.Dq multi-processor safe ;
+and the Giant lock will be acquired before calling the callout function
+and released when the callout function returns.
 .Pp
-.Ft void
-.Fn callout_init_mtx "struct callout *c" "struct mtx *mtx" "int flags"
-This function prepares a
-.Pq struct callout
-structure before it can be used.
-This function should not be used when the callout is pending a timeout.
 The
-.Fa mtx
-argument should be non-zero and should specify a pointer to a valid
-spinlock type of mutex or a valid regular non-sleepable mutex which
-the callback subsystem should lock before calling the callback
-function.
-Valid
+.Fn callout_init_mtx ,
+.Fn callout_init_rm ,
+and
+.Fn callout_init_rw
+functions initialize a callout structure in
+.Fa c
+that is associated with a specific lock.
+The lock is specified by the
+.Fa mtx ,
+.Fa rm ,
+or
+.Fa rw
+parameter.
+The associated lock must be held while stopping or rescheduling the
+callout.
+The callout subsystem acquires the associated lock before calling the
+callout function and releases it after the function returns.
+If the callout was cancelled while the callout subsystem waited for the
+associated lock,
+the callout function is not called,
+and the associated lock is released.
+This ensures that stopping or rescheduling the callout will abort any
+previously scheduled invocation.
+.Pp
+Only regular mutexes may be used with
+.Fn callout_init_mtx ;
+spin mutexes are not supported.
+A sleepable read-mostly lock
+.Po
+one initialized with the
+.Dv RM_SLEEPABLE
+flag
+.Pc
+may not be used with
+.Fn callout_init_rm .
+Similarly, other sleepable lock types such as
+.Xr sx 9
+and
+.Xr lockmgr 9
+cannot be used with callouts because sleeping is not permitted in
+the callout subsystem.
+.Pp
+These
 .Fa flags
-are:
+may be specified for
+.Fn callout_init_mtx ,
+.Fn callout_init_rm ,
+or
+.Fn callout_init_rw :
 .Bl -tag -width ".Dv CALLOUT_RETURNUNLOCKED"
 .It Dv CALLOUT_RETURNUNLOCKED
-It is assumed that the callout function has released the specified
-mutex before returning.
-Else the callout subsystem will release the specified mutex after the
-callout function has returned.
+The callout function will release the associated lock itself,
+so the callout subsystem should not attempt to unlock it
+after the callout function returns.
+.It Dv CALLOUT_SHAREDLOCK
+The lock is only acquired in read mode when running the callout handler.
+This flag is ignored by
+.Fn callout_init_mtx .
 .El
 .Pp
-.Ft void
-.Fn callout_init_rm "struct callout *c" "struct rmlock *rm" "int flags"
-This function is the same like the
-.Fn callout_init_mtx
-function except it accepts a read-mostly type of lock.
-The read-mostly lock must not be initialised with the
-.Dv RM_SLEEPABLE
-flag.
+The function
+.Fn callout_stop
+cancels a callout
+.Fa c
+if it is currently pending.
+If the callout is pending, then
+.Fn callout_stop
+returns a non-zero value.
+If the callout is not set,
+has already been serviced,
+or is currently being serviced,
+then zero will be returned.
+If the callout has an associated lock,
+then that lock must be held when this function is called.
 .Pp
-.Ft void
-.Fn callout_init_rw "struct callout *c" "struct rwlock *rw" "int flags"
-This function is the same like the
-.Fn callout_init_mtx
-function except it accepts a reader-writer type of lock.
-.Sh SCHEDULING CALLOUTS
-.Ft struct callout_handle
-.Fn timeout "timeout_t *func" "void *arg" "int ticks"
-This function is deprecated and schedules a call to the function given by the argument
-.Fa func
-to take place after
+The function
+.Fn callout_drain
+is identical to
+.Fn callout_stop
+except that it will wait for the callout
+.Fa c
+to complete if it is already in progress.
+This function MUST NOT be called while holding any
+locks on which the callout might block, or deadlock will result.
+Note that if the callout subsystem has already begun processing this
+callout, then the callout function may be invoked before
+.Fn callout_drain
+returns.
+However, the callout subsystem does guarantee that the callout will be
+fully stopped before
+.Fn callout_drain
+returns.
+.Pp
+The
+.Fn callout_reset
+and
+.Fn callout_schedule
+function families schedule a future function invocation for callout
+.Fa c .
+If
+.Fa c
+already has a pending callout,
+it is cancelled before the new invocation is scheduled.
+These functions return a non-zero value if a pending callout was cancelled
+and zero if there was no pending callout.
+If the callout has an associated lock,
+then that lock must be held when any of these functions are called.
+.Pp
+The time at which the callout function will be invoked is determined by
+either the
+.Fa ticks
+argument or the
+.Fa sbt ,
+.Fa pr ,
+and
+.Fa flags
+arguments.
+When
+.Fa ticks
+is used,
+the callout is scheduled to execute after
 .Fa ticks Ns No /hz
 seconds.
 Non-positive values of
 .Fa ticks
 are silently converted to the value
 .Sq 1 .
+.Pp
 The
-.Fa func
-argument should be a valid pointer to a function that takes a single
-.Fa void *
-argument.
-Upon invocation, the
-.Fa func
-function will receive
-.Fa arg
-as its only argument.
-The Giant lock is locked when the
-.Fa arg
-function is invoked and should not be unlocked by this function.
-The returned value from
-.Fn timeout
-is a
-.Ft struct callout_handle
-structure which can be used in conjunction with the
-.Fn untimeout
-function to request that a scheduled timeout be cancelled.
-As handles are recycled by the system, it is possible, although unlikely,
-that a handle from one invocation of
-.Fn timeout
-may match the handle of another invocation of
-.Fn timeout
-if both calls used the same function pointer and argument, and the first
-timeout is expired or canceled before the second call.
-Please ensure that the function and argument pointers are unique when using this function.
-.Pp
-.Ft int
-.Fn callout_reset "struct callout *c" "int ticks" "callout_func_t *func" "void *arg"
-This function is used to schedule or re-schedule a callout.
-This function at first stops the callout given by the
-.Fa c
-argument, if any.
-Then it will start the callout given by the
-.Fa c
-argument.
-The relative time until the timeout callback happens is given by the
-.Fa ticks
-argument.
-The number of ticks in a second is defined by
-.Dv hz
-and can vary from system to system.
-This function returns a non-zero value if the given callout was pending and
-the callback function was prevented from being called.
-Else a value of zero is returned.
-If a lock is associated with the callout given by the
-.Fa c
-argument and it is exclusivly locked when this function is called this
-function will always ensure that previous callback function, if any,
-is never reached.
-In other words the callout will be atomically restarted.
-Else there is no such guarantee.
-The callback function is given by the
-.Fa func
-argument and its function argument is given by the
-.Fa arg
-argument.
-.Pp
-.Ft int
-.Fn callout_reset_curcpu "struct callout *c" "int ticks" "callout_func_t *func" \
-"void *arg"
-This function works the same like the
-.Fn callout_reset
-function except the callback function given by the
-.Fa func
-argument will be executed on the same CPU which called this function.
-A change in the CPU selection can happen if the callout has a lock
-associated with it and is locked when this function is called.
-A change in the CPU selection cannot happen if this function is
-re-scheduled inside a callout function.
-Else the callback function given by the
-.Fa func
-argument will be executed on the same CPU like previously done.
-.Pp
-.Ft int
-.Fn callout_reset_on "struct callout *c" "int ticks" "callout_func_t *func" \
-"void *arg" "int cpu"
-This function works the same like the
-.Fn callout_reset
-function except the callback function given by the
-.Fa func
-argument will be executed on the CPU given by the
-.Fa cpu
-argument.
-A change in the CPU selection can happen if the callout has a lock
-associated with it and is locked when this function is called.
-A change in the CPU selection cannot happen if this function is
-re-scheduled inside a callout function.
-Else the callback function given by the
-.Fa func
-argument will be executed on the same CPU like previously done.
-.Pp
-.Ft int
-.Fn callout_reset_sbt "struct callout *c" "sbintime_t sbt" \
-"sbintime_t pr" "callout_func_t *func" "void *arg" "int flags"
-This function works the same like the
-.Fn callout_reset
-function except the relative or absolute time after which the timeout
-callback should happen is given by the
+.Fa sbt ,
+.Fa pr ,
+and
+.Fa flags
+arguments provide more control over the scheduled time including
+support for higher resolution times,
+specifying the precision of the scheduled time,
+and setting an absolute deadline instead of a relative timeout.
+The callout is scheduled to execute in a time window which begins at
+the time specified in
 .Fa sbt
-argument and extends for the amount of time specified in the
-.Fa pr
-argument.
-This function is used when you need high precision timeouts.
-If the
+and extends for the amount of time specified in
+.Fa pr .
+If
 .Fa sbt
-argument specifies a time in the past,
+specifies a time in the past,
 the window is adjusted to start at the current time.
 A non-zero value for
 .Fa pr
 allows the callout subsystem to coalesce callouts scheduled close to each
 other into fewer timer interrupts,
 reducing processing overhead and power consumption.
-The
+These
 .Fa flags
-argument may be non-zero to adjust the interpretation of the
+may be specified to adjust the interpretation of
 .Fa sbt
-and the
-.Fa pr
-arguments:
+and
+.Fa pr :
 .Bl -tag -width ".Dv C_DIRECT_EXEC"
 .It Dv C_ABSOLUTE
 Handle the
 .Fa sbt
 argument as an absolute time since boot.
-By default, the
+By default,
 .Fa sbt
-argument is treated like a relative amount of time,
+is treated as a relative amount of time,
 similar to
 .Fa ticks .
 .It Dv C_DIRECT_EXEC
@@ -301,7 +347,7 @@ Callout functions run in this context may use only spin mutexes for locking
 and should be as small as possible because they run with absolute priority.
 .It Fn C_PREL
 Specifies relative event time precision as binary logarithm of time interval
-divided by acceptable time deviation: 1 -- 1/2, 2 -- 1/4 and so on.
+divided by acceptable time deviation: 1 -- 1/2, 2 -- 1/4, etc.
 Note that the larger of
 .Fa pr
 or this value is used as the length of the time window.
@@ -314,215 +360,65 @@ Align the timeouts to
 calls if possible.
 .El
 .Pp
-.Ft int
-.Fn callout_reset_sbt_curcpu "struct callout *c" "sbintime_t sbt" \
-"sbintime_t pr" "callout_func_t *func" "void *arg" "int flags"
-This function works the same like the
-.Fn callout_reset_sbt
-function except the callback function given by the
-.Fa func
-argument will be executed on the same CPU which called this function.
-A change in the CPU selection can happen if the callout has a lock
-associated with it and is locked when this function is called.
-A change in the CPU selection cannot happen if this function is
-re-scheduled inside a callout function.
-Else the callback function given by the
-.Fa func
-argument will be executed on the same CPU like previously done.
-.Pp
-.Ft int
-.Fn callout_reset_sbt_on "struct callout *c" "sbintime_t sbt" \
-"sbintime_t pr" "callout_func_t *func" "void *arg" "int cpu" "int flags"
-This function works the same like the
-.Fn callout_reset_sbt
-function except the callback function given by the
-.Fa func
-argument will be executed on the CPU given by the
-.Fa cpu
-argument.
-A change in the CPU selection can happen if the callout has a lock
-associated with it and is locked when this function is called.
-A change in the CPU selection cannot happen if this function is
-re-scheduled inside a callout function.
-Else the callback function given by the
-.Fa func
-argument will be executed on the same CPU like previously done.
-.Pp
-.Ft int
-.Fn callout_schedule "struct callout *c" "int ticks"
-This function works the same like the
+The
 .Fn callout_reset
-function except it re-uses the callback function and the callback argument
-already stored in the
-.Pq struct callout
-structure.
-.Pp
-.Ft int
-.Fn callout_schedule_curcpu "struct callout *c" "int ticks"
-This function works the same like the
-.Fn callout_reset_curcpu
-function except it re-uses the callback function and the callback argument
-already stored in the
-.Pq struct callout
-structure.
-.Pp
-.Ft int
-.Fn callout_schedule_on "struct callout *c" "int ticks" "int cpu"
-This function works the same like the
-.Fn callout_reset_on
-function except it re-uses the callback function and the callback argument
-already stored in the
-.Pq struct callout
-structure.
-.Pp
-.Ft int
-.Fn callout_schedule_sbt "struct callout *c" "sbintime_t sbt" \
-"sbintime_t pr" "int flags"
-This function works the same like the
-.Fn callout_reset_sbt
-function except it re-uses the callback function and the callback argument
-already stored in the
-.Pq struct callout
-structure.
-.Pp
-.Ft int
-.Fn callout_schedule_sbt_curcpu "struct callout *c" "sbintime_t sbt" \
-"sbintime_t pr" "int flags"
-This function works the same like the
-.Fn callout_reset_sbt_curcpu
-function except it re-uses the callback function and the callback argument
-already stored in the
-.Pq struct callout
-structure.
-.Pp
-.Ft int
-.Fn callout_schedule_sbt_on "struct callout *c" "sbintime_t sbt" \
-"sbintime_t pr" "int cpu" "int flags"
-This function works the same like the
-.Fn callout_reset_sbt_on
-function except it re-uses the callback function and the callback argument
-already stored in the
-.Pq struct callout
-structure.
-.Sh CHECKING THE STATE OF CALLOUTS
-.Ft int
-.Fn callout_pending "struct callout *c"
-This function returns non-zero if the callout pointed to by the
-.Fa c
-argument is pending for callback.
-Else this function returns zero.
-This function returns zero when inside the callout function if the
-callout is not re-scheduled.
-.Pp
-.Ft int
-.Fn callout_active "struct callout *c"
-This function is deprecated and returns non-zero if the callout
-pointed to by the
-.Fa c
-argument was scheduled in the past.
-Else this function returns zero.
-This function also returns zero after the
-.Fn callout_deactivate
-or the
-.Fn callout_stop
-or the
-.Fn callout_drain
-or the
-.Fn callout_drain_async
-function is called on the same callout as given by the
-.Fa c
-argument.
-.Pp
-.Ft void
-.Fn callout_deactivate "struct callout *c"
-This function is deprecated and ensures that subsequent calls to the
-.Fn callout_activate
-function returns zero until the callout is scheduled again.
-.Sh STOPPING CALLOUTS
-.Ft void
-.Fn untimeout "timeout_t *func" "void *arg" "struct callout_handle handle"
-This function is deprecated and cancels the timeout associated with the
-.Fa handle
-argument using the function pointed to by the
+functions accept a
 .Fa func
-argument and having the
-.Fa arg
-arguments to validate the handle.
-If the handle does not correspond to a timeout with
-the function
-.Fa func
-taking the argument
-.Fa arg
-no action is taken. The
-.Fa handle
-must be initialised by a previous call to
-.Fn timeout ,
-.Fn callout_handle_init
-or assigned the value of
-.Fn CALLOUT_HANDLE_INITIALIZER "&handle"
-before being passed to
-.Fn untimeout .
-The behavior of calling
-.Fn untimeout
-with an uninitialised handle
-is undefined.
-.Pp
-.Ft int
-.Fn callout_stop "struct callout *c"
-This function is used to stop a timeout function invocation associated with the callout pointed to by the
-.Fa c
-argument, in a non-blocking fashion.
-This function can be called multiple times in a row with no side effects, even if the callout is already stopped. This function however should not be called before the callout has been initialised.
-This function returns a non-zero value if the given callout was pending and
-the callback function was prevented from being called.
-Else a value of zero is returned.
-If a lock is associated with the callout given by the
-.Fa c
-argument and it is exclusivly locked when this function is called, the
-.Fn callout_stop
-function will always ensure that the callback function is never reached.
-In other words the callout will be atomically stopped.
-Else there is no such guarantee.
-.Sh DRAINING CALLOUTS
-.Ft int
-.Fn callout_drain "struct callout *c"
-This function works the same like the
-.Fn callout_stop
-function except it ensures that all callback functions have returned and there are no more references to the callout pointed to by the
-.Fa c
-argument inside the callout subsystem before it returns.
-Also this function ensures that the lock, if any, associated with the
-callout is no longer being used.
-When this function returns, it is safe to free the callout structure pointed to by the
-.Fa c
+argument which identifies the function to be called when the time expires.
+It must be a pointer to a function that takes a single
+.Fa void *
 argument.
-.Pp
-.Ft int
-.Fn callout_drain_async "struct callout *c" "callout_func_t *fn" "void *arg"
-This function is non-blocking and works the same like the
-.Fn callout_stop
-function except if it returns non-zero it means the callback function pointed to by the
-.Fa fn
-argument will be called back with the
+Upon invocation,
+.Fa func
+will receive
 .Fa arg
-argument when all references to the callout pointed to by the
-.Fa c
-argument are gone.
-If this function returns zero, it is safe to free the callout structure pointed to by the
-.Fa c
-argument right away.
-.Sh CALLOUT FUNCTION RESTRICTIONS
-Callout functions must not sleep.
-They may not acquire sleepable locks, wait on condition variables,
-perform blocking allocation requests, or invoke any other action that
-might sleep.
-.Sh CALLOUT SUBSYSTEM INTERNALS
-The callout subsystem has its own set of spinlocks to protect its internal state.
-The callout subsystem provides a softclock thread for each CPU in the
-system.
-Callouts are assigned to a single CPU and are executed by the
-softclock thread for that CPU.
-Initially, callouts are assigned to CPU 0.
+as its only argument.
+The
+.Fn callout_schedule
+functions reuse the
+.Fa func
+and
+.Fa arg
+arguments from the previous callout.
+Note that one of the
+.Fn callout_reset
+functions must always be called to initialize
+.Fa func
+and
+.Fa arg
+before one of the
+.Fn callout_schedule
+functions can be used.
+.Pp
+The callout subsystem provides a softclock thread for each CPU in the system.
+Callouts are assigned to a single CPU and are executed by the softclock thread
+for that CPU.
+Initially,
+callouts are assigned to CPU 0.
+The
+.Fn callout_reset_on ,
+.Fn callout_reset_sbt_on ,
+.Fn callout_schedule_on
+and
+.Fn callout_schedule_sbt_on
+functions assign the callout to CPU
+.Fa cpu .
+The
+.Fn callout_reset_curcpu ,
+.Fn callout_reset_sbt_curpu ,
+.Fn callout_schedule_curcpu
+and
+.Fn callout_schedule_sbt_curcpu
+functions assign the callout to the current CPU.
+The
+.Fn callout_reset ,
+.Fn callout_reset_sbt ,
+.Fn callout_schedule
+and
+.Fn callout_schedule_sbt
+functions schedule the callout to execute in the softclock thread of the CPU
+to which it is currently assigned.
+.Pp
 Softclock threads are not pinned to their respective CPUs by default.
 The softclock thread for CPU 0 can be pinned to CPU 0 by setting the
 .Va kern.pin_default_swi
@@ -531,7 +427,50 @@ Softclock threads for CPUs other than zero can be pinned to their
 respective CPUs by setting the
 .Va kern.pin_pcpu_swi
 loader tunable to a non-zero value.
-.Sh "AVOIDING RACE CONDITIONS"
+.Pp
+The macros
+.Fn callout_pending ,
+.Fn callout_active
+and
+.Fn callout_deactivate
+provide access to the current state of the callout.
+The
+.Fn callout_pending
+macro checks whether a callout is
+.Em pending ;
+a callout is considered
+.Em pending
+when a timeout has been set but the time has not yet arrived.
+Note that once the timeout time arrives and the callout subsystem
+starts to process this callout,
+.Fn callout_pending
+will return
+.Dv FALSE
+even though the callout function may not have finished
+.Pq or even begun
+executing.
+The
+.Fn callout_active
+macro checks whether a callout is marked as
+.Em active ,
+and the
+.Fn callout_deactivate
+macro clears the callout's
+.Em active
+flag.
+The callout subsystem marks a callout as
+.Em active
+when a timeout is set and it clears the
+.Em active
+flag in
+.Fn callout_stop
+and
+.Fn callout_drain ,
+but it
+.Em does not
+clear it when a callout expires normally via the execution of the
+callout function.
+.Ss "Avoiding Race Conditions"
 The callout subsystem invokes callout functions from its own thread
 context.
 Without some kind of synchronization,
@@ -548,7 +487,7 @@ synchronization concerns.
 The first approach is preferred as it is the simplest:
 .Bl -enum -offset indent
 .It
-Callouts can be associated with a specific lock when they are initialised
+Callouts can be associated with a specific lock when they are initialized
 by
 .Fn callout_init_mtx ,
 .Fn callout_init_rm ,
@@ -569,7 +508,7 @@ or
 .Fn callout_schedule
 functions to provide this safety.
 .Pp
-A callout initialised via
+A callout initialized via
 .Fn callout_init
 with
 .Fa mpsafe
@@ -592,8 +531,9 @@ function families
 .Pc
 indicates whether or not the callout was removed.
 If it is known that the callout was set and the callout function has
-not yet executed, then a return value of zero indicates that the
-callout function is about to be called.
+not yet executed, then a return value of
+.Dv FALSE
+indicates that the callout function is about to be called.
 For example:
 .Bd -literal -offset indent
 if (sc->sc_flags & SCFLG_CALLOUT_RUNNING) {
@@ -649,14 +589,16 @@ The callout function should first check the
 .Em pending
 flag and return without action if
 .Fn callout_pending
-returns non-zero.
+returns
+.Dv TRUE .
 This indicates that the callout was rescheduled using
 .Fn callout_reset
 just before the callout function was invoked.
 If
 .Fn callout_active
-returns zero then the callout function should also return without
-action.
+returns
+.Dv FALSE
+then the callout function should also return without action.
 This indicates that the callout has been stopped.
 Finally, the callout function should call
 .Fn callout_deactivate
@@ -726,13 +668,129 @@ a callout should always be drained prior to destroying its associated lock
 or releasing the storage for the callout structure.
 .Sh LEGACY API
 .Bf Sy
-The
-.Fn timeout
-and
-.Fn untimeout
-functions are a legacy API that will be removed in a future release.
+The functions below are a legacy API that will be removed in a future release.
 New code should not use these routines.
 .Ef
+.Pp
+The function
+.Fn timeout
+schedules a call to the function given by the argument
+.Fa func
+to take place after
+.Fa ticks Ns No /hz
+seconds.
+Non-positive values of
+.Fa ticks
+are silently converted to the value
+.Sq 1 .
+.Fa func
+should be a pointer to a function that takes a
+.Fa void *
+argument.
+Upon invocation,
+.Fa func
+will receive
+.Fa arg
+as its only argument.
+The return value from
+.Fn timeout
+is a
+.Ft struct callout_handle
+which can be used in conjunction with the
+.Fn untimeout
+function to request that a scheduled timeout be canceled.
+.Pp
+The function
+.Fn callout_handle_init
+can be used to initialize a handle to a state which will cause
+any calls to
+.Fn untimeout
+with that handle to return with no side
+effects.
+.Pp
+Assigning a callout handle the value of
+.Fn CALLOUT_HANDLE_INITIALIZER
+performs the same function as
+.Fn callout_handle_init
+and is provided for use on statically declared or global callout handles.
+.Pp
+The function
+.Fn untimeout
+cancels the timeout associated with
+.Fa handle
+using the
+.Fa func
+and
+.Fa arg
+arguments to validate the handle.
+If the handle does not correspond to a timeout with
+the function
+.Fa func
+taking the argument
+.Fa arg
+no action is taken.
+.Fa handle
+must be initialized by a previous call to
+.Fn timeout ,
+.Fn callout_handle_init ,
+or assigned the value of
+.Fn CALLOUT_HANDLE_INITIALIZER "&handle"
+before being passed to
+.Fn untimeout .
+The behavior of calling
+.Fn untimeout
+with an uninitialized handle
+is undefined.
+.Pp
+As handles are recycled by the system, it is possible (although unlikely)
+that a handle from one invocation of
+.Fn timeout
+may match the handle of another invocation of
+.Fn timeout
+if both calls used the same function pointer and argument, and the first
+timeout is expired or canceled before the second call.
+The timeout facility offers O(1) running time for
+.Fn timeout
+and
+.Fn untimeout .
+Timeouts are executed from
+.Fn softclock
+with the
+.Va Giant
+lock held.
+Thus they are protected from re-entrancy.
+.Sh RETURN VALUES
+The
+.Fn callout_active
+macro returns the state of a callout's
+.Em active
+flag.
+.Pp
+The
+.Fn callout_pending
+macro returns the state of a callout's
+.Em pending
+flag.
+.Pp
+The
+.Fn callout_reset
+and
+.Fn callout_schedule
+function families return non-zero if the callout was pending before the new
+function invocation was scheduled.
+.Pp
+The
+.Fn callout_stop
+and
+.Fn callout_drain
+functions return non-zero if the callout was still pending when it was
+called or zero otherwise.
+The
+.Fn timeout
+function returns a
+.Ft struct callout_handle
+that can be passed to
+.Fn untimeout .
 .Sh HISTORY
 The current timeout and untimeout routines are based on the work of
 .An Adam M. Costello
@@ -757,4 +815,4 @@ The current implementation replaces the long standing
 .Bx
 linked list
 callout mechanism which offered O(n) insertion and removal running time
-and did not generate or require handles for untimeout operations.
+but did not generate or require handles for untimeout operations.
diff --git a/sys/kern/init_main.c b/sys/kern/init_main.c
index 922959f8fa5..beb49bc5696 100644
--- a/sys/kern/init_main.c
+++ b/sys/kern/init_main.c
@@ -504,8 +504,7 @@ proc0_init(void *dummy __unused)
 
 	callout_init_mtx(&p->p_itcallout, &p->p_mtx, 0);
 	callout_init_mtx(&p->p_limco, &p->p_mtx, 0);
-	mtx_init(&td->td_slpmutex, "td_slpmutex", NULL, MTX_SPIN);
-	callout_init_mtx(&td->td_slpcallout, &td->td_slpmutex, 0);
+	callout_init(&td->td_slpcallout, CALLOUT_MPSAFE);
 
 	/* Create credentials. */
 	p->p_ucred = crget();
diff --git a/sys/kern/kern_condvar.c b/sys/kern/kern_condvar.c
index 8c2691babb3..2700a25d477 100644
--- a/sys/kern/kern_condvar.c
+++ b/sys/kern/kern_condvar.c
@@ -313,13 +313,15 @@ _cv_timedwait_sbt(struct cv *cvp, struct lock_object *lock, sbintime_t sbt,
 	DROP_GIANT();
 
 	sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR, 0);
-	sleepq_release(cvp);
 	sleepq_set_timeout_sbt(cvp, sbt, pr, flags);
 	if (lock != &Giant.lock_object) {
+		if (class->lc_flags & LC_SLEEPABLE)
+			sleepq_release(cvp);
 		WITNESS_SAVE(lock, lock_witness);
 		lock_state = class->lc_unlock(lock);
+		if (class->lc_flags & LC_SLEEPABLE)
+			sleepq_lock(cvp);
 	}
-	sleepq_lock(cvp);
 	rval = sleepq_timedwait(cvp, 0);
 
 #ifdef KTRACE
@@ -381,13 +383,15 @@ _cv_timedwait_sig_sbt(struct cv *cvp, struct lock_object *lock,
 
 	sleepq_add(cvp, lock, cvp->cv_description, SLEEPQ_CONDVAR |
 	    SLEEPQ_INTERRUPTIBLE, 0);
-	sleepq_release(cvp);
 	sleepq_set_timeout_sbt(cvp, sbt, pr, flags);
 	if (lock != &Giant.lock_object) {
+		if (class->lc_flags & LC_SLEEPABLE)
+			sleepq_release(cvp);
 		WITNESS_SAVE(lock, lock_witness);
 		lock_state = class->lc_unlock(lock);
+		if (class->lc_flags & LC_SLEEPABLE)
+			sleepq_lock(cvp);
 	}
-	sleepq_lock(cvp);
 	rval = sleepq_timedwait_sig(cvp, 0);
 
 #ifdef KTRACE
diff --git a/sys/kern/kern_lock.c b/sys/kern/kern_lock.c
index 38c870709c3..36a8470d297 100644
--- a/sys/kern/kern_lock.c
+++ b/sys/kern/kern_lock.c
@@ -210,11 +210,9 @@ sleeplk(struct lock *lk, u_int flags, struct lock_object *ilk,
 	GIANT_SAVE();
 	sleepq_add(&lk->lock_object, NULL, wmesg, SLEEPQ_LK | (catch ?
 	    SLEEPQ_INTERRUPTIBLE : 0), queue);
-	if ((flags & LK_TIMELOCK) && timo) {
-		sleepq_release(&lk->lock_object);
+	if ((flags & LK_TIMELOCK) && timo)
 		sleepq_set_timeout(&lk->lock_object, timo);
-		sleepq_lock(&lk->lock_object);
-	}
+
 	/*
 	 * Decisional switch for real sleeping.
 	 */
diff --git a/sys/kern/kern_switch.c b/sys/kern/kern_switch.c
index 61d91494464..d0009b1042f 100644
--- a/sys/kern/kern_switch.c
+++ b/sys/kern/kern_switch.c
@@ -93,6 +93,8 @@ SCHED_STAT_DEFINE_VAR(turnstile,
     &DPCPU_NAME(sched_switch_stats[SWT_TURNSTILE]), "");
 SCHED_STAT_DEFINE_VAR(sleepq,
     &DPCPU_NAME(sched_switch_stats[SWT_SLEEPQ]), "");
+SCHED_STAT_DEFINE_VAR(sleepqtimo,
+    &DPCPU_NAME(sched_switch_stats[SWT_SLEEPQTIMO]), "");
 SCHED_STAT_DEFINE_VAR(relinquish, 
     &DPCPU_NAME(sched_switch_stats[SWT_RELINQUISH]), "");
 SCHED_STAT_DEFINE_VAR(needresched,
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 19bf4e8a094..9501ba2ba07 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -236,16 +236,12 @@ _sleep(void *ident, struct lock_object *lock, int priority,
 	 * return from cursig().
 	 */
 	sleepq_add(ident, lock, wmesg, sleepq_flags, 0);
+	if (sbt != 0)
+		sleepq_set_timeout_sbt(ident, sbt, pr, flags);
 	if (lock != NULL && class->lc_flags & LC_SLEEPABLE) {
 		sleepq_release(ident);
 		WITNESS_SAVE(lock, lock_witness);
 		lock_state = class->lc_unlock(lock);
-		if (sbt != 0)
-			sleepq_set_timeout_sbt(ident, sbt, pr, flags);
-		sleepq_lock(ident);
-	} else if (sbt != 0) {
-		sleepq_release(ident);
-		sleepq_set_timeout_sbt(ident, sbt, pr, flags);
 		sleepq_lock(ident);
 	}
 	if (sbt != 0 && catch)
@@ -310,11 +306,8 @@ msleep_spin_sbt(void *ident, struct mtx *mtx, const char *wmesg,
 	 * We put ourselves on the sleep queue and start our timeout.
 	 */
 	sleepq_add(ident, &mtx->lock_object, wmesg, SLEEPQ_SLEEP, 0);
-	if (sbt != 0) {
-		sleepq_release(ident);
+	if (sbt != 0)
 		sleepq_set_timeout_sbt(ident, sbt, pr, flags);
-		sleepq_lock(ident);
-	}
 
 	/*
 	 * Can't call ktrace with any spin locks held so it can lock the
diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c
index b1e1a126359..2d0b0d278a5 100644
--- a/sys/kern/kern_thread.c
+++ b/sys/kern/kern_thread.c
@@ -149,9 +149,6 @@ thread_ctor(void *mem, int size, void *arg, int flags)
 	audit_thread_alloc(td);
 #endif
 	umtx_thread_alloc(td);
-
-	mtx_init(&td->td_slpmutex, "td_slpmutex", NULL, MTX_SPIN);
-	callout_init_mtx(&td->td_slpcallout, &td->td_slpmutex, 0);
 	return (0);
 }
 
@@ -165,10 +162,6 @@ thread_dtor(void *mem, int size, void *arg)
 
 	td = (struct thread *)mem;
 
-	/* make sure to drain any use of the "td->td_slpcallout" */
-	callout_drain(&td->td_slpcallout);
-	mtx_destroy(&td->td_slpmutex);
-
 #ifdef INVARIANTS
 	/* Verify that this thread is in a safe state to free. */
 	switch (td->td_state) {
@@ -551,6 +544,7 @@ thread_link(struct thread *td, struct proc *p)
 	LIST_INIT(&td->td_lprof[0]);
 	LIST_INIT(&td->td_lprof[1]);
 	sigqueue_init(&td->td_sigqueue, p);
+	callout_init(&td->td_slpcallout, CALLOUT_MPSAFE);
 	TAILQ_INSERT_TAIL(&p->p_threads, td, td_plist);
 	p->p_numthreads++;
 }
diff --git a/sys/kern/kern_timeout.c b/sys/kern/kern_timeout.c
index 4336faab3bf..13822fd99ac 100644
--- a/sys/kern/kern_timeout.c
+++ b/sys/kern/kern_timeout.c
@@ -54,8 +54,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/malloc.h>
 #include <sys/mutex.h>
-#include <sys/rmlock.h>
-#include <sys/rwlock.h>
 #include <sys/proc.h>
 #include <sys/sdt.h>
 #include <sys/sleepqueue.h>
@@ -126,216 +124,37 @@ SYSCTL_INT(_kern, OID_AUTO, pin_pcpu_swi, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &pin_
  */
 u_int callwheelsize, callwheelmask;
 
-typedef void callout_mutex_op_t(struct lock_object *);
-typedef int callout_owned_op_t(struct lock_object *);
-
-struct callout_mutex_ops {
-	callout_mutex_op_t *lock;
-	callout_mutex_op_t *unlock;
-	callout_owned_op_t *owned;
-};
-
-enum {
-	CALLOUT_LC_UNUSED_0,
-	CALLOUT_LC_UNUSED_1,
-	CALLOUT_LC_UNUSED_2,
-	CALLOUT_LC_UNUSED_3,
-	CALLOUT_LC_SPIN,
-	CALLOUT_LC_MUTEX,
-	CALLOUT_LC_RW,
-	CALLOUT_LC_RM,
-};
-
-static void
-callout_mutex_op_none(struct lock_object *lock)
-{
-}
-
-static int
-callout_owned_op_none(struct lock_object *lock)
-{
-	return (0);
-}
-
-static void
-callout_mutex_lock(struct lock_object *lock)
-{
-	mtx_lock((struct mtx *)lock);
-}
-
-static void
-callout_mutex_unlock(struct lock_object *lock)
-{
-	mtx_unlock((struct mtx *)lock);
-}
-
-static void
-callout_mutex_lock_spin(struct lock_object *lock)
-{
-	mtx_lock_spin((struct mtx *)lock);
-}
-
-static void
-callout_mutex_unlock_spin(struct lock_object *lock)
-{
-	mtx_unlock_spin((struct mtx *)lock);
-}
-
-static int
-callout_mutex_owned(struct lock_object *lock)
-{
-	return (mtx_owned((struct mtx *)lock));
-}
-
-static void
-callout_rm_wlock(struct lock_object *lock)
-{
-	rm_wlock((struct rmlock *)lock);
-}
-
-static void
-callout_rm_wunlock(struct lock_object *lock)
-{
-	rm_wunlock((struct rmlock *)lock);
-}
-
-static int
-callout_rm_owned(struct lock_object *lock)
-{
-	return (rm_wowned((struct rmlock *)lock));
-}
-
-static void
-callout_rw_wlock(struct lock_object *lock)
-{
-	rw_wlock((struct rwlock *)lock);
-}
-
-static void
-callout_rw_wunlock(struct lock_object *lock)
-{
-	rw_wunlock((struct rwlock *)lock);
-}
-
-static int
-callout_rw_owned(struct lock_object *lock)
-{
-	return (rw_wowned((struct rwlock *)lock));
-}
-
-static const struct callout_mutex_ops callout_mutex_ops[8] = {
-	[CALLOUT_LC_UNUSED_0] = {
-		.lock = callout_mutex_op_none,
-		.unlock = callout_mutex_op_none,
-		.owned = callout_owned_op_none,
-	},
-	[CALLOUT_LC_UNUSED_1] = {
-		.lock = callout_mutex_op_none,
-		.unlock = callout_mutex_op_none,
-		.owned = callout_owned_op_none,
-	},
-	[CALLOUT_LC_UNUSED_2] = {
-		.lock = callout_mutex_op_none,
-		.unlock = callout_mutex_op_none,
-		.owned = callout_owned_op_none,
-	},
-	[CALLOUT_LC_UNUSED_3] = {
-		.lock = callout_mutex_op_none,
-		.unlock = callout_mutex_op_none,
-		.owned = callout_owned_op_none,
-	},
-	[CALLOUT_LC_SPIN] = {
-		.lock = callout_mutex_lock_spin,
-		.unlock = callout_mutex_unlock_spin,
-		.owned = callout_mutex_owned,
-	},
-	[CALLOUT_LC_MUTEX] = {
-		.lock = callout_mutex_lock,
-		.unlock = callout_mutex_unlock,
-		.owned = callout_mutex_owned,
-	},
-	[CALLOUT_LC_RW] = {
-		.lock = callout_rw_wlock,
-		.unlock = callout_rw_wunlock,
-		.owned = callout_rw_owned,
-	},
-	[CALLOUT_LC_RM] = {
-		.lock = callout_rm_wlock,
-		.unlock = callout_rm_wunlock,
-		.owned = callout_rm_owned,
-	},
-};
-
-static void
-callout_lock_client(int c_flags, struct lock_object *c_lock)
-{
-	callout_mutex_ops[CALLOUT_GET_LC(c_flags)].lock(c_lock);
-}
-
-static void
-callout_unlock_client(int c_flags, struct lock_object *c_lock)
-{
-	callout_mutex_ops[CALLOUT_GET_LC(c_flags)].unlock(c_lock);
-}
-
-#ifdef SMP
-static int
-callout_lock_owned_client(int c_flags, struct lock_object *c_lock)
-{
-	return (callout_mutex_ops[CALLOUT_GET_LC(c_flags)].owned(c_lock));
-}
-#endif
-
 /*
- * The callout CPU exec structure represent information necessary for
- * describing the state of callouts currently running on the CPU and
- * for handling deferred callout restarts.
- *
- * In particular, the first entry of the array cc_exec_entity holds
- * information for callouts running from the SWI thread context, while
- * the second one holds information for callouts running directly from
- * the hardware interrupt context.
+ * The callout cpu exec entities represent informations necessary for
+ * describing the state of callouts currently running on the CPU and the ones
+ * necessary for migrating callouts to the new callout cpu. In particular,
+ * the first entry of the array cc_exec_entity holds informations for callout
+ * running in SWI thread context, while the second one holds informations
+ * for callout running directly from hardware interrupt context.
+ * The cached informations are very important for deferring migration when
+ * the migrating callout is already running.
  */
 struct cc_exec {
-	/*
-	 * The "cc_curr" points to the currently executing callout and
-	 * is protected by the "cc_lock" spinlock. If no callback is
-	 * currently executing it is equal to "NULL".
-	 */
+	struct callout		*cc_next;
 	struct callout		*cc_curr;
-	/*
-	 * The "cc_restart_args" structure holds the argument for a
-	 * deferred callback restart and is protected by the "cc_lock"
-	 * spinlock. The structure is only valid if "cc_restart" is
-	 * "true". If "cc_restart" is "false" the information in the
-	 * "cc_restart_args" structure shall be ignored.
-	 */
-	struct callout_args	cc_restart_args;
-	bool			cc_restart;
-	/*
-	 * The "cc_cancel" variable allows the currently pending
-	 * callback to be atomically cancelled. This field is write
-	 * protected by the "cc_lock" spinlock.
-	 */
-	bool cc_cancel;
-	/*
-	 * The "cc_drain_fn" points to a function which shall be
-	 * called with the argument stored in "cc_drain_arg" when an
-	 * asynchronous drain is performed. This field is write
-	 * protected by the "cc_lock" spinlock.
-	 */
-	callout_func_t *cc_drain_fn;
-	void *cc_drain_arg;
+#ifdef SMP
+	void			(*ce_migration_func)(void *);
+	void			*ce_migration_arg;
+	int			ce_migration_cpu;
+	sbintime_t		ce_migration_time;
+	sbintime_t		ce_migration_prec;
+#endif
+	bool			cc_cancel;
+	bool			cc_waiting;
 };
 
 /*
- * There is one "struct callout_cpu" per CPU, holding all relevant
+ * There is one struct callout_cpu per cpu, holding all relevant
  * state for the callout processing thread on the individual CPU.
  */
 struct callout_cpu {
 	struct mtx_padalign	cc_lock;
 	struct cc_exec 		cc_exec_entity[2];
-	struct callout		*cc_exec_next_dir;
 	struct callout		*cc_callout;
 	struct callout_list	*cc_callwheel;
 	struct callout_tailq	cc_expireq;
@@ -347,7 +166,27 @@ struct callout_cpu {
 	char			cc_ktr_event_name[20];
 };
 
+#define	cc_exec_curr		cc_exec_entity[0].cc_curr
+#define	cc_exec_next		cc_exec_entity[0].cc_next
+#define	cc_exec_cancel		cc_exec_entity[0].cc_cancel
+#define	cc_exec_waiting		cc_exec_entity[0].cc_waiting
+#define	cc_exec_curr_dir	cc_exec_entity[1].cc_curr
+#define	cc_exec_next_dir	cc_exec_entity[1].cc_next
+#define	cc_exec_cancel_dir	cc_exec_entity[1].cc_cancel
+#define	cc_exec_waiting_dir	cc_exec_entity[1].cc_waiting
+
 #ifdef SMP
+#define	cc_migration_func	cc_exec_entity[0].ce_migration_func
+#define	cc_migration_arg	cc_exec_entity[0].ce_migration_arg
+#define	cc_migration_cpu	cc_exec_entity[0].ce_migration_cpu
+#define	cc_migration_time	cc_exec_entity[0].ce_migration_time
+#define	cc_migration_prec	cc_exec_entity[0].ce_migration_prec
+#define	cc_migration_func_dir	cc_exec_entity[1].ce_migration_func
+#define	cc_migration_arg_dir	cc_exec_entity[1].ce_migration_arg
+#define	cc_migration_cpu_dir	cc_exec_entity[1].ce_migration_cpu
+#define	cc_migration_time_dir	cc_exec_entity[1].ce_migration_time
+#define	cc_migration_prec_dir	cc_exec_entity[1].ce_migration_prec
+
 struct callout_cpu cc_cpu[MAXCPU];
 #define	CPUBLOCK	MAXCPU
 #define	CC_CPU(cpu)	(&cc_cpu[(cpu)])
@@ -372,9 +211,60 @@ static void	softclock_call_cc(struct callout *c, struct callout_cpu *cc,
 
 static MALLOC_DEFINE(M_CALLOUT, "callout", "Callout datastructures");
 
+/**
+ * Locked by cc_lock:
+ *   cc_curr         - If a callout is in progress, it is cc_curr.
+ *                     If cc_curr is non-NULL, threads waiting in
+ *                     callout_drain() will be woken up as soon as the
+ *                     relevant callout completes.
+ *   cc_cancel       - Changing to 1 with both callout_lock and cc_lock held
+ *                     guarantees that the current callout will not run.
+ *                     The softclock() function sets this to 0 before it
+ *                     drops callout_lock to acquire c_lock, and it calls
+ *                     the handler only if curr_cancelled is still 0 after
+ *                     cc_lock is successfully acquired.
+ *   cc_waiting      - If a thread is waiting in callout_drain(), then
+ *                     callout_wait is nonzero.  Set only when
+ *                     cc_curr is non-NULL.
+ */
+
 /*
- * Kernel low level callwheel initialization called from cpu0 during
- * kernel startup:
+ * Resets the execution entity tied to a specific callout cpu.
+ */
+static void
+cc_cce_cleanup(struct callout_cpu *cc, int direct)
+{
+
+	cc->cc_exec_entity[direct].cc_curr = NULL;
+	cc->cc_exec_entity[direct].cc_next = NULL;
+	cc->cc_exec_entity[direct].cc_cancel = false;
+	cc->cc_exec_entity[direct].cc_waiting = false;
+#ifdef SMP
+	cc->cc_exec_entity[direct].ce_migration_cpu = CPUBLOCK;
+	cc->cc_exec_entity[direct].ce_migration_time = 0;
+	cc->cc_exec_entity[direct].ce_migration_prec = 0;
+	cc->cc_exec_entity[direct].ce_migration_func = NULL;
+	cc->cc_exec_entity[direct].ce_migration_arg = NULL;
+#endif
+}
+
+/*
+ * Checks if migration is requested by a specific callout cpu.
+ */
+static int
+cc_cce_migrating(struct callout_cpu *cc, int direct)
+{
+
+#ifdef SMP
+	return (cc->cc_exec_entity[direct].ce_migration_cpu != CPUBLOCK);
+#else
+	return (0);
+#endif
+}
+
+/*
+ * Kernel low level callwheel initialization
+ * called on cpu0 during kernel startup.
  */
 static void
 callout_callwheel_init(void *dummy)
@@ -434,6 +324,8 @@ callout_cpu_init(struct callout_cpu *cc, int cpu)
 		LIST_INIT(&cc->cc_callwheel[i]);
 	TAILQ_INIT(&cc->cc_expireq);
 	cc->cc_firstevent = SBT_MAX;
+	for (i = 0; i < 2; i++)
+		cc_cce_cleanup(cc, i);
 	snprintf(cc->cc_ktr_event_name, sizeof(cc->cc_ktr_event_name),
 	    "callwheel cpu %d", cpu);
 	if (cc->cc_callout == NULL)	/* Only cpu0 handles timeout(9) */
@@ -441,11 +333,41 @@ callout_cpu_init(struct callout_cpu *cc, int cpu)
 	for (i = 0; i < ncallout; i++) {
 		c = &cc->cc_callout[i];
 		callout_init(c, 0);
-		c->c_flags |= CALLOUT_LOCAL_ALLOC;
+		c->c_flags = CALLOUT_LOCAL_ALLOC;
 		SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);
 	}
 }
 
+#ifdef SMP
+/*
+ * Switches the cpu tied to a specific callout.
+ * The function expects a locked incoming callout cpu and returns with
+ * locked outcoming callout cpu.
+ */
+static struct callout_cpu *
+callout_cpu_switch(struct callout *c, struct callout_cpu *cc, int new_cpu)
+{
+	struct callout_cpu *new_cc;
+
+	MPASS(c != NULL && cc != NULL);
+	CC_LOCK_ASSERT(cc);
+
+	/*
+	 * Avoid interrupts and preemption firing after the callout cpu
+	 * is blocked in order to avoid deadlocks as the new thread
+	 * may be willing to acquire the callout cpu lock.
+	 */
+	c->c_cpu = CPUBLOCK;
+	spinlock_enter();
+	CC_UNLOCK(cc);
+	new_cc = CC_CPU(new_cpu);
+	CC_LOCK(new_cc);
+	spinlock_exit();
+	c->c_cpu = new_cpu;
+	return (new_cc);
+}
+#endif
+
 /*
  * Start standard softclock thread.
  */
@@ -522,8 +444,9 @@ callout_process(sbintime_t now)
 #ifdef CALLOUT_PROFILING
 	int depth_dir = 0, mpcalls_dir = 0, lockcalls_dir = 0;
 #endif
+
 	cc = CC_SELF();
-	CC_LOCK(cc);
+	mtx_lock_spin_flags(&cc->cc_lock, MTX_QUIET);
 
 	/* Compute the buckets of the last scan and present times. */
 	firstb = callout_hash(cc->cc_lastscan);
@@ -626,7 +549,7 @@ next:
 	avg_mpcalls_dir += (mpcalls_dir * 1000 - avg_mpcalls_dir) >> 8;
 	avg_lockcalls_dir += (lockcalls_dir * 1000 - avg_lockcalls_dir) >> 8;
 #endif
-	CC_UNLOCK(cc);
+	mtx_unlock_spin_flags(&cc->cc_lock, MTX_QUIET);
 	/*
 	 * swi_sched acquires the thread lock, so we don't want to call it
 	 * with cc_lock held; incorrect locking order.
@@ -639,55 +562,49 @@ static struct callout_cpu *
 callout_lock(struct callout *c)
 {
 	struct callout_cpu *cc;
-	cc = CC_CPU(c->c_cpu);
-	CC_LOCK(cc);
+	int cpu;
+
+	for (;;) {
+		cpu = c->c_cpu;
+#ifdef SMP
+		if (cpu == CPUBLOCK) {
+			while (c->c_cpu == CPUBLOCK)
+				cpu_spinwait();
+			continue;
+		}
+#endif
+		cc = CC_CPU(cpu);
+		CC_LOCK(cc);
+		if (cpu == c->c_cpu)
+			break;
+		CC_UNLOCK(cc);
+	}
 	return (cc);
 }
 
-static struct callout_cpu *
-callout_cc_add_locked(struct callout *c, struct callout_cpu *cc,
-    struct callout_args *coa, bool can_swap_cpu)
+static void
+callout_cc_add(struct callout *c, struct callout_cpu *cc,
+    sbintime_t sbt, sbintime_t precision, void (*func)(void *),
+    void *arg, int cpu, int flags)
 {
-#ifndef NO_EVENTTIMERS
-	sbintime_t sbt;
-#endif
 	int bucket;
 
 	CC_LOCK_ASSERT(cc);
-
-	/* update flags before swapping locks, if any */
-	c->c_flags &= ~(CALLOUT_PROCESSED | CALLOUT_DIRECT | CALLOUT_DEFRESTART);
-	if (coa->flags & C_DIRECT_EXEC)
-		c->c_flags |= (CALLOUT_ACTIVE | CALLOUT_PENDING | CALLOUT_DIRECT);
-	else
-		c->c_flags |= (CALLOUT_ACTIVE | CALLOUT_PENDING);
-
-#ifdef SMP
-	/*
-	 * Check if we are changing the CPU on which the callback
-	 * should be executed and if we have a lock protecting us:
-	 */
-	if (can_swap_cpu != false && coa->cpu != c->c_cpu &&
-	    callout_lock_owned_client(c->c_flags, c->c_lock) != 0) {
-		CC_UNLOCK(cc);
-		c->c_cpu = coa->cpu;
-		cc = callout_lock(c);
-	}
-#endif
-	if (coa->time < cc->cc_lastscan)
-		coa->time = cc->cc_lastscan;
-	c->c_arg = coa->arg;
-	c->c_func = coa->func;
-	c->c_time = coa->time;
-	c->c_precision = coa->precision;
-
+	if (sbt < cc->cc_lastscan)
+		sbt = cc->cc_lastscan;
+	c->c_arg = arg;
+	c->c_flags |= (CALLOUT_ACTIVE | CALLOUT_PENDING);
+	if (flags & C_DIRECT_EXEC)
+		c->c_flags |= CALLOUT_DIRECT;
+	c->c_flags &= ~CALLOUT_PROCESSED;
+	c->c_func = func;
+	c->c_time = sbt;
+	c->c_precision = precision;
 	bucket = callout_get_bucket(c->c_time);
 	CTR3(KTR_CALLOUT, "precision set for %p: %d.%08x",
 	    c, (int)(c->c_precision >> 32),
 	    (u_int)(c->c_precision & 0xffffffff));
 	LIST_INSERT_HEAD(&cc->cc_callwheel[bucket], c, c_links.le);
-
-	/* Ensure we are first to be scanned, if called via a callback */
 	if (cc->cc_bucket == bucket)
 		cc->cc_exec_next_dir = c;
 #ifndef NO_EVENTTIMERS
@@ -700,16 +617,17 @@ callout_cc_add_locked(struct callout *c, struct callout_cpu *cc,
 	sbt = c->c_time + c->c_precision;
 	if (sbt < cc->cc_firstevent) {
 		cc->cc_firstevent = sbt;
-		cpu_new_callout(coa->cpu, sbt, c->c_time);
+		cpu_new_callout(cpu, sbt, c->c_time);
 	}
 #endif
-	return (cc);
 }
 
 static void
 callout_cc_del(struct callout *c, struct callout_cpu *cc)
 {
 
+	if ((c->c_flags & CALLOUT_LOCAL_ALLOC) == 0)
+		return;
 	c->c_func = NULL;
 	SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);
 }
@@ -721,10 +639,20 @@ softclock_call_cc(struct callout *c, struct callout_cpu *cc,
 #endif
     int direct)
 {
-	callout_func_t *c_func;
+	struct rm_priotracker tracker;
+	void (*c_func)(void *);
 	void *c_arg;
+	struct lock_class *class;
 	struct lock_object *c_lock;
+	uintptr_t lock_status;
 	int c_flags;
+#ifdef SMP
+	struct callout_cpu *new_cc;
+	void (*new_func)(void *);
+	void *new_arg;
+	int flags, new_cpu;
+	sbintime_t new_prec, new_time;
+#endif
 #if defined(DIAGNOSTIC) || defined(CALLOUT_PROFILING) 
 	sbintime_t sbt1, sbt2;
 	struct timespec ts2;
@@ -735,39 +663,37 @@ softclock_call_cc(struct callout *c, struct callout_cpu *cc,
 	KASSERT((c->c_flags & (CALLOUT_PENDING | CALLOUT_ACTIVE)) ==
 	    (CALLOUT_PENDING | CALLOUT_ACTIVE),
 	    ("softclock_call_cc: pend|act %p %x", c, c->c_flags));
+	class = (c->c_lock != NULL) ? LOCK_CLASS(c->c_lock) : NULL;
+	lock_status = 0;
+	if (c->c_flags & CALLOUT_SHAREDLOCK) {
+		if (class == &lock_class_rm)
+			lock_status = (uintptr_t)&tracker;
+		else
+			lock_status = 1;
+	}
 	c_lock = c->c_lock;
 	c_func = c->c_func;
 	c_arg = c->c_arg;
 	c_flags = c->c_flags;
-
-	/* remove pending bit */
-	c->c_flags &= ~CALLOUT_PENDING;
-
-	/* reset our local state */
+	if (c->c_flags & CALLOUT_LOCAL_ALLOC)
+		c->c_flags = CALLOUT_LOCAL_ALLOC;
+	else
+		c->c_flags &= ~CALLOUT_PENDING;
 	cc->cc_exec_entity[direct].cc_curr = c;
-	cc->cc_exec_entity[direct].cc_restart = false;
-	cc->cc_exec_entity[direct].cc_drain_fn = NULL;
-	cc->cc_exec_entity[direct].cc_drain_arg = NULL;
-
+	cc->cc_exec_entity[direct].cc_cancel = false;
+	CC_UNLOCK(cc);
 	if (c_lock != NULL) {
-		cc->cc_exec_entity[direct].cc_cancel = false;
-		CC_UNLOCK(cc);
-
-		/* unlocked region for switching locks */
-
-		callout_lock_client(c_flags, c_lock);
-
+		class->lc_lock(c_lock, lock_status);
 		/*
-		 * Check if the callout may have been cancelled while
-		 * we were switching locks. Even though the callout is
-		 * specifying a lock, it might not be certain this
-		 * lock is locked when starting and stopping callouts.
+		 * The callout may have been cancelled
+		 * while we switched locks.
 		 */
-		CC_LOCK(cc);
 		if (cc->cc_exec_entity[direct].cc_cancel) {
-			callout_unlock_client(c_flags, c_lock);
-			goto skip_cc_locked;
+			class->lc_unlock(c_lock);
+			goto skip;
 		}
+		/* The callout cannot be stopped now. */
+		cc->cc_exec_entity[direct].cc_cancel = true;
 		if (c_lock == &Giant.lock_object) {
 #ifdef CALLOUT_PROFILING
 			(*gcalls)++;
@@ -788,11 +714,6 @@ softclock_call_cc(struct callout *c, struct callout_cpu *cc,
 		CTR3(KTR_CALLOUT, "callout %p func %p arg %p",
 		    c, c_func, c_arg);
 	}
-	/* The callout cannot be stopped now! */
-	cc->cc_exec_entity[direct].cc_cancel = true;
-	CC_UNLOCK(cc);
-
-	/* unlocked region */
 	KTR_STATE3(KTR_SCHED, "callout", cc->cc_ktr_event_name, "running",
 	    "func:%p", c_func, "arg:%p", c_arg, "direct:%d", direct);
 #if defined(DIAGNOSTIC) || defined(CALLOUT_PROFILING)
@@ -819,40 +740,85 @@ softclock_call_cc(struct callout *c, struct callout_cpu *cc,
 #endif
 	KTR_STATE0(KTR_SCHED, "callout", cc->cc_ktr_event_name, "idle");
 	CTR1(KTR_CALLOUT, "callout %p finished", c);
-
-	/*
-	 * At this point the callback structure might have been freed,
-	 * so we need to check the previously copied value of
-	 * "c->c_flags":
-	 */
 	if ((c_flags & CALLOUT_RETURNUNLOCKED) == 0)
-		callout_unlock_client(c_flags, c_lock);
-
+		class->lc_unlock(c_lock);
+skip:
 	CC_LOCK(cc);
-
-skip_cc_locked:
 	KASSERT(cc->cc_exec_entity[direct].cc_curr == c, ("mishandled cc_curr"));
 	cc->cc_exec_entity[direct].cc_curr = NULL;
-
-	/* Check if there is anything which needs draining */
-	if (cc->cc_exec_entity[direct].cc_drain_fn != NULL) {
+	if (cc->cc_exec_entity[direct].cc_waiting) {
 		/*
-		 * Unlock the CPU callout last, so that any use of
-		 * structures belonging to the callout are complete:
+		 * There is someone waiting for the
+		 * callout to complete.
+		 * If the callout was scheduled for
+		 * migration just cancel it.
 		 */
+		if (cc_cce_migrating(cc, direct)) {
+			cc_cce_cleanup(cc, direct);
+
+			/*
+			 * It should be assert here that the callout is not
+			 * destroyed but that is not easy.
+			 */
+			c->c_flags &= ~CALLOUT_DFRMIGRATION;
+		}
+		cc->cc_exec_entity[direct].cc_waiting = false;
 		CC_UNLOCK(cc);
-		/* call drain function unlocked */
-		cc->cc_exec_entity[direct].cc_drain_fn(
-		    cc->cc_exec_entity[direct].cc_drain_arg);
+		wakeup(&cc->cc_exec_entity[direct].cc_waiting);
 		CC_LOCK(cc);
-	} else if (c_flags & CALLOUT_LOCAL_ALLOC) {
-		/* return callout back to freelist */
-		callout_cc_del(c, cc);
-	} else if (cc->cc_exec_entity[direct].cc_restart) {
-		/* [re-]schedule callout, if any */
-		cc = callout_cc_add_locked(c, cc,
-		    &cc->cc_exec_entity[direct].cc_restart_args, false);
+	} else if (cc_cce_migrating(cc, direct)) {
+		KASSERT((c_flags & CALLOUT_LOCAL_ALLOC) == 0,
+		    ("Migrating legacy callout %p", c));
+#ifdef SMP
+		/*
+		 * If the callout was scheduled for
+		 * migration just perform it now.
+		 */
+		new_cpu = cc->cc_exec_entity[direct].ce_migration_cpu;
+		new_time = cc->cc_exec_entity[direct].ce_migration_time;
+		new_prec = cc->cc_exec_entity[direct].ce_migration_prec;
+		new_func = cc->cc_exec_entity[direct].ce_migration_func;
+		new_arg = cc->cc_exec_entity[direct].ce_migration_arg;
+		cc_cce_cleanup(cc, direct);
+
+		/*
+		 * It should be assert here that the callout is not destroyed
+		 * but that is not easy.
+		 *
+		 * As first thing, handle deferred callout stops.
+		 */
+		if ((c->c_flags & CALLOUT_DFRMIGRATION) == 0) {
+			CTR3(KTR_CALLOUT,
+			     "deferred cancelled %p func %p arg %p",
+			     c, new_func, new_arg);
+			callout_cc_del(c, cc);
+			return;
+		}
+		c->c_flags &= ~CALLOUT_DFRMIGRATION;
+
+		new_cc = callout_cpu_switch(c, cc, new_cpu);
+		flags = (direct) ? C_DIRECT_EXEC : 0;
+		callout_cc_add(c, new_cc, new_time, new_prec, new_func,
+		    new_arg, new_cpu, flags);
+		CC_UNLOCK(new_cc);
+		CC_LOCK(cc);
+#else
+		panic("migration should not happen");
+#endif
 	}
+	/*
+	 * If the current callout is locally allocated (from
+	 * timeout(9)) then put it on the freelist.
+	 *
+	 * Note: we need to check the cached copy of c_flags because
+	 * if it was not local, then it's not safe to deref the
+	 * callout pointer.
+	 */
+	KASSERT((c_flags & CALLOUT_LOCAL_ALLOC) == 0 ||
+	    c->c_flags == CALLOUT_LOCAL_ALLOC,
+	    ("corrupted callout"));
+	if (c_flags & CALLOUT_LOCAL_ALLOC)
+		callout_cc_del(c, cc);
 }
 
 /*
@@ -933,11 +899,10 @@ timeout(timeout_t *ftn, void *arg, int to_ticks)
 		/* XXX Attempt to malloc first */
 		panic("timeout table full");
 	SLIST_REMOVE_HEAD(&cc->cc_callfree, c_links.sle);
+	callout_reset(new, to_ticks, ftn, arg);
 	handle.callout = new;
 	CC_UNLOCK(cc);
 
-	callout_reset(new, to_ticks, ftn, arg);
-
 	return (handle);
 }
 
@@ -945,7 +910,6 @@ void
 untimeout(timeout_t *ftn, void *arg, struct callout_handle handle)
 {
 	struct callout_cpu *cc;
-	bool match;
 
 	/*
 	 * Check for a handle that was initialized
@@ -956,11 +920,9 @@ untimeout(timeout_t *ftn, void *arg, struct callout_handle handle)
 		return;
 
 	cc = callout_lock(handle.callout);
-	match = (handle.callout->c_func == ftn && handle.callout->c_arg == arg);
-	CC_UNLOCK(cc);
-
-	if (match)
+	if (handle.callout->c_func == ftn && handle.callout->c_arg == arg)
 		callout_stop(handle.callout);
+	CC_UNLOCK(cc);
 }
 
 void
@@ -969,119 +931,6 @@ callout_handle_init(struct callout_handle *handle)
 	handle->callout = NULL;
 }
 
-static int
-callout_restart_async(struct callout *c, struct callout_args *coa,
-    callout_func_t *drain_fn, void *drain_arg)
-{
-	struct callout_cpu *cc;
-	int cancelled;
-	int direct;
-
-	cc = callout_lock(c);
-
-	/* Figure out if the callout is direct or not */
-	direct = ((c->c_flags & CALLOUT_DIRECT) != 0);
-
-	/*
-	 * Check if the callback is currently scheduled for
-	 * completion:
-	 */
-	if (cc->cc_exec_entity[direct].cc_curr == c) {
-		/*
-		 * Try to prevent the callback from running by setting
-		 * the "cc_cancel" variable to "true". Also check if
-		 * the callout was previously subject to a deferred
-		 * callout restart:
-		 */
-		if (cc->cc_exec_entity[direct].cc_cancel == false ||
-		    (c->c_flags & CALLOUT_DEFRESTART) != 0) {
-			cc->cc_exec_entity[direct].cc_cancel = true;
-			cancelled = 1;
-		} else {
-			cancelled = 0;
-		}
-
-		/*
-		 * Prevent callback restart if "callout_drain_xxx()"
-		 * is being called or we are stopping the callout or
-		 * the callback was preallocated by us:
-		 */
-		if (cc->cc_exec_entity[direct].cc_drain_fn != NULL ||
-		    coa == NULL || (c->c_flags & CALLOUT_LOCAL_ALLOC) != 0) {
-			CTR4(KTR_CALLOUT, "%s %p func %p arg %p",
-			    cancelled ? "cancelled and draining" : "draining",
-			    c, c->c_func, c->c_arg);
-
-			/* clear old flags, if any */
-			c->c_flags &= ~(CALLOUT_ACTIVE | CALLOUT_PENDING |
-			    CALLOUT_DEFRESTART | CALLOUT_PROCESSED);
-
-			/* clear restart flag, if any */
-			cc->cc_exec_entity[direct].cc_restart = false;
-
-			/* set drain function, if any */
-			if (drain_fn != NULL) {
-				cc->cc_exec_entity[direct].cc_drain_fn = drain_fn;
-				cc->cc_exec_entity[direct].cc_drain_arg = drain_arg;
-				cancelled |= 2;		/* XXX define the value */
-			}
-		} else {
-			CTR4(KTR_CALLOUT, "%s %p func %p arg %p",
-			    cancelled ? "cancelled and restarting" : "restarting",
-			    c, c->c_func, c->c_arg);
-
-			/* get us back into the game */
-			c->c_flags |= (CALLOUT_ACTIVE | CALLOUT_PENDING |
-			    CALLOUT_DEFRESTART);
-			c->c_flags &= ~CALLOUT_PROCESSED;
-
-			/* enable deferred restart */
-			cc->cc_exec_entity[direct].cc_restart = true;
-
-			/* store arguments for the deferred restart, if any */
-			cc->cc_exec_entity[direct].cc_restart_args = *coa;
-		}
-	} else {
-		/* stop callout */
-		if (c->c_flags & CALLOUT_PENDING) {
-			/*
-			 * The callback has not yet been executed, and
-			 * we simply just need to unlink it:
-			 */
-			if ((c->c_flags & CALLOUT_PROCESSED) == 0) {
-				if (cc->cc_exec_next_dir == c)
-					cc->cc_exec_next_dir = LIST_NEXT(c, c_links.le);
-				LIST_REMOVE(c, c_links.le);
-			} else {
-				TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
-			}
-			cancelled = 1;
-		} else {
-			cancelled = 0;
-		}
-
-		CTR4(KTR_CALLOUT, "%s %p func %p arg %p",
-		    cancelled ? "rescheduled" : "scheduled",
-		    c, c->c_func, c->c_arg);
-
-		/* [re-]schedule callout, if any */
-		if (coa != NULL) {
-			cc = callout_cc_add_locked(c, cc, coa, true);
-		} else {
-			/* clear old flags, if any */
-			c->c_flags &= ~(CALLOUT_ACTIVE | CALLOUT_PENDING |
-			    CALLOUT_DEFRESTART | CALLOUT_PROCESSED);
-
-			/* return callback to pre-allocated list, if any */
-			if ((c->c_flags & CALLOUT_LOCAL_ALLOC) && cancelled != 0) {
-				callout_cc_del(c, cc);
-			}
-		}
-	}
-	CC_UNLOCK(cc);
-	return (cancelled);
-}
-
 /*
  * New interface; clients allocate their own callout structures.
  *
@@ -1100,32 +949,25 @@ callout_restart_async(struct callout *c, struct callout_args *coa,
  */
 int
 callout_reset_sbt_on(struct callout *c, sbintime_t sbt, sbintime_t precision,
-    callout_func_t *ftn, void *arg, int cpu, int flags)
+    void (*ftn)(void *), void *arg, int cpu, int flags)
 {
-	struct callout_args coa;
+	sbintime_t to_sbt, pr;
+	struct callout_cpu *cc;
+	int cancelled, direct;
 
-	/* store arguments for callout add function */
-	coa.func = ftn;
-	coa.arg = arg;
-	coa.precision = precision;
-	coa.flags = flags;
-	coa.cpu = cpu;
-
-	/* compute the rest of the arguments needed */
-	if (coa.flags & C_ABSOLUTE) {
-		coa.time = sbt;
+	cancelled = 0;
+	if (flags & C_ABSOLUTE) {
+		to_sbt = sbt;
 	} else {
-		sbintime_t pr;
-
-		if ((coa.flags & C_HARDCLOCK) && (sbt < tick_sbt))
+		if ((flags & C_HARDCLOCK) && (sbt < tick_sbt))
 			sbt = tick_sbt;
-		if ((coa.flags & C_HARDCLOCK) ||
+		if ((flags & C_HARDCLOCK) ||
 #ifdef NO_EVENTTIMERS
 		    sbt >= sbt_timethreshold) {
-			coa.time = getsbinuptime();
+			to_sbt = getsbinuptime();
 
 			/* Add safety belt for the case of hz > 1000. */
-			coa.time += tc_tick_sbt - tick_sbt;
+			to_sbt += tc_tick_sbt - tick_sbt;
 #else
 		    sbt >= sbt_tickthreshold) {
 			/*
@@ -1135,29 +977,101 @@ callout_reset_sbt_on(struct callout *c, sbintime_t sbt, sbintime_t precision,
 			 * active ones.
 			 */
 #ifdef __LP64__
-			coa.time = DPCPU_GET(hardclocktime);
+			to_sbt = DPCPU_GET(hardclocktime);
 #else
 			spinlock_enter();
-			coa.time = DPCPU_GET(hardclocktime);
+			to_sbt = DPCPU_GET(hardclocktime);
 			spinlock_exit();
 #endif
 #endif
-			if ((coa.flags & C_HARDCLOCK) == 0)
-				coa.time += tick_sbt;
+			if ((flags & C_HARDCLOCK) == 0)
+				to_sbt += tick_sbt;
 		} else
-			coa.time = sbinuptime();
-		if (SBT_MAX - coa.time < sbt)
-			coa.time = SBT_MAX;
+			to_sbt = sbinuptime();
+		if (SBT_MAX - to_sbt < sbt)
+			to_sbt = SBT_MAX;
 		else
-			coa.time += sbt;
-		pr = ((C_PRELGET(coa.flags) < 0) ? sbt >> tc_precexp :
-		    sbt >> C_PRELGET(coa.flags));
-		if (pr > coa.precision)
-			coa.precision = pr;
+			to_sbt += sbt;
+		pr = ((C_PRELGET(flags) < 0) ? sbt >> tc_precexp :
+		    sbt >> C_PRELGET(flags));
+		if (pr > precision)
+			precision = pr;
+	}
+	/*
+	 * Don't allow migration of pre-allocated callouts lest they
+	 * become unbalanced.
+	 */
+	if (c->c_flags & CALLOUT_LOCAL_ALLOC)
+		cpu = c->c_cpu;
+	direct = (c->c_flags & CALLOUT_DIRECT) != 0;
+	KASSERT(!direct || c->c_lock == NULL,
+	    ("%s: direct callout %p has lock", __func__, c));
+	cc = callout_lock(c);
+	if (cc->cc_exec_entity[direct].cc_curr == c) {
+		/*
+		 * We're being asked to reschedule a callout which is
+		 * currently in progress.  If there is a lock then we
+		 * can cancel the callout if it has not really started.
+		 */
+		if (c->c_lock != NULL && !cc->cc_exec_entity[direct].cc_cancel)
+			cancelled = cc->cc_exec_entity[direct].cc_cancel = true;
+		if (cc->cc_exec_entity[direct].cc_waiting) {
+			/*
+			 * Someone has called callout_drain to kill this
+			 * callout.  Don't reschedule.
+			 */
+			CTR4(KTR_CALLOUT, "%s %p func %p arg %p",
+			    cancelled ? "cancelled" : "failed to cancel",
+			    c, c->c_func, c->c_arg);
+			CC_UNLOCK(cc);
+			return (cancelled);
+		}
+	}
+	if (c->c_flags & CALLOUT_PENDING) {
+		if ((c->c_flags & CALLOUT_PROCESSED) == 0) {
+			if (cc->cc_exec_next_dir == c)
+				cc->cc_exec_next_dir = LIST_NEXT(c, c_links.le);
+			LIST_REMOVE(c, c_links.le);
+		} else
+			TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
+		cancelled = 1;
+		c->c_flags &= ~(CALLOUT_ACTIVE | CALLOUT_PENDING);
 	}
 
-	/* get callback started, if any */
-	return (callout_restart_async(c, &coa, NULL, NULL));
+#ifdef SMP
+	/*
+	 * If the callout must migrate try to perform it immediately.
+	 * If the callout is currently running, just defer the migration
+	 * to a more appropriate moment.
+	 */
+	if (c->c_cpu != cpu) {
+		if (cc->cc_exec_entity[direct].cc_curr == c) {
+			cc->cc_exec_entity[direct].ce_migration_cpu = cpu;
+			cc->cc_exec_entity[direct].ce_migration_time
+			    = to_sbt;
+			cc->cc_exec_entity[direct].ce_migration_prec 
+			    = precision;
+			cc->cc_exec_entity[direct].ce_migration_func = ftn;
+			cc->cc_exec_entity[direct].ce_migration_arg = arg;
+			c->c_flags |= CALLOUT_DFRMIGRATION;
+			CTR6(KTR_CALLOUT,
+		    "migration of %p func %p arg %p in %d.%08x to %u deferred",
+			    c, c->c_func, c->c_arg, (int)(to_sbt >> 32),
+			    (u_int)(to_sbt & 0xffffffff), cpu);
+			CC_UNLOCK(cc);
+			return (cancelled);
+		}
+		cc = callout_cpu_switch(c, cc, cpu);
+	}
+#endif
+
+	callout_cc_add(c, cc, to_sbt, precision, ftn, arg, cpu, flags);
+	CTR6(KTR_CALLOUT, "%sscheduled %p func %p arg %p in %d.%08x",
+	    cancelled ? "re" : "", c, c->c_func, c->c_arg, (int)(to_sbt >> 32),
+	    (u_int)(to_sbt & 0xffffffff));
+	CC_UNLOCK(cc);
+
+	return (cancelled);
 }
 
 /*
@@ -1176,105 +1090,204 @@ callout_schedule(struct callout *c, int to_ticks)
 }
 
 int
-callout_stop(struct callout *c)
+_callout_stop_safe(struct callout *c, int safe)
 {
-	/* get callback stopped, if any */
-	return (callout_restart_async(c, NULL, NULL, NULL));
-}
+	struct callout_cpu *cc, *old_cc;
+	struct lock_class *class;
+	int direct, sq_locked, use_lock;
 
-static void
-callout_drain_function(void *arg)
-{
-	wakeup(arg);
-}
+	/*
+	 * Some old subsystems don't hold Giant while running a callout_stop(),
+	 * so just discard this check for the moment.
+	 */
+	if (!safe && c->c_lock != NULL) {
+		if (c->c_lock == &Giant.lock_object)
+			use_lock = mtx_owned(&Giant);
+		else {
+			use_lock = 1;
+			class = LOCK_CLASS(c->c_lock);
+			class->lc_assert(c->c_lock, LA_XLOCKED);
+		}
+	} else
+		use_lock = 0;
+	direct = (c->c_flags & CALLOUT_DIRECT) != 0;
+	sq_locked = 0;
+	old_cc = NULL;
+again:
+	cc = callout_lock(c);
 
-int
-callout_drain_async(struct callout *c, callout_func_t *fn, void *arg)
-{
-	/* get callback stopped, if any */
-	return (callout_restart_async(c, NULL, fn, arg) & 2);
-}
+	/*
+	 * If the callout was migrating while the callout cpu lock was
+	 * dropped,  just drop the sleepqueue lock and check the states
+	 * again.
+	 */
+	if (sq_locked != 0 && cc != old_cc) {
+#ifdef SMP
+		CC_UNLOCK(cc);
+		sleepq_release(&old_cc->cc_exec_entity[direct].cc_waiting);
+		sq_locked = 0;
+		old_cc = NULL;
+		goto again;
+#else
+		panic("migration should not happen");
+#endif
+	}
 
-int
-callout_drain(struct callout *c)
-{
-	int cancelled;
-
-	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
-	    "Draining callout");
-
-	callout_lock_client(c->c_flags, c->c_lock);
-
-	/* at this point the "c->c_cpu" field is not changing */
-
-	cancelled = callout_drain_async(c, &callout_drain_function, c);
-
-	if (cancelled != 0) {
-		struct callout_cpu *cc;
-		int direct;
-
-		CTR3(KTR_CALLOUT, "need to drain %p func %p arg %p",
-		    c, c->c_func, c->c_arg);
-
-		cc = callout_lock(c);
-		direct = ((c->c_flags & CALLOUT_DIRECT) != 0);
+	/*
+	 * If the callout isn't pending, it's not on the queue, so
+	 * don't attempt to remove it from the queue.  We can try to
+	 * stop it by other means however.
+	 */
+	if (!(c->c_flags & CALLOUT_PENDING)) {
+		c->c_flags &= ~CALLOUT_ACTIVE;
 
 		/*
-		 * We've gotten our callout CPU lock, it is safe to
-		 * drop the initial lock:
+		 * If it wasn't on the queue and it isn't the current
+		 * callout, then we can't stop it, so just bail.
 		 */
-		callout_unlock_client(c->c_flags, c->c_lock);
+		if (cc->cc_exec_entity[direct].cc_curr != c) {
+			CTR3(KTR_CALLOUT, "failed to stop %p func %p arg %p",
+			    c, c->c_func, c->c_arg);
+			CC_UNLOCK(cc);
+			if (sq_locked)
+				sleepq_release(
+				    &cc->cc_exec_entity[direct].cc_waiting);
+			return (0);
+		}
 
-		/* Wait for drain to complete */
+		if (safe) {
+			/*
+			 * The current callout is running (or just
+			 * about to run) and blocking is allowed, so
+			 * just wait for the current invocation to
+			 * finish.
+			 */
+			while (cc->cc_exec_entity[direct].cc_curr == c) {
+				/*
+				 * Use direct calls to sleepqueue interface
+				 * instead of cv/msleep in order to avoid
+				 * a LOR between cc_lock and sleepqueue
+				 * chain spinlocks.  This piece of code
+				 * emulates a msleep_spin() call actually.
+				 *
+				 * If we already have the sleepqueue chain
+				 * locked, then we can safely block.  If we
+				 * don't already have it locked, however,
+				 * we have to drop the cc_lock to lock
+				 * it.  This opens several races, so we
+				 * restart at the beginning once we have
+				 * both locks.  If nothing has changed, then
+				 * we will end up back here with sq_locked
+				 * set.
+				 */
+				if (!sq_locked) {
+					CC_UNLOCK(cc);
+					sleepq_lock(
+					&cc->cc_exec_entity[direct].cc_waiting);
+					sq_locked = 1;
+					old_cc = cc;
+					goto again;
+				}
 
-		while (cc->cc_exec_entity[direct].cc_curr == c)
-			msleep_spin(c, (struct mtx *)&cc->cc_lock, "codrain", 0);
+				/*
+				 * Migration could be cancelled here, but
+				 * as long as it is still not sure when it
+				 * will be packed up, just let softclock()
+				 * take care of it.
+				 */
+				cc->cc_exec_entity[direct].cc_waiting = true;
+				DROP_GIANT();
+				CC_UNLOCK(cc);
+				sleepq_add(
+				    &cc->cc_exec_entity[direct].cc_waiting,
+				    &cc->cc_lock.lock_object, "codrain",
+				    SLEEPQ_SLEEP, 0);
+				sleepq_wait(
+				    &cc->cc_exec_entity[direct].cc_waiting,
+					     0);
+				sq_locked = 0;
+				old_cc = NULL;
 
+				/* Reacquire locks previously released. */
+				PICKUP_GIANT();
+				CC_LOCK(cc);
+			}
+		} else if (use_lock &&
+			    !cc->cc_exec_entity[direct].cc_cancel) {
+			/*
+			 * The current callout is waiting for its
+			 * lock which we hold.  Cancel the callout
+			 * and return.  After our caller drops the
+			 * lock, the callout will be skipped in
+			 * softclock().
+			 */
+			cc->cc_exec_entity[direct].cc_cancel = true;
+			CTR3(KTR_CALLOUT, "cancelled %p func %p arg %p",
+			    c, c->c_func, c->c_arg);
+			KASSERT(!cc_cce_migrating(cc, direct),
+			    ("callout wrongly scheduled for migration"));
+			CC_UNLOCK(cc);
+			KASSERT(!sq_locked, ("sleepqueue chain locked"));
+			return (1);
+		} else if ((c->c_flags & CALLOUT_DFRMIGRATION) != 0) {
+			c->c_flags &= ~CALLOUT_DFRMIGRATION;
+			CTR3(KTR_CALLOUT, "postponing stop %p func %p arg %p",
+			    c, c->c_func, c->c_arg);
+			CC_UNLOCK(cc);
+			return (1);
+		}
+		CTR3(KTR_CALLOUT, "failed to stop %p func %p arg %p",
+		    c, c->c_func, c->c_arg);
 		CC_UNLOCK(cc);
-	} else {
-		callout_unlock_client(c->c_flags, c->c_lock);
+		KASSERT(!sq_locked, ("sleepqueue chain still locked"));
+		return (0);
 	}
+	if (sq_locked)
+		sleepq_release(&cc->cc_exec_entity[direct].cc_waiting);
+
+	c->c_flags &= ~(CALLOUT_ACTIVE | CALLOUT_PENDING);
 
 	CTR3(KTR_CALLOUT, "cancelled %p func %p arg %p",
 	    c, c->c_func, c->c_arg);
+	if ((c->c_flags & CALLOUT_PROCESSED) == 0) {
+		if (cc->cc_exec_next_dir == c)
+			cc->cc_exec_next_dir = LIST_NEXT(c, c_links.le);
+		LIST_REMOVE(c, c_links.le);
+	} else
+		TAILQ_REMOVE(&cc->cc_expireq, c, c_links.tqe);
+	callout_cc_del(c, cc);
 
-	return (cancelled & 1);
+	CC_UNLOCK(cc);
+	return (1);
 }
 
 void
 callout_init(struct callout *c, int mpsafe)
 {
+	bzero(c, sizeof *c);
 	if (mpsafe) {
-		_callout_init_lock(c, NULL, CALLOUT_RETURNUNLOCKED);
+		c->c_lock = NULL;
+		c->c_flags = CALLOUT_RETURNUNLOCKED;
 	} else {
-		_callout_init_lock(c, &Giant.lock_object, 0);
+		c->c_lock = &Giant.lock_object;
+		c->c_flags = 0;
 	}
+	c->c_cpu = timeout_cpu;
 }
 
 void
 _callout_init_lock(struct callout *c, struct lock_object *lock, int flags)
 {
 	bzero(c, sizeof *c);
-	KASSERT((flags & ~CALLOUT_RETURNUNLOCKED) == 0,
-	    ("callout_init_lock: bad flags 0x%08x", flags));
-	flags &= CALLOUT_RETURNUNLOCKED;
-	if (lock != NULL) {
-		struct lock_class *class = LOCK_CLASS(lock);
-		if (class == &lock_class_mtx_sleep)
-			flags |= CALLOUT_SET_LC(CALLOUT_LC_MUTEX);
-		else if (class == &lock_class_mtx_spin)
-			flags |= CALLOUT_SET_LC(CALLOUT_LC_SPIN);
-		else if (class == &lock_class_rm)
-			flags |= CALLOUT_SET_LC(CALLOUT_LC_RM);
-		else if (class == &lock_class_rw)
-			flags |= CALLOUT_SET_LC(CALLOUT_LC_RW);
-		else
-			panic("callout_init_lock: Unsupported lock class '%s'\n", class->lc_name);
-	} else {
-		flags |= CALLOUT_SET_LC(CALLOUT_LC_UNUSED_0);
-	}
 	c->c_lock = lock;
-	c->c_flags = flags;
+	KASSERT((flags & ~(CALLOUT_RETURNUNLOCKED | CALLOUT_SHAREDLOCK)) == 0,
+	    ("callout_init_lock: bad flags %d", flags));
+	KASSERT(lock != NULL || (flags & CALLOUT_RETURNUNLOCKED) == 0,
+	    ("callout_init_lock: CALLOUT_RETURNUNLOCKED with no lock"));
+	KASSERT(lock == NULL || !(LOCK_CLASS(lock)->lc_flags &
+	    (LC_SPINLOCK | LC_SLEEPABLE)), ("%s: invalid lock class",
+	    __func__));
+	c->c_flags = flags & (CALLOUT_RETURNUNLOCKED | CALLOUT_SHAREDLOCK);
 	c->c_cpu = timeout_cpu;
 }
 
diff --git a/sys/kern/subr_sleepqueue.c b/sys/kern/subr_sleepqueue.c
index 18dc2a058f5..bbbec920ef0 100644
--- a/sys/kern/subr_sleepqueue.c
+++ b/sys/kern/subr_sleepqueue.c
@@ -152,8 +152,7 @@ static uma_zone_t sleepq_zone;
  */
 static int	sleepq_catch_signals(void *wchan, int pri);
 static int	sleepq_check_signals(void);
-static int	sleepq_check_timeout(struct thread *);
-static void	sleepq_stop_timeout(struct thread *);
+static int	sleepq_check_timeout(void);
 #ifdef INVARIANTS
 static void	sleepq_dtor(void *mem, int size, void *arg);
 #endif
@@ -374,14 +373,17 @@ void
 sleepq_set_timeout_sbt(void *wchan, sbintime_t sbt, sbintime_t pr,
     int flags)
 {
+	struct sleepqueue_chain *sc;
 	struct thread *td;
 
 	td = curthread;
-
-	mtx_lock_spin(&td->td_slpmutex);
+	sc = SC_LOOKUP(wchan);
+	mtx_assert(&sc->sc_lock, MA_OWNED);
+	MPASS(TD_ON_SLEEPQ(td));
+	MPASS(td->td_sleepqueue == NULL);
+	MPASS(wchan != NULL);
 	callout_reset_sbt_on(&td->td_slpcallout, sbt, pr,
 	    sleepq_timeout, td, PCPU_GET(cpuid), flags | C_DIRECT_EXEC);
-	mtx_unlock_spin(&td->td_slpmutex);
 }
 
 /*
@@ -557,8 +559,11 @@ sleepq_switch(void *wchan, int pri)
  * Check to see if we timed out.
  */
 static int
-sleepq_check_timeout(struct thread *td)
+sleepq_check_timeout(void)
 {
+	struct thread *td;
+
+	td = curthread;
 	THREAD_LOCK_ASSERT(td, MA_OWNED);
 
 	/*
@@ -568,18 +573,25 @@ sleepq_check_timeout(struct thread *td)
 		td->td_flags &= ~TDF_TIMEOUT;
 		return (EWOULDBLOCK);
 	}
-	return (0);
-}
 
-/*
- * Atomically stop the timeout by using a mutex.
- */
-static void
-sleepq_stop_timeout(struct thread *td)
-{
-	mtx_lock_spin(&td->td_slpmutex);
-	callout_stop(&td->td_slpcallout);
-	mtx_unlock_spin(&td->td_slpmutex);
+	/*
+	 * If TDF_TIMOFAIL is set, the timeout ran after we had
+	 * already been woken up.
+	 */
+	if (td->td_flags & TDF_TIMOFAIL)
+		td->td_flags &= ~TDF_TIMOFAIL;
+
+	/*
+	 * If callout_stop() fails, then the timeout is running on
+	 * another CPU, so synchronize with it to avoid having it
+	 * accidentally wake up a subsequent sleep.
+	 */
+	else if (callout_stop(&td->td_slpcallout) == 0) {
+		td->td_flags |= TDF_TIMEOUT;
+		TD_SET_SLEEPING(td);
+		mi_switch(SW_INVOL | SWT_SLEEPQTIMO, NULL);
+	}
+	return (0);
 }
 
 /*
@@ -652,11 +664,9 @@ sleepq_timedwait(void *wchan, int pri)
 	MPASS(!(td->td_flags & TDF_SINTR));
 	thread_lock(td);
 	sleepq_switch(wchan, pri);
-	rval = sleepq_check_timeout(td);
+	rval = sleepq_check_timeout();
 	thread_unlock(td);
 
-	sleepq_stop_timeout(td);
-
 	return (rval);
 }
 
@@ -667,18 +677,12 @@ sleepq_timedwait(void *wchan, int pri)
 int
 sleepq_timedwait_sig(void *wchan, int pri)
 {
-	struct thread *td;
 	int rcatch, rvalt, rvals;
 
-	td = curthread;
-
 	rcatch = sleepq_catch_signals(wchan, pri);
-	rvalt = sleepq_check_timeout(td);
+	rvalt = sleepq_check_timeout();
 	rvals = sleepq_check_signals();
-	thread_unlock(td);
-
-	sleepq_stop_timeout(td);
-
+	thread_unlock(curthread);
 	if (rcatch)
 		return (rcatch);
 	if (rvals)
@@ -885,49 +889,64 @@ sleepq_broadcast(void *wchan, int flags, int pri, int queue)
 static void
 sleepq_timeout(void *arg)
 {
-	struct thread *td = arg;
-	int wakeup_swapper = 0;
+	struct sleepqueue_chain *sc;
+	struct sleepqueue *sq;
+	struct thread *td;
+	void *wchan;
+	int wakeup_swapper;
 
+	td = arg;
+	wakeup_swapper = 0;
 	CTR3(KTR_PROC, "sleepq_timeout: thread %p (pid %ld, %s)",
 	    (void *)td, (long)td->td_proc->p_pid, (void *)td->td_name);
 
-	/* Handle the three cases which can happen */
-
+	/*
+	 * First, see if the thread is asleep and get the wait channel if
+	 * it is.
+	 */
 	thread_lock(td);
-	if (TD_ON_SLEEPQ(td)) {
-		if (TD_IS_SLEEPING(td)) {
-			struct sleepqueue_chain *sc;
-			struct sleepqueue *sq;
-			void *wchan;
-
-			/*
-			 * Case I - thread is asleep and needs to be
-			 * awoken:
-			 */
-			wchan = td->td_wchan;
-			sc = SC_LOOKUP(wchan);
-			THREAD_LOCKPTR_ASSERT(td, &sc->sc_lock);
-			sq = sleepq_lookup(wchan);
-			MPASS(sq != NULL);
-			td->td_flags |= TDF_TIMEOUT;
-			wakeup_swapper = sleepq_resume_thread(sq, td, 0);
-		} else {
-			/*
-			 * Case II - cancel going to sleep by setting
-			 * the timeout flag because the target thread
-			 * is not asleep yet. It can be on another CPU
-			 * in between sleepq_add() and one of the
-			 * sleepq_*wait*() routines or it can be in
-			 * sleepq_catch_signals().
-			 */
-			td->td_flags |= TDF_TIMEOUT;
-		}
-	} else {
-		/*
-		 * Case III - thread is already woken up by a wakeup
-		 * call and should not timeout. Nothing to do!
-		 */
+	if (TD_IS_SLEEPING(td) && TD_ON_SLEEPQ(td)) {
+		wchan = td->td_wchan;
+		sc = SC_LOOKUP(wchan);
+		THREAD_LOCKPTR_ASSERT(td, &sc->sc_lock);
+		sq = sleepq_lookup(wchan);
+		MPASS(sq != NULL);
+		td->td_flags |= TDF_TIMEOUT;
+		wakeup_swapper = sleepq_resume_thread(sq, td, 0);
+		thread_unlock(td);
+		if (wakeup_swapper)
+			kick_proc0();
+		return;
 	}
+
+	/*
+	 * If the thread is on the SLEEPQ but isn't sleeping yet, it
+	 * can either be on another CPU in between sleepq_add() and
+	 * one of the sleepq_*wait*() routines or it can be in
+	 * sleepq_catch_signals().
+	 */
+	if (TD_ON_SLEEPQ(td)) {
+		td->td_flags |= TDF_TIMEOUT;
+		thread_unlock(td);
+		return;
+	}
+
+	/*
+	 * Now check for the edge cases.  First, if TDF_TIMEOUT is set,
+	 * then the other thread has already yielded to us, so clear
+	 * the flag and resume it.  If TDF_TIMEOUT is not set, then the
+	 * we know that the other thread is not on a sleep queue, but it
+	 * hasn't resumed execution yet.  In that case, set TDF_TIMOFAIL
+	 * to let it know that the timeout has already run and doesn't
+	 * need to be canceled.
+	 */
+	if (td->td_flags & TDF_TIMEOUT) {
+		MPASS(TD_IS_SLEEPING(td));
+		td->td_flags &= ~TDF_TIMEOUT;
+		TD_CLR_SLEEPING(td);
+		wakeup_swapper = setrunnable(td);
+	} else
+		td->td_flags |= TDF_TIMOFAIL;
 	thread_unlock(td);
 	if (wakeup_swapper)
 		kick_proc0();
diff --git a/sys/ofed/include/linux/completion.h b/sys/ofed/include/linux/completion.h
index 088828399c7..df4aec3595e 100644
--- a/sys/ofed/include/linux/completion.h
+++ b/sys/ofed/include/linux/completion.h
@@ -105,9 +105,7 @@ _wait_for_timeout_common(struct completion *c, long timeout, int flags)
 		if (c->done)
 			break;
 		sleepq_add(c, NULL, "completion", flags, 0);
-		sleepq_release(c);
 		sleepq_set_timeout(c, end - ticks);
-		sleepq_lock(c);
 		if (flags & SLEEPQ_INTERRUPTIBLE) {
 			if (sleepq_timedwait_sig(c, 0) != 0)
 				return (-ERESTARTSYS);
diff --git a/sys/sys/_callout.h b/sys/sys/_callout.h
index f58a3835c97..e186aecb8dd 100644
--- a/sys/sys/_callout.h
+++ b/sys/sys/_callout.h
@@ -46,30 +46,19 @@ LIST_HEAD(callout_list, callout);
 SLIST_HEAD(callout_slist, callout);
 TAILQ_HEAD(callout_tailq, callout);
 
-typedef void callout_func_t(void *);
-
-struct callout_args {
-	sbintime_t time;		/* absolute time for the event */
-	sbintime_t precision;		/* delta allowed wrt opt */
-	void	*arg;			/* function argument */
-	callout_func_t *func;		/* function to call */
-	int	flags;			/* flags passed to callout_reset() */
-	int	cpu;			/* CPU we're scheduled on */
-};
-
 struct callout {
 	union {
 		LIST_ENTRY(callout) le;
 		SLIST_ENTRY(callout) sle;
 		TAILQ_ENTRY(callout) tqe;
 	} c_links;
-	sbintime_t c_time;			/* absolute time for the event */
+	sbintime_t c_time;			/* ticks to the event */
 	sbintime_t c_precision;			/* delta allowed wrt opt */
 	void	*c_arg;				/* function argument */
-	callout_func_t *c_func;			/* function to call */
-	struct lock_object *c_lock;		/* callback lock */
+	void	(*c_func)(void *);		/* function to call */
+	struct lock_object *c_lock;		/* lock to handle */
 	int	c_flags;			/* state of this entry */
-	int	c_cpu;				/* CPU we're scheduled on */
+	volatile int c_cpu;			/* CPU we're scheduled on */
 };
 
 #endif
diff --git a/sys/sys/callout.h b/sys/sys/callout.h
index 235da99f039..1096cb26ff9 100644
--- a/sys/sys/callout.h
+++ b/sys/sys/callout.h
@@ -45,12 +45,10 @@
 #define	CALLOUT_PENDING		0x0004 /* callout is waiting for timeout */
 #define	CALLOUT_MPSAFE		0x0008 /* callout handler is mp safe */
 #define	CALLOUT_RETURNUNLOCKED	0x0010 /* handler returns with mtx unlocked */
-#define	CALLOUT_UNUSED_5	0x0020 /* --available-- */
-#define	CALLOUT_DEFRESTART	0x0040 /* callout restart is deferred */
+#define	CALLOUT_SHAREDLOCK	0x0020 /* callout lock held in shared mode */
+#define	CALLOUT_DFRMIGRATION	0x0040 /* callout in deferred migration mode */
 #define	CALLOUT_PROCESSED	0x0080 /* callout in wheel or processing list? */
 #define	CALLOUT_DIRECT 		0x0100 /* allow exec from hw int context */
-#define	CALLOUT_SET_LC(x)	(((x) & 7) << 16) /* set lock class */
-#define	CALLOUT_GET_LC(x)	(((x) >> 16) & 7) /* get lock class */
 
 #define	C_DIRECT_EXEC		0x0001 /* direct execution of callout */
 #define	C_PRELBITS		7
@@ -67,8 +65,7 @@ struct callout_handle {
 #ifdef _KERNEL
 #define	callout_active(c)	((c)->c_flags & CALLOUT_ACTIVE)
 #define	callout_deactivate(c)	((c)->c_flags &= ~CALLOUT_ACTIVE)
-int	callout_drain(struct callout *);
-int	callout_drain_async(struct callout *, callout_func_t *, void *);
+#define	callout_drain(c)	_callout_stop_safe(c, 1)
 void	callout_init(struct callout *, int);
 void	_callout_init_lock(struct callout *, struct lock_object *, int);
 #define	callout_init_mtx(c, mtx, flags)					\
@@ -82,7 +79,7 @@ void	_callout_init_lock(struct callout *, struct lock_object *, int);
 	   NULL, (flags))
 #define	callout_pending(c)	((c)->c_flags & CALLOUT_PENDING)
 int	callout_reset_sbt_on(struct callout *, sbintime_t, sbintime_t,
-	    callout_func_t *, void *, int, int);
+	    void (*)(void *), void *, int, int);
 #define	callout_reset_sbt(c, sbt, pr, fn, arg, flags)			\
     callout_reset_sbt_on((c), (sbt), (pr), (fn), (arg), (c)->c_cpu, (flags))
 #define	callout_reset_sbt_curcpu(c, sbt, pr, fn, arg, flags)		\
@@ -106,7 +103,8 @@ int	callout_schedule(struct callout *, int);
 int	callout_schedule_on(struct callout *, int, int);
 #define	callout_schedule_curcpu(c, on_tick)				\
     callout_schedule_on((c), (on_tick), PCPU_GET(cpuid))
-int	callout_stop(struct callout *);
+#define	callout_stop(c)		_callout_stop_safe(c, 0)
+int	_callout_stop_safe(struct callout *, int);
 void	callout_process(sbintime_t now);
 
 #endif
diff --git a/sys/sys/param.h b/sys/sys/param.h
index 0babf847870..bf59b0bf722 100644
--- a/sys/sys/param.h
+++ b/sys/sys/param.h
@@ -58,7 +58,7 @@
  *		in the range 5 to 9.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 1100054	/* Master, propagated to newvers */
+#define __FreeBSD_version 1100055	/* Master, propagated to newvers */
 
 /*
  * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index e694d918c45..9b6c695d37f 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -308,7 +308,6 @@ struct thread {
 	} td_uretoff;			/* (k) Syscall aux returns. */
 #define td_retval	td_uretoff.tdu_retval
 	struct callout	td_slpcallout;	/* (h) Callout for sleep. */
-	struct mtx	td_slpmutex;	/* (h) Mutex for sleep callout */
 	struct trapframe *td_frame;	/* (k) */
 	struct vm_object *td_kstack_obj;/* (a) Kstack object. */
 	vm_offset_t	td_kstack;	/* (a) Kernel VA of kstack. */
@@ -365,7 +364,7 @@ do {									\
 #define	TDF_ALLPROCSUSP	0x00000200 /* suspended by SINGLE_ALLPROC */
 #define	TDF_BOUNDARY	0x00000400 /* Thread suspended at user boundary */
 #define	TDF_ASTPENDING	0x00000800 /* Thread has some asynchronous events. */
-#define	TDF_UNUSED12	0x00001000 /* --available-- */
+#define	TDF_TIMOFAIL	0x00001000 /* Timeout from sleep after we were awake. */
 #define	TDF_SBDRY	0x00002000 /* Stop only on usermode boundary. */
 #define	TDF_UPIBLOCKED	0x00004000 /* Thread blocked on user PI mutex. */
 #define	TDF_NEEDSUSPCHK	0x00008000 /* Thread may need to suspend. */
@@ -707,7 +706,7 @@ struct proc {
 #define	SWT_OWEPREEMPT		2	/* Switching due to opepreempt. */
 #define	SWT_TURNSTILE		3	/* Turnstile contention. */
 #define	SWT_SLEEPQ		4	/* Sleepq wait. */
-#define	SWT_UNUSED5		5	/* --available-- */
+#define	SWT_SLEEPQTIMO		5	/* Sleepq timeout wait. */
 #define	SWT_RELINQUISH		6	/* yield call. */
 #define	SWT_NEEDRESCHED		7	/* NEEDRESCHED was set. */
 #define	SWT_IDLE		8	/* Switching from the idle thread. */

From 379694e3d35fac3ec3c39811aee5a14dc57403f7 Mon Sep 17 00:00:00 2001
From: Alexander Motin <mav@FreeBSD.org>
Date: Thu, 22 Jan 2015 14:49:36 +0000
Subject: [PATCH 187/279] Don't count requests with status sent as overlapping.

While those requests are still in target OOA queue, for initiator they are
already completed, so tags can be reused.

MFC after:	1 week
---
 sys/cam/ctl/ctl.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/sys/cam/ctl/ctl.c b/sys/cam/ctl/ctl.c
index e90a029b359..35651779d41 100644
--- a/sys/cam/ctl/ctl.c
+++ b/sys/cam/ctl/ctl.c
@@ -10826,7 +10826,8 @@ ctl_check_for_blockage(struct ctl_lun *lun, union ctl_io *pending_io,
 	      ooa_io->io_hdr.nexus.targ_port)
 	  && (pending_io->io_hdr.nexus.initid.id ==
 	      ooa_io->io_hdr.nexus.initid.id))
-	 && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
+	 && ((ooa_io->io_hdr.flags & (CTL_FLAG_ABORT |
+	      CTL_FLAG_STATUS_SENT)) == 0))
 		return (CTL_ACTION_OVERLAP);
 
 	/*
@@ -10847,7 +10848,8 @@ ctl_check_for_blockage(struct ctl_lun *lun, union ctl_io *pending_io,
 	      ooa_io->io_hdr.nexus.targ_port)
 	  && (pending_io->io_hdr.nexus.initid.id ==
 	      ooa_io->io_hdr.nexus.initid.id))
-	 && ((ooa_io->io_hdr.flags & CTL_FLAG_ABORT) == 0))
+	 && ((ooa_io->io_hdr.flags & (CTL_FLAG_ABORT |
+	      CTL_FLAG_STATUS_SENT)) == 0))
 		return (CTL_ACTION_OVERLAP_TAG);
 
 	/*

From d01641e2c1201c89ce1be4acfd90c17814630690 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Thu, 22 Jan 2015 17:09:54 +0000
Subject: [PATCH 188/279] Improve CARP logging so that all state transitions
 are logged.

sys/netinet/ip_carp.c:
	Add a "reason" string parameter to carp_set_state() and
	carp_master_down_locked() allowing more specific logging
	information to be passed into these apis.

	Refactor existing state transition logging into a single
	log call in carp_set_state().

	Update all calls to carp_set_state() and
	carp_master_down_locked() to pass an appropriate reason
	string.  For state transitions that were previously logged,
	the output should be unchanged.

Submitted by:	gibbs (original), asomers (updated)
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1039697 on 2014/02/11 (original)
		1049992 on 2014/03/21 (updated)
---
 sys/netinet/ip_carp.c | 57 +++++++++++++++++--------------------------
 1 file changed, 23 insertions(+), 34 deletions(-)

diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c
index 4ecc58a0dfb..5c449c0ddd5 100644
--- a/sys/netinet/ip_carp.c
+++ b/sys/netinet/ip_carp.c
@@ -304,11 +304,12 @@ static void	carp_destroy(struct carp_softc *);
 static struct carp_if
 		*carp_alloc_if(struct ifnet *);
 static void	carp_free_if(struct carp_if *);
-static void	carp_set_state(struct carp_softc *, int);
+static void	carp_set_state(struct carp_softc *, int, const char* reason);
 static void	carp_sc_state(struct carp_softc *);
 static void	carp_setrun(struct carp_softc *, sa_family_t);
 static void	carp_master_down(void *);
-static void	carp_master_down_locked(struct carp_softc *);
+static void	carp_master_down_locked(struct carp_softc *,
+    		    const char* reason);
 static void	carp_send_ad(void *);
 static void	carp_send_ad_locked(struct carp_softc *);
 static void	carp_addroute(struct carp_softc *);
@@ -654,11 +655,8 @@ carp_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af)
 		if (timevalcmp(&sc_tv, &ch_tv, >) ||
 		    timevalcmp(&sc_tv, &ch_tv, ==)) {
 			callout_stop(&sc->sc_ad_tmo);
-			CARP_LOG("VHID %u@%s: MASTER -> BACKUP "
-			    "(more frequent advertisement received)\n",
-			    sc->sc_vhid,
-			    sc->sc_carpdev->if_xname);
-			carp_set_state(sc, BACKUP);
+			carp_set_state(sc, BACKUP,
+			    "more frequent advertisement received");
 			carp_setrun(sc, 0);
 			carp_delroute(sc);
 		}
@@ -669,11 +667,8 @@ carp_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af)
 		 * and this one claims to be slower, treat him as down.
 		 */
 		if (V_carp_preempt && timevalcmp(&sc_tv, &ch_tv, <)) {
-			CARP_LOG("VHID %u@%s: BACKUP -> MASTER "
-			    "(preempting a slower master)\n",
-			    sc->sc_vhid,
-			    sc->sc_carpdev->if_xname);
-			carp_master_down_locked(sc);
+			carp_master_down_locked(sc,
+			    "preempting a slower master");
 			break;
 		}
 
@@ -684,11 +679,7 @@ carp_input_c(struct mbuf *m, struct carp_header *ch, sa_family_t af)
 		 */
 		sc_tv.tv_sec = sc->sc_advbase * 3;
 		if (timevalcmp(&sc_tv, &ch_tv, <)) {
-			CARP_LOG("VHID %u@%s: BACKUP -> MASTER "
-			    "(master timed out)\n",
-			    sc->sc_vhid,
-			    sc->sc_carpdev->if_xname);
-			carp_master_down_locked(sc);
+			carp_master_down_locked(sc, "master will time out");
 			break;
 		}
 
@@ -1153,10 +1144,7 @@ carp_master_down(void *v)
 
 	CURVNET_SET(sc->sc_carpdev->if_vnet);
 	if (sc->sc_state == BACKUP) {
-		CARP_LOG("VHID %u@%s: BACKUP -> MASTER (master down)\n",
-		    sc->sc_vhid,
-		    sc->sc_carpdev->if_xname);
-		carp_master_down_locked(sc);
+		carp_master_down_locked(sc, "master timed out");
 	}
 	CURVNET_RESTORE();
 
@@ -1164,14 +1152,14 @@ carp_master_down(void *v)
 }
 
 static void
-carp_master_down_locked(struct carp_softc *sc)
+carp_master_down_locked(struct carp_softc *sc, const char *reason)
 {
 
 	CARP_LOCK_ASSERT(sc);
 
 	switch (sc->sc_state) {
 	case BACKUP:
-		carp_set_state(sc, MASTER);
+		carp_set_state(sc, MASTER, reason);
 		carp_send_ad_locked(sc);
 #ifdef INET
 		carp_send_arp(sc);
@@ -1212,10 +1200,7 @@ carp_setrun(struct carp_softc *sc, sa_family_t af)
 
 	switch (sc->sc_state) {
 	case INIT:
-		CARP_LOG("VHID %u@%s: INIT -> BACKUP\n",
-		    sc->sc_vhid,
-		    sc->sc_carpdev->if_xname);
-		carp_set_state(sc, BACKUP);
+		carp_set_state(sc, BACKUP, "initialization complete");
 		carp_setrun(sc, 0);
 		break;
 	case BACKUP:
@@ -1717,12 +1702,12 @@ carp_ioctl(struct ifreq *ifr, u_long cmd, struct thread *td)
 			switch (carpr.carpr_state) {
 			case BACKUP:
 				callout_stop(&sc->sc_ad_tmo);
-				carp_set_state(sc, BACKUP);
+				carp_set_state(sc, BACKUP, "SIOCSVH");
 				carp_setrun(sc, 0);
 				carp_delroute(sc);
 				break;
 			case MASTER:
-				carp_master_down_locked(sc);
+				carp_master_down_locked(sc, "SIOCSVH");
 				break;
 			default:
 				break;
@@ -1964,7 +1949,7 @@ carp_detach_locked(struct ifaddr *ifa)
 }
 
 static void
-carp_set_state(struct carp_softc *sc, int state)
+carp_set_state(struct carp_softc *sc, int state, const char *reason)
 {
 
 	CARP_LOCK_ASSERT(sc);
@@ -1973,10 +1958,14 @@ carp_set_state(struct carp_softc *sc, int state)
 		const char *carp_states[] = { CARP_STATES };
 		char subsys[IFNAMSIZ+5];
 
-		sc->sc_state = state;
-
 		snprintf(subsys, IFNAMSIZ+5, "%u@%s", sc->sc_vhid,
 		    sc->sc_carpdev->if_xname);
+
+		CARP_LOG("%s: %s -> %s (%s)\n", subsys,
+		    carp_states[sc->sc_state], carp_states[state], reason);
+
+		sc->sc_state = state;
+
 		devctl_notify("CARP", subsys, carp_states[state], NULL);
 	}
 }
@@ -2010,13 +1999,13 @@ carp_sc_state(struct carp_softc *sc)
 #ifdef INET6
 		callout_stop(&sc->sc_md6_tmo);
 #endif
-		carp_set_state(sc, INIT);
+		carp_set_state(sc, INIT, "hw interface down");
 		carp_setrun(sc, 0);
 		if (!sc->sc_suppress)
 			carp_demote_adj(V_carp_ifdown_adj, "interface down");
 		sc->sc_suppress = 1;
 	} else {
-		carp_set_state(sc, INIT);
+		carp_set_state(sc, INIT, "hw interface up");
 		carp_setrun(sc, 0);
 		if (sc->sc_suppress)
 			carp_demote_adj(-V_carp_ifdown_adj, "interface up");

From f79998c8fcddd4b95c861425de50f4596eae3ed9 Mon Sep 17 00:00:00 2001
From: Hajimu UMEMOTO <ume@FreeBSD.org>
Date: Thu, 22 Jan 2015 17:13:43 +0000
Subject: [PATCH 189/279] Don't include newvers.sh into freebsd-version.

---
 bin/freebsd-version/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bin/freebsd-version/Makefile b/bin/freebsd-version/Makefile
index aac030686c3..e515d0c225d 100644
--- a/bin/freebsd-version/Makefile
+++ b/bin/freebsd-version/Makefile
@@ -11,7 +11,7 @@ freebsd-version.sh: ${.CURDIR}/freebsd-version.sh.in ${NEWVERS}
 			s/@@TYPE@@/$${TYPE}/g; \
 			s/@@REVISION@@/$${REVISION}/g; \
 			s/@@BRANCH@@/$${BRANCH}/g; \
-		" ${.ALLSRC} >${.TARGET} ; then \
+		" ${.CURDIR}/freebsd-version.sh.in >${.TARGET} ; then \
 		rm -f ${.TARGET} ; \
 		exit 1 ; \
 	fi

From f62ae5518e09587629167b76ad10ac42acc92eae Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Thu, 22 Jan 2015 17:42:30 +0000
Subject: [PATCH 190/279] Add the Maxmem global and set it to the highest
 physical page number plus 1.

---
 sys/arm/arm/physmem.c | 29 +++++++++++++++++++++++------
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/sys/arm/arm/physmem.c b/sys/arm/arm/physmem.c
index 8618e46e27a..eae03f7ab5d 100644
--- a/sys/arm/arm/physmem.c
+++ b/sys/arm/arm/physmem.c
@@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <vm/vm.h>
+#include <machine/md_var.h>
 #include <machine/physmem.h>
 
 /*
@@ -86,8 +87,12 @@ static size_t excnt;
 vm_paddr_t phys_avail[MAX_AVAIL_ENTRIES + 2]; /* +2 to allow for a pair  */
 vm_paddr_t dump_avail[MAX_AVAIL_ENTRIES + 2]; /* of zeroes to terminate. */
 
-/* This is the total number of hardware pages, excluded or not. */
+/*
+ * realmem is the total number of hardware pages, excluded or not.
+ * Maxmem is one greater than the last physical page number.
+ */
 long realmem;
+long Maxmem;
 
 /* The address at which the kernel was loaded.  Set early in initarm(). */
 vm_paddr_t arm_physmem_kernaddr;
@@ -152,8 +157,8 @@ arm_physmem_print_tables()
  *
  * Returns the number of pages of non-excluded memory added to the avail list.
  */
-static long
-regions_to_avail(vm_paddr_t *avail, uint32_t exflags)
+static size_t
+regions_to_avail(vm_paddr_t *avail, uint32_t exflags, long *pavail)
 {
 	size_t acnt, exi, hwi;
 	vm_paddr_t end, start, xend, xstart;
@@ -236,7 +241,9 @@ regions_to_avail(vm_paddr_t *avail, uint32_t exflags)
 			panic("Not enough space in the dump/phys_avail arrays");
 	}
 
-	return (availmem);
+	if (pavail)
+		*pavail = availmem;
+	return (acnt);
 }
 
 /*
@@ -311,13 +318,23 @@ void arm_physmem_exclude_region(vm_paddr_t pa, vm_size_t sz, uint32_t exflags)
 
 /*
  * Process all the regions added earlier into the global avail lists.
+ *
+ * Updates the kernel global 'physmem' with the number of physical pages
+ * available for use (all pages not in any exclusion region).
+ *
+ * Updates the kernel global 'Maxmem' with the page number one greater then the
+ * last page of physical memory in the system.
  */
 void
 arm_physmem_init_kernel_globals(void)
 {
+	size_t nextidx;
 
-	regions_to_avail(dump_avail, EXFLAG_NODUMP);
-	physmem = regions_to_avail(phys_avail, EXFLAG_NOALLOC);
+	regions_to_avail(dump_avail, EXFLAG_NODUMP, NULL);
+	nextidx = regions_to_avail(phys_avail, EXFLAG_NOALLOC, &physmem);
+	if (nextidx == 0)
+		panic("No memory entries in phys_avail");
+	Maxmem = atop(phys_avail[nextidx - 1]);
 }
 
 #ifdef DDB

From 8436f7fe62df63e3f5461e721349eee93d0fa780 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Thu, 22 Jan 2015 17:46:05 +0000
Subject: [PATCH 191/279] Declare Maxmem on arm.  This should have been part of
 r277532.

---
 sys/arm/include/md_var.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sys/arm/include/md_var.h b/sys/arm/include/md_var.h
index d54a4259ef3..030b48bf056 100644
--- a/sys/arm/include/md_var.h
+++ b/sys/arm/include/md_var.h
@@ -33,6 +33,7 @@
 #ifndef	_MACHINE_MD_VAR_H_
 #define	_MACHINE_MD_VAR_H_
 
+extern long Maxmem;
 extern char sigcode[];
 extern int szsigcode;
 extern uint32_t *vm_page_dump;

From e9c93198bc886d4c41c8bfb4e732800b626e8bb6 Mon Sep 17 00:00:00 2001
From: Ed Maste <emaste@FreeBSD.org>
Date: Thu, 22 Jan 2015 17:53:30 +0000
Subject: [PATCH 192/279] Update reported elftoolchain version

The last elftoolchain update is upstream rev 3136. Update our reported
version and add M (Modified) suffix as our elftoolchain contains
additional fixes not yet committed upstream.

Sponsored by:	The FreeBSD Foundation
---
 lib/libelftc/elftc_version.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/libelftc/elftc_version.c b/lib/libelftc/elftc_version.c
index 9f278e61e91..a6bf57155c1 100644
--- a/lib/libelftc/elftc_version.c
+++ b/lib/libelftc/elftc_version.c
@@ -6,5 +6,5 @@
 const char *
 elftc_version(void)
 {
-	return "libelftc r3130";
+	return "elftoolchain r3136M";
 }

From 4afe7cf500c71e278eba5b91192d024b3a211d85 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Thu, 22 Jan 2015 19:43:05 +0000
Subject: [PATCH 193/279] Rename the 'release' target to 'real-release', and
 add two targets, 'vm-release' and 'cloudware-release', that are invoked if
 WITH_VMIMAGES and WITH_CLOUDWARE are not empty.

This fixes an issue where 'make release' would not
build the cloud provider targets because CLOUDWARE was
not yet set. [1]

Move the WITH_VMIMAGES and WITH_CLOUDWARE targets to
Makefile.vm.

Note: There is no 'cloudware-install' target yet, since
some of the disk image names may need to be specific to
the provider, so this is probably best handled by the
build scripts.

Reported by:	cperciva [1]
MFC after:	1 month
X-MFC-with:	r277458
Sponsored by:	The FreeBSD Foundation
---
 release/Makefile    | 15 +++++++--------
 release/Makefile.vm | 10 ++++++++++
 2 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/release/Makefile b/release/Makefile
index e462bf3533a..a601921b330 100644
--- a/release/Makefile
+++ b/release/Makefile
@@ -8,7 +8,10 @@
 #  memstick: Builds memory stick image (memstick.img)
 #  mini-memstick: Builds minimal memory stick image (mini-memstick.img)
 #  ftp: Sets up FTP distribution area (ftp)
-#  release: Build all media and FTP distribution area
+#  release: Invokes real-release, vm-release, and cloudware-release targets
+#  real-release: Build all media and FTP distribution area
+#  vm-release: Build all virtual machine image targets
+#  cloudware-release: Build all cloud hosting provider targets
 #  install: Invokes the release-install and vm-install targets
 #  release-install: Copies all release installation media into ${DESTDIR}
 #  vm-install: Copies all virtual machine images into ${DESTDIR}
@@ -275,15 +278,11 @@ ftp: packagesystem
 	mkdir -p ftp
 	cp *.txz MANIFEST ftp
 
-release:
+release:	real-release vm-release cloudware-release
+
+real-release:
 	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} obj
 	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} ${RELEASE_TARGETS}
-.if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
-	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} ${VMTARGETS}
-.endif
-.if defined(WITH_CLOUDWARE) && !empty(WITH_CLOUDWARE) && !empty(CLOUDWARE)
-	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} ${CLOUDTARGETS}
-.endif
 
 install:	release-install vm-install
 
diff --git a/release/Makefile.vm b/release/Makefile.vm
index 9b5f543dd65..17229bfb279 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -109,3 +109,13 @@ vm-install:
 	cd ${DESTDIR}/vmimages && md5 ${OSRELEASE}* > \
 		${DESTDIR}/vmimages/CHECKSUM.MD5
 .endif
+
+vm-release:
+.if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
+	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} ${VMTARGETS}
+.endif
+
+cloudware-release:
+.if defined(WITH_CLOUDWARE) && !empty(WITH_CLOUDWARE) && !empty(CLOUDWARE)
+	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} ${CLOUDTARGETS}
+.endif

From f70dd52fe67a77baa8402a557d80d2f019aba9cd Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Thu, 22 Jan 2015 22:04:43 +0000
Subject: [PATCH 194/279] Allow use of a pre-instantiated RTAS as well as a
 self-instantiated one. This lets the kernel boot on RTAS-based systems by
 being kexec'ed from Linux.

---
 sys/powerpc/ofw/rtas.c | 43 ++++++++++++++++++++++++++----------------
 1 file changed, 27 insertions(+), 16 deletions(-)

diff --git a/sys/powerpc/ofw/rtas.c b/sys/powerpc/ofw/rtas.c
index 3e751b3ec08..15cb58e8353 100644
--- a/sys/powerpc/ofw/rtas.c
+++ b/sys/powerpc/ofw/rtas.c
@@ -86,12 +86,6 @@ rtas_setup(void *junk)
 		return;
 	}
 	OF_package_to_path(rtas, path, sizeof(path));
-	rtasi = OF_open(path);
-	if (rtasi == 0) {
-		rtas = 0;
-		printf("Error initializing RTAS: could not open node\n");
-		return;
-	}
 
 	mtx_init(&rtas_mtx, "RTAS", NULL, MTX_SPIN);
 
@@ -110,7 +104,7 @@ rtas_setup(void *junk)
 	 * It must be 4KB-aligned and not cross a 256 MB boundary.
 	 */
 
-	OF_getprop(rtas, "rtas-size", &rtas_size, sizeof(rtas_size));
+	OF_getencprop(rtas, "rtas-size", &rtas_size, sizeof(rtas_size));
 	rtas_size = round_page(rtas_size);
 	rtas_bounce_virt = contigmalloc(rtas_size + PAGE_SIZE, M_RTAS, 0, 0,
 	    ulmin(platform_real_maxaddr(), BUS_SPACE_MAXADDR_32BIT),
@@ -125,15 +119,32 @@ rtas_setup(void *junk)
 	 * Instantiate RTAS. We always use the 32-bit version.
 	 */
 
-	result = OF_call_method("instantiate-rtas", rtasi, 1, 1,
-	    (cell_t)rtas_private_data, &rtas_ptr);
-	OF_close(rtasi);
+	if (OF_hasprop(rtas, "linux,rtas-entry") &&
+	    OF_hasprop(rtas, "linux,rtas-base")) {
+		OF_getencprop(rtas, "linux,rtas-base", &rtas_ptr,
+		    sizeof(rtas_ptr));
+		rtas_private_data = rtas_ptr;
+		OF_getencprop(rtas, "linux,rtas-entry", &rtas_ptr,
+		    sizeof(rtas_ptr));
+	} else {
+		rtasi = OF_open(path);
+		if (rtasi == 0) {
+			rtas = 0;
+			printf("Error initializing RTAS: could not open "
+			    "node\n");
+			return;
+		}
 
-	if (result != 0) {
-		rtas = 0;
-		rtas_ptr = 0;
-		printf("Error initializing RTAS (%d)\n", result);
-		return;
+		result = OF_call_method("instantiate-rtas", rtasi, 1, 1,
+		    (cell_t)rtas_private_data, &rtas_ptr);
+		OF_close(rtasi);
+
+		if (result != 0) {
+			rtas = 0;
+			rtas_ptr = 0;
+			printf("Error initializing RTAS (%d)\n", result);
+			return;
+		}
 	}
 
 	rtas_entry = (uintptr_t)(rtas_ptr);
@@ -252,7 +263,7 @@ rtas_token_lookup(const char *method)
 	if (!rtas_exists())
 		return (-1);
 
-	if (OF_getprop(rtas, method, &token, sizeof(token)) == -1)
+	if (OF_getencprop(rtas, method, &token, sizeof(token)) == -1)
 		return (-1);
 
 	return (token);

From 5e497231273fbe5e786d080f140dd66c44fedde8 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Thu, 22 Jan 2015 23:50:47 +0000
Subject: [PATCH 195/279] Add sample log rotation support for opensm

Up to 7 archives of the log will be kept (just for consistency with the other
log rotation rules)

PR: 196788
MFC after: 1 week
Reviewed by: hselasky
Sponsored by: EMC / Isilon Storage Division
---
 etc/Makefile                     |  5 ++++-
 etc/newsyslog.conf.d/Makefile    | 13 +++++++++++++
 etc/newsyslog.conf.d/opensm.conf |  3 +++
 3 files changed, 20 insertions(+), 1 deletion(-)
 create mode 100644 etc/newsyslog.conf.d/Makefile
 create mode 100644 etc/newsyslog.conf.d/opensm.conf

diff --git a/etc/Makefile b/etc/Makefile
index 76db02a0cb2..62b837fa49e 100644
--- a/etc/Makefile
+++ b/etc/Makefile
@@ -3,8 +3,11 @@
 
 .include <src.opts.mk>
 
+SUBDIR=	\
+	newsyslog.conf.d
+
 .if ${MK_SENDMAIL} != "no"
-SUBDIR=	sendmail
+SUBDIR+=sendmail
 .endif
 
 .if ${MK_TESTS} != "no"
diff --git a/etc/newsyslog.conf.d/Makefile b/etc/newsyslog.conf.d/Makefile
new file mode 100644
index 00000000000..c2574b74c43
--- /dev/null
+++ b/etc/newsyslog.conf.d/Makefile
@@ -0,0 +1,13 @@
+# $FreeBSD$
+
+.include <src.opts.mk>
+
+BINDIR=		/etc/newsyslog.conf.d
+
+FILES=
+
+.if ${MK_OFED} != "no"
+FILES+=		opensm.conf
+.endif
+
+.include <bsd.prog.mk>
diff --git a/etc/newsyslog.conf.d/opensm.conf b/etc/newsyslog.conf.d/opensm.conf
new file mode 100644
index 00000000000..36248bde5fd
--- /dev/null
+++ b/etc/newsyslog.conf.d/opensm.conf
@@ -0,0 +1,3 @@
+# $FreeBSD$
+
+/var/log/opensm.log			600  7	*	*	Z	/var/run/opensm.pid	30

From 0e5f55bb9557b80a00a11c0f4b848e33680c082d Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Fri, 23 Jan 2015 00:06:35 +0000
Subject: [PATCH 196/279] Improve the distribution of LAGG port traffic.

I edited the original change to retain the use of arc4random() as a seed for
the hashing as a very basic defense against intentional lagg port selection.

The author's original commit message (edited slightly):

sys/net/ieee8023ad_lacp.c
sys/net/if_lagg.c
	In lagg_hashmbuf, use the FNV hash instead of the old
	hash32_buf.  The hash32 family of functions operate one octet
	at a time, and when run on a string s of length n, their output
	is equivalent to :

		   ----- i=n-1
		   \
	       n    \           (n-i-1)              32
	( seed^  +  /        33^        * s[i] ) % 2^
		   /
		   ----- i=0

	The problem is that the last five bytes of input don't get
	multiplied by sufficiently many powers of 33 to rollover 2^32.
	That means that changing the last few bytes (but obviously not
	the very last) of input will always change the value of the
	hash by a multiple of 33.  In the case of lagg_hashmbuf() with
	ipv4 input, the last four bytes are the TCP or UDP port
	numbers.  Since the output of lagg_hashmbuf is always taken
	modulo the port count, and 3 is a common port count for a lagg,
	that's bad.  It means that the UDP or TCP source port will
	never affect which lagg member is selected on a 3-port lagg.

	At 10Gbps, I was not able to measure any difference in CPU
	consumption between the old and new hash.

Submitted by:	asomers (original commit)
Reviewed by:	emaste, glebius
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1001723 on 2013/08/28 (original)
		1114258 on 2015/01/22 (edit)
---
 sys/net/ieee8023ad_lacp.c |  6 +++++-
 sys/net/if_lagg.c         | 27 +++++++++++++++------------
 2 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/sys/net/ieee8023ad_lacp.c b/sys/net/ieee8023ad_lacp.c
index 75f6366e788..3052e4b526e 100644
--- a/sys/net/ieee8023ad_lacp.c
+++ b/sys/net/ieee8023ad_lacp.c
@@ -35,6 +35,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/eventhandler.h>
 #include <sys/mbuf.h>
 #include <sys/systm.h>
+#include <sys/fnv_hash.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h> /* hz */
 #include <sys/socket.h> /* for net/if.h */
@@ -757,13 +758,16 @@ void
 lacp_attach(struct lagg_softc *sc)
 {
 	struct lacp_softc *lsc;
+	uint32_t seed;
 
 	lsc = malloc(sizeof(struct lacp_softc), M_DEVBUF, M_WAITOK | M_ZERO);
 
 	sc->sc_psc = lsc;
 	lsc->lsc_softc = sc;
 
-	lsc->lsc_hashkey = arc4random();
+	seed = arc4random();
+	lsc->lsc_hashkey = FNV1_32_INIT;
+	lsc->lsc_hashkey = fnv_32_buf(&seed, sizeof(seed), lsc->lsc_hashkey);
 	lsc->lsc_active_aggregator = NULL;
 	lsc->lsc_strict_mode = 1;
 	LACP_LOCK_INIT(lsc);
diff --git a/sys/net/if_lagg.c b/sys/net/if_lagg.c
index a6e5b3c65a1..ec828fa8a94 100644
--- a/sys/net/if_lagg.c
+++ b/sys/net/if_lagg.c
@@ -36,7 +36,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/priv.h>
 #include <sys/systm.h>
 #include <sys/proc.h>
-#include <sys/hash.h>
+#include <sys/fnv_hash.h>
 #include <sys/lock.h>
 #include <sys/rmlock.h>
 #include <sys/taskqueue.h>
@@ -1853,13 +1853,13 @@ lagg_hashmbuf(struct lagg_softc *sc, struct mbuf *m, uint32_t key)
 	eh = mtod(m, struct ether_header *);
 	etype = ntohs(eh->ether_type);
 	if (sc->sc_flags & LAGG_F_HASHL2) {
-		p = hash32_buf(&eh->ether_shost, ETHER_ADDR_LEN, p);
-		p = hash32_buf(&eh->ether_dhost, ETHER_ADDR_LEN, p);
+		p = fnv_32_buf(&eh->ether_shost, ETHER_ADDR_LEN, p);
+		p = fnv_32_buf(&eh->ether_dhost, ETHER_ADDR_LEN, p);
 	}
 
 	/* Special handling for encapsulating VLAN frames */
 	if ((m->m_flags & M_VLANTAG) && (sc->sc_flags & LAGG_F_HASHL2)) {
-		p = hash32_buf(&m->m_pkthdr.ether_vtag,
+		p = fnv_32_buf(&m->m_pkthdr.ether_vtag,
 		    sizeof(m->m_pkthdr.ether_vtag), p);
 	} else if (etype == ETHERTYPE_VLAN) {
 		vlan = lagg_gethdr(m, off,  sizeof(*vlan), &buf);
@@ -1867,7 +1867,7 @@ lagg_hashmbuf(struct lagg_softc *sc, struct mbuf *m, uint32_t key)
 			goto out;
 
 		if (sc->sc_flags & LAGG_F_HASHL2)
-			p = hash32_buf(&vlan->evl_tag, sizeof(vlan->evl_tag), p);
+			p = fnv_32_buf(&vlan->evl_tag, sizeof(vlan->evl_tag), p);
 		etype = ntohs(vlan->evl_proto);
 		off += sizeof(*vlan) - sizeof(*eh);
 	}
@@ -1880,8 +1880,8 @@ lagg_hashmbuf(struct lagg_softc *sc, struct mbuf *m, uint32_t key)
 			goto out;
 
 		if (sc->sc_flags & LAGG_F_HASHL3) {
-			p = hash32_buf(&ip->ip_src, sizeof(struct in_addr), p);
-			p = hash32_buf(&ip->ip_dst, sizeof(struct in_addr), p);
+			p = fnv_32_buf(&ip->ip_src, sizeof(struct in_addr), p);
+			p = fnv_32_buf(&ip->ip_dst, sizeof(struct in_addr), p);
 		}
 		if (!(sc->sc_flags & LAGG_F_HASHL4))
 			break;
@@ -1896,7 +1896,7 @@ lagg_hashmbuf(struct lagg_softc *sc, struct mbuf *m, uint32_t key)
 				ports = lagg_gethdr(m, off, sizeof(*ports), &buf);
 				if (ports == NULL)
 					break;
-				p = hash32_buf(ports, sizeof(*ports), p);
+				p = fnv_32_buf(ports, sizeof(*ports), p);
 				break;
 		}
 		break;
@@ -1909,10 +1909,10 @@ lagg_hashmbuf(struct lagg_softc *sc, struct mbuf *m, uint32_t key)
 		if (ip6 == NULL)
 			goto out;
 
-		p = hash32_buf(&ip6->ip6_src, sizeof(struct in6_addr), p);
-		p = hash32_buf(&ip6->ip6_dst, sizeof(struct in6_addr), p);
+		p = fnv_32_buf(&ip6->ip6_src, sizeof(struct in6_addr), p);
+		p = fnv_32_buf(&ip6->ip6_dst, sizeof(struct in6_addr), p);
 		flow = ip6->ip6_flow & IPV6_FLOWLABEL_MASK;
-		p = hash32_buf(&flow, sizeof(flow), p);	/* IPv6 flow label */
+		p = fnv_32_buf(&flow, sizeof(flow), p);	/* IPv6 flow label */
 		break;
 #endif
 	}
@@ -2087,12 +2087,15 @@ lagg_lb_attach(struct lagg_softc *sc)
 {
 	struct lagg_port *lp;
 	struct lagg_lb *lb;
+	uint32_t seed;
 
 	lb = malloc(sizeof(struct lagg_lb), M_DEVBUF, M_WAITOK | M_ZERO);
 
 	sc->sc_capabilities = IFCAP_LAGG_FULLDUPLEX;
 
-	lb->lb_key = arc4random();
+	seed = arc4random();
+	lb->lb_key = FNV1_32_INIT;
+	lb->lb_key = fnv_32_buf(&seed, sizeof(seed), lb->lb_key);
 	sc->sc_psc = lb;
 
 	SLIST_FOREACH(lp, &sc->sc_ports, lp_entries)

From 5087ad02126a15380d587ba1ce0c38a520da01d9 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Fri, 23 Jan 2015 01:18:08 +0000
Subject: [PATCH 197/279] Enable all the mmc/sd controllers, because there's no
 telling which ones any given board will be using.

Yes, changing source is an unusually primitive power control implementation.
---
 sys/arm/freescale/imx/imx6_ccm.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/arm/freescale/imx/imx6_ccm.c b/sys/arm/freescale/imx/imx6_ccm.c
index 769a84108be..2c80bd9ee03 100644
--- a/sys/arm/freescale/imx/imx6_ccm.c
+++ b/sys/arm/freescale/imx/imx6_ccm.c
@@ -95,7 +95,7 @@ ccm_init_gates(struct ccm_softc *sc)
 	WR4(sc, CCM_CCGR3, 0x3ff00000); /* DDR memory controller */
 	WR4(sc, CCM_CCGR4, 0x0000f300); /* pl301 bus crossbar */
 	WR4(sc, CCM_CCGR5, 0x0f000000); /* uarts */
-	WR4(sc, CCM_CCGR6, 0x000000cc); /* usdhc 1 & 3 */
+	WR4(sc, CCM_CCGR6, 0x000000ff); /* usdhc 1-4 */
 }
 
 static int

From 43682290a52f95652c89ae0f724ba29f0d1ccb36 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Fri, 23 Jan 2015 06:01:34 +0000
Subject: [PATCH 198/279] Remove the duplicate zzz entry

---
 usr.sbin/Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile
index da16ba25c00..edb3a16c379 100644
--- a/usr.sbin/Makefile
+++ b/usr.sbin/Makefile
@@ -88,7 +88,6 @@ SUBDIR+=	sa
 
 .if ${MK_ACPI} != "no" || ${MK_APM} != "no"
 SUBDIR+=	powerd
-SUBDIR+=	zzz
 .endif
 
 .if ${MK_AMD} != "no"

From c5f282daad8b9d7350ede24c4dd63a7291f7e37d Mon Sep 17 00:00:00 2001
From: Alexey Dokuchaev <danfe@FreeBSD.org>
Date: Fri, 23 Jan 2015 07:30:57 +0000
Subject: [PATCH 199/279] Fix usage example in kvprintf(9) and its copy in
 libstand(3): trailing '\n' in bitfield argument is wrong, as it will be
 treated as bit 10, causing any code printing >=10 bits with bit 10 on as
 having a trailing comma.

Newline (intended one) should be part of the format string (already present
in the examples).

Also fix grammar and kill EOL whitespace in comment while here.

PR:		195005
Approved by:	bdrewery
---
 lib/libstand/libstand.3 | 2 +-
 lib/libstand/printf.c   | 4 ++--
 share/man/man9/printf.9 | 2 +-
 sys/kern/subr_prf.c     | 4 ++--
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/libstand/libstand.3 b/lib/libstand/libstand.3
index b42c8b5bdd7..42289c1bce4 100644
--- a/lib/libstand/libstand.3
+++ b/lib/libstand/libstand.3
@@ -288,7 +288,7 @@ Thus
 printf(
 .Qq reg=%b\en ,
 3,
-.Qq \e10\e2BITTWO\e1BITONE\en
+.Qq \e10\e2BITTWO\e1BITONE
 );
 .Ed
 .Pp
diff --git a/lib/libstand/printf.c b/lib/libstand/printf.c
index 157b327ead8..fef53416561 100644
--- a/lib/libstand/printf.c
+++ b/lib/libstand/printf.c
@@ -187,7 +187,7 @@ ksprintn(char *nbuf, uintmax_t num, int base, int *lenp, int upper)
  * the next characters (up to a control character, i.e. a character <= 32),
  * give the name of the register.  Thus:
  *
- *	kvprintf("reg=%b\n", 3, "\10\2BITTWO\1BITONE\n");
+ *	kvprintf("reg=%b\n", 3, "\10\2BITTWO\1BITONE");
  *
  * would produce output:
  *
@@ -500,7 +500,7 @@ number:
 			while (percent < fmt)
 				PCHAR(*percent++);
 			/*
-			 * Since we ignore an formatting argument it is no 
+			 * Since we ignore a formatting argument it is no
 			 * longer safe to obey the remaining formatting
 			 * arguments as the arguments will no longer match
 			 * the format specs.
diff --git a/share/man/man9/printf.9 b/share/man/man9/printf.9
index 571e7e638c3..84ac822da02 100644
--- a/share/man/man9/printf.9
+++ b/share/man/man9/printf.9
@@ -151,7 +151,7 @@ void
 printf_test(void)
 {
 
-	printf("reg=%b\en", 3, "\e10\e2BITTWO\e1BITONE\en");
+	printf("reg=%b\en", 3, "\e10\e2BITTWO\e1BITONE");
 	printf("out: %4D\en", "AAAA", ":");
 }
 .Ed
diff --git a/sys/kern/subr_prf.c b/sys/kern/subr_prf.c
index 814e57f4794..cc30de1f645 100644
--- a/sys/kern/subr_prf.c
+++ b/sys/kern/subr_prf.c
@@ -598,7 +598,7 @@ ksprintn(char *nbuf, uintmax_t num, int base, int *lenp, int upper)
  * the next characters (up to a control character, i.e. a character <= 32),
  * give the name of the register.  Thus:
  *
- *	kvprintf("reg=%b\n", 3, "\10\2BITTWO\1BITONE\n");
+ *	kvprintf("reg=%b\n", 3, "\10\2BITTWO\1BITONE");
  *
  * would produce output:
  *
@@ -911,7 +911,7 @@ number:
 			while (percent < fmt)
 				PCHAR(*percent++);
 			/*
-			 * Since we ignore a formatting argument it is no 
+			 * Since we ignore a formatting argument it is no
 			 * longer safe to obey the remaining formatting
 			 * arguments as the arguments will no longer match
 			 * the format specs.

From ec336f0f0c3b5a9383009306a5933de8f4d794a0 Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Fri, 23 Jan 2015 07:36:51 +0000
Subject: [PATCH 200/279] Use relocation-safe methods to determine the sizes of
 the exception handlers. A "size" symbol with its address set to the length of
 handler would be shifted forward with all other addresses when relocations
 are processed. Instead, just note the end and do the subtraction at runtime.

---
 sys/powerpc/aim/machdep.c     | 69 ++++++++++++++++++++---------------
 sys/powerpc/aim/trap_subr32.S | 20 +++++-----
 sys/powerpc/aim/trap_subr64.S | 24 ++++++------
 3 files changed, 61 insertions(+), 52 deletions(-)

diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c
index 82a72328d63..a6d254b3f14 100644
--- a/sys/powerpc/aim/machdep.c
+++ b/sys/powerpc/aim/machdep.c
@@ -237,14 +237,14 @@ extern void	*rfid_patch, *rfi_patch1, *rfi_patch2;
 extern void	*trapcode64;
 #endif
 
-extern void	*rstcode, *rstsize;
-extern void	*trapcode, *trapsize, *trapcode2;
-extern void	*slbtrap, *slbtrapsize;
-extern void	*alitrap, *alisize;
-extern void	*dsitrap, *dsisize;
+extern void	*rstcode, *rstcodeend;
+extern void	*trapcode, *trapcodeend, *trapcode2;
+extern void	*slbtrap, *slbtrapend;
+extern void	*alitrap, *aliend;
+extern void	*dsitrap, *dsiend;
 extern void	*decrint, *decrsize;
 extern void     *extint, *extsize;
-extern void	*dblow, *dbsize;
+extern void	*dblow, *dbend;
 extern void	*imisstrap, *imisssize;
 extern void	*dlmisstrap, *dlmisssize;
 extern void	*dsmisstrap, *dsmisssize;
@@ -255,7 +255,7 @@ powerpc_init(vm_offset_t fdt, vm_offset_t toc, vm_offset_t ofentry, void *mdp)
 	struct		pcpu *pc;
 	vm_offset_t	startkernel, endkernel;
 	void		*generictrap;
-	size_t		trap_offset;
+	size_t		trap_offset, trapsize;
 	void		*kmdp;
         char		*env;
 	register_t	msr, scratch;
@@ -513,35 +513,44 @@ powerpc_init(vm_offset_t fdt, vm_offset_t toc, vm_offset_t ofentry, void *mdp)
 	*((register_t *)TRAP_TOCBASE) = toc;
 	#endif
 
-	bcopy(&rstcode, (void *)(EXC_RST + trap_offset),  (size_t)&rstsize);
+	trapsize = (size_t)&trapcodeend - (size_t)&trapcode;
+
+	bcopy(&rstcode, (void *)(EXC_RST + trap_offset), (size_t)&rstcodeend -
+	    (size_t)&rstcode);
 
 #ifdef KDB
-	bcopy(&dblow,	(void *)(EXC_MCHK + trap_offset), (size_t)&dbsize);
-	bcopy(&dblow,   (void *)(EXC_PGM + trap_offset),  (size_t)&dbsize);
-	bcopy(&dblow,   (void *)(EXC_TRC + trap_offset),  (size_t)&dbsize);
-	bcopy(&dblow,   (void *)(EXC_BPT + trap_offset),  (size_t)&dbsize);
+	bcopy(&dblow, (void *)(EXC_MCHK + trap_offset), (size_t)&dbend -
+	    (size_t)&dblow);
+	bcopy(&dblow, (void *)(EXC_PGM + trap_offset), (size_t)&dbend -
+	    (size_t)&dblow);
+	bcopy(&dblow, (void *)(EXC_TRC + trap_offset), (size_t)&dbend -
+	    (size_t)&dblow);
+	bcopy(&dblow, (void *)(EXC_BPT + trap_offset), (size_t)&dbend -
+	    (size_t)&dblow);
 #else
-	bcopy(generictrap, (void *)EXC_MCHK, (size_t)&trapsize);
-	bcopy(generictrap, (void *)EXC_PGM,  (size_t)&trapsize);
-	bcopy(generictrap, (void *)EXC_TRC,  (size_t)&trapsize);
-	bcopy(generictrap, (void *)EXC_BPT,  (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_MCHK, trapsize);
+	bcopy(generictrap, (void *)EXC_PGM,  trapsize);
+	bcopy(generictrap, (void *)EXC_TRC,  trapsize);
+	bcopy(generictrap, (void *)EXC_BPT,  trapsize);
 #endif
-	bcopy(&alitrap,  (void *)(EXC_ALI + trap_offset),  (size_t)&alisize);
-	bcopy(&dsitrap,  (void *)(EXC_DSI + trap_offset),  (size_t)&dsisize);
-	bcopy(generictrap, (void *)EXC_ISI,  (size_t)&trapsize);
+	bcopy(&alitrap,  (void *)(EXC_ALI + trap_offset),  (size_t)&aliend -
+	    (size_t)&alitrap);
+	bcopy(&dsitrap,  (void *)(EXC_DSI + trap_offset),  (size_t)&dsitrap -
+	    (size_t)&dsitrap);
+	bcopy(generictrap, (void *)EXC_ISI,  trapsize);
 	#ifdef __powerpc64__
-	bcopy(&slbtrap, (void *)EXC_DSE,  (size_t)&slbtrapsize);
-	bcopy(&slbtrap, (void *)EXC_ISE,  (size_t)&slbtrapsize);
+	bcopy(&slbtrap, (void *)EXC_DSE,(size_t)&slbtrapend - (size_t)&slbtrap);
+	bcopy(&slbtrap, (void *)EXC_ISE,(size_t)&slbtrapend - (size_t)&slbtrap);
 	#endif
-	bcopy(generictrap, (void *)EXC_EXI,  (size_t)&trapsize);
-	bcopy(generictrap, (void *)EXC_FPU,  (size_t)&trapsize);
-	bcopy(generictrap, (void *)EXC_DECR, (size_t)&trapsize);
-	bcopy(generictrap, (void *)EXC_SC,   (size_t)&trapsize);
-	bcopy(generictrap, (void *)EXC_FPA,  (size_t)&trapsize);
-	bcopy(generictrap, (void *)EXC_VEC,  (size_t)&trapsize);
-	bcopy(generictrap, (void *)EXC_PERF,  (size_t)&trapsize);
-	bcopy(generictrap, (void *)EXC_VECAST_G4, (size_t)&trapsize);
-	bcopy(generictrap, (void *)EXC_VECAST_G5, (size_t)&trapsize);
+	bcopy(generictrap, (void *)EXC_EXI,  trapsize);
+	bcopy(generictrap, (void *)EXC_FPU,  trapsize);
+	bcopy(generictrap, (void *)EXC_DECR, trapsize);
+	bcopy(generictrap, (void *)EXC_SC,   trapsize);
+	bcopy(generictrap, (void *)EXC_FPA,  trapsize);
+	bcopy(generictrap, (void *)EXC_VEC,  trapsize);
+	bcopy(generictrap, (void *)EXC_PERF,  trapsize);
+	bcopy(generictrap, (void *)EXC_VECAST_G4, trapsize);
+	bcopy(generictrap, (void *)EXC_VECAST_G5, trapsize);
 	#ifndef __powerpc64__
 	/* G2-specific TLB miss helper handlers */
 	bcopy(&imisstrap, (void *)EXC_IMISS,  (size_t)&imisssize);
diff --git a/sys/powerpc/aim/trap_subr32.S b/sys/powerpc/aim/trap_subr32.S
index 2aae2a00abb..7c753b541dd 100644
--- a/sys/powerpc/aim/trap_subr32.S
+++ b/sys/powerpc/aim/trap_subr32.S
@@ -299,10 +299,10 @@ CNAME(restorebridgesize) = .-CNAME(restorebridge)
  * not still hanging around in the trap handling region
  * once the MMU is turned on.
  */
-	.globl	CNAME(rstcode), CNAME(rstsize)
+	.globl	CNAME(rstcode), CNAME(rstcodeend)
 CNAME(rstcode):
 	ba	cpu_reset
-CNAME(rstsize) = . - CNAME(rstcode)
+CNAME(rstcodeend):
 
 cpu_reset:
 	bl	1f
@@ -339,14 +339,14 @@ cpu_reset:
  * (except ISI/DSI, ALI, and the interrupts)
  */
 
-	.globl	CNAME(trapcode),CNAME(trapsize)
+	.globl	CNAME(trapcode),CNAME(trapcodeend)
 CNAME(trapcode):
 	mtsprg1	%r1			/* save SP */
 	mflr	%r1			/* Save the old LR in r1 */
 	mtsprg2 %r1			/* And then in SPRG2 */
 	li	%r1, 0x20		/* How to get the vector from LR */
 	bla	generictrap		/* LR & SPRG3 is exception # */
-CNAME(trapsize) = .-CNAME(trapcode)
+CNAME(trapcodeend):
 
 /*
  * 64-bit version of trapcode. Identical, except it calls generictrap64.
@@ -362,7 +362,7 @@ CNAME(trapcode64):
 /*
  * For ALI: has to save DSISR and DAR
  */
-	.globl	CNAME(alitrap),CNAME(alisize)
+	.globl	CNAME(alitrap),CNAME(aliend)
 CNAME(alitrap):
 	mtsprg1	%r1			/* save SP */
 	GET_CPUINFO(%r1)
@@ -386,7 +386,7 @@ CNAME(alitrap):
 	mfsrr1	%r31
 	mtcr	%r31
 	bla	s_trap
-CNAME(alisize) = .-CNAME(alitrap)
+CNAME(aliend):
 
 /*
  * G2 specific: instuction TLB miss.
@@ -594,7 +594,7 @@ CNAME(dsmisssize) = .-CNAME(dsmisstrap)
  * Has to handle BAT spills
  * and standard pagetable spills
  */
-	.globl	CNAME(dsitrap),CNAME(dsisize)
+	.globl	CNAME(dsitrap),CNAME(dsiend)
 CNAME(dsitrap):
 	mtsprg1	%r1			/* save SP */
 	GET_CPUINFO(%r1)
@@ -645,7 +645,7 @@ CNAME(dsitrap):
 1:
 	mflr	%r28			/* save LR (SP already saved) */
 	bla	disitrap
-CNAME(dsisize) = .-CNAME(dsitrap)
+CNAME(dsiend):
 
 /*
  * Preamble code for DSI/ISI traps
@@ -883,7 +883,7 @@ CNAME(rfi_patch2):
 /*
  * In case of KDB we want a separate trap catcher for it
  */
-	.globl	CNAME(dblow),CNAME(dbsize)
+	.globl	CNAME(dblow),CNAME(dbend)
 CNAME(dblow):
 	mtsprg1	%r1			/* save SP */
 	mtsprg2	%r29			/* save r29 */
@@ -909,5 +909,5 @@ CNAME(dblow):
         stw	%r31,(PC_DBSAVE+CPUSAVE_R31)(%r1)	/* free r31 */
         mflr	%r28					/* save LR */
 	bla	dbtrap
-CNAME(dbsize) = .-CNAME(dblow)
+CNAME(dbend):
 #endif /* KDB */
diff --git a/sys/powerpc/aim/trap_subr64.S b/sys/powerpc/aim/trap_subr64.S
index a2d5bc5b1b4..de79845eded 100644
--- a/sys/powerpc/aim/trap_subr64.S
+++ b/sys/powerpc/aim/trap_subr64.S
@@ -294,7 +294,7 @@ dtrace_invop_calltrap_addr:
  * not still hanging around in the trap handling region
  * once the MMU is turned on.
  */
-	.globl	CNAME(rstcode), CNAME(rstsize)
+	.globl	CNAME(rstcode), CNAME(rstcodeend)
 CNAME(rstcode):
 	/* Explicitly set MSR[SF] */
 	mfmsr	%r9
@@ -309,7 +309,7 @@ CNAME(rstcode):
 	mtlr	%r9
 
 	blr
-CNAME(rstsize) = . - CNAME(rstcode)
+CNAME(rstcodeend):
 
 cpu_reset:
 	GET_TOCBASE(%r2)
@@ -350,7 +350,7 @@ cpu_reset:
  * (except ISI/DSI, ALI, and the interrupts). Has to fit in 8 instructions!
  */
 
-	.globl	CNAME(trapcode),CNAME(trapsize)
+	.globl	CNAME(trapcode),CNAME(trapcodeend)
 	.p2align 3
 CNAME(trapcode):
 	mtsprg1	%r1			/* save SP */
@@ -361,7 +361,7 @@ CNAME(trapcode):
 	mtlr	%r1
 	li	%r1, 0xA0		/* How to get the vector from LR */
 	blrl				/* Branch to generictrap */
-CNAME(trapsize) = .-CNAME(trapcode)
+CNAME(trapcodeend):
 
 /*
  * For SLB misses: do special things for the kernel
@@ -369,7 +369,7 @@ CNAME(trapsize) = .-CNAME(trapcode)
  * Note: SPRG1 is always safe to overwrite any time the MMU is on, which is
  * the only time this can be called.
  */
-	.globl	CNAME(slbtrap),CNAME(slbtrapsize)
+	.globl	CNAME(slbtrap),CNAME(slbtrapend)
 	.p2align 3
 CNAME(slbtrap):
 	mtsprg1	%r1			/* save SP */
@@ -404,7 +404,7 @@ CNAME(slbtrap):
 	mtlr	%r1
 	GET_CPUINFO(%r1)
 	blrl				/* 124 bytes -- 4 to spare */
-CNAME(slbtrapsize) = .-CNAME(slbtrap)
+CNAME(slbtrapend):
 
 kern_slbtrap:
 	std	%r2,(PC_SLBSAVE+136)(%r1) /* old LR */
@@ -525,7 +525,7 @@ kern_slbtrap:
 /*
  * For ALI: has to save DSISR and DAR
  */
-	.globl	CNAME(alitrap),CNAME(alisize)
+	.globl	CNAME(alitrap),CNAME(aliend)
 CNAME(alitrap):
 	mtsprg1	%r1			/* save SP */
 	GET_CPUINFO(%r1)
@@ -560,13 +560,13 @@ CNAME(alitrap):
 	mfsrr1	%r31
 	mtcr	%r31
 	blrl
-CNAME(alisize) = .-CNAME(alitrap)
+CNAME(aliend):
 
 /*
  * Similar to the above for DSI
  * Has to handle standard pagetable spills
  */
-	.globl	CNAME(dsitrap),CNAME(dsisize)
+	.globl	CNAME(dsitrap),CNAME(dsiend)
 CNAME(dsitrap):
 	mtsprg1	%r1			/* save SP */
 	GET_CPUINFO(%r1)
@@ -587,7 +587,7 @@ CNAME(dsitrap):
 	ld	%r1,0(%r1)
 	mtlr	%r1
 	blrl				/* Branch to generictrap */
-CNAME(dsisize) = .-CNAME(dsitrap)
+CNAME(dsiend):
 
 /*
  * Preamble code for DSI/ISI traps
@@ -830,7 +830,7 @@ dbleave:
 /*
  * In case of KDB we want a separate trap catcher for it
  */
-	.globl	CNAME(dblow),CNAME(dbsize)
+	.globl	CNAME(dblow),CNAME(dbend)
 CNAME(dblow):
 	mtsprg1	%r1			/* save SP */
 	mtsprg2	%r29			/* save r29 */
@@ -869,5 +869,5 @@ CNAME(dblow):
 	ld	%r1,0(%r1)
 	mtlr	%r1
 	blrl				/* Branch to generictrap */
-CNAME(dbsize) = .-CNAME(dblow)
+CNAME(dbend):
 #endif /* KDB */

From ffc610b221e853e3246be960bc9007de751ccf7e Mon Sep 17 00:00:00 2001
From: "Andrey V. Elsukov" <ae@FreeBSD.org>
Date: Fri, 23 Jan 2015 13:26:35 +0000
Subject: [PATCH 201/279] After r270929 RAW IP code assumes that all IP fields
 are in network byte order. Fix ping(8) to pass an IP header with converted
 ip_off and ip_len fields, when IP_HDRINCL socket option used.

---
 sbin/ping/ping.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sbin/ping/ping.c b/sbin/ping/ping.c
index aeb301430fb..f24ecdeea0c 100644
--- a/sbin/ping/ping.c
+++ b/sbin/ping/ping.c
@@ -713,7 +713,7 @@ main(int argc, char *const *argv)
 		ip->ip_hl = sizeof(struct ip) >> 2;
 		ip->ip_tos = tos;
 		ip->ip_id = 0;
-		ip->ip_off = df ? IP_DF : 0;
+		ip->ip_off = htons(df ? IP_DF : 0);
 		ip->ip_ttl = ttl;
 		ip->ip_p = IPPROTO_ICMP;
 		ip->ip_src.s_addr = source ? sock_in.sin_addr.s_addr : INADDR_ANY;
@@ -1078,7 +1078,7 @@ pinger(void)
 	if (options & F_HDRINCL) {
 		cc += sizeof(struct ip);
 		ip = (struct ip *)outpackhdr;
-		ip->ip_len = cc;
+		ip->ip_len = htons(cc);
 		ip->ip_sum = in_cksum((u_short *)outpackhdr, cc);
 		packet = outpackhdr;
 	}

From 369a67085766126543941e688dc91ccff073d8ca Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Fri, 23 Jan 2015 14:28:12 +0000
Subject: [PATCH 202/279] When a CARP state change is caused by an ifconfig
 request, log it accordingly.

Suggested by:	glebius
MFC after:	1 week
MFC with:	277530
---
 sys/netinet/ip_carp.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c
index 5c449c0ddd5..b8881044f16 100644
--- a/sys/netinet/ip_carp.c
+++ b/sys/netinet/ip_carp.c
@@ -1702,12 +1702,14 @@ carp_ioctl(struct ifreq *ifr, u_long cmd, struct thread *td)
 			switch (carpr.carpr_state) {
 			case BACKUP:
 				callout_stop(&sc->sc_ad_tmo);
-				carp_set_state(sc, BACKUP, "SIOCSVH");
+				carp_set_state(sc, BACKUP,
+				    "user requested via ifconfig");
 				carp_setrun(sc, 0);
 				carp_delroute(sc);
 				break;
 			case MASTER:
-				carp_master_down_locked(sc, "SIOCSVH");
+				carp_master_down_locked(sc,
+				    "user requested via ifconfig");
 				break;
 			default:
 				break;

From bb269f3ae49724b7b1e550d6b005b152ed10d360 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Fri, 23 Jan 2015 14:30:24 +0000
Subject: [PATCH 203/279] Log hardware interface up/down as "hardware" rather
 than just "hw".

Suggested by:	glebius
MFC after:	1 week
MFC with:	277530
---
 sys/netinet/ip_carp.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sys/netinet/ip_carp.c b/sys/netinet/ip_carp.c
index b8881044f16..524ebd8443b 100644
--- a/sys/netinet/ip_carp.c
+++ b/sys/netinet/ip_carp.c
@@ -2001,13 +2001,13 @@ carp_sc_state(struct carp_softc *sc)
 #ifdef INET6
 		callout_stop(&sc->sc_md6_tmo);
 #endif
-		carp_set_state(sc, INIT, "hw interface down");
+		carp_set_state(sc, INIT, "hardware interface down");
 		carp_setrun(sc, 0);
 		if (!sc->sc_suppress)
 			carp_demote_adj(V_carp_ifdown_adj, "interface down");
 		sc->sc_suppress = 1;
 	} else {
-		carp_set_state(sc, INIT, "hw interface up");
+		carp_set_state(sc, INIT, "hardware interface up");
 		carp_setrun(sc, 0);
 		if (sc->sc_suppress)
 			carp_demote_adj(-V_carp_ifdown_adj, "interface up");

From 128236c022a441231caf31be9fff4f0290d67ba1 Mon Sep 17 00:00:00 2001
From: Kevin Lo <kevlo@FreeBSD.org>
Date: Fri, 23 Jan 2015 15:14:30 +0000
Subject: [PATCH 204/279] Remove break after return.

---
 sys/dev/agp/agp_nvidia.c | 10 +++++-----
 sys/dev/cx/cxddk.c       |  6 +++---
 sys/dev/dc/if_dc.c       |  5 -----
 3 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/sys/dev/agp/agp_nvidia.c b/sys/dev/agp/agp_nvidia.c
index bb153f9cc49..a0098f0252e 100644
--- a/sys/dev/agp/agp_nvidia.c
+++ b/sys/dev/agp/agp_nvidia.c
@@ -274,11 +274,11 @@ static u_int32_t
 agp_nvidia_get_aperture(device_t dev)
 {
 	switch (pci_read_config(dev, AGP_NVIDIA_0_APSIZE, 1) & 0x0f) {
-	case 0: return (512 * 1024 * 1024); break;
-	case 8: return (256 * 1024 * 1024); break;
-	case 12: return (128 * 1024 * 1024); break;
-	case 14: return (64 * 1024 * 1024); break;
-	case 15: return (32 * 1024 * 1024); break;
+	case 0: return (512 * 1024 * 1024);
+	case 8: return (256 * 1024 * 1024);
+	case 12: return (128 * 1024 * 1024);
+	case 14: return (64 * 1024 * 1024);
+	case 15: return (32 * 1024 * 1024);
 	default:
 		device_printf(dev, "Invalid aperture setting 0x%x\n",
 		    pci_read_config(dev, AGP_NVIDIA_0_APSIZE, 1));
diff --git a/sys/dev/cx/cxddk.c b/sys/dev/cx/cxddk.c
index bbcc904051e..94780fb3591 100644
--- a/sys/dev/cx/cxddk.c
+++ b/sys/dev/cx/cxddk.c
@@ -875,9 +875,9 @@ int cx_get_port (cx_chan_t *c)
 
 		if (iftype)
 			switch (c->type) {
-			case T_UNIV_V35:   return 1; break;
-			case T_UNIV_RS449: return 2; break;
-			default:	   return -1; break;
+			case T_UNIV_V35:   return 1;
+			case T_UNIV_RS449: return 2;
+			default:	   return -1;
 			}
 		else
 			return 0;
diff --git a/sys/dev/dc/if_dc.c b/sys/dev/dc/if_dc.c
index 9458a444430..421e326d256 100644
--- a/sys/dev/dc/if_dc.c
+++ b/sys/dev/dc/if_dc.c
@@ -671,20 +671,16 @@ dc_miibus_readreg(device_t dev, int phy, int reg)
 			 * code think there's a PHY here.
 			 */
 				return (BMSR_MEDIAMASK);
-				break;
 			case MII_PHYIDR1:
 				if (DC_IS_PNIC(sc))
 					return (DC_VENDORID_LO);
 				return (DC_VENDORID_DEC);
-				break;
 			case MII_PHYIDR2:
 				if (DC_IS_PNIC(sc))
 					return (DC_DEVICEID_82C168);
 				return (DC_DEVICEID_21143);
-				break;
 			default:
 				return (0);
-				break;
 			}
 		} else
 			return (0);
@@ -748,7 +744,6 @@ dc_miibus_readreg(device_t dev, int phy, int reg)
 			device_printf(dev, "phy_read: bad phy register %x\n",
 			    reg);
 			return (0);
-			break;
 		}
 
 		rval = CSR_READ_4(sc, phy_reg) & 0x0000FFFF;

From abf607ed8f5d5f5981be326d7173d48c185d8ef3 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Fri, 23 Jan 2015 15:55:03 +0000
Subject: [PATCH 205/279] Print transfer times for read, write, & overall
 independently.

Round up calculated values for iops and average time per io to avoid a
shifting display if there are 1000+ (or even 10000+) iops, or if an
average time per io column is 1000+ ms.

Reviewed by:	ken
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1089947 on 2014/09/26
		1093625 on 2014/09/29
		1093650 on 2014/09/29
		1095662 on 2014/10/09
---
 usr.sbin/iostat/iostat.c | 32 +++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/usr.sbin/iostat/iostat.c b/usr.sbin/iostat/iostat.c
index f566f39036b..195e59c540a 100644
--- a/usr.sbin/iostat/iostat.c
+++ b/usr.sbin/iostat/iostat.c
@@ -726,15 +726,17 @@ static void
 devstats(int perf_select, long double etime, int havelast)
 {
 	int dn;
-	long double transfers_per_second, transfers_per_second_read, transfers_per_second_write;
-	long double kb_per_transfer, mb_per_second, mb_per_second_read, mb_per_second_write;
+	long double transfers_per_second, transfers_per_second_read;
+	long double transfers_per_second_write;
+	long double kb_per_transfer, mb_per_second, mb_per_second_read;
+	long double mb_per_second_write;
 	u_int64_t total_bytes, total_transfers, total_blocks;
 	u_int64_t total_bytes_read, total_transfers_read;
 	u_int64_t total_bytes_write, total_transfers_write;
 	long double busy_pct, busy_time;
 	u_int64_t queue_len;
-	long double total_mb;
-	long double blocks_per_second, ms_per_transaction, total_duration;
+	long double total_mb, blocks_per_second, total_duration;
+	long double ms_per_other, ms_per_read, ms_per_write, ms_per_transaction;
 	int firstline = 1;
 	char *devname;
 
@@ -746,8 +748,8 @@ devstats(int perf_select, long double etime, int havelast)
 			printf("           cpu ");
 		printf("\n");
 		if (Iflag == 0) {
-			printf("device     r/s   w/s    kr/s    kw/s qlen "
-			    "svc_t  %%b  ");
+			printf("device     r/s   w/s     kr/s     kw/s "
+			    " ms/r  ms/w  ms/o  ms/t qlen  %%b  ");
 		} else {
 			printf("device           r/i         w/i         kr/i"
 			    "         kw/i qlen   tsvc_t/i      sb/i  ");
@@ -786,6 +788,9 @@ devstats(int perf_select, long double etime, int havelast)
 		    DSM_MB_PER_SECOND_WRITE, &mb_per_second_write,
 		    DSM_BLOCKS_PER_SECOND, &blocks_per_second,
 		    DSM_MS_PER_TRANSACTION, &ms_per_transaction,
+		    DSM_MS_PER_TRANSACTION_READ, &ms_per_read,
+		    DSM_MS_PER_TRANSACTION_WRITE, &ms_per_write,
+		    DSM_MS_PER_TRANSACTION_OTHER, &ms_per_other,
 		    DSM_BUSY_PCT, &busy_pct,
 		    DSM_QUEUE_LENGTH, &queue_len,
 		    DSM_TOTAL_DURATION, &total_duration,
@@ -820,13 +825,18 @@ devstats(int perf_select, long double etime, int havelast)
 			    mb_per_second_write > ((long double).0005)/1024 ||
 			    busy_pct > 0.5) {
 				if (Iflag == 0)
-					printf("%-8.8s %5.1Lf %5.1Lf %7.1Lf %7.1Lf %4" PRIu64 " %5.1Lf %3.0Lf ",
-					    devname, transfers_per_second_read,
-					    transfers_per_second_write,
+					printf("%-8.8s %5d %5d %8.1Lf "
+					    "%8.1Lf %5d %5d %5d %5d "
+					    "%4" PRIu64 " %3.0Lf ",
+					    devname,
+					    (int)transfers_per_second_read,
+					    (int)transfers_per_second_write,
 					    mb_per_second_read * 1024,
 					    mb_per_second_write * 1024,
-					    queue_len,
-					    ms_per_transaction, busy_pct);
+					    (int)ms_per_read, (int)ms_per_write,
+					    (int)ms_per_other,
+					    (int)ms_per_transaction,
+					    queue_len, busy_pct);
 				else
 					printf("%-8.8s %11.1Lf %11.1Lf "
 					    "%12.1Lf %12.1Lf %4" PRIu64

From ca71b908cc363f80f9080974eeafdcbf9465bf69 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Fri, 23 Jan 2015 16:03:02 +0000
Subject: [PATCH 206/279] Allow lists for "nooption" and "nooptions" keywords.

usr.sbin/config/config.y
	According to config(5), the "device", "devices",
	"nodevice", "nodevices", "option", "options",
	"nooption", and "nooptions" keywords can all take a
	comma-separated list of values.  However, the yacc code
	did not allow lists for "nooption" and "nooptions", only
	single values.  This commit fixes the yacc code to allow
	comma separated values for all the above keywords.

Submitted by:	asomers
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1095296 on 2014/10/07
---
 usr.sbin/config/config.y | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/usr.sbin/config/config.y b/usr.sbin/config/config.y
index 81329d922af..c7198a027ba 100644
--- a/usr.sbin/config/config.y
+++ b/usr.sbin/config/config.y
@@ -178,7 +178,7 @@ Config_spec:
 	      } |
 	OPTIONS Opt_list
 		|
-	NOOPTION Save_id { rmopt_schedule(&opt, $2); } |
+	NOOPTION NoOpt_list |
 	MAKEOPTIONS Mkopt_list
 		|
 	NOMAKEOPTION Save_id { rmopt_schedule(&mkopt, $2); } |
@@ -225,6 +225,11 @@ Opt_list:
 	Option
 		;
 
+NoOpt_list:
+	NoOpt_list COMMA NoOption
+	  	|
+	NoOption
+		;
 Option:
 	Save_id {
 		newopt(&opt, $1, NULL, 0);
@@ -236,6 +241,11 @@ Option:
 		newopt(&opt, $1, $3, 0);
 	      } ;
 
+NoOption:
+	Save_id {
+		rmopt_schedule(&opt, $1);
+		};
+
 Opt_value:
 	ID { $$ = $1; } |
 	NUMBER {

From 9e8ab8f7310220748d2d759cb7fc7ced2dc0617d Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Fri, 23 Jan 2015 16:05:47 +0000
Subject: [PATCH 207/279] Add pinctrl driver support for the encoded input
 register config words that the linux guys made up on the fly (but didn't
 document) last August. This type of encoded config now appears in the imx6
 fdt data.

---
 sys/arm/freescale/imx/imx_iomux.c | 31 ++++++++++++++++++++++++++++---
 1 file changed, 28 insertions(+), 3 deletions(-)

diff --git a/sys/arm/freescale/imx/imx_iomux.c b/sys/arm/freescale/imx/imx_iomux.c
index 7786b76e0a4..d1ac6cbc265 100644
--- a/sys/arm/freescale/imx/imx_iomux.c
+++ b/sys/arm/freescale/imx/imx_iomux.c
@@ -117,10 +117,36 @@ WR4(struct iomux_softc *sc, bus_size_t off, uint32_t val)
 	bus_write_4(sc->mem_res, off, val);
 }
 
+static void
+iomux_configure_input(struct iomux_softc *sc, uint32_t reg, uint32_t val)
+{
+	u_int select, mask, shift, width;
+
+	/* If register and value are zero, there is nothing to configure. */
+	if (reg == 0 && val == 0)
+		return;
+
+	/*
+	 * If the config value has 0xff in the high byte it is encoded:
+	 * 	31     23      15      7        0
+	 *      | 0xff | shift | width | select |
+	 * We need to mask out the old select value and OR in the new, using a
+	 * mask of the given width and shifting the values up by shift.
+	 */
+	if ((val & 0xff000000) == 0xff000000) {
+		select = val & 0x000000ff;
+		width = (val & 0x0000ff00) >> 8;
+		shift = (val & 0x00ff0000) >> 16;
+		mask  = ((1u << width) - 1) << shift;
+		val = (RD4(sc, reg) & ~mask) | (select << shift);
+	}
+	WR4(sc, reg, val);
+}
+
 static int
 iomux_configure_pins(device_t dev, phandle_t cfgxref)
 {
-	struct iomux_softc * sc;
+	struct iomux_softc *sc;
 	struct pincfg *cfgtuples, *cfg;
 	phandle_t cfgnode;
 	int i, ntuples;
@@ -137,8 +163,7 @@ iomux_configure_pins(device_t dev, phandle_t cfgxref)
 	for (i = 0, cfg = cfgtuples; i < ntuples; i++, cfg++) {
 		sion = (cfg->padconf_val & PADCONF_SION) ? PADMUX_SION : 0;
 		WR4(sc, cfg->mux_reg, cfg->mux_val | sion);
-		if (cfg->input_reg != 0)
-			WR4(sc, cfg->input_reg, cfg->input_val);
+		iomux_configure_input(sc, cfg->input_reg, cfg->input_val);
 		if ((cfg->padconf_val & PADCONF_NONE) == 0)
 			WR4(sc, cfg->padconf_reg, cfg->padconf_val);
 		if (bootverbose) {

From 961a5dae5c11523a2bd41baee9416eed11073ca0 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Fri, 23 Jan 2015 16:15:55 +0000
Subject: [PATCH 208/279] Make "vmstat -i" respect the -c <count> and -i
 <interval> options together.

Submitted by:	asomers
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1066735 on 2014/06/04
---
 usr.bin/vmstat/vmstat.c | 149 +++++++++++++++++++++++++++++-----------
 1 file changed, 110 insertions(+), 39 deletions(-)

diff --git a/usr.bin/vmstat/vmstat.c b/usr.bin/vmstat/vmstat.c
index 760dce83593..a54a0dd45b2 100644
--- a/usr.bin/vmstat/vmstat.c
+++ b/usr.bin/vmstat/vmstat.c
@@ -148,7 +148,7 @@ static void	cpustats(void);
 static void	pcpustats(int, u_long, int);
 static void	devstats(void);
 static void	doforkst(void);
-static void	dointr(void);
+static void	dointr(unsigned int, int);
 static void	dosum(void);
 static void	dovmstat(unsigned int, int);
 static void	domemstat_malloc(void);
@@ -325,7 +325,7 @@ retry_nlist:
 		dotimes();
 #endif
 	if (todo & INTRSTAT)
-		dointr();
+		dointr(interval, reps);
 	if (todo & VMSTAT)
 		dovmstat(interval, reps);
 	exit(0);
@@ -1165,61 +1165,132 @@ pcpustats(int ncpus, u_long cpumask, int maxid)
 	}
 }
 
-static void
-dointr(void)
+static unsigned int
+read_intrcnts(unsigned long **intrcnts)
 {
-	unsigned long *intrcnt, uptime;
-	uint64_t inttotal;
-	size_t clen, inamlen, intrcntlen, istrnamlen;
-	unsigned int i, nintr;
-	char *intrname, *tintrname;
+	size_t intrcntlen;
 
-	uptime = getuptime();
 	if (kd != NULL) {
 		kread(X_SINTRCNT, &intrcntlen, sizeof(intrcntlen));
-		kread(X_SINTRNAMES, &inamlen, sizeof(inamlen));
-		if ((intrcnt = malloc(intrcntlen)) == NULL ||
-		    (intrname = malloc(inamlen)) == NULL)
+		if ((*intrcnts = malloc(intrcntlen)) == NULL)
 			err(1, "malloc()");
-		kread(X_INTRCNT, intrcnt, intrcntlen);
-		kread(X_INTRNAMES, intrname, inamlen);
+		kread(X_INTRCNT, *intrcnts, intrcntlen);
 	} else {
-		for (intrcnt = NULL, intrcntlen = 1024; ; intrcntlen *= 2) {
-			if ((intrcnt = reallocf(intrcnt, intrcntlen)) == NULL)
+		for (*intrcnts = NULL, intrcntlen = 1024; ; intrcntlen *= 2) {
+			*intrcnts = reallocf(*intrcnts, intrcntlen);
+			if (*intrcnts == NULL)
 				err(1, "reallocf()");
 			if (mysysctl("hw.intrcnt",
-			    intrcnt, &intrcntlen, NULL, 0) == 0)
-				break;
-		}
-		for (intrname = NULL, inamlen = 1024; ; inamlen *= 2) {
-			if ((intrname = reallocf(intrname, inamlen)) == NULL)
-				err(1, "reallocf()");
-			if (mysysctl("hw.intrnames",
-			    intrname, &inamlen, NULL, 0) == 0)
+			    *intrcnts, &intrcntlen, NULL, 0) == 0)
 				break;
 		}
 	}
-	nintr = intrcntlen / sizeof(unsigned long);
-	tintrname = intrname;
+
+	return (intrcntlen / sizeof(unsigned long));
+}
+
+static void
+print_intrcnts(unsigned long *intrcnts, unsigned long *old_intrcnts,
+		char *intrnames, unsigned int nintr,
+		size_t istrnamlen, unsigned long long period)
+{
+	unsigned long *intrcnt, *old_intrcnt;
+	uint64_t inttotal, old_inttotal, total_count, total_rate;
+	char* intrname;
+	unsigned int i;
+
+	inttotal = 0;
+	old_inttotal = 0;
+	intrname = intrnames;
+	for (i = 0, intrcnt=intrcnts, old_intrcnt=old_intrcnts; i < nintr; i++) {
+		if (intrname[0] != '\0' && (*intrcnt != 0 || aflag)) {
+			unsigned long count, rate;
+
+			count = *intrcnt - *old_intrcnt;
+			rate = (count * 1000 + period/2) / period;
+			(void)printf("%-*s %20lu %10lu\n", (int)istrnamlen,
+			    intrname, count, rate);
+		}
+		intrname += strlen(intrname) + 1;
+		inttotal += *intrcnt++;
+		old_inttotal += *old_intrcnt++;
+	}
+	total_count = inttotal - old_inttotal;
+	total_rate = (total_count * 1000 + period/2) / period;
+	(void)printf("%-*s %20" PRIu64 " %10" PRIu64 "\n", (int)istrnamlen,
+	    "Total", total_count, total_rate);
+}
+
+static void
+dointr(unsigned int interval, int reps)
+{
+	unsigned long *intrcnts;
+	unsigned long long uptime, period;
+	unsigned long *old_intrcnts = NULL;
+	size_t clen, inamlen, istrnamlen;
+	unsigned int rep;
+	char *intrnames, *intrname;
+
+	uptime = getuptime();
+
+	/* Get the names of each interrupt source */
+	if (kd != NULL) {
+		kread(X_SINTRNAMES, &inamlen, sizeof(inamlen));
+		if ((intrnames = malloc(inamlen)) == NULL)
+			err(1, "malloc()");
+		kread(X_INTRNAMES, intrnames, inamlen);
+	} else {
+		for (intrnames = NULL, inamlen = 1024; ; inamlen *= 2) {
+			if ((intrnames = reallocf(intrnames, inamlen)) == NULL)
+				err(1, "reallocf()");
+			if (mysysctl("hw.intrnames",
+			    intrnames, &inamlen, NULL, 0) == 0)
+				break;
+		}
+	}
+
+	/* Determine the length of the longest interrupt name */
+	intrname = intrnames;
 	istrnamlen = strlen("interrupt");
-	for (i = 0; i < nintr; i++) {
-		clen = strlen(tintrname);
+	while(*intrname != '\0') {
+		clen = strlen(intrname);
 		if (clen > istrnamlen)
 			istrnamlen = clen;
-		tintrname += clen + 1;
+		intrname += strlen(intrname) + 1;
 	}
 	(void)printf("%-*s %20s %10s\n", (int)istrnamlen, "interrupt", "total",
 	    "rate");
-	inttotal = 0;
-	for (i = 0; i < nintr; i++) {
-		if (intrname[0] != '\0' && (*intrcnt != 0 || aflag))
-			(void)printf("%-*s %20lu %10lu\n", (int)istrnamlen,
-			    intrname, *intrcnt, *intrcnt / uptime);
-		intrname += strlen(intrname) + 1;
-		inttotal += *intrcnt++;
+
+	/* 
+	 * Loop reps times printing differential interrupt counts.  If reps is
+	 * zero, then run just once, printing total counts
+	 */
+	period = uptime * 1000;
+	while(1) {
+		char *intrname;
+		unsigned int nintr;
+
+		nintr = read_intrcnts(&intrcnts);
+		/* 
+		 * Initialize old_intrcnts to 0 for the first pass, so
+		 * print_intrcnts will print total interrupts since boot
+		 */
+		if (old_intrcnts == NULL) {
+			old_intrcnts = calloc(nintr, sizeof(unsigned long));
+			if (old_intrcnts == NULL)
+				err(1, "calloc()");
+		}
+
+		print_intrcnts(intrcnts, old_intrcnts, intrnames, nintr,
+		    istrnamlen, period);
+
+		free(old_intrcnts);
+		old_intrcnts = intrcnts;
+		if (reps >= 0 && --reps <= 0)
+			break;
+		usleep(interval * 1000);
+		period = interval;
 	}
-	(void)printf("%-*s %20" PRIu64 " %10" PRIu64 "\n", (int)istrnamlen,
-	    "Total", inttotal, inttotal / uptime);
 }
 
 static void

From 157b158ea9e4a289663c000d02ea8037e3fd717f Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Fri, 23 Jan 2015 16:18:39 +0000
Subject: [PATCH 209/279] Use CLOCK_UPTIME to get the uptime instead of
 CLOCK_MONOTONIC.

Submitted by:	asomers
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1066740 on 2014/06/04
---
 usr.bin/vmstat/vmstat.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/usr.bin/vmstat/vmstat.c b/usr.bin/vmstat/vmstat.c
index a54a0dd45b2..925967c7f43 100644
--- a/usr.bin/vmstat/vmstat.c
+++ b/usr.bin/vmstat/vmstat.c
@@ -411,7 +411,7 @@ getuptime(void)
 {
 	struct timespec sp;
 
-	(void)clock_gettime(CLOCK_MONOTONIC, &sp);
+	(void)clock_gettime(CLOCK_UPTIME, &sp);
 
 	return(sp.tv_sec);
 }

From d0fde85e656f7e841972e5c8ba2e3868c974cdf3 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Fri, 23 Jan 2015 16:21:31 +0000
Subject: [PATCH 210/279] Use clock_gettime to measure the time that we spent
 asleep during "vmstat -i" instead of assuming it's what we asked for.

Submitted by:	asomers
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1066751 on 2014/06/04
---
 usr.bin/vmstat/vmstat.c | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/usr.bin/vmstat/vmstat.c b/usr.bin/vmstat/vmstat.c
index 925967c7f43..9049e442227 100644
--- a/usr.bin/vmstat/vmstat.c
+++ b/usr.bin/vmstat/vmstat.c
@@ -163,7 +163,7 @@ static void	printhdr(int, u_long);
 static void	usage(void);
 
 static long	pct(long, long);
-static long	getuptime(void);
+static long long	getuptime(void);
 
 static char   **getdrivedata(char **);
 
@@ -406,14 +406,15 @@ getdrivedata(char **argv)
 	return(argv);
 }
 
-static long
+/* Return system uptime in nanoseconds */
+static long long
 getuptime(void)
 {
 	struct timespec sp;
 
 	(void)clock_gettime(CLOCK_UPTIME, &sp);
 
-	return(sp.tv_sec);
+	return((long long)sp.tv_sec * 1000000000LL + sp.tv_nsec);
 }
 
 static void
@@ -654,7 +655,7 @@ dovmstat(unsigned int interval, int reps)
 	u_long cpumask;
 	int rate_adj;
 
-	uptime = getuptime();
+	uptime = getuptime() / 1000000000LL;
 	halfuptime = uptime / 2;
 	rate_adj = 1;
 	ncpus = 1;
@@ -1192,7 +1193,7 @@ read_intrcnts(unsigned long **intrcnts)
 static void
 print_intrcnts(unsigned long *intrcnts, unsigned long *old_intrcnts,
 		char *intrnames, unsigned int nintr,
-		size_t istrnamlen, unsigned long long period)
+		size_t istrnamlen, long long period_ms)
 {
 	unsigned long *intrcnt, *old_intrcnt;
 	uint64_t inttotal, old_inttotal, total_count, total_rate;
@@ -1207,7 +1208,7 @@ print_intrcnts(unsigned long *intrcnts, unsigned long *old_intrcnts,
 			unsigned long count, rate;
 
 			count = *intrcnt - *old_intrcnt;
-			rate = (count * 1000 + period/2) / period;
+			rate = (count * 1000 + period_ms / 2) / period_ms;
 			(void)printf("%-*s %20lu %10lu\n", (int)istrnamlen,
 			    intrname, count, rate);
 		}
@@ -1216,7 +1217,7 @@ print_intrcnts(unsigned long *intrcnts, unsigned long *old_intrcnts,
 		old_inttotal += *old_intrcnt++;
 	}
 	total_count = inttotal - old_inttotal;
-	total_rate = (total_count * 1000 + period/2) / period;
+	total_rate = (total_count * 1000 + period_ms / 2) / period_ms;
 	(void)printf("%-*s %20" PRIu64 " %10" PRIu64 "\n", (int)istrnamlen,
 	    "Total", total_count, total_rate);
 }
@@ -1225,10 +1226,9 @@ static void
 dointr(unsigned int interval, int reps)
 {
 	unsigned long *intrcnts;
-	unsigned long long uptime, period;
+	long long uptime, period_ms;
 	unsigned long *old_intrcnts = NULL;
 	size_t clen, inamlen, istrnamlen;
-	unsigned int rep;
 	char *intrnames, *intrname;
 
 	uptime = getuptime();
@@ -1265,10 +1265,10 @@ dointr(unsigned int interval, int reps)
 	 * Loop reps times printing differential interrupt counts.  If reps is
 	 * zero, then run just once, printing total counts
 	 */
-	period = uptime * 1000;
+	period_ms = uptime / 1000000;
 	while(1) {
-		char *intrname;
 		unsigned int nintr;
+		long long old_uptime;
 
 		nintr = read_intrcnts(&intrcnts);
 		/* 
@@ -1282,14 +1282,16 @@ dointr(unsigned int interval, int reps)
 		}
 
 		print_intrcnts(intrcnts, old_intrcnts, intrnames, nintr,
-		    istrnamlen, period);
+		    istrnamlen, period_ms);
 
 		free(old_intrcnts);
 		old_intrcnts = intrcnts;
 		if (reps >= 0 && --reps <= 0)
 			break;
 		usleep(interval * 1000);
-		period = interval;
+		old_uptime = uptime;
+		uptime = getuptime();
+		period_ms = (uptime - old_uptime) / 1000000;
 	}
 }
 

From 4552fccb4b3e019ad21364969278b11a04f52542 Mon Sep 17 00:00:00 2001
From: Jung-uk Kim <jkim@FreeBSD.org>
Date: Fri, 23 Jan 2015 18:12:44 +0000
Subject: [PATCH 211/279] Revert r216942.  This commit was premature and caused
 too many complaints.

PR:		162859
MFC after:	3 days
---
 sys/dev/acpica/acpi_ec.c | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/sys/dev/acpica/acpi_ec.c b/sys/dev/acpica/acpi_ec.c
index 6537e4062b4..2261b355d9f 100644
--- a/sys/dev/acpica/acpi_ec.c
+++ b/sys/dev/acpica/acpi_ec.c
@@ -620,7 +620,7 @@ EcGpeQueryHandler(void *Context)
     struct acpi_ec_softc	*sc = (struct acpi_ec_softc *)Context;
     UINT8			Data;
     ACPI_STATUS			Status;
-    int				retry, sci_enqueued;
+    int				retry;
     char			qxx[5];
 
     ACPI_FUNCTION_TRACE((char *)(uintptr_t)__func__);
@@ -641,7 +641,6 @@ EcGpeQueryHandler(void *Context)
      * that may arise from running the query from causing another query
      * to be queued, we clear the pending flag only after running it.
      */
-    sci_enqueued = sc->ec_sci_pend;
     for (retry = 0; retry < 2; retry++) {
 	Status = EcCommand(sc, EC_COMMAND_QUERY);
 	if (ACPI_SUCCESS(Status))
@@ -681,14 +680,6 @@ EcGpeQueryHandler(void *Context)
 	device_printf(sc->ec_dev, "evaluation of query method %s failed: %s\n",
 	    qxx, AcpiFormatException(Status));
     }
-
-    /* Reenable runtime GPE if its execution was deferred. */
-    if (sci_enqueued) {
-	Status = AcpiFinishGpe(sc->ec_gpehandle, sc->ec_gpebit);
-	if (ACPI_FAILURE(Status))
-	    device_printf(sc->ec_dev, "reenabling runtime GPE failed: %s\n",
-		AcpiFormatException(Status));
-    }
 }
 
 /*
@@ -722,10 +713,9 @@ EcGpeHandler(ACPI_HANDLE GpeDevice, UINT32 GpeNumber, void *Context)
     if ((EcStatus & EC_EVENT_SCI) && !sc->ec_sci_pend) {
 	CTR0(KTR_ACPI, "ec gpe queueing query handler");
 	Status = AcpiOsExecute(OSL_GPE_HANDLER, EcGpeQueryHandler, Context);
-	if (ACPI_SUCCESS(Status)) {
+	if (ACPI_SUCCESS(Status))
 	    sc->ec_sci_pend = TRUE;
-	    return (0);
-	} else
+	else
 	    printf("EcGpeHandler: queuing GPE query handler failed\n");
     }
     return (ACPI_REENABLE_GPE);

From ae1ef64fe4714d164bfc85b64625db89bd63d2c3 Mon Sep 17 00:00:00 2001
From: Jung-uk Kim <jkim@FreeBSD.org>
Date: Fri, 23 Jan 2015 18:55:04 +0000
Subject: [PATCH 212/279] Simplify retry loops.  No functional change.

---
 sys/dev/acpica/acpi_ec.c | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/sys/dev/acpica/acpi_ec.c b/sys/dev/acpica/acpi_ec.c
index 2261b355d9f..8a8cb6ea4f3 100644
--- a/sys/dev/acpica/acpi_ec.c
+++ b/sys/dev/acpica/acpi_ec.c
@@ -645,10 +645,8 @@ EcGpeQueryHandler(void *Context)
 	Status = EcCommand(sc, EC_COMMAND_QUERY);
 	if (ACPI_SUCCESS(Status))
 	    break;
-	if (ACPI_SUCCESS(EcCheckStatus(sc, "retr_check",
+	if (ACPI_FAILURE(EcCheckStatus(sc, "retr_check",
 	    EC_EVENT_INPUT_BUFFER_EMPTY)))
-	    continue;
-	else
 	    break;
     }
     sc->ec_sci_pend = FALSE;
@@ -963,15 +961,13 @@ EcRead(struct acpi_ec_softc *sc, UINT8 Address, UINT8 *Data)
 	gen_count = sc->ec_gencount;
 	EC_SET_DATA(sc, Address);
 	status = EcWaitEvent(sc, EC_EVENT_OUTPUT_BUFFER_FULL, gen_count);
-	if (ACPI_FAILURE(status)) {
-	    if (ACPI_SUCCESS(EcCheckStatus(sc, "retr_check",
-		EC_EVENT_INPUT_BUFFER_EMPTY)))
-		continue;
-	    else
-		break;
+	if (ACPI_SUCCESS(status)) {
+	    *Data = EC_GET_DATA(sc);
+	    return (AE_OK);
 	}
-	*Data = EC_GET_DATA(sc);
-	return (AE_OK);
+	if (ACPI_FAILURE(EcCheckStatus(sc, "retr_check",
+	    EC_EVENT_INPUT_BUFFER_EMPTY)))
+	    break;
     }
     device_printf(sc->ec_dev, "EcRead: failed waiting to get data\n");
     return (status);

From c14e0e5390eff621dc857e82e5a7b294c00614bb Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Fri, 23 Jan 2015 19:33:03 +0000
Subject: [PATCH 213/279] Document r276881, libedit UTF-8 support.

Sponsored by:	The FreeBSD Foundation
---
 release/doc/en_US.ISO8859-1/relnotes/article.xml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/release/doc/en_US.ISO8859-1/relnotes/article.xml b/release/doc/en_US.ISO8859-1/relnotes/article.xml
index 4a5b4d6d28d..1522f61da1b 100644
--- a/release/doc/en_US.ISO8859-1/relnotes/article.xml
+++ b/release/doc/en_US.ISO8859-1/relnotes/article.xml
@@ -791,6 +791,11 @@
 	<application>strings</application> were switched to the
 	versions from the ELF Tool Chain project.</para>
 
+      <para revision="276881">The <literal>libedit</literal> library
+	has been updated to include <acronym>UTF-8</acronym> support,
+	adding <acronym>UTF-8</acronym> support to the &man.sh.1;
+	shell.</para>
+
       <para revision="277270"><application>OpenSSL</application> has
 	been updated to version 1.0.1l.</para>
     </sect2>

From 2950933a5a5b02c8cd3676b7c5ca1bd81b628dc9 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Fri, 23 Jan 2015 19:33:05 +0000
Subject: [PATCH 214/279] Document r277166, ptrace(2) Altivec register support.

Sponsored by:	The FreeBSD Foundation
---
 release/doc/en_US.ISO8859-1/relnotes/article.xml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/release/doc/en_US.ISO8859-1/relnotes/article.xml b/release/doc/en_US.ISO8859-1/relnotes/article.xml
index 1522f61da1b..ac30f3b00a0 100644
--- a/release/doc/en_US.ISO8859-1/relnotes/article.xml
+++ b/release/doc/en_US.ISO8859-1/relnotes/article.xml
@@ -691,6 +691,10 @@
 	has been updated to support <acronym>UTF</acronym>-8, which
 	additionally provides unicode support to &man.sh.1;.</para>
 
+      <para revision="277166" arch="powerpc">The &man.ptrace.2; system
+	call has been updated include support for Altivec registers on
+	&os;/&arch.powerpc;.</para>
+
       <sect3 xml:id="abi-compat">
 	<title><acronym>ABI</acronym> Compatibility</title>
 

From ee21c8b77afe9b487d16b2ad8987afb9f2c0de45 Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Fri, 23 Jan 2015 19:33:08 +0000
Subject: [PATCH 215/279] Document r277458, support for cloud hosting providers
 added to the Release Engineering build tools.

Sponsored by:	The FreeBSD Foundation
---
 release/doc/en_US.ISO8859-1/relnotes/article.xml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/release/doc/en_US.ISO8859-1/relnotes/article.xml b/release/doc/en_US.ISO8859-1/relnotes/article.xml
index ac30f3b00a0..756e74a8081 100644
--- a/release/doc/en_US.ISO8859-1/relnotes/article.xml
+++ b/release/doc/en_US.ISO8859-1/relnotes/article.xml
@@ -813,7 +813,10 @@
     <sect2 xml:id="releng">
       <title>Release Engineering and Integration</title>
 
-      <para>&nbsp;</para>
+      <para revision="277458" contrib="sponsor" sponsor="&ff;">The
+	Release Engineering build tools have been updated to include
+	support for producing virtual machine disk images for various
+	cloud hosting providers.</para>
     </sect2>
 
     <sect2 xml:id="doc">

From ade9ba645590607ac91b716a5b343ccfdf1e4e7d Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Fri, 23 Jan 2015 19:33:46 +0000
Subject: [PATCH 216/279] Update the latest revision.

Sponsored by:	The FreeBSD Foundation
---
 release/doc/en_US.ISO8859-1/relnotes/article.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/release/doc/en_US.ISO8859-1/relnotes/article.xml b/release/doc/en_US.ISO8859-1/relnotes/article.xml
index 756e74a8081..e0625f302c6 100644
--- a/release/doc/en_US.ISO8859-1/relnotes/article.xml
+++ b/release/doc/en_US.ISO8859-1/relnotes/article.xml
@@ -20,7 +20,7 @@
 
     <pubdate>$FreeBSD$</pubdate>
 
-    <!-- Last rev: 276551 -->
+    <!-- Last rev: 277458 -->
 
     <copyright>
       <year>2015</year>

From 81be95ceb4bff64a9d3e6f0a76499b7c29bf2595 Mon Sep 17 00:00:00 2001
From: Colin Percival <cperciva@FreeBSD.org>
Date: Fri, 23 Jan 2015 20:24:13 +0000
Subject: [PATCH 217/279] Provide a recipe of "true" for building the "release"
 target once its prerequisites are satisfied, in order to avoid having an
 implicit rule triggered by the presence of release.sh.

Approved by:	gjb
---
 release/Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/release/Makefile b/release/Makefile
index a601921b330..9b80f8afa8f 100644
--- a/release/Makefile
+++ b/release/Makefile
@@ -279,6 +279,7 @@ ftp: packagesystem
 	cp *.txz MANIFEST ftp
 
 release:	real-release vm-release cloudware-release
+	@true
 
 real-release:
 	${MAKE} -C ${.CURDIR} ${.MAKEFLAGS} obj

From 9d4a2eed2c6c26ecd7ad1b7a0f90d0380c2af001 Mon Sep 17 00:00:00 2001
From: "Pedro F. Giffuni" <pfg@FreeBSD.org>
Date: Fri, 23 Jan 2015 20:32:19 +0000
Subject: [PATCH 218/279] GDB: Fix memset thinkos.

2005-03-25  Anthony Green  <green@redhat.com>

       * remote.c (remote_store_registers): Fix memset usage.
       * std-regs.c (value_of_builtin_frame_reg): Ditto.
       (value_of_builtin_frame_fp_reg): Ditto.
       (value_of_builtin_frame_reg): Ditto.

Sourceware commit:	170cd118a9e2169798247e70160f5a638b516661
---
 gdb/remote.c   | 2 +-
 gdb/std-regs.c | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/gdb/remote.c b/gdb/remote.c
index 36da449973d..bf4e6d96956 100644
--- a/gdb/remote.c
+++ b/gdb/remote.c
@@ -3463,7 +3463,7 @@ remote_store_registers (int regnum)
   {
     int i;
     regs = alloca (rs->sizeof_g_packet);
-    memset (regs, rs->sizeof_g_packet, 0);
+    memset (regs, 0, rs->sizeof_g_packet);
     for (i = 0; i < NUM_REGS + NUM_PSEUDO_REGS; i++)
       {
 	struct packet_reg *r = &rs->regs[i];
diff --git a/gdb/std-regs.c b/gdb/std-regs.c
index 368720dd95b..bd134b515b3 100644
--- a/gdb/std-regs.c
+++ b/gdb/std-regs.c
@@ -61,7 +61,7 @@ value_of_builtin_frame_reg (struct frame_info *frame)
   val = allocate_value (builtin_type_frame_reg);
   VALUE_LVAL (val) = not_lval;
   buf = VALUE_CONTENTS_RAW (val);
-  memset (buf, TYPE_LENGTH (VALUE_TYPE (val)), 0);
+  memset (buf, 0, TYPE_LENGTH (VALUE_TYPE (val)));
   /* frame.base.  */
   if (frame != NULL)
     ADDRESS_TO_POINTER (builtin_type_void_data_ptr, buf,
@@ -87,7 +87,7 @@ value_of_builtin_frame_fp_reg (struct frame_info *frame)
       struct value *val = allocate_value (builtin_type_void_data_ptr);
       char *buf = VALUE_CONTENTS_RAW (val);
       if (frame == NULL)
-	memset (buf, TYPE_LENGTH (VALUE_TYPE (val)), 0);
+	memset (buf, 0, TYPE_LENGTH (VALUE_TYPE (val)));
       else
 	ADDRESS_TO_POINTER (builtin_type_void_data_ptr, buf,
 			    get_frame_base_address (frame));
@@ -105,7 +105,7 @@ value_of_builtin_frame_pc_reg (struct frame_info *frame)
       struct value *val = allocate_value (builtin_type_void_data_ptr);
       char *buf = VALUE_CONTENTS_RAW (val);
       if (frame == NULL)
-	memset (buf, TYPE_LENGTH (VALUE_TYPE (val)), 0);
+	memset (buf, 0, TYPE_LENGTH (VALUE_TYPE (val)));
       else
 	ADDRESS_TO_POINTER (builtin_type_void_data_ptr, buf,
 			    get_frame_pc (frame));

From e7c9030c0d6fb04f90f1087f2eab97c1bcf6f67a Mon Sep 17 00:00:00 2001
From: Glen Barber <gjb@FreeBSD.org>
Date: Fri, 23 Jan 2015 21:04:59 +0000
Subject: [PATCH 219/279] When iterating through VMFORMATS, the VMBASE file is
 not removed or truncated to a zero-size file, which if used to create more
 than one disk image format, can result in accidental pollution of the target
 formatted disk image.

Instead of using a single VMBASE image (vm.img, by
default), use a single base file for each format, named
as VMFORMAT.img, which produces VMBASE.VMFORMAT as the
final formatted image.

Reported by:	cperciva
MFC after:	1 month
X-MFC-with:	r277458, r277536
Sponsored by:	The FreeBSD Foundation
---
 release/Makefile.vm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/release/Makefile.vm b/release/Makefile.vm
index 17229bfb279..fe955d30d8c 100644
--- a/release/Makefile.vm
+++ b/release/Makefile.vm
@@ -50,8 +50,8 @@ vm-${_CW:tl}:
 
 .if defined(WITH_VMIMAGES) && !empty(WITH_VMIMAGES)
 CLEANDIRS+=	${VMTARGETS}
-CLEANFILES+=	${VMBASE}.img
 . for FORMAT in ${VMFORMATS}
+CLEANFILES+=	${FORMAT}.img
 CLEANFILES+=	${VMBASE}.${FORMAT}
 . endfor
 .endif
@@ -65,7 +65,7 @@ vm-image:
 	env TARGET=${TARGET} TARGET_ARCH=${TARGET_ARCH} \
 		${.CURDIR}/scripts/mk-vmimage.sh \
 		-C ${.CURDIR}/tools/vmimage.subr -d ${.OBJDIR}/${.TARGET} \
-		-i ${.OBJDIR}/${VMBASE}.img -s ${VMSIZE} -f ${FORMAT} \
+		-i ${.OBJDIR}/${FORMAT}.img -s ${VMSIZE} -f ${FORMAT} \
 		-S ${WORLDDIR} -o ${.OBJDIR}/${VMBASE}.${FORMAT}
 . endfor
 .endif

From 2205e0d1bd2813b1fad5bee39b3365ee2dc8a1ec Mon Sep 17 00:00:00 2001
From: Jilles Tjoelker <jilles@FreeBSD.org>
Date: Fri, 23 Jan 2015 21:07:08 +0000
Subject: [PATCH 220/279] Add futimens and utimensat system calls.

The core kernel part is patch file utimes.2008.4.diff from
pluknet@FreeBSD.org. I updated the code for API changes, added the manual
page and added compatibility code for old kernels. There is also audit and
Capsicum support.

A new UTIME_* constant might allow setting birthtimes in future.

Differential Revision:	https://reviews.freebsd.org/D1426
Submitted by:	pluknet (partially)
Reviewed by:	delphij, pluknet, rwatson
Relnotes:	yes
---
 bin/ln/symlink.7                      |   7 +-
 lib/libc/include/libc_private.h       |   4 +
 lib/libc/sys/Makefile.inc             |   6 +
 lib/libc/sys/Symbol.map               |   2 +
 lib/libc/sys/futimens.c               |  97 +++++++++
 lib/libc/sys/utimensat.2              | 292 ++++++++++++++++++++++++++
 lib/libc/sys/utimensat.c              | 109 ++++++++++
 share/man/man4/rights.4               |   8 +-
 sys/compat/freebsd32/freebsd32_misc.c |  43 ++++
 sys/compat/freebsd32/syscalls.master  |   5 +
 sys/kern/capabilities.conf            |   4 +-
 sys/kern/syscalls.master              |   5 +
 sys/kern/vfs_syscalls.c               | 126 ++++++++++-
 sys/sys/capsicum.h                    |   4 +-
 sys/sys/param.h                       |   2 +-
 sys/sys/stat.h                        |   8 +
 sys/sys/syscallsubr.h                 |   5 +
 usr.bin/kdump/kdump.c                 |   1 +
 18 files changed, 717 insertions(+), 11 deletions(-)
 create mode 100644 lib/libc/sys/futimens.c
 create mode 100644 lib/libc/sys/utimensat.2
 create mode 100644 lib/libc/sys/utimensat.c

diff --git a/bin/ln/symlink.7 b/bin/ln/symlink.7
index 6aba773db7e..821a3128c91 100644
--- a/bin/ln/symlink.7
+++ b/bin/ln/symlink.7
@@ -29,7 +29,7 @@
 .\"	@(#)symlink.7	8.3 (Berkeley) 3/31/94
 .\" $FreeBSD$
 .\"
-.Dd December 29, 2014
+.Dd January 23, 2015
 .Dt SYMLINK 7
 .Os
 .Sh NAME
@@ -147,9 +147,10 @@ unless given the
 .Dv AT_SYMLINK_NOFOLLOW
 flag:
 .Xr fchmodat 2 ,
-.Xr fchownat 2
+.Xr fchownat 2 ,
+.Xr fstatat 2
 and
-.Xr fstatat 2 .
+.Xr utimensat 2 .
 .Pp
 The owner and group of an existing symbolic link can be changed by
 means of the
diff --git a/lib/libc/include/libc_private.h b/lib/libc/include/libc_private.h
index 347b46305d4..bfcd3d060e9 100644
--- a/lib/libc/include/libc_private.h
+++ b/lib/libc/include/libc_private.h
@@ -357,6 +357,10 @@ int		__libc_system(const char *);
 int		__libc_tcdrain(int);
 int		__fcntl_compat(int fd, int cmd, ...);
 
+int		__sys_futimens(int fd, const struct timespec *times) __hidden;
+int		__sys_utimensat(int fd, const char *path,
+		    const struct timespec *times, int flag) __hidden;
+
 /* execve() with PATH processing to implement posix_spawnp() */
 int _execvpe(const char *, char * const *, char * const *);
 
diff --git a/lib/libc/sys/Makefile.inc b/lib/libc/sys/Makefile.inc
index 0a3c13f8e3b..e8ec58e7a08 100644
--- a/lib/libc/sys/Makefile.inc
+++ b/lib/libc/sys/Makefile.inc
@@ -38,6 +38,10 @@ SRCS+=	${SYSCALL_COMPAT_SRCS}
 NOASM+=	${SYSCALL_COMPAT_SRCS:S/.c/.o/}
 .endif
 
+SRCS+= futimens.c utimensat.c
+NOASM+= futimens.o utimensat.o
+PSEUDO+= _futimens.o _utimensat.o
+
 INTERPOSED = \
 	accept \
 	accept4 \
@@ -310,6 +314,7 @@ MAN+=	sctp_generic_recvmsg.2 \
 	umask.2 \
 	undelete.2 \
 	unlink.2 \
+	utimensat.2 \
 	utimes.2 \
 	utrace.2 \
 	uuidgen.2 \
@@ -442,6 +447,7 @@ MLINKS+=timer_settime.2 timer_getoverrun.2 \
 	timer_settime.2 timer_gettime.2
 MLINKS+=truncate.2 ftruncate.2
 MLINKS+=unlink.2 unlinkat.2
+MLINKS+=utimensat.2 futimens.2
 MLINKS+=utimes.2 futimes.2 \
 	utimes.2 futimesat.2 \
 	utimes.2 lutimes.2
diff --git a/lib/libc/sys/Symbol.map b/lib/libc/sys/Symbol.map
index f1f57ba9fb7..194aa5bf099 100644
--- a/lib/libc/sys/Symbol.map
+++ b/lib/libc/sys/Symbol.map
@@ -397,7 +397,9 @@ FBSD_1.3 {
 };
 
 FBSD_1.4 {
+	futimens;
 	ppoll;
+	utimensat;
 };
 
 FBSDprivate_1.0 {
diff --git a/lib/libc/sys/futimens.c b/lib/libc/sys/futimens.c
new file mode 100644
index 00000000000..2014cc5cecf
--- /dev/null
+++ b/lib/libc/sys/futimens.c
@@ -0,0 +1,97 @@
+/*-
+ * Copyright (c) 2015 Jilles Tjoelker
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "namespace.h"
+#include <sys/stat.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <time.h>
+#include "un-namespace.h"
+
+#include "libc_private.h"
+
+int
+futimens(int fd, const struct timespec times[2])
+{
+	struct timeval now, tv[2], *tvp;
+	struct stat sb;
+
+	if (__getosreldate() >= 1100056)
+		return (__sys_futimens(fd, times));
+
+	if (times == NULL || (times[0].tv_nsec == UTIME_NOW &&
+	    times[1].tv_nsec == UTIME_NOW))
+		tvp = NULL;
+	else if (times[0].tv_nsec == UTIME_OMIT &&
+	    times[1].tv_nsec == UTIME_OMIT)
+		return (0);
+	else {
+		if ((times[0].tv_nsec < 0 || times[0].tv_nsec > 999999999) &&
+		    times[0].tv_nsec != UTIME_NOW &&
+		    times[0].tv_nsec != UTIME_OMIT) {
+			errno = EINVAL;
+			return (-1);
+		}
+		if ((times[1].tv_nsec < 0 || times[1].tv_nsec > 999999999) &&
+		    times[1].tv_nsec != UTIME_NOW &&
+		    times[1].tv_nsec != UTIME_OMIT) {
+			errno = EINVAL;
+			return (-1);
+		}
+		tv[0].tv_sec = times[0].tv_sec;
+		tv[0].tv_usec = times[0].tv_nsec / 1000;
+		tv[1].tv_sec = times[1].tv_sec;
+		tv[1].tv_usec = times[1].tv_nsec / 1000;
+		tvp = tv;
+		if (times[0].tv_nsec == UTIME_OMIT ||
+		    times[1].tv_nsec == UTIME_OMIT) {
+			if (_fstat(fd, &sb) == -1)
+				return (-1);
+			if (times[0].tv_nsec == UTIME_OMIT) {
+				tv[0].tv_sec = sb.st_atim.tv_sec;
+				tv[0].tv_usec = sb.st_atim.tv_nsec / 1000;
+			}
+			if (times[1].tv_nsec == UTIME_OMIT) {
+				tv[1].tv_sec = sb.st_mtim.tv_sec;
+				tv[1].tv_usec = sb.st_mtim.tv_nsec / 1000;
+			}
+		}
+		if (times[0].tv_nsec == UTIME_NOW ||
+		    times[1].tv_nsec == UTIME_NOW) {
+			if (gettimeofday(&now, NULL) == -1)
+				return (-1);
+			if (times[0].tv_nsec == UTIME_NOW)
+				tv[0] = now;
+			if (times[1].tv_nsec == UTIME_NOW)
+				tv[1] = now;
+		}
+	}
+	return (futimes(fd, tvp));
+}
diff --git a/lib/libc/sys/utimensat.2 b/lib/libc/sys/utimensat.2
new file mode 100644
index 00000000000..0f397c6a99a
--- /dev/null
+++ b/lib/libc/sys/utimensat.2
@@ -0,0 +1,292 @@
+.\"	$NetBSD: utimes.2,v 1.13 1999/03/22 19:45:11 garbled Exp $
+.\"
+.\" Copyright (c) 1990, 1993
+.\"	The Regents of the University of California.  All rights reserved.
+.\" Copyright (c) 2012, Jilles Tjoelker
+.\"
+.\" Redistribution and use in source and binary forms, with or without
+.\" modification, are permitted provided that the following conditions
+.\" are met:
+.\" 1. Redistributions of source code must retain the above copyright
+.\"    notice, this list of conditions and the following disclaimer.
+.\" 2. Redistributions in binary form must reproduce the above copyright
+.\"    notice, this list of conditions and the following disclaimer in the
+.\"    documentation and/or other materials provided with the distribution.
+.\" 4. Neither the name of the University nor the names of its contributors
+.\"    may be used to endorse or promote products derived from this software
+.\"    without specific prior written permission.
+.\"
+.\" THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+.\" ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+.\" SUCH DAMAGE.
+.\"
+.\"     @(#)utimes.2	8.1 (Berkeley) 6/4/93
+.\" $FreeBSD$
+.\"
+.Dd January 23, 2015
+.Dt UTIMENSAT 2
+.Os
+.Sh NAME
+.Nm futimens ,
+.Nm utimensat
+.Nd set file access and modification times
+.Sh LIBRARY
+.Lb libc
+.Sh SYNOPSIS
+.In sys/stat.h
+.Ft int
+.Fn futimens "int fd" "const struct timespec times[2]"
+.Ft int
+.Fo utimensat
+.Fa "int fd"
+.Fa "const char *path"
+.Fa "const struct timespec times[2]"
+.Fa "int flag"
+.Fc
+.Sh DESCRIPTION
+The access and modification times of the file named by
+.Fa path
+or referenced by
+.Fa fd
+are changed as specified by the argument
+.Fa times .
+The inode-change-time of the file is set to the current time.
+.Pp
+If
+.Fa path
+specifies a relative path,
+it is relative to the current working directory if
+.Fa fd
+is
+.Dv AT_FDCWD
+and otherwise relative to the directory associated with the file descriptor
+.Fa fd .
+.Pp
+The
+.Va tv_nsec
+field of a
+.Vt timespec
+structure
+can be set to the special value
+.Dv UTIME_NOW
+to set the current time, or to
+.Dv UTIME_OMIT
+to leave the time unchanged.
+In either case, the
+.Va tv_sec
+field is ignored.
+.Pp
+If
+.Fa times
+is
+.No non- Ns Dv NULL ,
+it is assumed to point to an array of two timespec structures.
+The access time is set to the value of the first element, and the
+modification time is set to the value of the second element.
+For file systems that support file birth (creation) times (such as
+.Dv UFS2 ) ,
+the birth time will be set to the value of the second element
+if the second element is older than the currently set birth time.
+To set both a birth time and a modification time,
+two calls are required; the first to set the birth time
+and the second to set the (presumably newer) modification time.
+Ideally a new system call will be added that allows the setting
+of all three times at once.
+If
+.Fa times
+is
+.Dv NULL ,
+this is equivalent to passing
+a pointer to an array of two timespec structures
+with both
+.Va tv_nsec
+fields set to
+.Dv UTIME_NOW .
+.Pp
+If both
+.Va tv_nsec
+fields are
+.Dv UTIME_OMIT ,
+the timestamps remain unchanged and
+no permissions are needed for the file itself,
+although search permissions may be required for the path prefix.
+The call may or may not succeed if the named file does not exist.
+.Pp
+If both
+.Va tv_nsec
+fields are
+.Dv UTIME_NOW ,
+the caller must be the owner of the file, have permission to
+write the file, or be the super-user.
+.Pp
+For all other values of the timestamps,
+the caller must be the owner of the file or be the super-user.
+.Pp
+The values for the
+.Fa flag
+argument of the
+.Fn utimensat
+system call
+are constructed by a bitwise-inclusive OR of flags from the following list,
+defined in
+.In fcntl.h :
+.Bl -tag -width indent
+.It Dv AT_SYMLINK_NOFOLLOW
+If
+.Fa path
+names a symbolic link, the symbolic link's times are changed.
+By default,
+.Fn utimensat
+changes the times of the file referenced by the symbolic link.
+.El
+.Sh RETURN VALUES
+.Rv -std
+.Sh COMPATIBILITY
+If the running kernel does not support this system call,
+a wrapper emulates it using
+.Xr fstatat 2 ,
+.Xr futimesat 2
+and
+.Xr lutimes 2 .
+As a result, timestamps will be rounded down to the nearest microsecond,
+.Dv UTIME_OMIT
+is not atomic and
+.Dv AT_SYMLINK_NOFOLLOW
+is not available with a path relative to a file descriptor.
+.Sh ERRORS
+These system calls will fail if:
+.Bl -tag -width Er
+.It Bq Er EACCES
+The
+.Fa times
+argument is
+.Dv NULL ,
+or both
+.Va tv_nsec
+values are
+.Dv UTIME_NOW ,
+and the effective user ID of the process does not
+match the owner of the file, and is not the super-user, and write
+access is denied.
+.It Bq Er EFAULT
+The
+.Fa times
+argument
+points outside the process's allocated address space.
+.It Bq Er EINVAL
+The
+.Va tv_usec
+component of at least one of the values specified by the
+.Fa times
+argument has a value less than 0 or greater than 999999.
+.It Bq Er EIO
+An I/O error occurred while reading or writing the affected inode.
+.It Bq Er EPERM
+The
+.Fa times
+argument is not
+.Dv NULL
+nor are both
+.Va tv_nsec
+values
+.Dv UTIME_NOW ,
+nor are both
+.Va tv_nsec
+values
+.Dv UTIME_OMIT
+and the calling process's effective user ID
+does not match the owner of the file and is not the super-user.
+.It Bq Er EPERM
+The named file has its immutable or append-only flag set, see the
+.Xr chflags 2
+manual page for more information.
+.It Bq Er EROFS
+The file system containing the file is mounted read-only.
+.El
+.Pp
+The
+.Fn futimens
+system call
+will fail if:
+.Bl -tag -width Er
+.It Bq Er EBADF
+The
+.Fa fd
+argument
+does not refer to a valid descriptor.
+.El
+.Pp
+The
+.Fn utimensat
+system call
+will fail if:
+.Bl -tag -width Er
+.It Bq Er EACCES
+Search permission is denied for a component of the path prefix.
+.It Bq Er EBADF
+The
+.Fa path
+argument does not specify an absolute path and the
+.Fa fd
+argument is neither
+.Dv AT_FDCWD
+nor a valid file descriptor.
+.It Bq Er EFAULT
+The
+.Fa path
+argument
+points outside the process's allocated address space.
+.It Bq Er ELOOP
+Too many symbolic links were encountered in translating the pathname.
+.It Bq Er ENAMETOOLONG
+A component of a pathname exceeded
+.Dv NAME_MAX
+characters, or an entire path name exceeded
+.Dv PATH_MAX
+characters.
+.It Bq Er ENOENT
+The named file does not exist.
+.It Bq Er ENOTDIR
+A component of the path prefix is not a directory.
+.It Bq Er ENOTDIR
+The
+.Fa path
+argument is not an absolute path and
+.Fa fd
+is neither
+.Dv AT_FDCWD
+nor a file descriptor associated with a directory.
+.It Bq Er ENOTSUP
+The running kernel does not support this system call and
+.Dv AT_SYMLINK_NOFOLLOW
+is used with a path relative to a file descriptor.
+.El
+.Sh SEE ALSO
+.Xr chflags 2 ,
+.Xr stat 2 ,
+.Xr symlink 2 ,
+.Xr utimes 2 ,
+.Xr utime 3 ,
+.Xr symlink 7
+.Sh STANDARDS
+The
+.Fn futimens
+and
+.Fn utimensat
+system calls are expected to conform to
+.St -p1003.1-2008 .
+.Sh HISTORY
+The
+.Fn futimens
+and
+.Fn utimensat
+system calls appeared in
+.Fx 11.0 .
diff --git a/lib/libc/sys/utimensat.c b/lib/libc/sys/utimensat.c
new file mode 100644
index 00000000000..67d19cb2f9b
--- /dev/null
+++ b/lib/libc/sys/utimensat.c
@@ -0,0 +1,109 @@
+/*-
+ * Copyright (c) 2015 Jilles Tjoelker
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include "namespace.h"
+#include <sys/stat.h>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <time.h>
+#include "un-namespace.h"
+
+#include "libc_private.h"
+
+int
+utimensat(int fd, const char *path, const struct timespec times[2], int flag)
+{
+	struct timeval now, tv[2], *tvp;
+	struct stat sb;
+
+	if (__getosreldate() >= 1100056)
+		return (__sys_utimensat(fd, path, times, flag));
+
+	if ((flag & ~AT_SYMLINK_NOFOLLOW) != 0) {
+		errno = EINVAL;
+		return (-1);
+	}
+	if (times == NULL || (times[0].tv_nsec == UTIME_NOW &&
+	    times[1].tv_nsec == UTIME_NOW))
+		tvp = NULL;
+	else if (times[0].tv_nsec == UTIME_OMIT &&
+	    times[1].tv_nsec == UTIME_OMIT)
+		return (0);
+	else {
+		if ((times[0].tv_nsec < 0 || times[0].tv_nsec > 999999999) &&
+		    times[0].tv_nsec != UTIME_NOW &&
+		    times[0].tv_nsec != UTIME_OMIT) {
+			errno = EINVAL;
+			return (-1);
+		}
+		if ((times[1].tv_nsec < 0 || times[1].tv_nsec > 999999999) &&
+		    times[1].tv_nsec != UTIME_NOW &&
+		    times[1].tv_nsec != UTIME_OMIT) {
+			errno = EINVAL;
+			return (-1);
+		}
+		tv[0].tv_sec = times[0].tv_sec;
+		tv[0].tv_usec = times[0].tv_nsec / 1000;
+		tv[1].tv_sec = times[1].tv_sec;
+		tv[1].tv_usec = times[1].tv_nsec / 1000;
+		tvp = tv;
+		if (times[0].tv_nsec == UTIME_OMIT ||
+		    times[1].tv_nsec == UTIME_OMIT) {
+			if (fstatat(fd, path, &sb, flag) == -1)
+				return (-1);
+			if (times[0].tv_nsec == UTIME_OMIT) {
+				tv[0].tv_sec = sb.st_atim.tv_sec;
+				tv[0].tv_usec = sb.st_atim.tv_nsec / 1000;
+			}
+			if (times[1].tv_nsec == UTIME_OMIT) {
+				tv[1].tv_sec = sb.st_mtim.tv_sec;
+				tv[1].tv_usec = sb.st_mtim.tv_nsec / 1000;
+			}
+		}
+		if (times[0].tv_nsec == UTIME_NOW ||
+		    times[1].tv_nsec == UTIME_NOW) {
+			if (gettimeofday(&now, NULL) == -1)
+				return (-1);
+			if (times[0].tv_nsec == UTIME_NOW)
+				tv[0] = now;
+			if (times[1].tv_nsec == UTIME_NOW)
+				tv[1] = now;
+		}
+	}
+	if ((flag & AT_SYMLINK_NOFOLLOW) == 0)
+		return (futimesat(fd, path, tvp));
+	else if ((flag & AT_SYMLINK_NOFOLLOW) != 0 &&
+	    (fd == AT_FDCWD || path[0] == '/'))
+		return (lutimes(path, tvp));
+	else {
+		errno = ENOTSUP;
+		return (-1);
+	}
+}
diff --git a/share/man/man4/rights.4 b/share/man/man4/rights.4
index 476b15bad1d..16025c179e6 100644
--- a/share/man/man4/rights.4
+++ b/share/man/man4/rights.4
@@ -32,7 +32,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd September 23, 2013
+.Dd January 23, 2015
 .Dt RIGHTS 4
 .Os
 .Sh NAME
@@ -272,9 +272,13 @@ with the
 flag.
 .It Dv CAP_FUTIMES
 Permit
-.Xr futimes 2
+.Xr futimens 2
 and
+.Xr futimes 2 ,
+and permit
 .Xr futimesat 2
+and
+.Xr utimensat 2
 if the
 .Dv CAP_LOOKUP
 right is also present.
diff --git a/sys/compat/freebsd32/freebsd32_misc.c b/sys/compat/freebsd32/freebsd32_misc.c
index ffea2837104..a72612b53c1 100644
--- a/sys/compat/freebsd32/freebsd32_misc.c
+++ b/sys/compat/freebsd32/freebsd32_misc.c
@@ -1299,6 +1299,49 @@ freebsd32_futimesat(struct thread *td, struct freebsd32_futimesat_args *uap)
 		sp, UIO_SYSSPACE));
 }
 
+int
+freebsd32_futimens(struct thread *td, struct freebsd32_futimens_args *uap)
+{
+	struct timespec32 ts32[2];
+	struct timespec ts[2], *tsp;
+	int error;
+
+	if (uap->times != NULL) {
+		error = copyin(uap->times, ts32, sizeof(ts32));
+		if (error)
+			return (error);
+		CP(ts32[0], ts[0], tv_sec);
+		CP(ts32[0], ts[0], tv_nsec);
+		CP(ts32[1], ts[1], tv_sec);
+		CP(ts32[1], ts[1], tv_nsec);
+		tsp = ts;
+	} else
+		tsp = NULL;
+	return (kern_futimens(td, uap->fd, tsp, UIO_SYSSPACE));
+}
+
+int
+freebsd32_utimensat(struct thread *td, struct freebsd32_utimensat_args *uap)
+{
+	struct timespec32 ts32[2];
+	struct timespec ts[2], *tsp;
+	int error;
+
+	if (uap->times != NULL) {
+		error = copyin(uap->times, ts32, sizeof(ts32));
+		if (error)
+			return (error);
+		CP(ts32[0], ts[0], tv_sec);
+		CP(ts32[0], ts[0], tv_nsec);
+		CP(ts32[1], ts[1], tv_sec);
+		CP(ts32[1], ts[1], tv_nsec);
+		tsp = ts;
+	} else
+		tsp = NULL;
+	return (kern_utimensat(td, uap->fd, uap->path, UIO_USERSPACE,
+	    tsp, UIO_SYSSPACE, uap->flag));
+}
+
 int
 freebsd32_adjtime(struct thread *td, struct freebsd32_adjtime_args *uap)
 {
diff --git a/sys/compat/freebsd32/syscalls.master b/sys/compat/freebsd32/syscalls.master
index ce655bcfa58..91d1727a7f9 100644
--- a/sys/compat/freebsd32/syscalls.master
+++ b/sys/compat/freebsd32/syscalls.master
@@ -1069,3 +1069,8 @@
 545	AUE_POLL	STD	{ int freebsd32_ppoll(struct pollfd *fds, \
 				    u_int nfds, const struct timespec32 *ts, \
 				    const sigset_t *set); }
+546	AUE_FUTIMES	STD	{ int freebsd32_futimens(int fd, \
+				    struct timespec *times); }
+547	AUE_FUTIMESAT	STD	{ int freebsd32_utimensat(int fd, \
+				    char *path, \
+				    struct timespec *times, int flag); }
diff --git a/sys/kern/capabilities.conf b/sys/kern/capabilities.conf
index f7a46aebce2..d0ea97c4e83 100644
--- a/sys/kern/capabilities.conf
+++ b/sys/kern/capabilities.conf
@@ -220,8 +220,9 @@ fsync
 ftruncate
 
 ##
-## Allow futimes(2), subject to capability rights.
+## Allow futimens(2) and futimes(2), subject to capability rights.
 ##
+futimens
 futimes
 
 ##
@@ -453,6 +454,7 @@ readlinkat
 renameat
 symlinkat
 unlinkat
+utimensat
 
 ##
 ## Allow entry into open(2). This system call will fail, since access to the
diff --git a/sys/kern/syscalls.master b/sys/kern/syscalls.master
index 089896b7375..09d38d40101 100644
--- a/sys/kern/syscalls.master
+++ b/sys/kern/syscalls.master
@@ -983,5 +983,10 @@
 545	AUE_POLL	STD	{ int ppoll(struct pollfd *fds, u_int nfds, \
 				    const struct timespec *ts, \
 				    const sigset_t *set); }
+546	AUE_FUTIMES	STD	{ int futimens(int fd, \
+				    struct timespec *times); }
+547	AUE_FUTIMESAT	STD	{ int utimensat(int fd, \
+				    char *path, \
+				    struct timespec *times, int flag); }
 ; Please copy any additions and changes to the following compatability tables:
 ; sys/compat/freebsd32/syscalls.master
diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c
index 3105c2f4bcd..0f248199d0d 100644
--- a/sys/kern/vfs_syscalls.c
+++ b/sys/kern/vfs_syscalls.c
@@ -95,10 +95,12 @@ SDT_PROBE_DEFINE2(vfs, , stat, mode, "char *", "int");
 SDT_PROBE_DEFINE2(vfs, , stat, reg, "char *", "int");
 
 static int chroot_refuse_vdir_fds(struct filedesc *fdp);
-static int getutimes(const struct timeval *, enum uio_seg, struct timespec *);
 static int kern_chflagsat(struct thread *td, int fd, const char *path,
     enum uio_seg pathseg, u_long flags, int atflag);
 static int setfflags(struct thread *td, struct vnode *, u_long);
+static int getutimes(const struct timeval *, enum uio_seg, struct timespec *);
+static int getutimens(const struct timespec *, enum uio_seg,
+    struct timespec *, int *);
 static int setutimes(struct thread *td, struct vnode *,
     const struct timespec *, int, int);
 static int vn_access(struct vnode *vp, int user_flags, struct ucred *cred,
@@ -3007,7 +3009,53 @@ getutimes(usrtvp, tvpseg, tsp)
 }
 
 /*
- * Common implementation code for utimes(), lutimes(), and futimes().
+ * Common implementation code for futimens(), utimensat().
+ */
+#define	UTIMENS_NULL	0x1
+#define	UTIMENS_EXIT	0x2
+static int
+getutimens(const struct timespec *usrtsp, enum uio_seg tspseg,
+    struct timespec *tsp, int *retflags)
+{
+	struct timespec tsnow;
+	int error;
+
+	vfs_timestamp(&tsnow);
+	*retflags = 0;
+	if (usrtsp == NULL) {
+		tsp[0] = tsnow;
+		tsp[1] = tsnow;
+		*retflags |= UTIMENS_NULL;
+		return (0);
+	}
+	if (tspseg == UIO_SYSSPACE) {
+		tsp[0] = usrtsp[0];
+		tsp[1] = usrtsp[1];
+	} else if ((error = copyin(usrtsp, tsp, sizeof(*tsp) * 2)) != 0)
+		return (error);
+	if (tsp[0].tv_nsec == UTIME_OMIT && tsp[1].tv_nsec == UTIME_OMIT)
+		*retflags |= UTIMENS_EXIT;
+	if (tsp[0].tv_nsec == UTIME_NOW && tsp[1].tv_nsec == UTIME_NOW)
+		*retflags |= UTIMENS_NULL;
+	if (tsp[0].tv_nsec == UTIME_OMIT)
+		tsp[0].tv_sec = VNOVAL;
+	else if (tsp[0].tv_nsec == UTIME_NOW)
+		tsp[0] = tsnow;
+	else if (tsp[0].tv_nsec < 0 || tsp[0].tv_nsec >= 1000000000L)
+		return (EINVAL);
+	if (tsp[1].tv_nsec == UTIME_OMIT)
+		tsp[1].tv_sec = VNOVAL;
+	else if (tsp[1].tv_nsec == UTIME_NOW)
+		tsp[1] = tsnow;
+	else if (tsp[1].tv_nsec < 0 || tsp[1].tv_nsec >= 1000000000L)
+		return (EINVAL);
+
+	return (0);
+}
+
+/*
+ * Common implementation code for utimes(), lutimes(), futimes(), futimens(),
+ * and utimensat().
  */
 static int
 setutimes(td, vp, ts, numtimes, nullflag)
@@ -3196,6 +3244,80 @@ kern_futimes(struct thread *td, int fd, struct timeval *tptr,
 	return (error);
 }
 
+int
+sys_futimens(struct thread *td, struct futimens_args *uap)
+{
+
+	return (kern_futimens(td, uap->fd, uap->times, UIO_USERSPACE));
+}
+
+int
+kern_futimens(struct thread *td, int fd, struct timespec *tptr,
+    enum uio_seg tptrseg)
+{
+	struct timespec ts[2];
+	struct file *fp;
+	cap_rights_t rights;
+	int error, flags;
+
+	AUDIT_ARG_FD(fd);
+	error = getutimens(tptr, tptrseg, ts, &flags);
+	if (error != 0)
+		return (error);
+	if (flags & UTIMENS_EXIT)
+		return (0);
+	error = getvnode(td->td_proc->p_fd, fd,
+	    cap_rights_init(&rights, CAP_FUTIMES), &fp);
+	if (error != 0)
+		return (error);
+#ifdef AUDIT
+	vn_lock(fp->f_vnode, LK_SHARED | LK_RETRY);
+	AUDIT_ARG_VNODE1(fp->f_vnode);
+	VOP_UNLOCK(fp->f_vnode, 0);
+#endif
+	error = setutimes(td, fp->f_vnode, ts, 2, flags & UTIMENS_NULL);
+	fdrop(fp, td);
+	return (error);
+}
+
+int
+sys_utimensat(struct thread *td, struct utimensat_args *uap)
+{
+
+	return (kern_utimensat(td, uap->fd, uap->path, UIO_USERSPACE,
+	    uap->times, UIO_USERSPACE, uap->flag));
+}
+
+int
+kern_utimensat(struct thread *td, int fd, char *path, enum uio_seg pathseg,
+    struct timespec *tptr, enum uio_seg tptrseg, int flag)
+{
+	struct nameidata nd;
+	struct timespec ts[2];
+	int error, flags;
+
+	if (flag & ~AT_SYMLINK_NOFOLLOW)
+		return (EINVAL);
+
+	if ((error = getutimens(tptr, tptrseg, ts, &flags)) != 0)
+		return (error);
+	NDINIT_AT(&nd, LOOKUP, ((flag & AT_SYMLINK_NOFOLLOW) ? NOFOLLOW :
+	    FOLLOW) | AUDITVNODE1, pathseg, path, fd, td);
+	if ((error = namei(&nd)) != 0)
+		return (error);
+	/*
+	 * We are allowed to call namei() regardless of 2xUTIME_OMIT.
+	 * POSIX states:
+	 * "If both tv_nsec fields are UTIME_OMIT... EACCESS may be detected."
+	 * "Search permission is denied by a component of the path prefix."
+	 */
+	NDFREE(&nd, NDF_ONLY_PNBUF);
+	if ((flags & UTIMENS_EXIT) == 0)
+		error = setutimes(td, nd.ni_vp, ts, 2, flags & UTIMENS_NULL);
+	vrele(nd.ni_vp);
+	return (error);
+}
+
 /*
  * Truncate a file given its path name.
  */
diff --git a/sys/sys/capsicum.h b/sys/sys/capsicum.h
index de113c62fef..8502a967498 100644
--- a/sys/sys/capsicum.h
+++ b/sys/sys/capsicum.h
@@ -146,9 +146,9 @@
 #define	CAP_FSTATAT		(CAP_FSTAT | CAP_LOOKUP)
 /* Allows for fstatfs(2). */
 #define	CAP_FSTATFS		CAPRIGHT(0, 0x0000000000100000ULL)
-/* Allows for futimes(2). */
+/* Allows for futimens(2) and futimes(2). */
 #define	CAP_FUTIMES		CAPRIGHT(0, 0x0000000000200000ULL)
-/* Allows for futimes(2) and futimesat(2). */
+/* Allows for futimens(2), futimes(2), futimesat(2) and utimensat(2). */
 #define	CAP_FUTIMESAT		(CAP_FUTIMES | CAP_LOOKUP)
 /* Allows for linkat(2) and renameat(2) (destination directory descriptor). */
 #define	CAP_LINKAT		(CAP_LOOKUP | 0x0000000000400000ULL)
diff --git a/sys/sys/param.h b/sys/sys/param.h
index bf59b0bf722..eea93c957ab 100644
--- a/sys/sys/param.h
+++ b/sys/sys/param.h
@@ -58,7 +58,7 @@
  *		in the range 5 to 9.
  */
 #undef __FreeBSD_version
-#define __FreeBSD_version 1100055	/* Master, propagated to newvers */
+#define __FreeBSD_version 1100056	/* Master, propagated to newvers */
 
 /*
  * __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,
diff --git a/sys/sys/stat.h b/sys/sys/stat.h
index 13dea2dc72b..7a4370d64a2 100644
--- a/sys/sys/stat.h
+++ b/sys/sys/stat.h
@@ -307,6 +307,11 @@ struct nstat {
 
 #endif /* __BSD_VISIBLE */
 
+#if __POSIX_VISIBLE >= 200809
+#define	UTIME_NOW	-1
+#define	UTIME_OMIT	-2
+#endif
+
 #ifndef _KERNEL
 __BEGIN_DECLS
 #if __BSD_VISIBLE
@@ -322,6 +327,9 @@ int	fchmod(int, mode_t);
 #endif
 #if __POSIX_VISIBLE >= 200809
 int	fchmodat(int, const char *, mode_t, int);
+int	futimens(int fd, const struct timespec times[2]);
+int	utimensat(int fd, const char *path, const struct timespec times[2],
+		int flag);
 #endif
 int	fstat(int, struct stat *);
 #if __BSD_VISIBLE
diff --git a/sys/sys/syscallsubr.h b/sys/sys/syscallsubr.h
index f4131a48032..cb6d4e36cc9 100644
--- a/sys/sys/syscallsubr.h
+++ b/sys/sys/syscallsubr.h
@@ -99,6 +99,8 @@ int	kern_fstatfs(struct thread *td, int fd, struct statfs *buf);
 int	kern_ftruncate(struct thread *td, int fd, off_t length);
 int	kern_futimes(struct thread *td, int fd, struct timeval *tptr,
 	    enum uio_seg tptrseg);
+int	kern_futimens(struct thread *td, int fd, struct timespec *tptr,
+	    enum uio_seg tptrseg);
 int	kern_getdirentries(struct thread *td, int fd, char *buf, u_int count,
 	    long *basep, ssize_t *residp, enum uio_seg bufseg);
 int	kern_getfsstat(struct thread *td, struct statfs **buf, size_t bufsize,
@@ -220,6 +222,9 @@ int	kern_unlinkat(struct thread *td, int fd, char *path,
 	    enum uio_seg pathseg, ino_t oldinum);
 int	kern_utimesat(struct thread *td, int fd, char *path,
 	    enum uio_seg pathseg, struct timeval *tptr, enum uio_seg tptrseg);
+int	kern_utimensat(struct thread *td, int fd, char *path,
+	    enum uio_seg pathseg, struct timespec *tptr, enum uio_seg tptrseg,
+	    int follow);
 int	kern_wait(struct thread *td, pid_t pid, int *status, int options,
 	    struct rusage *rup);
 int	kern_wait6(struct thread *td, enum idtype idtype, id_t id, int *status,
diff --git a/usr.bin/kdump/kdump.c b/usr.bin/kdump/kdump.c
index ad29bfde74c..0daf7373cd1 100644
--- a/usr.bin/kdump/kdump.c
+++ b/usr.bin/kdump/kdump.c
@@ -714,6 +714,7 @@ ktrsyscall(struct ktr_syscall *ktr, u_int flags)
 			case SYS_readlinkat:
 			case SYS_renameat:
 			case SYS_unlinkat:
+			case SYS_utimensat:
 				putchar('(');
 				atfdname(*ip, decimal);
 				c = ',';

From 2b35e6a9f21abb8dfa51968375d29826cf25e55a Mon Sep 17 00:00:00 2001
From: Jilles Tjoelker <jilles@FreeBSD.org>
Date: Fri, 23 Jan 2015 21:08:24 +0000
Subject: [PATCH 221/279] Run make sysent.

---
 sys/compat/freebsd32/freebsd32_proto.h        | 16 ++++-
 sys/compat/freebsd32/freebsd32_syscall.h      |  6 +-
 sys/compat/freebsd32/freebsd32_syscalls.c     |  4 +-
 sys/compat/freebsd32/freebsd32_sysent.c       |  4 +-
 .../freebsd32/freebsd32_systrace_args.c       | 60 +++++++++++++++++++
 sys/kern/init_sysent.c                        |  4 +-
 sys/kern/syscalls.c                           |  4 +-
 sys/kern/systrace_args.c                      | 60 +++++++++++++++++++
 sys/sys/syscall.h                             |  6 +-
 sys/sys/syscall.mk                            |  6 +-
 sys/sys/sysproto.h                            | 16 ++++-
 11 files changed, 174 insertions(+), 12 deletions(-)

diff --git a/sys/compat/freebsd32/freebsd32_proto.h b/sys/compat/freebsd32/freebsd32_proto.h
index 305b76c2d73..bf9253dc347 100644
--- a/sys/compat/freebsd32/freebsd32_proto.h
+++ b/sys/compat/freebsd32/freebsd32_proto.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 276654 2015-01-04 10:34:02Z dchagin 
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 277610 2015-01-23 21:07:08Z jilles 
  */
 
 #ifndef _FREEBSD32_SYSPROTO_H_
@@ -693,6 +693,16 @@ struct freebsd32_ppoll_args {
 	char ts_l_[PADL_(const struct timespec32 *)]; const struct timespec32 * ts; char ts_r_[PADR_(const struct timespec32 *)];
 	char set_l_[PADL_(const sigset_t *)]; const sigset_t * set; char set_r_[PADR_(const sigset_t *)];
 };
+struct freebsd32_futimens_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char times_l_[PADL_(struct timespec *)]; struct timespec * times; char times_r_[PADR_(struct timespec *)];
+};
+struct freebsd32_utimensat_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char path_l_[PADL_(char *)]; char * path; char path_r_[PADR_(char *)];
+	char times_l_[PADL_(struct timespec *)]; struct timespec * times; char times_r_[PADR_(struct timespec *)];
+	char flag_l_[PADL_(int)]; int flag; char flag_r_[PADR_(int)];
+};
 #if !defined(PAD64_REQUIRED) && (defined(__powerpc__) || defined(__mips__))
 #define PAD64_REQUIRED
 #endif
@@ -825,6 +835,8 @@ int	freebsd32_procctl(struct thread *, struct freebsd32_procctl_args *);
 int	freebsd32_procctl(struct thread *, struct freebsd32_procctl_args *);
 #endif
 int	freebsd32_ppoll(struct thread *, struct freebsd32_ppoll_args *);
+int	freebsd32_futimens(struct thread *, struct freebsd32_futimens_args *);
+int	freebsd32_utimensat(struct thread *, struct freebsd32_utimensat_args *);
 
 #ifdef COMPAT_43
 
@@ -1240,6 +1252,8 @@ int	freebsd7_freebsd32_shmctl(struct thread *, struct freebsd7_freebsd32_shmctl_
 #define	FREEBSD32_SYS_AUE_freebsd32_procctl	AUE_NULL
 #define	FREEBSD32_SYS_AUE_freebsd32_procctl	AUE_NULL
 #define	FREEBSD32_SYS_AUE_freebsd32_ppoll	AUE_POLL
+#define	FREEBSD32_SYS_AUE_freebsd32_futimens	AUE_FUTIMES
+#define	FREEBSD32_SYS_AUE_freebsd32_utimensat	AUE_FUTIMESAT
 
 #undef PAD_
 #undef PADL_
diff --git a/sys/compat/freebsd32/freebsd32_syscall.h b/sys/compat/freebsd32/freebsd32_syscall.h
index 370c0706fee..5e5bb0ad6ed 100644
--- a/sys/compat/freebsd32/freebsd32_syscall.h
+++ b/sys/compat/freebsd32/freebsd32_syscall.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 276654 2015-01-04 10:34:02Z dchagin 
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 277610 2015-01-23 21:07:08Z jilles 
  */
 
 #define	FREEBSD32_SYS_syscall	0
@@ -453,4 +453,6 @@
 #define	FREEBSD32_SYS_freebsd32_procctl	544
 #define	FREEBSD32_SYS_freebsd32_procctl	544
 #define	FREEBSD32_SYS_freebsd32_ppoll	545
-#define	FREEBSD32_SYS_MAXSYSCALL	546
+#define	FREEBSD32_SYS_freebsd32_futimens	546
+#define	FREEBSD32_SYS_freebsd32_utimensat	547
+#define	FREEBSD32_SYS_MAXSYSCALL	548
diff --git a/sys/compat/freebsd32/freebsd32_syscalls.c b/sys/compat/freebsd32/freebsd32_syscalls.c
index 9cad2f135f2..143ed72107e 100644
--- a/sys/compat/freebsd32/freebsd32_syscalls.c
+++ b/sys/compat/freebsd32/freebsd32_syscalls.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 276654 2015-01-04 10:34:02Z dchagin 
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 277610 2015-01-23 21:07:08Z jilles 
  */
 
 const char *freebsd32_syscallnames[] = {
@@ -579,4 +579,6 @@ const char *freebsd32_syscallnames[] = {
 	"freebsd32_procctl",			/* 544 = freebsd32_procctl */
 #endif
 	"freebsd32_ppoll",			/* 545 = freebsd32_ppoll */
+	"freebsd32_futimens",			/* 546 = freebsd32_futimens */
+	"freebsd32_utimensat",			/* 547 = freebsd32_utimensat */
 };
diff --git a/sys/compat/freebsd32/freebsd32_sysent.c b/sys/compat/freebsd32/freebsd32_sysent.c
index 195ee940010..6d2a7d48bd3 100644
--- a/sys/compat/freebsd32/freebsd32_sysent.c
+++ b/sys/compat/freebsd32/freebsd32_sysent.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 276654 2015-01-04 10:34:02Z dchagin 
+ * created from FreeBSD: head/sys/compat/freebsd32/syscalls.master 277610 2015-01-23 21:07:08Z jilles 
  */
 
 #include "opt_compat.h"
@@ -616,4 +616,6 @@ struct sysent freebsd32_sysent[] = {
 	{ AS(freebsd32_procctl_args), (sy_call_t *)freebsd32_procctl, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 544 = freebsd32_procctl */
 #endif
 	{ AS(freebsd32_ppoll_args), (sy_call_t *)freebsd32_ppoll, AUE_POLL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 545 = freebsd32_ppoll */
+	{ AS(freebsd32_futimens_args), (sy_call_t *)freebsd32_futimens, AUE_FUTIMES, NULL, 0, 0, 0, SY_THR_STATIC },	/* 546 = freebsd32_futimens */
+	{ AS(freebsd32_utimensat_args), (sy_call_t *)freebsd32_utimensat, AUE_FUTIMESAT, NULL, 0, 0, 0, SY_THR_STATIC },	/* 547 = freebsd32_utimensat */
 };
diff --git a/sys/compat/freebsd32/freebsd32_systrace_args.c b/sys/compat/freebsd32/freebsd32_systrace_args.c
index 6f6b0ac97c1..a2668aa7cb0 100644
--- a/sys/compat/freebsd32/freebsd32_systrace_args.c
+++ b/sys/compat/freebsd32/freebsd32_systrace_args.c
@@ -3309,6 +3309,24 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 		*n_args = 4;
 		break;
 	}
+	/* freebsd32_futimens */
+	case 546: {
+		struct freebsd32_futimens_args *p = params;
+		iarg[0] = p->fd; /* int */
+		uarg[1] = (intptr_t) p->times; /* struct timespec * */
+		*n_args = 2;
+		break;
+	}
+	/* freebsd32_utimensat */
+	case 547: {
+		struct freebsd32_utimensat_args *p = params;
+		iarg[0] = p->fd; /* int */
+		uarg[1] = (intptr_t) p->path; /* char * */
+		uarg[2] = (intptr_t) p->times; /* struct timespec * */
+		iarg[3] = p->flag; /* int */
+		*n_args = 4;
+		break;
+	}
 	default:
 		*n_args = 0;
 		break;
@@ -8873,6 +8891,38 @@ systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 			break;
 		};
 		break;
+	/* freebsd32_futimens */
+	case 546:
+		switch(ndx) {
+		case 0:
+			p = "int";
+			break;
+		case 1:
+			p = "struct timespec *";
+			break;
+		default:
+			break;
+		};
+		break;
+	/* freebsd32_utimensat */
+	case 547:
+		switch(ndx) {
+		case 0:
+			p = "int";
+			break;
+		case 1:
+			p = "char *";
+			break;
+		case 2:
+			p = "struct timespec *";
+			break;
+		case 3:
+			p = "int";
+			break;
+		default:
+			break;
+		};
+		break;
 	default:
 		break;
 	};
@@ -10751,6 +10801,16 @@ systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 		if (ndx == 0 || ndx == 1)
 			p = "int";
 		break;
+	/* freebsd32_futimens */
+	case 546:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* freebsd32_utimensat */
+	case 547:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
 	default:
 		break;
 	};
diff --git a/sys/kern/init_sysent.c b/sys/kern/init_sysent.c
index 9d044784863..8aefa3f7be1 100644
--- a/sys/kern/init_sysent.c
+++ b/sys/kern/init_sysent.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/kern/syscalls.master 276654 2015-01-04 10:34:02Z dchagin 
+ * created from FreeBSD: head/sys/kern/syscalls.master 277610 2015-01-23 21:07:08Z jilles 
  */
 
 #include "opt_compat.h"
@@ -580,4 +580,6 @@ struct sysent sysent[] = {
 	{ AS(aio_mlock_args), (sy_call_t *)lkmressys, AUE_NULL, NULL, 0, 0, 0, SY_THR_ABSENT },	/* 543 = aio_mlock */
 	{ AS(procctl_args), (sy_call_t *)sys_procctl, AUE_NULL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 544 = procctl */
 	{ AS(ppoll_args), (sy_call_t *)sys_ppoll, AUE_POLL, NULL, 0, 0, 0, SY_THR_STATIC },	/* 545 = ppoll */
+	{ AS(futimens_args), (sy_call_t *)sys_futimens, AUE_FUTIMES, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 546 = futimens */
+	{ AS(utimensat_args), (sy_call_t *)sys_utimensat, AUE_FUTIMESAT, NULL, 0, 0, SYF_CAPENABLED, SY_THR_STATIC },	/* 547 = utimensat */
 };
diff --git a/sys/kern/syscalls.c b/sys/kern/syscalls.c
index 41f5cce8f1b..e99ae94a2e5 100644
--- a/sys/kern/syscalls.c
+++ b/sys/kern/syscalls.c
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/kern/syscalls.master 276654 2015-01-04 10:34:02Z dchagin 
+ * created from FreeBSD: head/sys/kern/syscalls.master 277610 2015-01-23 21:07:08Z jilles 
  */
 
 const char *syscallnames[] = {
@@ -553,4 +553,6 @@ const char *syscallnames[] = {
 	"aio_mlock",			/* 543 = aio_mlock */
 	"procctl",			/* 544 = procctl */
 	"ppoll",			/* 545 = ppoll */
+	"futimens",			/* 546 = futimens */
+	"utimensat",			/* 547 = utimensat */
 };
diff --git a/sys/kern/systrace_args.c b/sys/kern/systrace_args.c
index d5c368df4d4..c5d3b85fcd1 100644
--- a/sys/kern/systrace_args.c
+++ b/sys/kern/systrace_args.c
@@ -3382,6 +3382,24 @@ systrace_args(int sysnum, void *params, uint64_t *uarg, int *n_args)
 		*n_args = 4;
 		break;
 	}
+	/* futimens */
+	case 546: {
+		struct futimens_args *p = params;
+		iarg[0] = p->fd; /* int */
+		uarg[1] = (intptr_t) p->times; /* struct timespec * */
+		*n_args = 2;
+		break;
+	}
+	/* utimensat */
+	case 547: {
+		struct utimensat_args *p = params;
+		iarg[0] = p->fd; /* int */
+		uarg[1] = (intptr_t) p->path; /* char * */
+		uarg[2] = (intptr_t) p->times; /* struct timespec * */
+		iarg[3] = p->flag; /* int */
+		*n_args = 4;
+		break;
+	}
 	default:
 		*n_args = 0;
 		break;
@@ -9019,6 +9037,38 @@ systrace_entry_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 			break;
 		};
 		break;
+	/* futimens */
+	case 546:
+		switch(ndx) {
+		case 0:
+			p = "int";
+			break;
+		case 1:
+			p = "struct timespec *";
+			break;
+		default:
+			break;
+		};
+		break;
+	/* utimensat */
+	case 547:
+		switch(ndx) {
+		case 0:
+			p = "int";
+			break;
+		case 1:
+			p = "char *";
+			break;
+		case 2:
+			p = "struct timespec *";
+			break;
+		case 3:
+			p = "int";
+			break;
+		default:
+			break;
+		};
+		break;
 	default:
 		break;
 	};
@@ -10962,6 +11012,16 @@ systrace_return_setargdesc(int sysnum, int ndx, char *desc, size_t descsz)
 		if (ndx == 0 || ndx == 1)
 			p = "int";
 		break;
+	/* futimens */
+	case 546:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
+	/* utimensat */
+	case 547:
+		if (ndx == 0 || ndx == 1)
+			p = "int";
+		break;
 	default:
 		break;
 	};
diff --git a/sys/sys/syscall.h b/sys/sys/syscall.h
index 42e52909fb5..54a450ff8c8 100644
--- a/sys/sys/syscall.h
+++ b/sys/sys/syscall.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/kern/syscalls.master 276654 2015-01-04 10:34:02Z dchagin 
+ * created from FreeBSD: head/sys/kern/syscalls.master 277610 2015-01-23 21:07:08Z jilles 
  */
 
 #define	SYS_syscall	0
@@ -463,4 +463,6 @@
 #define	SYS_aio_mlock	543
 #define	SYS_procctl	544
 #define	SYS_ppoll	545
-#define	SYS_MAXSYSCALL	546
+#define	SYS_futimens	546
+#define	SYS_utimensat	547
+#define	SYS_MAXSYSCALL	548
diff --git a/sys/sys/syscall.mk b/sys/sys/syscall.mk
index 294a1140b92..4e8e891c0d5 100644
--- a/sys/sys/syscall.mk
+++ b/sys/sys/syscall.mk
@@ -1,7 +1,7 @@
 # FreeBSD system call names.
 # DO NOT EDIT-- this file is automatically generated.
 # $FreeBSD$
-# created from FreeBSD: head/sys/kern/syscalls.master 276654 2015-01-04 10:34:02Z dchagin 
+# created from FreeBSD: head/sys/kern/syscalls.master 277610 2015-01-23 21:07:08Z jilles 
 MIASM =  \
 	syscall.o \
 	exit.o \
@@ -410,4 +410,6 @@ MIASM =  \
 	pipe2.o \
 	aio_mlock.o \
 	procctl.o \
-	ppoll.o
+	ppoll.o \
+	futimens.o \
+	utimensat.o
diff --git a/sys/sys/sysproto.h b/sys/sys/sysproto.h
index ecaaa593cdf..71dc979fd91 100644
--- a/sys/sys/sysproto.h
+++ b/sys/sys/sysproto.h
@@ -3,7 +3,7 @@
  *
  * DO NOT EDIT-- this file is automatically generated.
  * $FreeBSD$
- * created from FreeBSD: head/sys/kern/syscalls.master 276654 2015-01-04 10:34:02Z dchagin 
+ * created from FreeBSD: head/sys/kern/syscalls.master 277610 2015-01-23 21:07:08Z jilles 
  */
 
 #ifndef _SYS_SYSPROTO_H_
@@ -1819,6 +1819,16 @@ struct ppoll_args {
 	char ts_l_[PADL_(const struct timespec *)]; const struct timespec * ts; char ts_r_[PADR_(const struct timespec *)];
 	char set_l_[PADL_(const sigset_t *)]; const sigset_t * set; char set_r_[PADR_(const sigset_t *)];
 };
+struct futimens_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char times_l_[PADL_(struct timespec *)]; struct timespec * times; char times_r_[PADR_(struct timespec *)];
+};
+struct utimensat_args {
+	char fd_l_[PADL_(int)]; int fd; char fd_r_[PADR_(int)];
+	char path_l_[PADL_(char *)]; char * path; char path_r_[PADR_(char *)];
+	char times_l_[PADL_(struct timespec *)]; struct timespec * times; char times_r_[PADR_(struct timespec *)];
+	char flag_l_[PADL_(int)]; int flag; char flag_r_[PADR_(int)];
+};
 int	nosys(struct thread *, struct nosys_args *);
 void	sys_sys_exit(struct thread *, struct sys_exit_args *);
 int	sys_fork(struct thread *, struct fork_args *);
@@ -2211,6 +2221,8 @@ int	sys_pipe2(struct thread *, struct pipe2_args *);
 int	sys_aio_mlock(struct thread *, struct aio_mlock_args *);
 int	sys_procctl(struct thread *, struct procctl_args *);
 int	sys_ppoll(struct thread *, struct ppoll_args *);
+int	sys_futimens(struct thread *, struct futimens_args *);
+int	sys_utimensat(struct thread *, struct utimensat_args *);
 
 #ifdef COMPAT_43
 
@@ -2917,6 +2929,8 @@ int	freebsd7_shmctl(struct thread *, struct freebsd7_shmctl_args *);
 #define	SYS_AUE_aio_mlock	AUE_NULL
 #define	SYS_AUE_procctl	AUE_NULL
 #define	SYS_AUE_ppoll	AUE_POLL
+#define	SYS_AUE_futimens	AUE_FUTIMES
+#define	SYS_AUE_utimensat	AUE_FUTIMESAT
 
 #undef PAD_
 #undef PADL_

From 27051ca14b0bf72310d5cd58f9c8e7355080a3e7 Mon Sep 17 00:00:00 2001
From: Ed Maste <emaste@FreeBSD.org>
Date: Fri, 23 Jan 2015 21:34:08 +0000
Subject: [PATCH 222/279] Restore addr2line to cross tools

Addr2line is not required for the build, and a per-arch binary is no
longer required with the switch to the ELF Tool Chain. However, building
these tools during the cross tools stage can be useful for developers
who cross build HEAD from stable/10, and adds very little to the build
time.

Reviewed by:	ian, imp
Differential Revision:	https://reviews.freebsd.org/D1583
---
 Makefile.inc1 | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/Makefile.inc1 b/Makefile.inc1
index afa49408e68..0d0aea47691 100644
--- a/Makefile.inc1
+++ b/Makefile.inc1
@@ -1437,6 +1437,9 @@ _elftctools=	lib/libelftc \
 		usr.bin/nm \
 		usr.bin/size \
 		usr.bin/strings
+# These are not required by the build, but can be useful for developers who
+# cross-build on a FreeBSD 10 host:
+_elftctools+=	usr.bin/addr2line
 .endif
 .endif
 

From fda4e2154926094fb06ac910a62f7476b36b251d Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Fri, 23 Jan 2015 23:53:56 +0000
Subject: [PATCH 223/279] Fix panic in firewire and creation of invalid config
 ROM. (This change was supposed to be included in r277508.)

sys/boot/i386/libfirewire/firewire.c:
	Fix configuration ROM generation count wrapping logic
	so that the generation count is never outside of
	allowed limits (0x2 -> 0xF).

Submitted by:	gibbs
MFC after:	1 week
MFC with:	277508
Sponsored by:	Spectra Logic
MFSpectraBSD:	1110685 on 2015/01/05
---
 sys/boot/i386/libfirewire/firewire.c | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/sys/boot/i386/libfirewire/firewire.c b/sys/boot/i386/libfirewire/firewire.c
index 5edb67a4ecb..8191a63d8b3 100644
--- a/sys/boot/i386/libfirewire/firewire.c
+++ b/sys/boot/i386/libfirewire/firewire.c
@@ -233,7 +233,8 @@ fw_init_crom(struct fwohci_softc *sc)
 	src->businfo.cyc_clk_acc = 100;
 	src->businfo.max_rec = sc->maxrec;
 	src->businfo.max_rom = MAXROM_4;
-	src->businfo.generation = 1;
+#define FW_GENERATION_CHANGEABLE 2
+	src->businfo.generation = FW_GENERATION_CHANGEABLE;
 	src->businfo.link_spd = sc->speed;
 
 	src->businfo.eui64.hi = sc->eui.hi;
@@ -313,11 +314,14 @@ fw_crom(struct fwohci_softc *sc)
 	src = &sc->crom_src_buf->src;
 	crom_load(src, (uint32_t *)newrom, CROMSIZE);
 	if (bcmp(newrom, sc->config_rom, CROMSIZE) != 0) {
-		/* bump generation and reload */
-		src->businfo.generation ++;
-		/* generation must be between 0x2 and 0xF */
+		/* Bump generation and reload. */
+		src->businfo.generation++;
+
+		/* Handle generation count wraps. */
 		if (src->businfo.generation < 2)
-			src->businfo.generation ++;
+			src->businfo.generation = 2;
+
+		/* Recalculate CRC to account for generation change. */
 		crom_load(src, (uint32_t *)newrom, CROMSIZE);
 		bcopy(newrom, (void *)sc->config_rom, CROMSIZE);
 	}

From 7cd27e6394967e11a984cf15448b6706cb6e9422 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Sat, 24 Jan 2015 00:04:34 +0000
Subject: [PATCH 224/279] Bump config(8) minor version for r277567.

Config files built using this version that take advantage of the bugfix
won't be buildable using older config(8).

Submitted by:	imp
MFC after:	1 week
MFC with:	277567
---
 usr.sbin/config/configvers.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/usr.sbin/config/configvers.h b/usr.sbin/config/configvers.h
index 56ca85cd692..5c2937311d7 100644
--- a/usr.sbin/config/configvers.h
+++ b/usr.sbin/config/configvers.h
@@ -49,5 +49,5 @@
  *
  * $FreeBSD$
  */
-#define	CONFIGVERS	600013
+#define	CONFIGVERS	600014
 #define	MAJOR_VERS(x)	((x) / 100000)

From e09ff1712993b61a4eb556aad094618e2b049eae Mon Sep 17 00:00:00 2001
From: Neel Natu <neel@FreeBSD.org>
Date: Sat, 24 Jan 2015 00:35:49 +0000
Subject: [PATCH 225/279] Add macro to identify AVIC capability (advanced
 virtual interrupt controller) in AMD processors.

Submitted by:	Dmitry Luhtionov (dmitryluhtionov@gmail.com)
---
 sys/amd64/vmm/amd/svm.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/sys/amd64/vmm/amd/svm.c b/sys/amd64/vmm/amd/svm.c
index 31164d7c58d..bcd9efc4b20 100644
--- a/sys/amd64/vmm/amd/svm.c
+++ b/sys/amd64/vmm/amd/svm.c
@@ -80,6 +80,7 @@ SYSCTL_NODE(_hw_vmm, OID_AUTO, svm, CTLFLAG_RW, NULL, NULL);
 #define AMD_CPUID_SVM_DECODE_ASSIST	BIT(7)  /* Decode assist */
 #define AMD_CPUID_SVM_PAUSE_INC		BIT(10) /* Pause intercept filter. */
 #define AMD_CPUID_SVM_PAUSE_FTH		BIT(12) /* Pause filter threshold */
+#define	AMD_CPUID_SVM_AVIC		BIT(13)	/* AVIC present */
 
 #define	VMCB_CACHE_DEFAULT	(VMCB_CACHE_ASID 	|	\
 				VMCB_CACHE_IOPM		|	\

From ff6fcf6e0bade26b80d8486a180a7775a33ce53f Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Sat, 24 Jan 2015 00:37:41 +0000
Subject: [PATCH 226/279] Add
 routing_test:static_ipv6_loopback_route_for_each_fib. It tests that all FIBs
 get a static IPv6 loopback route.

Submitted by:	asomers
MFC after:	1 week
Sponsored by:	Spectra Logic
MFSpectraBSD:	1048456 on 2014/03/13
		1114523 on 2015/01/23
---
 etc/tests/Makefile             |   2 +
 etc/tests/rc.d/Makefile        |   7 ++
 etc/tests/rc.d/routing_test.sh | 138 +++++++++++++++++++++++++++++++++
 3 files changed, 147 insertions(+)
 create mode 100644 etc/tests/rc.d/Makefile
 create mode 100755 etc/tests/rc.d/routing_test.sh

diff --git a/etc/tests/Makefile b/etc/tests/Makefile
index 5aacd5b19e5..fd9efdaa566 100644
--- a/etc/tests/Makefile
+++ b/etc/tests/Makefile
@@ -7,4 +7,6 @@ TESTSDIR=	${TESTSBASE}/etc
 .PATH:		${.CURDIR:H:H}/tests
 KYUAFILE=	yes
 
+TESTS_SUBDIRS+=	rc.d
+
 .include <bsd.test.mk>
diff --git a/etc/tests/rc.d/Makefile b/etc/tests/rc.d/Makefile
new file mode 100644
index 00000000000..368e8f4fc87
--- /dev/null
+++ b/etc/tests/rc.d/Makefile
@@ -0,0 +1,7 @@
+# $FreeBSD$
+
+TESTSDIR=	${TESTSBASE}/etc/rc.d
+
+ATF_TESTS_SH+=	routing_test
+
+.include <bsd.test.mk>
diff --git a/etc/tests/rc.d/routing_test.sh b/etc/tests/rc.d/routing_test.sh
new file mode 100755
index 00000000000..693af232ea3
--- /dev/null
+++ b/etc/tests/rc.d/routing_test.sh
@@ -0,0 +1,138 @@
+#
+#  Copyright (c) 2014 Spectra Logic Corporation
+#  All rights reserved.
+# 
+#  Redistribution and use in source and binary forms, with or without
+#  modification, are permitted provided that the following conditions
+#  are met:
+#  1. Redistributions of source code must retain the above copyright
+#     notice, this list of conditions, and the following disclaimer,
+#     without modification.
+#  2. Redistributions in binary form must reproduce at minimum a disclaimer
+#     substantially similar to the "NO WARRANTY" disclaimer below
+#     ("Disclaimer") and any redistribution must be conditioned upon
+#     including a substantially similar Disclaimer requirement for further
+#     binary redistribution.
+# 
+#  NO WARRANTY
+#  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+#  "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+#  LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTIBILITY AND FITNESS FOR
+#  A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+#  HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+#  DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+#  OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+#  HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+#  STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+#  IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+#  POSSIBILITY OF SUCH DAMAGES.
+# 
+#  Authors: Alan Somers         (Spectra Logic Corporation)
+#
+# $FreeBSD$
+
+atf_test_case static_ipv6_loopback_route_for_each_fib cleanup
+static_ipv6_loopback_route_for_each_fib_head()
+{
+	atf_set "descr" "Every FIB should have a static IPv6 loopback route"
+	atf_set "require.user" "root"
+	atf_set "require.config" "fibs"
+	atf_set "require.progs" "sysrc"
+}
+static_ipv6_loopback_route_for_each_fib_body()
+{
+	# Configure the TAP interface to use an RFC5737 nonrouteable address
+	# and a non-default fib
+	ADDR="192.0.2.2"
+	SUBNET="192.0.2.0"
+	MASK="24"
+
+	# Check system configuration
+	if [ 0 != `sysctl -n net.add_addr_allfibs` ]; then
+		atf_skip "This test requires net.add_addr_allfibs=0"
+	fi
+
+	get_fibs 1
+	get_tap
+	
+	# Configure a TAP interface in /etc/rc.conf.  Register the sysrc
+	# variable for cleanup.
+	echo "ifconfig_${TAP}" >> "sysrc_vars_to_cleanup"
+	sysrc ifconfig_${TAP}="${ADDR}/${MASK} fib ${FIB0}"
+
+	# Start the interface
+	service netif start ${TAP}
+	# Check for an IPv6 loopback route
+	setfib ${FIB0} netstat -rn -f inet6 | grep -q "^::1.*lo0$"
+	if [ 0 -eq $? ]; then
+		atf_pass
+	else
+		setfib ${FIB0} netstat -rn -f inet6
+		atf_fail "Did not find an IPv6 loopback route"
+	fi
+}
+static_ipv6_loopback_route_for_each_fib_cleanup()
+{
+	cleanup_sysrc
+	cleanup_tap
+}
+
+atf_init_test_cases()
+{
+	atf_add_test_case static_ipv6_loopback_route_for_each_fib
+}
+
+# Looks up one or more fibs from the configuration data and validates them.
+# Returns the results in the env varilables FIB0, FIB1, etc.
+# parameter numfibs	The number of fibs to lookup
+get_fibs()
+{
+	NUMFIBS=$1
+	net_fibs=`sysctl -n net.fibs`
+	i=0
+	while [ $i -lt "$NUMFIBS" ]; do
+		fib=`atf_config_get "fibs" | \
+			awk -v i=$(( i + 1 )) '{print $i}'`
+		echo "fib is ${fib}"
+		eval FIB${i}=${fib}
+		if [ "$fib" -ge "$net_fibs" ]; then
+			msg="The ${i}th configured fib is ${fub}, which is "
+			msg="$msg not less than net.fibs (${net_fibs})"
+			atf_skip "$msg"
+		fi
+		i=$(( $i + 1 ))
+	done
+}
+
+
+# Creates a new tap(4) interface, registers it for cleanup, and returns the
+# name via the environment variable TAP
+get_tap()
+{
+	local TAPN=0
+	while ! ifconfig tap${TAPN} create > /dev/null 2>&1; do
+		if [ "$TAPN" -ge 8 ]; then
+			atf_skip "Could not create a tap(4) interface"
+		else
+			TAPN=$(($TAPN + 1))
+		fi
+	done
+	local TAPD=tap${TAPN}
+	# Record the TAP device so we can clean it up later
+	echo ${TAPD} >> "tap_devices_to_cleanup"
+	TAP=${TAPD}
+}
+
+cleanup_sysrc()
+{
+	for var in `cat "sysrc_vars_to_cleanup"`; do
+		sysrc -x $var
+	done
+}
+
+cleanup_tap()
+{
+	for TAPD in `cat "tap_devices_to_cleanup"`; do
+		ifconfig ${TAPD} destroy
+	done
+}

From b4e360d2392414dcb3860fbbc01b23fd989fe2ce Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Sat, 24 Jan 2015 00:43:02 +0000
Subject: [PATCH 227/279] When creating or updating a node, use vfs_timestamp()
 for "now" instead of gethrestime(), to allow the administrator to decide the
 appropriate timestamp precision instead of always using nanosecond precision.

---
 sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c
index 44b782096b2..d9257129332 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_znode.c
@@ -794,7 +794,7 @@ zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr,
 		gen = vap->va_nblocks;		/* ditto */
 	} else {
 		obj = 0;
-		gethrestime(&now);
+		vfs_timestamp(&now);
 		gen = dmu_tx_get_txg(tx);
 	}
 
@@ -1426,7 +1426,7 @@ zfs_tstamp_update_setup(znode_t *zp, uint_t flag, uint64_t mtime[2],
 {
 	timestruc_t	now;
 
-	gethrestime(&now);
+	vfs_timestamp(&now);
 
 	if (have_tx) {	/* will sa_bulk_update happen really soon? */
 		zp->z_atime_dirty = 0;

From 5967db8dcaf95e0be3059d487efb1e944080e8d4 Mon Sep 17 00:00:00 2001
From: Oleksandr Tymoshenko <gonzo@FreeBSD.org>
Date: Sat, 24 Jan 2015 01:53:28 +0000
Subject: [PATCH 228/279] After interrupt read value from register to make sure
 it reached hardware. This should fix "Spurious inteerupt" message

Suggested by ian@
---
 sys/arm/ti/am335x/am335x_lcd.c | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/sys/arm/ti/am335x/am335x_lcd.c b/sys/arm/ti/am335x/am335x_lcd.c
index ada08011b64..4fd811cc2ea 100644
--- a/sys/arm/ti/am335x/am335x_lcd.c
+++ b/sys/arm/ti/am335x/am335x_lcd.c
@@ -356,6 +356,8 @@ am335x_lcd_intr(void *arg)
 
 	reg = LCD_READ4(sc, LCD_IRQSTATUS);
 	LCD_WRITE4(sc, LCD_IRQSTATUS, reg);
+	/* Read value back to make sure it reached the hardware */
+	reg = LCD_READ4(sc, LCD_IRQSTATUS);
 
 	if (reg & IRQ_SYNC_LOST) {
 		reg = LCD_READ4(sc, LCD_RASTER_CTRL);
@@ -401,6 +403,8 @@ am335x_lcd_intr(void *arg)
 
 done:
 	LCD_WRITE4(sc, LCD_END_OF_INT_IND, 0);
+	/* Read value back to make sure it reached the hardware */
+	reg = LCD_READ4(sc, LCD_END_OF_INT_IND);
 }
 
 static int

From 01fc52e76db89d023c1cc762be8efc7f95ef3994 Mon Sep 17 00:00:00 2001
From: Nathan Whitehorn <nwhitehorn@FreeBSD.org>
Date: Sat, 24 Jan 2015 01:58:15 +0000
Subject: [PATCH 229/279] Fix typo in r277561.

---
 sys/powerpc/aim/machdep.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c
index a6d254b3f14..020c07ab48f 100644
--- a/sys/powerpc/aim/machdep.c
+++ b/sys/powerpc/aim/machdep.c
@@ -535,7 +535,7 @@ powerpc_init(vm_offset_t fdt, vm_offset_t toc, vm_offset_t ofentry, void *mdp)
 #endif
 	bcopy(&alitrap,  (void *)(EXC_ALI + trap_offset),  (size_t)&aliend -
 	    (size_t)&alitrap);
-	bcopy(&dsitrap,  (void *)(EXC_DSI + trap_offset),  (size_t)&dsitrap -
+	bcopy(&dsitrap,  (void *)(EXC_DSI + trap_offset),  (size_t)&dsiend -
 	    (size_t)&dsitrap);
 	bcopy(generictrap, (void *)EXC_ISI,  trapsize);
 	#ifdef __powerpc64__

From 7cf3e8e4c353e86d04df84d38b6489b048a6cd2c Mon Sep 17 00:00:00 2001
From: Navdeep Parhar <np@FreeBSD.org>
Date: Sat, 24 Jan 2015 04:41:14 +0000
Subject: [PATCH 230/279] Make sure the compiler flag to get cxgbe(4) to
 compile with gcc is used only when gcc is being used.  This is what r277225
 should have been.

Suggested by:	dim@
---
 sys/conf/files                      | 2 +-
 sys/modules/cxgbe/if_cxgbe/Makefile | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/sys/conf/files b/sys/conf/files
index 5895cce88ab..ee8a5381be9 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1141,7 +1141,7 @@ dev/cxgb/sys/uipc_mvec.c	optional cxgb pci \
 dev/cxgb/cxgb_t3fw.c		optional cxgb cxgb_t3fw \
 	compile-with "${NORMAL_C} -I$S/dev/cxgb"
 dev/cxgbe/t4_mp_ring.c		optional cxgbe pci \
-	compile-with "${NORMAL_C} -I$S/dev/cxgbe -fms-extensions"
+	compile-with "${NORMAL_C} -I$S/dev/cxgbe ${GCC_MS_EXTENSIONS}"
 dev/cxgbe/t4_main.c		optional cxgbe pci \
 	compile-with "${NORMAL_C} -I$S/dev/cxgbe"
 dev/cxgbe/t4_netmap.c		optional cxgbe pci \
diff --git a/sys/modules/cxgbe/if_cxgbe/Makefile b/sys/modules/cxgbe/if_cxgbe/Makefile
index 9c0c857a8e9..32347f468b1 100644
--- a/sys/modules/cxgbe/if_cxgbe/Makefile
+++ b/sys/modules/cxgbe/if_cxgbe/Makefile
@@ -23,6 +23,7 @@ SRCS+=	t4_tracer.c
 # Provide the timestamp of a packet in its header mbuf.
 #CFLAGS+= -DT4_PKT_TIMESTAMP
 
-CFLAGS+= -I${CXGBE} -fms-extensions
+CFLAGS+= -I${CXGBE}
 
 .include <bsd.kmod.mk>
+CFLAGS+= ${GCC_MS_EXTENSIONS}

From 2fae2ab4c7a265561f9c2e9b22cb44e98db7cd4f Mon Sep 17 00:00:00 2001
From: Xin LI <delphij@FreeBSD.org>
Date: Sat, 24 Jan 2015 06:06:46 +0000
Subject: [PATCH 231/279] Don't include libcapsicum headers when requested.

Reported by:	luigi
MFC after:	14 days
X-MFC-with:	r276788
---
 contrib/tcpdump/tcpdump.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/contrib/tcpdump/tcpdump.c b/contrib/tcpdump/tcpdump.c
index 9c9609b9325..b1630818f61 100644
--- a/contrib/tcpdump/tcpdump.c
+++ b/contrib/tcpdump/tcpdump.c
@@ -77,12 +77,12 @@ extern int SIZE_BUF;
 #ifdef __FreeBSD__
 #include <sys/capsicum.h>
 #include <sys/sysctl.h>
+#endif /* __FreeBSD__ */
+#ifdef HAVE_CAPSICUM
 #include <libcapsicum.h>
 #include <libcapsicum_dns.h>
 #include <libcapsicum_service.h>
 #include <nv.h>
-#endif /* __FreeBSD__ */
-#ifdef HAVE_CAPSICUM
 #include <sys/capability.h>
 #include <sys/ioccom.h>
 #include <net/bpf.h>
@@ -966,8 +966,8 @@ main(int argc, char **argv)
 	FILE *VFile;
 #ifdef HAVE_CAPSICUM
 	cap_rights_t rights;
-	int cansandbox;
 #endif	/* HAVE_CAPSICUM */
+	int cansandbox;
 
 #ifdef WIN32
 	if(wsockinit() != 0) return 1;

From f43f9c89f2599e574d5e812b916b2c46131d4a34 Mon Sep 17 00:00:00 2001
From: Luigi Rizzo <luigi@FreeBSD.org>
Date: Sat, 24 Jan 2015 08:48:05 +0000
Subject: [PATCH 232/279] do not strip /stand

This is a temporary workaround until the elftoolchain's version
of strip is fixed:

The previous (GNU) strip, when acting on a file with multiple links,
would modify the one and only file in place (which means creating
a new stripped copy, and then writing it back to the original).

The new version from elftoolchain creates the new file and then
unlinks the old one and renames the new.
With multiple hard links, the original remains alive.  In the /stand
directory, this ends up creating 80+ copies of the same file.
---
 release/picobsd/build/picobsd | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/release/picobsd/build/picobsd b/release/picobsd/build/picobsd
index 19a98394927..728e9f4e3d6 100755
--- a/release/picobsd/build/picobsd
+++ b/release/picobsd/build/picobsd
@@ -734,8 +734,8 @@ populate_mfs_tree() {
     [ -n "${copy_files}" ] && do_copyfiles ${dst} copy_files
     do_copyfiles_user ${dst} || true
     [ -n "${links}" ] && do_links ${dst} links
-    strip ${dst}/libexec/* ${dst}/lib/* ${dst}/stand/* 2> /dev/null || true
-
+    strip ${dst}/libexec/* ${dst}/lib/* 2> /dev/null || true
+    # strip ${dst}/stand/* 2> /dev/null || true
     # The 'import_files' mechanism is deprecated, as it requires
     # root permissions to follow the symlinks, and also does
     # not let you rename the entries.
@@ -756,7 +756,7 @@ populate_mfs_tree() {
 	# override the owner
 	echo "/set uid=0 gid=0" > mtree.out
 	mtree -ic -p ${dst} -k "" >> mtree.out
-	log "mtre.out at ${BUILDDIR}/mtree.out"
+	log "mtree.out at ${BUILDDIR}/mtree.out size  ${MFS_SIZE}k"
 	makefs -t ffs -o bsize=4096 -o fsize=512 \
 		-s ${MFS_SIZE}k -f 1000 -F mtree.out ${c_fs} ${dst}
 	ls -l ${c_fs} )

From 51782e3a89a7dd45234d598ae0872ff2e9e16da7 Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Sat, 24 Jan 2015 12:43:36 +0000
Subject: [PATCH 233/279] Provide individual prototype and generate macros for
 the red-black tree. This helps to reduce code size in statically linked
 applications.

Submitted by:	Sebastian Huber <sebastian.huber@embedded-brains.de>
MFC after:	2 weeks
---
 share/man/man3/tree.3 | 71 ++++++++++++++++++++++++++++++++++-
 sys/sys/tree.h        | 86 ++++++++++++++++++++++++++++++-------------
 2 files changed, 131 insertions(+), 26 deletions(-)

diff --git a/share/man/man3/tree.3 b/share/man/man3/tree.3
index 2e9c63cf751..ea70506ea02 100644
--- a/share/man/man3/tree.3
+++ b/share/man/man3/tree.3
@@ -30,7 +30,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd November 10, 2013
+.Dd January 24, 2015
 .Dt TREE 3
 .Os
 .Sh NAME
@@ -53,8 +53,26 @@
 .Nm SPLAY_REMOVE ,
 .Nm RB_PROTOTYPE ,
 .Nm RB_PROTOTYPE_STATIC ,
+.Nm RB_PROTOTYPE_INSERT ,
+.Nm RB_PROTOTYPE_INSERT_COLOR ,
+.Nm RB_PROTOTYPE_REMOVE ,
+.Nm RB_PROTOTYPE_REMOVE_COLOR ,
+.Nm RB_PROTOTYPE_FIND ,
+.Nm RB_PROTOTYPE_NFIND ,
+.Nm RB_PROTOTYPE_NEXT ,
+.Nm RB_PROTOTYPE_PREV ,
+.Nm RB_PROTOTYPE_MINMAX ,
 .Nm RB_GENERATE ,
 .Nm RB_GENERATE_STATIC ,
+.Nm RB_GENERATE_INSERT ,
+.Nm RB_GENERATE_INSERT_COLOR ,
+.Nm RB_GENERATE_REMOVE ,
+.Nm RB_GENERATE_REMOVE_COLOR ,
+.Nm RB_GENERATE_FIND ,
+.Nm RB_GENERATE_NFIND ,
+.Nm RB_GENERATE_NEXT ,
+.Nm RB_GENERATE_PREV ,
+.Nm RB_GENERATE_MINMAX ,
 .Nm RB_ENTRY ,
 .Nm RB_HEAD ,
 .Nm RB_INITIALIZER ,
@@ -111,8 +129,26 @@
 .Fn SPLAY_REMOVE NAME "SPLAY_HEAD *head" "struct TYPE *elm"
 .Fn RB_PROTOTYPE NAME TYPE FIELD CMP
 .Fn RB_PROTOTYPE_STATIC NAME TYPE FIELD CMP
+.Fn RB_PROTOTYPE_INSERT NAME TYPE ATTR
+.Fn RB_PROTOTYPE_INSERT_COLOR NAME TYPE ATTR
+.Fn RB_PROTOTYPE_REMOVE NAME TYPE ATTR
+.Fn RB_PROTOTYPE_REMOVE_COLOR NAME TYPE ATTR
+.Fn RB_PROTOTYPE_FIND NAME TYPE ATTR
+.Fn RB_PROTOTYPE_NFIND NAME TYPE ATTR
+.Fn RB_PROTOTYPE_NEXT NAME TYPE ATTR
+.Fn RB_PROTOTYPE_PREV NAME TYPE ATTR
+.Fn RB_PROTOTYPE_MINMAX NAME TYPE ATTR
 .Fn RB_GENERATE NAME TYPE FIELD CMP
 .Fn RB_GENERATE_STATIC NAME TYPE FIELD CMP
+.Fn RB_GENERATE_INSERT NAME TYPE FIELD CMP ATTR
+.Fn RB_GENERATE_INSERT_COLOR NAME TYPE FIELD ATTR
+.Fn RB_GENERATE_REMOVE NAME TYPE FIELD ATTR
+.Fn RB_GENERATE_REMOVE_COLOR NAME TYPE FIELD ATTR
+.Fn RB_GENERATE_FIND NAME TYPE FIELD CMP ATTR
+.Fn RB_GENERATE_NFIND NAME TYPE FIELD CMP ATTR
+.Fn RB_GENERATE_NEXT NAME TYPE FIELD ATTR
+.Fn RB_GENERATE_PREV NAME TYPE FIELD ATTR
+.Fn RB_GENERATE_MINMAX NAME TYPE FIELD ATTR
 .Fn RB_ENTRY TYPE
 .Fn RB_HEAD HEADNAME TYPE
 .Fn RB_INITIALIZER "RB_HEAD *head"
@@ -377,6 +413,27 @@ The
 .Fa FIELD
 argument is the name of the element defined by
 .Fn RB_ENTRY .
+Individual prototypes can be declared with
+.Fn RB_PROTOTYPE_INSERT ,
+.Fn RB_PROTOTYPE_INSERT_COLOR ,
+.Fn RB_PROTOTYPE_REMOVE ,
+.Fn RB_PROTOTYPE_REMOVE_COLOR ,
+.Fn RB_PROTOTYPE_FIND ,
+.Fn RB_PROTOTYPE_NFIND ,
+.Fn RB_PROTOTYPE_NEXT ,
+.Fn RB_PROTOTYPE_PREV ,
+and
+.Fn RB_PROTOTYPE_MINMAX
+in case not all functions are required.  The individual prototype macros expect
+.Fa NAME ,
+.Fa TYPE ,
+and
+.Fa ATTR
+arguments.  The
+.Fa ATTR
+argument must be empty for global functions or
+.Fa static
+for static functions.
 .Pp
 The function bodies are generated with the
 .Fn RB_GENERATE
@@ -388,6 +445,18 @@ These macros take the same arguments as the
 and
 .Fn RB_PROTOTYPE_STATIC
 macros, but should be used only once.
+As an alternative individual function bodies are generated with the
+.Fn RB_GENERATE_INSERT ,
+.Fn RB_GENERATE_INSERT_COLOR ,
+.Fn RB_GENERATE_REMOVE ,
+.Fn RB_GENERATE_REMOVE_COLOR ,
+.Fn RB_GENERATE_FIND ,
+.Fn RB_GENERATE_NFIND ,
+.Fn RB_GENERATE_NEXT ,
+.Fn RB_GENERATE_PREV ,
+and
+.Fn RB_GENERATE_MINMAX
+macros.
 .Pp
 Finally,
 the
diff --git a/sys/sys/tree.h b/sys/sys/tree.h
index 1cce727868a..c9df686fbb1 100644
--- a/sys/sys/tree.h
+++ b/sys/sys/tree.h
@@ -383,16 +383,33 @@ struct {								\
 #define	RB_PROTOTYPE_STATIC(name, type, field, cmp)			\
 	RB_PROTOTYPE_INTERNAL(name, type, field, cmp, __unused static)
 #define RB_PROTOTYPE_INTERNAL(name, type, field, cmp, attr)		\
-attr void name##_RB_INSERT_COLOR(struct name *, struct type *);		\
-attr void name##_RB_REMOVE_COLOR(struct name *, struct type *, struct type *);\
-attr struct type *name##_RB_REMOVE(struct name *, struct type *);	\
-attr struct type *name##_RB_INSERT(struct name *, struct type *);	\
-attr struct type *name##_RB_FIND(struct name *, struct type *);		\
-attr struct type *name##_RB_NFIND(struct name *, struct type *);	\
-attr struct type *name##_RB_NEXT(struct type *);			\
-attr struct type *name##_RB_PREV(struct type *);			\
-attr struct type *name##_RB_MINMAX(struct name *, int);			\
-									\
+	RB_PROTOTYPE_INSERT_COLOR(name, type, attr);			\
+	RB_PROTOTYPE_REMOVE_COLOR(name, type, attr);			\
+	RB_PROTOTYPE_INSERT(name, type, attr);				\
+	RB_PROTOTYPE_REMOVE(name, type, attr);				\
+	RB_PROTOTYPE_FIND(name, type, attr);				\
+	RB_PROTOTYPE_NFIND(name, type, attr);				\
+	RB_PROTOTYPE_NEXT(name, type, attr);				\
+	RB_PROTOTYPE_PREV(name, type, attr);				\
+	RB_PROTOTYPE_MINMAX(name, type, attr);
+#define RB_PROTOTYPE_INSERT_COLOR(name, type, attr)			\
+	attr void name##_RB_INSERT_COLOR(struct name *, struct type *)
+#define RB_PROTOTYPE_REMOVE_COLOR(name, type, attr)			\
+	attr void name##_RB_REMOVE_COLOR(struct name *, struct type *, struct type *)
+#define RB_PROTOTYPE_REMOVE(name, type, attr)				\
+	attr struct type *name##_RB_REMOVE(struct name *, struct type *)
+#define RB_PROTOTYPE_INSERT(name, type, attr)				\
+	attr struct type *name##_RB_INSERT(struct name *, struct type *)
+#define RB_PROTOTYPE_FIND(name, type, attr)				\
+	attr struct type *name##_RB_FIND(struct name *, struct type *)
+#define RB_PROTOTYPE_NFIND(name, type, attr)				\
+	attr struct type *name##_RB_NFIND(struct name *, struct type *)
+#define RB_PROTOTYPE_NEXT(name, type, attr)				\
+	attr struct type *name##_RB_NEXT(struct type *)
+#define RB_PROTOTYPE_PREV(name, type, attr)				\
+	attr struct type *name##_RB_PREV(struct type *)
+#define RB_PROTOTYPE_MINMAX(name, type, attr)				\
+	attr struct type *name##_RB_MINMAX(struct name *, int)
 
 /* Main rb operation.
  * Moves node close to the key of elm to top
@@ -402,6 +419,17 @@ attr struct type *name##_RB_MINMAX(struct name *, int);			\
 #define	RB_GENERATE_STATIC(name, type, field, cmp)			\
 	RB_GENERATE_INTERNAL(name, type, field, cmp, __unused static)
 #define RB_GENERATE_INTERNAL(name, type, field, cmp, attr)		\
+	RB_GENERATE_INSERT_COLOR(name, type, field, attr)		\
+	RB_GENERATE_REMOVE_COLOR(name, type, field, attr)		\
+	RB_GENERATE_INSERT(name, type, field, cmp, attr)		\
+	RB_GENERATE_REMOVE(name, type, field, attr)			\
+	RB_GENERATE_FIND(name, type, field, cmp, attr)			\
+	RB_GENERATE_NFIND(name, type, field, cmp, attr)			\
+	RB_GENERATE_NEXT(name, type, field, attr)			\
+	RB_GENERATE_PREV(name, type, field, attr)			\
+	RB_GENERATE_MINMAX(name, type, field, attr)
+
+#define RB_GENERATE_INSERT_COLOR(name, type, field, attr)		\
 attr void								\
 name##_RB_INSERT_COLOR(struct name *head, struct type *elm)		\
 {									\
@@ -444,8 +472,9 @@ name##_RB_INSERT_COLOR(struct name *head, struct type *elm)		\
 		}							\
 	}								\
 	RB_COLOR(head->rbh_root, field) = RB_BLACK;			\
-}									\
-									\
+}
+
+#define RB_GENERATE_REMOVE_COLOR(name, type, field, attr)		\
 attr void								\
 name##_RB_REMOVE_COLOR(struct name *head, struct type *parent, struct type *elm) \
 {									\
@@ -522,8 +551,9 @@ name##_RB_REMOVE_COLOR(struct name *head, struct type *parent, struct type *elm)
 	}								\
 	if (elm)							\
 		RB_COLOR(elm, field) = RB_BLACK;			\
-}									\
-									\
+}
+
+#define RB_GENERATE_REMOVE(name, type, field, attr)			\
 attr struct type *							\
 name##_RB_REMOVE(struct name *head, struct type *elm)			\
 {									\
@@ -590,7 +620,8 @@ color:									\
 		name##_RB_REMOVE_COLOR(head, parent, child);		\
 	return (old);							\
 }									\
-									\
+
+#define RB_GENERATE_INSERT(name, type, field, cmp, attr)		\
 /* Inserts a node into the RB tree */					\
 attr struct type *							\
 name##_RB_INSERT(struct name *head, struct type *elm)			\
@@ -620,8 +651,9 @@ name##_RB_INSERT(struct name *head, struct type *elm)			\
 		RB_ROOT(head) = elm;					\
 	name##_RB_INSERT_COLOR(head, elm);				\
 	return (NULL);							\
-}									\
-									\
+}
+
+#define RB_GENERATE_FIND(name, type, field, cmp, attr)			\
 /* Finds the node with the same key as elm */				\
 attr struct type *							\
 name##_RB_FIND(struct name *head, struct type *elm)			\
@@ -638,8 +670,9 @@ name##_RB_FIND(struct name *head, struct type *elm)			\
 			return (tmp);					\
 	}								\
 	return (NULL);							\
-}									\
-									\
+}
+
+#define RB_GENERATE_NFIND(name, type, field, cmp, attr)			\
 /* Finds the first node greater than or equal to the search key */	\
 attr struct type *							\
 name##_RB_NFIND(struct name *head, struct type *elm)			\
@@ -659,8 +692,9 @@ name##_RB_NFIND(struct name *head, struct type *elm)			\
 			return (tmp);					\
 	}								\
 	return (res);							\
-}									\
-									\
+}
+
+#define RB_GENERATE_NEXT(name, type, field, attr)			\
 /* ARGSUSED */								\
 attr struct type *							\
 name##_RB_NEXT(struct type *elm)					\
@@ -681,8 +715,9 @@ name##_RB_NEXT(struct type *elm)					\
 		}							\
 	}								\
 	return (elm);							\
-}									\
-									\
+}
+
+#define RB_GENERATE_PREV(name, type, field, attr)			\
 /* ARGSUSED */								\
 attr struct type *							\
 name##_RB_PREV(struct type *elm)					\
@@ -703,8 +738,9 @@ name##_RB_PREV(struct type *elm)					\
 		}							\
 	}								\
 	return (elm);							\
-}									\
-									\
+}
+
+#define RB_GENERATE_MINMAX(name, type, field, attr)			\
 attr struct type *							\
 name##_RB_MINMAX(struct name *head, int val)				\
 {									\

From 8f4548ff25674316d5b8fe06cd3423c8eb27caf5 Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Sat, 24 Jan 2015 12:51:15 +0000
Subject: [PATCH 234/279] Remove Giant from /dev/mem and /dev/kmem.  It is
 definitely not needed for i386, and from the code inspection, nothing in the
 arm/mips/sparc64 implementations depends on it.

Discussed with:	imp, nwhitehorn
Sponsored by:	The FreeBSD Foundation
MFC after:	3 weeks
---
 sys/arm/arm/mem.c         | 2 --
 sys/dev/mem/memdev.c      | 2 +-
 sys/i386/i386/mem.c       | 4 ----
 sys/mips/mips/mem.c       | 2 --
 sys/sparc64/sparc64/mem.c | 2 --
 5 files changed, 1 insertion(+), 11 deletions(-)

diff --git a/sys/arm/arm/mem.c b/sys/arm/arm/mem.c
index 460a0049510..30d4b1d8ab0 100644
--- a/sys/arm/arm/mem.c
+++ b/sys/arm/arm/mem.c
@@ -82,8 +82,6 @@ memrw(struct cdev *dev, struct uio *uio, int flags)
 	int error = 0;
 	vm_offset_t addr, eaddr;
 
-	GIANT_REQUIRED;
-
 	while (uio->uio_resid > 0 && error == 0) {
 		iov = uio->uio_iov;
 		if (iov->iov_len == 0) {
diff --git a/sys/dev/mem/memdev.c b/sys/dev/mem/memdev.c
index 37bad1538f7..c3d1af1ac3a 100644
--- a/sys/dev/mem/memdev.c
+++ b/sys/dev/mem/memdev.c
@@ -52,7 +52,7 @@ static struct cdev *memdev, *kmemdev;
 
 static struct cdevsw mem_cdevsw = {
 	.d_version =	D_VERSION,
-	.d_flags =	D_MEM|D_NEEDGIANT,
+	.d_flags =	D_MEM,
 	.d_open =	memopen,
 	.d_read =	memrw,
 	.d_write =	memrw,
diff --git a/sys/i386/i386/mem.c b/sys/i386/i386/mem.c
index 9c83f47428f..b036bd3ec3c 100644
--- a/sys/i386/i386/mem.c
+++ b/sys/i386/i386/mem.c
@@ -86,10 +86,6 @@ memrw(struct cdev *dev, struct uio *uio, int flags)
 	int error = 0;
 	vm_offset_t addr;
 
-	/* XXX UPS Why ? */
-	GIANT_REQUIRED;
-
-
 	if (dev2unit(dev) != CDEV_MINOR_MEM && dev2unit(dev) != CDEV_MINOR_KMEM)
 		return EIO;
 
diff --git a/sys/mips/mips/mem.c b/sys/mips/mips/mem.c
index d40c424888a..08bb6b094c4 100644
--- a/sys/mips/mips/mem.c
+++ b/sys/mips/mips/mem.c
@@ -85,8 +85,6 @@ memrw(struct cdev *dev, struct uio *uio, int flags)
 	cnt = 0;
 	error = 0;
 
-	GIANT_REQUIRED;
-
 	pmap_page_init(&m);
 	while (uio->uio_resid > 0 && !error) {
 		iov = uio->uio_iov;
diff --git a/sys/sparc64/sparc64/mem.c b/sys/sparc64/sparc64/mem.c
index d09f6b86771..6bd52253666 100644
--- a/sys/sparc64/sparc64/mem.c
+++ b/sys/sparc64/sparc64/mem.c
@@ -99,8 +99,6 @@ memrw(struct cdev *dev, struct uio *uio, int flags)
 	error = 0;
 	ova = 0;
 
-	GIANT_REQUIRED;
-
 	while (uio->uio_resid > 0 && error == 0) {
 		iov = uio->uio_iov;
 		if (iov->iov_len == 0) {

From f0583578a1f2577f999016a51f0d01fbe7851898 Mon Sep 17 00:00:00 2001
From: Ruslan Bukin <br@FreeBSD.org>
Date: Sat, 24 Jan 2015 13:07:07 +0000
Subject: [PATCH 235/279] Add support for audio transmitting, include drivers
 for: o Digital Audio Multiplexer (AUDMUX) o Smart Direct Memory Access
 Controller (SDMA) o Synchronous Serial Interface (SSI)

Disable by default as it depends on SDMA firmware.

Sponsored by:	Machdep, Inc.
---
 sys/arm/freescale/imx/files.imx6       |  19 +
 sys/arm/freescale/imx/imx6_anatop.c    |  21 +
 sys/arm/freescale/imx/imx6_anatopreg.h |   3 +
 sys/arm/freescale/imx/imx6_anatopvar.h |   2 +
 sys/arm/freescale/imx/imx6_audmux.c    | 159 +++++
 sys/arm/freescale/imx/imx6_ccm.c       |  54 +-
 sys/arm/freescale/imx/imx6_ccmreg.h    |  20 +
 sys/arm/freescale/imx/imx6_sdma.c      | 518 +++++++++++++++
 sys/arm/freescale/imx/imx6_sdma.h      | 245 +++++++
 sys/arm/freescale/imx/imx6_ssi.c       | 855 +++++++++++++++++++++++++
 sys/arm/freescale/imx/imx_ccmvar.h     |   1 +
 sys/boot/fdt/dts/arm/apalis-imx6.dts   |  20 +-
 sys/boot/fdt/dts/arm/imx6.dtsi         |   8 +-
 13 files changed, 1920 insertions(+), 5 deletions(-)
 create mode 100644 sys/arm/freescale/imx/imx6_audmux.c
 create mode 100644 sys/arm/freescale/imx/imx6_sdma.c
 create mode 100644 sys/arm/freescale/imx/imx6_sdma.h
 create mode 100644 sys/arm/freescale/imx/imx6_ssi.c

diff --git a/sys/arm/freescale/imx/files.imx6 b/sys/arm/freescale/imx/files.imx6
index df86ef9d596..de1fc41000b 100644
--- a/sys/arm/freescale/imx/files.imx6
+++ b/sys/arm/freescale/imx/files.imx6
@@ -28,6 +28,9 @@ arm/freescale/imx/imx_machdep.c		standard
 arm/freescale/imx/imx_gpt.c		standard
 arm/freescale/imx/imx_gpio.c		optional gpio
 arm/freescale/imx/imx_i2c.c		optional fsliic
+arm/freescale/imx/imx6_sdma.c		optional sdma
+arm/freescale/imx/imx6_audmux.c		optional sound
+arm/freescale/imx/imx6_ssi.c		optional sound
 
 #
 # Optional devices.
@@ -52,3 +55,19 @@ arm/freescale/imx/imx6_usbphy.c		optional ehci
 # Not ready yet...
 #
 #arm/freescale/imx/imx51_ipuv3.c  	optional sc
+
+# SDMA firmware
+sdma_fw.c				optional sdma_fw		\
+	compile-with	"${AWK} -f $S/tools/fw_stub.awk sdma-imx6q-to1.bin:sdma_fw -msdma -c${.TARGET}" \
+	no-implicit-rule before-depend local				\
+	clean		"sdma_fw.c"
+sdma-imx6q-to1.fwo			optional sdma_fw		\
+	dependency	"sdma-imx6q-to1.bin"				\
+	compile-with	"${LD} -b binary -d -warn-common -r -d -o ${.TARGET} sdma-imx6q-to1.bin" \
+	no-implicit-rule						\
+	clean		"sdma-imx6q-to1.fwo"
+sdma-imx6q-to1.bin			optional sdma_fw		\
+	dependency	"$S/contrib/dev/imx/sdma-imx6q-to1.bin.uu"	\
+	compile-with	"uudecode < $S/contrib/dev/imx/sdma-imx6q-to1.bin.uu" \
+	no-obj no-implicit-rule						\
+	clean		"sdma-imx6q-to1.bin"
diff --git a/sys/arm/freescale/imx/imx6_anatop.c b/sys/arm/freescale/imx/imx6_anatop.c
index fab28b3732b..644ac7fa777 100644
--- a/sys/arm/freescale/imx/imx6_anatop.c
+++ b/sys/arm/freescale/imx/imx6_anatop.c
@@ -710,6 +710,27 @@ out:
 	return (err);
 }
 
+uint32_t
+pll4_configure_output(uint32_t mfi, uint32_t mfn, uint32_t mfd)
+{
+	int reg;
+
+	/*
+	 * Audio PLL (PLL4).
+	 * PLL output frequency = Fref * (DIV_SELECT + NUM/DENOM)
+	 */
+
+	reg = (IMX6_ANALOG_CCM_PLL_AUDIO_ENABLE);
+	reg &= ~(IMX6_ANALOG_CCM_PLL_AUDIO_DIV_SELECT_MASK << \
+		IMX6_ANALOG_CCM_PLL_AUDIO_DIV_SELECT_SHIFT);
+	reg |= (mfi << IMX6_ANALOG_CCM_PLL_AUDIO_DIV_SELECT_SHIFT);
+	imx6_anatop_write_4(IMX6_ANALOG_CCM_PLL_AUDIO, reg);
+	imx6_anatop_write_4(IMX6_ANALOG_CCM_PLL_AUDIO_NUM, mfn);
+	imx6_anatop_write_4(IMX6_ANALOG_CCM_PLL_AUDIO_DENOM, mfd);
+
+	return (0);
+}
+
 static int
 imx6_anatop_probe(device_t dev)
 {
diff --git a/sys/arm/freescale/imx/imx6_anatopreg.h b/sys/arm/freescale/imx/imx6_anatopreg.h
index bb75735c521..a155249ec00 100644
--- a/sys/arm/freescale/imx/imx6_anatopreg.h
+++ b/sys/arm/freescale/imx/imx6_anatopreg.h
@@ -58,6 +58,9 @@
 #define	IMX6_ANALOG_CCM_PLL_SYS_NUM			0x050
 #define	IMX6_ANALOG_CCM_PLL_SYS_DENOM			0x060
 #define	IMX6_ANALOG_CCM_PLL_AUDIO			0x070
+#define	   IMX6_ANALOG_CCM_PLL_AUDIO_ENABLE		  (1 << 13)
+#define	   IMX6_ANALOG_CCM_PLL_AUDIO_DIV_SELECT_SHIFT	  0
+#define	   IMX6_ANALOG_CCM_PLL_AUDIO_DIV_SELECT_MASK	  0x7f
 #define	IMX6_ANALOG_CCM_PLL_AUDIO_SET			0x074
 #define	IMX6_ANALOG_CCM_PLL_AUDIO_CLR			0x078
 #define	IMX6_ANALOG_CCM_PLL_AUDIO_TOG			0x07C
diff --git a/sys/arm/freescale/imx/imx6_anatopvar.h b/sys/arm/freescale/imx/imx6_anatopvar.h
index 5ede71187e6..4ebe672dfda 100644
--- a/sys/arm/freescale/imx/imx6_anatopvar.h
+++ b/sys/arm/freescale/imx/imx6_anatopvar.h
@@ -42,4 +42,6 @@ void imx6_anatop_write_4(bus_size_t _offset, uint32_t _value);
 
 uint32_t imx6_get_cpu_clock(void);
 
+uint32_t pll4_configure_output(uint32_t mfi, uint32_t mfn, uint32_t mfd);
+
 #endif
diff --git a/sys/arm/freescale/imx/imx6_audmux.c b/sys/arm/freescale/imx/imx6_audmux.c
new file mode 100644
index 00000000000..4ea21094784
--- /dev/null
+++ b/sys/arm/freescale/imx/imx6_audmux.c
@@ -0,0 +1,159 @@
+/*-
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * i.MX6 Digital Audio Multiplexer (AUDMUX)
+ * Chapter 16, i.MX 6Dual/6Quad Applications Processor Reference Manual,
+ * Rev. 1, 04/2013
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/malloc.h>
+#include <sys/endian.h>
+#include <sys/rman.h>
+#include <sys/timeet.h>
+#include <sys/timetc.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <machine/bus.h>
+#include <machine/fdt.h>
+#include <machine/cpu.h>
+#include <machine/intr.h>
+
+#define	READ4(_sc, _reg)	\
+	bus_space_read_4(_sc->bst, _sc->bsh, _reg)
+#define	WRITE4(_sc, _reg, _val)	\
+	bus_space_write_4(_sc->bst, _sc->bsh, _reg, _val)
+
+#define	AUDMUX_PTCR(n)	(0x8 * (n - 1))	/* Port Timing Control Register */
+#define	 PTCR_TFS_DIR	(1 << 31)	/* Transmit Frame Sync Direction Control */
+#define	 PTCR_TFSEL_S	27		/* Transmit Frame Sync Select */
+#define	 PTCR_TFSEL_M	0xf
+#define	 PTCR_TCLKDIR	(1 << 26)	/* Transmit Clock Direction Control */
+#define	 PTCR_TCSEL_S	22		/* Transmit Clock Select. */
+#define	 PTCR_TCSEL_M	0xf
+#define	 PTCR_RFS_DIR	(1 << 21)	/* Receive Frame Sync Direction Control */
+#define	 PTCR_SYN	(1 << 11)
+#define	AUDMUX_PDCR(n)	(0x8 * (n - 1) + 0x4)	/* Port Data Control Reg */
+#define	 PDCR_RXDSEL_S		13	/* Receive Data Select */
+#define	 PDCR_RXDSEL_M		0x3
+#define	 PDCR_RXDSEL_PORT(n)	(n - 1)
+
+struct audmux_softc {
+	struct resource		*res[1];
+	bus_space_tag_t		bst;
+	bus_space_handle_t	bsh;
+	void			*ih;
+};
+
+static struct resource_spec audmux_spec[] = {
+	{ SYS_RES_MEMORY,	0,	RF_ACTIVE },
+	{ -1, 0 }
+};
+
+static int
+audmux_probe(device_t dev)
+{
+
+	if (!ofw_bus_status_okay(dev))
+		return (ENXIO);
+
+	if (!ofw_bus_is_compatible(dev, "fsl,imx6q-audmux"))
+		return (ENXIO);
+
+	device_set_desc(dev, "i.MX6 Digital Audio Multiplexer");
+	return (BUS_PROBE_DEFAULT);
+}
+
+static int
+audmux_configure(struct audmux_softc *sc,
+	int ssi_port, int audmux_port)
+{
+	uint32_t reg;
+
+	/* Direction: output */
+	reg = (PTCR_TFS_DIR | PTCR_TCLKDIR | PTCR_SYN);
+	WRITE4(sc, AUDMUX_PTCR(audmux_port), reg);
+
+	/* Select source */
+	reg = (PDCR_RXDSEL_PORT(ssi_port) << PDCR_RXDSEL_S);
+	WRITE4(sc, AUDMUX_PDCR(audmux_port), reg);
+
+	return (0);
+}
+
+static int
+audmux_attach(device_t dev)
+{
+	struct audmux_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	if (bus_alloc_resources(dev, audmux_spec, sc->res)) {
+		device_printf(dev, "could not allocate resources\n");
+		return (ENXIO);
+	}
+
+	/* Memory interface */
+	sc->bst = rman_get_bustag(sc->res[0]);
+	sc->bsh = rman_get_bushandle(sc->res[0]);
+
+	/*
+	 * Direct SSI1 output to AUDMUX5 pins.
+	 * TODO: dehardcore this.
+	 */
+	audmux_configure(sc, 1, 5);
+
+	return (0);
+};
+
+static device_method_t audmux_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		audmux_probe),
+	DEVMETHOD(device_attach,	audmux_attach),
+	{ 0, 0 }
+};
+
+static driver_t audmux_driver = {
+	"audmux",
+	audmux_methods,
+	sizeof(struct audmux_softc),
+};
+
+static devclass_t audmux_devclass;
+
+DRIVER_MODULE(audmux, simplebus, audmux_driver, audmux_devclass, 0, 0);
diff --git a/sys/arm/freescale/imx/imx6_ccm.c b/sys/arm/freescale/imx/imx6_ccm.c
index 2c80bd9ee03..8d7f14f1307 100644
--- a/sys/arm/freescale/imx/imx6_ccm.c
+++ b/sys/arm/freescale/imx/imx6_ccm.c
@@ -94,7 +94,7 @@ ccm_init_gates(struct ccm_softc *sc)
 	WR4(sc, CCM_CCGR2, 0x0fffffc0); /* ipmux & ipsync (bridges), iomux, i2c */
 	WR4(sc, CCM_CCGR3, 0x3ff00000); /* DDR memory controller */
 	WR4(sc, CCM_CCGR4, 0x0000f300); /* pl301 bus crossbar */
-	WR4(sc, CCM_CCGR5, 0x0f000000); /* uarts */
+	WR4(sc, CCM_CCGR5, 0x0ffc00c0); /* uarts, ssi, sdma */
 	WR4(sc, CCM_CCGR6, 0x000000ff); /* usdhc 1-4 */
 }
 
@@ -179,6 +179,58 @@ ccm_probe(device_t dev)
 	return (BUS_PROBE_DEFAULT);
 }
 
+void
+imx_ccm_ssi_configure(device_t _ssidev)
+{
+	struct ccm_softc *sc;
+	uint32_t reg;
+
+	sc = ccm_sc;
+
+	/*
+	 * Select PLL4 (Audio PLL) clock multiplexer as source.
+	 * PLL output frequency = Fref * (DIV_SELECT + NUM/DENOM).
+	 */
+
+	reg = RD4(sc, CCM_CSCMR1);
+	reg &= ~(SSI_CLK_SEL_M << SSI1_CLK_SEL_S);
+	reg |= (SSI_CLK_SEL_PLL4 << SSI1_CLK_SEL_S);
+	reg &= ~(SSI_CLK_SEL_M << SSI2_CLK_SEL_S);
+	reg |= (SSI_CLK_SEL_PLL4 << SSI2_CLK_SEL_S);
+	reg &= ~(SSI_CLK_SEL_M << SSI3_CLK_SEL_S);
+	reg |= (SSI_CLK_SEL_PLL4 << SSI3_CLK_SEL_S);
+	WR4(sc, CCM_CSCMR1, reg);
+
+	/*
+	 * Ensure we have set hardware-default values
+	 * for pre and post dividers.
+	 */
+
+	/* SSI1 and SSI3 */
+	reg = RD4(sc, CCM_CS1CDR);
+	/* Divide by 2 */
+	reg &= ~(SSI_CLK_PODF_MASK << SSI1_CLK_PODF_SHIFT);
+	reg &= ~(SSI_CLK_PODF_MASK << SSI3_CLK_PODF_SHIFT);
+	reg |= (0x1 << SSI1_CLK_PODF_SHIFT);
+	reg |= (0x1 << SSI3_CLK_PODF_SHIFT);
+	/* Divide by 4 */
+	reg &= ~(SSI_CLK_PRED_MASK << SSI1_CLK_PRED_SHIFT);
+	reg &= ~(SSI_CLK_PRED_MASK << SSI3_CLK_PRED_SHIFT);
+	reg |= (0x3 << SSI1_CLK_PRED_SHIFT);
+	reg |= (0x3 << SSI3_CLK_PRED_SHIFT);
+	WR4(sc, CCM_CS1CDR, reg);
+
+	/* SSI2 */
+	reg = RD4(sc, CCM_CS2CDR);
+	/* Divide by 2 */
+	reg &= ~(SSI_CLK_PODF_MASK << SSI2_CLK_PODF_SHIFT);
+	reg |= (0x1 << SSI2_CLK_PODF_SHIFT);
+	/* Divide by 4 */
+	reg &= ~(SSI_CLK_PRED_MASK << SSI2_CLK_PRED_SHIFT);
+	reg |= (0x3 << SSI2_CLK_PRED_SHIFT);
+	WR4(sc, CCM_CS2CDR, reg);
+}
+
 void
 imx_ccm_usb_enable(device_t _usbdev)
 {
diff --git a/sys/arm/freescale/imx/imx6_ccmreg.h b/sys/arm/freescale/imx/imx6_ccmreg.h
index d50c190c736..31d92caa6d4 100644
--- a/sys/arm/freescale/imx/imx6_ccmreg.h
+++ b/sys/arm/freescale/imx/imx6_ccmreg.h
@@ -29,6 +29,26 @@
 #ifndef	IMX6_CCMREG_H
 #define	IMX6_CCMREG_H
 
+#define	CCM_CSCMR1			0x01C
+#define	  SSI1_CLK_SEL_S		  10
+#define	  SSI2_CLK_SEL_S		  12
+#define	  SSI3_CLK_SEL_S		  14
+#define	  SSI_CLK_SEL_M			  0x3
+#define	  SSI_CLK_SEL_508_PFD		  0
+#define	  SSI_CLK_SEL_454_PFD		  1
+#define	  SSI_CLK_SEL_PLL4		  2
+#define	CCM_CSCMR2			0x020
+#define	CCM_CS1CDR			0x028
+#define	  SSI1_CLK_PODF_SHIFT		  0
+#define	  SSI1_CLK_PRED_SHIFT		  6
+#define	  SSI3_CLK_PODF_SHIFT		  16
+#define	  SSI3_CLK_PRED_SHIFT		  22
+#define	  SSI_CLK_PODF_MASK		  0x3f
+#define	  SSI_CLK_PRED_MASK		  0x7
+#define	CCM_CS2CDR			0x02C
+#define	  SSI2_CLK_PODF_SHIFT		  0
+#define	  SSI2_CLK_PRED_SHIFT		  6
+#define	CCM_CSCDR2			0x038
 #define	CCM_CLPCR			0x054
 #define	  CCM_CLPCR_LPM_MASK		  0x03
 #define	  CCM_CLPCR_LPM_RUN		  0x00
diff --git a/sys/arm/freescale/imx/imx6_sdma.c b/sys/arm/freescale/imx/imx6_sdma.c
new file mode 100644
index 00000000000..4a4c2872550
--- /dev/null
+++ b/sys/arm/freescale/imx/imx6_sdma.c
@@ -0,0 +1,518 @@
+/*-
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * i.MX6 Smart Direct Memory Access Controller (sDMA)
+ * Chapter 41, i.MX 6Dual/6Quad Applications Processor Reference Manual,
+ * Rev. 1, 04/2013
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/malloc.h>
+#include <sys/endian.h>
+#include <sys/rman.h>
+#include <sys/timeet.h>
+#include <sys/timetc.h>
+#include <sys/firmware.h>
+
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <machine/bus.h>
+#include <machine/fdt.h>
+#include <machine/cpu.h>
+#include <machine/intr.h>
+
+#include <arm/freescale/imx/imx6_sdma.h>
+
+#define	MAX_BD	(PAGE_SIZE / sizeof(struct sdma_buffer_descriptor))
+
+#define	READ4(_sc, _reg)	\
+	bus_space_read_4(_sc->bst, _sc->bsh, _reg)
+#define	WRITE4(_sc, _reg, _val)	\
+	bus_space_write_4(_sc->bst, _sc->bsh, _reg, _val)
+
+struct sdma_softc *sdma_sc;
+
+static struct resource_spec sdma_spec[] = {
+	{ SYS_RES_MEMORY,	0,	RF_ACTIVE },
+	{ SYS_RES_IRQ,		0,	RF_ACTIVE },
+	{ -1, 0 }
+};
+
+static void
+sdma_intr(void *arg)
+{
+	struct sdma_buffer_descriptor *bd;
+	struct sdma_channel *channel;
+	struct sdma_conf *conf;
+	struct sdma_softc *sc;
+	int pending;
+	int i;
+	int j;
+
+	sc = arg;
+
+	pending = READ4(sc, SDMAARM_INTR);
+
+	/* Ack intr */
+	WRITE4(sc, SDMAARM_INTR, pending);
+
+	for (i = 0; i < SDMA_N_CHANNELS; i++) {
+		if ((pending & (1 << i)) == 0)
+			continue;
+		channel = &sc->channel[i];
+		conf = channel->conf;
+		if (!conf)
+			continue;
+		for (j = 0; j < conf->num_bd; j++) {
+			bd = &channel->bd[j];
+			bd->mode.status |= BD_DONE;
+			if (bd->mode.status & BD_RROR)
+				printf("sDMA error\n");
+		}
+
+		conf->ih(conf->ih_user, 1);
+
+		WRITE4(sc, SDMAARM_HSTART, (1 << i));
+	}
+}
+
+static int
+sdma_probe(device_t dev)
+{
+
+	if (!ofw_bus_status_okay(dev))
+		return (ENXIO);
+
+	if (!ofw_bus_is_compatible(dev, "fsl,imx6q-sdma"))
+		return (ENXIO);
+
+	device_set_desc(dev, "i.MX6 Smart Direct Memory Access Controller");
+	return (BUS_PROBE_DEFAULT);
+}
+
+int
+sdma_start(int chn)
+{
+	struct sdma_softc *sc;
+
+	sc = sdma_sc;
+
+	WRITE4(sc, SDMAARM_HSTART, (1 << chn));
+
+	return (0);
+}
+
+int
+sdma_stop(int chn)
+{
+	struct sdma_softc *sc;
+
+	sc = sdma_sc;
+
+	WRITE4(sc, SDMAARM_STOP_STAT, (1 << chn));
+
+	return (0);
+}
+
+int
+sdma_alloc(void)
+{
+	struct sdma_channel *channel;
+	struct sdma_softc *sc;
+	int found;
+	int chn;
+	int i;
+
+	sc = sdma_sc;
+	found = 0;
+
+	/* Channel 0 can't be used */
+	for (i = 1; i < SDMA_N_CHANNELS; i++) {
+		channel = &sc->channel[i];
+		if (channel->in_use == 0) {
+			channel->in_use = 1;
+			found = 1;
+			break;
+		}
+	}
+
+	if (!found)
+		return (-1);
+
+	chn = i;
+
+	/* Allocate area for buffer descriptors */
+	channel->bd = (void *)kmem_alloc_contig(kernel_arena,
+	    PAGE_SIZE, M_ZERO, 0, ~0, PAGE_SIZE, 0,
+	    VM_MEMATTR_UNCACHEABLE);
+
+	return (chn);
+}
+
+int
+sdma_free(int chn)
+{
+	struct sdma_channel *channel;
+	struct sdma_softc *sc;
+
+	sc = sdma_sc;
+
+	channel = &sc->channel[chn];
+	channel->in_use = 0;
+
+	kmem_free(kernel_arena, (vm_offset_t)channel->bd,
+			PAGE_SIZE);
+
+	return (0);
+}
+
+static int
+sdma_overrides(struct sdma_softc *sc, int chn,
+		int evt, int host, int dsp)
+{
+	int reg;
+
+	/* Ignore sDMA requests */
+	reg = READ4(sc, SDMAARM_EVTOVR);
+	if (evt)
+		reg |= (1 << chn);
+	else
+		reg &= ~(1 << chn);
+	WRITE4(sc, SDMAARM_EVTOVR, reg);
+
+	/* Ignore enable bit (HE) */
+	reg = READ4(sc, SDMAARM_HOSTOVR);
+	if (host)
+		reg |= (1 << chn);
+	else
+		reg &= ~(1 << chn);
+	WRITE4(sc, SDMAARM_HOSTOVR, reg);
+
+	/* Prevent sDMA channel from starting */
+	reg = READ4(sc, SDMAARM_DSPOVR);
+	if (!dsp)
+		reg |= (1 << chn);
+	else
+		reg &= ~(1 << chn);
+	WRITE4(sc, SDMAARM_DSPOVR, reg);
+
+	return (0);
+}
+
+int
+sdma_configure(int chn, struct sdma_conf *conf)
+{
+	struct sdma_buffer_descriptor *bd0;
+	struct sdma_buffer_descriptor *bd;
+	struct sdma_context_data *context;
+	struct sdma_channel *channel;
+	struct sdma_softc *sc;
+#if 0
+	int timeout;
+	int ret;
+#endif
+	int i;
+
+	sc = sdma_sc;
+
+	channel = &sc->channel[chn];
+	channel->conf = conf;
+
+	/* Ensure operation has stopped */
+	sdma_stop(chn);
+
+	/* Set priority and enable the channel */
+	WRITE4(sc, SDMAARM_SDMA_CHNPRI(chn), 1);
+	WRITE4(sc, SDMAARM_CHNENBL(conf->event), (1 << chn));
+
+	sdma_overrides(sc, chn, 0, 0, 0);
+
+	if (conf->num_bd > MAX_BD) {
+		device_printf(sc->dev, "Error: too much buffer"
+				" descriptors requested\n");
+		return (-1);
+	}
+
+	for (i = 0; i < conf->num_bd; i++) {
+		bd = &channel->bd[i];
+		bd->mode.command = conf->command;
+		bd->mode.status = BD_DONE | BD_EXTD | BD_CONT | BD_INTR;
+		if (i == (conf->num_bd - 1))
+			bd->mode.status |= BD_WRAP;
+		bd->mode.count = conf->period;
+		bd->buffer_addr = conf->saddr + (conf->period * i);
+		bd->ext_buffer_addr = 0;
+	}
+
+	sc->ccb[chn].base_bd_ptr = vtophys(channel->bd);
+	sc->ccb[chn].current_bd_ptr = vtophys(channel->bd);
+
+	/*
+	 * Load context.
+	 *
+	 * i.MX6 Reference Manual: Appendix A SDMA Scripts
+	 * A.3.1.7.1 (mcu_2_app)
+	 */
+
+	/*
+	 * TODO: allow using other scripts
+	 */
+	context = sc->context;
+	memset(context, 0, sizeof(*context));
+	context->channel_state.pc = sc->fw_scripts->mcu_2_app_addr;
+
+	/*
+	 * Tx FIFO 0 address (r6)
+	 * Event_mask (r1)
+	 * Event2_mask (r0)
+	 * Watermark level (r7)
+	 */
+
+	if (conf->event > 32) {
+		context->gReg[0] = (1 << (conf->event % 32));
+		context->gReg[1] = 0;
+	} else {
+		context->gReg[0] = 0;
+		context->gReg[1] = (1 << conf->event);
+	}
+
+	context->gReg[6] = conf->daddr;
+	context->gReg[7] = conf->word_length;
+
+	bd0 = sc->bd0;
+	bd0->mode.command = C0_SETDM;
+	bd0->mode.status = BD_DONE | BD_INTR | BD_WRAP | BD_EXTD;
+	bd0->mode.count = sizeof(*context) / 4;
+	bd0->buffer_addr = sc->context_phys;
+	bd0->ext_buffer_addr = 2048 + (sizeof(*context) / 4) * chn;
+
+	WRITE4(sc, SDMAARM_HSTART, 1);
+
+#if 0
+	/* Debug purposes */
+
+	timeout = 1000;
+	while (!(ret = READ4(sc, SDMAARM_INTR) & 1)) {
+		if (timeout-- <= 0)
+			break;
+		DELAY(10);
+	};
+
+	if (!ret) {
+		device_printf(sc->dev, "Failed to load context.\n");
+		return (-1);
+	}
+
+	WRITE4(sc, SDMAARM_INTR, ret);
+
+	device_printf(sc->dev, "Context loaded successfully.\n");
+#endif
+
+	return (0);
+}
+
+static int
+load_firmware(struct sdma_softc *sc)
+{
+	struct sdma_firmware_header *header;
+	const struct firmware *fp;
+
+	fp = firmware_get("sdma_fw");
+	if (fp == NULL) {
+		device_printf(sc->dev, "Can't get firmware.\n");
+		return (-1);
+	}
+
+	header = (struct sdma_firmware_header *)fp->data;
+	if (header->magic != FW_HEADER_MAGIC) {
+		device_printf(sc->dev, "Can't use firmware.\n");
+		return (-1);
+	}
+
+	sc->fw_header = header;
+	sc->fw_scripts = (void *)((char *)header +
+				header->script_addrs_start);
+
+	return (0);
+}
+
+static int
+boot_firmware(struct sdma_softc *sc)
+{
+	struct sdma_buffer_descriptor *bd0;
+	uint32_t *ram_code;
+	int timeout;
+	int ret;
+	int chn;
+	int sz;
+	int i;
+
+	ram_code = (void *)((char *)sc->fw_header +
+			sc->fw_header->ram_code_start);
+
+	/* Make sure SDMA has not started yet */
+	WRITE4(sc, SDMAARM_MC0PTR, 0);
+
+	sz = SDMA_N_CHANNELS * sizeof(struct sdma_channel_control) + \
+	    sizeof(struct sdma_context_data);
+	sc->ccb = (void *)kmem_alloc_contig(kernel_arena,
+	    sz, M_ZERO, 0, ~0, PAGE_SIZE, 0, VM_MEMATTR_UNCACHEABLE);
+	sc->ccb_phys = vtophys(sc->ccb);
+
+	sc->context = (void *)((char *)sc->ccb + \
+	    SDMA_N_CHANNELS * sizeof(struct sdma_channel_control));
+	sc->context_phys = vtophys(sc->context);
+
+	/* Disable all the channels */
+	for (i = 0; i < SDMA_N_EVENTS; i++)
+		WRITE4(sc, SDMAARM_CHNENBL(i), 0);
+
+	/* All channels have priority 0 */
+	for (i = 0; i < SDMA_N_CHANNELS; i++)
+		WRITE4(sc, SDMAARM_SDMA_CHNPRI(i), 0);
+
+	/* Channel 0 is used for booting firmware */
+	chn = 0;
+
+	sc->bd0 = (void *)kmem_alloc_contig(kernel_arena,
+	    PAGE_SIZE, M_ZERO, 0, ~0, PAGE_SIZE, 0,
+	    VM_MEMATTR_UNCACHEABLE);
+	bd0 = sc->bd0;
+	sc->ccb[chn].base_bd_ptr = vtophys(bd0);
+	sc->ccb[chn].current_bd_ptr = vtophys(bd0);
+
+	WRITE4(sc, SDMAARM_SDMA_CHNPRI(chn), 1);
+
+	sdma_overrides(sc, chn, 1, 0, 0);
+
+	/* XXX: not sure what is that */
+	WRITE4(sc, SDMAARM_CHN0ADDR, 0x4050);
+
+	WRITE4(sc, SDMAARM_CONFIG, 0);
+	WRITE4(sc, SDMAARM_MC0PTR, sc->ccb_phys);
+	WRITE4(sc, SDMAARM_CONFIG, CONFIG_CSM);
+	WRITE4(sc, SDMAARM_SDMA_CHNPRI(chn), 1);
+
+	bd0->mode.command = C0_SETPM;
+	bd0->mode.status = BD_DONE | BD_INTR | BD_WRAP | BD_EXTD;
+	bd0->mode.count = sc->fw_header->ram_code_size / 2;
+	bd0->buffer_addr = vtophys(ram_code);
+	bd0->ext_buffer_addr = sc->fw_scripts->ram_code_start_addr;
+
+	WRITE4(sc, SDMAARM_HSTART, 1);
+
+	timeout = 100;
+	while (!(ret = READ4(sc, SDMAARM_INTR) & 1)) {
+		if (timeout-- <= 0)
+			break;
+		DELAY(10);
+	};
+
+	if (ret == 0) {
+		device_printf(sc->dev, "SDMA failed to boot\n");
+		return (-1);
+	}
+
+	WRITE4(sc, SDMAARM_INTR, ret);
+
+#if 0
+	device_printf(sc->dev, "SDMA booted successfully.\n");
+#endif
+
+	/* Debug is disabled */
+	WRITE4(sc, SDMAARM_ONCE_ENB, 0);
+
+	return (0);
+}
+
+static int
+sdma_attach(device_t dev)
+{
+	struct sdma_softc *sc;
+	int err;
+
+	sc = device_get_softc(dev);
+	sc->dev = dev;
+
+	if (bus_alloc_resources(dev, sdma_spec, sc->res)) {
+		device_printf(dev, "could not allocate resources\n");
+		return (ENXIO);
+	}
+
+	/* Memory interface */
+	sc->bst = rman_get_bustag(sc->res[0]);
+	sc->bsh = rman_get_bushandle(sc->res[0]);
+
+	sdma_sc = sc;
+
+	/* Setup interrupt handler */
+	err = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC | INTR_MPSAFE,
+	    NULL, sdma_intr, sc, &sc->ih);
+	if (err) {
+		device_printf(dev, "Unable to alloc interrupt resource.\n");
+		return (ENXIO);
+	}
+
+	if (load_firmware(sc) == -1)
+		return (ENXIO);
+
+	if (boot_firmware(sc) == -1)
+		return (ENXIO);
+
+	return (0);
+};
+
+static device_method_t sdma_methods[] = {
+	/* Device interface */
+	DEVMETHOD(device_probe,		sdma_probe),
+	DEVMETHOD(device_attach,	sdma_attach),
+	{ 0, 0 }
+};
+
+static driver_t sdma_driver = {
+	"sdma",
+	sdma_methods,
+	sizeof(struct sdma_softc),
+};
+
+static devclass_t sdma_devclass;
+
+DRIVER_MODULE(sdma, simplebus, sdma_driver, sdma_devclass, 0, 0);
diff --git a/sys/arm/freescale/imx/imx6_sdma.h b/sys/arm/freescale/imx/imx6_sdma.h
new file mode 100644
index 00000000000..0f970de4b02
--- /dev/null
+++ b/sys/arm/freescale/imx/imx6_sdma.h
@@ -0,0 +1,245 @@
+/*-
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#define	SDMAARM_MC0PTR		0x00	/* ARM platform Channel 0 Pointer */
+#define	SDMAARM_INTR		0x04	/* Channel Interrupts */
+#define	SDMAARM_STOP_STAT	0x08	/* Channel Stop/Channel Status */
+#define	SDMAARM_HSTART		0x0C	/* Channel Start */
+#define	SDMAARM_EVTOVR		0x10	/* Channel Event Override */
+#define	SDMAARM_DSPOVR		0x14	/* Channel BP Override */
+#define	SDMAARM_HOSTOVR		0x18	/* Channel ARM platform Override */
+#define	SDMAARM_EVTPEND		0x1C	/* Channel Event Pending */
+#define	SDMAARM_RESET		0x24	/* Reset Register */
+#define	SDMAARM_EVTERR		0x28	/* DMA Request Error Register */
+#define	SDMAARM_INTRMASK	0x2C	/* Channel ARM platform Interrupt Mask */
+#define	SDMAARM_PSW		0x30	/* Schedule Status */
+#define	SDMAARM_EVTERRDBG	0x34	/* DMA Request Error Register */
+#define	SDMAARM_CONFIG		0x38	/* Configuration Register */
+#define	 CONFIG_CSM		0x3
+#define	SDMAARM_SDMA_LOCK	0x3C	/* SDMA LOCK */
+#define	SDMAARM_ONCE_ENB	0x40	/* OnCE Enable */
+#define	SDMAARM_ONCE_DATA	0x44	/* OnCE Data Register */
+#define	SDMAARM_ONCE_INSTR	0x48	/* OnCE Instruction Register */
+#define	SDMAARM_ONCE_STAT	0x4C	/* OnCE Status Register */
+#define	SDMAARM_ONCE_CMD	0x50	/* OnCE Command Register */
+#define	SDMAARM_ILLINSTADDR	0x58	/* Illegal Instruction Trap Address */
+#define	SDMAARM_CHN0ADDR	0x5C	/* Channel 0 Boot Address */
+#define	SDMAARM_EVT_MIRROR	0x60	/* DMA Requests */
+#define	SDMAARM_EVT_MIRROR2	0x64	/* DMA Requests 2 */
+#define	SDMAARM_XTRIG_CONF1	0x70	/* Cross-Trigger Events Configuration Register 1 */
+#define	SDMAARM_XTRIG_CONF2	0x74	/* Cross-Trigger Events Configuration Register 2 */
+#define	SDMAARM_SDMA_CHNPRI(n)	(0x100 + 0x4 * n)	/* Channel Priority Registers */
+#define	SDMAARM_CHNENBL(n)	(0x200 + 0x4 * n)	/* Channel Enable RAM */
+
+/* SDMA Event Mappings */
+#define	SSI1_RX_1	35
+#define	SSI1_TX_1	36
+#define	SSI1_RX_0	37
+#define	SSI1_TX_0	38
+#define	SSI2_RX_1	39
+#define	SSI2_TX_1	40
+#define	SSI2_RX_0	41
+#define	SSI2_TX_0	42
+#define	SSI3_RX_1	43
+#define	SSI3_TX_1	44
+#define	SSI3_RX_0	45
+#define	SSI3_TX_0	46
+
+#define	C0_ADDR		0x01
+#define	C0_LOAD		0x02
+#define	C0_DUMP		0x03
+#define	C0_SETCTX	0x07
+#define	C0_GETCTX	0x03
+#define	C0_SETDM	0x01
+#define	C0_SETPM	0x04
+#define	C0_GETDM	0x02
+#define	C0_GETPM	0x08
+
+#define	BD_DONE		0x01
+#define	BD_WRAP		0x02
+#define	BD_CONT		0x04
+#define	BD_INTR		0x08
+#define	BD_RROR		0x10
+#define	BD_LAST		0x20
+#define	BD_EXTD		0x80
+
+/* sDMA data transfer length */
+#define	CMD_4BYTES	0
+#define	CMD_3BYTES	3
+#define	CMD_2BYTES	2
+#define	CMD_1BYTES	1
+
+struct sdma_firmware_header {
+	uint32_t	magic;
+	uint32_t	version_major;
+	uint32_t	version_minor;
+	uint32_t	script_addrs_start;
+	uint32_t	num_script_addrs;
+	uint32_t	ram_code_start;
+	uint32_t	ram_code_size;
+};
+
+struct sdma_mode_count {
+	uint16_t count;
+	uint8_t status;
+	uint8_t command;
+};
+
+struct sdma_buffer_descriptor {
+	struct sdma_mode_count	mode;
+	uint32_t		buffer_addr;
+	uint32_t		ext_buffer_addr;
+} __packed;
+
+struct sdma_channel_control {
+	uint32_t	current_bd_ptr;
+	uint32_t	base_bd_ptr;
+	uint32_t	unused[2];
+} __packed;
+
+struct sdma_state_registers {
+	uint32_t	pc     :14;
+	uint32_t	unused1: 1;
+	uint32_t	t      : 1;
+	uint32_t	rpc    :14;
+	uint32_t	unused0: 1;
+	uint32_t	sf     : 1;
+	uint32_t	spc    :14;
+	uint32_t	unused2: 1;
+	uint32_t	df     : 1;
+	uint32_t	epc    :14;
+	uint32_t	lm     : 2;
+} __packed;
+
+struct sdma_context_data {
+	struct sdma_state_registers	channel_state;
+	uint32_t	gReg[8];
+	uint32_t	mda;
+	uint32_t	msa;
+	uint32_t	ms;
+	uint32_t	md;
+	uint32_t	pda;
+	uint32_t	psa;
+	uint32_t	ps;
+	uint32_t	pd;
+	uint32_t	ca;
+	uint32_t	cs;
+	uint32_t	dda;
+	uint32_t	dsa;
+	uint32_t	ds;
+	uint32_t	dd;
+	uint32_t	unused[8];
+} __packed;
+
+/* SDMA firmware script pointers */
+struct sdma_script_start_addrs {
+	int32_t	ap_2_ap_addr;
+	int32_t	ap_2_bp_addr;
+	int32_t	ap_2_ap_fixed_addr;
+	int32_t	bp_2_ap_addr;
+	int32_t	loopback_on_dsp_side_addr;
+	int32_t	mcu_interrupt_only_addr;
+	int32_t	firi_2_per_addr;
+	int32_t	firi_2_mcu_addr;
+	int32_t	per_2_firi_addr;
+	int32_t	mcu_2_firi_addr;
+	int32_t	uart_2_per_addr;
+	int32_t	uart_2_mcu_addr;
+	int32_t	per_2_app_addr;
+	int32_t	mcu_2_app_addr;
+	int32_t	per_2_per_addr;
+	int32_t	uartsh_2_per_addr;
+	int32_t	uartsh_2_mcu_addr;
+	int32_t	per_2_shp_addr;
+	int32_t	mcu_2_shp_addr;
+	int32_t	ata_2_mcu_addr;
+	int32_t	mcu_2_ata_addr;
+	int32_t	app_2_per_addr;
+	int32_t	app_2_mcu_addr;
+	int32_t	shp_2_per_addr;
+	int32_t	shp_2_mcu_addr;
+	int32_t	mshc_2_mcu_addr;
+	int32_t	mcu_2_mshc_addr;
+	int32_t	spdif_2_mcu_addr;
+	int32_t	mcu_2_spdif_addr;
+	int32_t	asrc_2_mcu_addr;
+	int32_t	ext_mem_2_ipu_addr;
+	int32_t	descrambler_addr;
+	int32_t	dptc_dvfs_addr;
+	int32_t	utra_addr;
+	int32_t	ram_code_start_addr;
+	int32_t	mcu_2_ssish_addr;
+	int32_t	ssish_2_mcu_addr;
+	int32_t	hdmi_dma_addr;
+};
+
+#define	SDMA_N_CHANNELS 32
+#define	SDMA_N_EVENTS	48
+#define	FW_HEADER_MAGIC	0x414d4453
+
+struct sdma_channel {
+	struct sdma_conf		*conf;
+	struct sdma_buffer_descriptor	*bd;
+	uint8_t 			in_use;
+};
+
+struct sdma_softc {
+	struct resource			*res[2];
+	bus_space_tag_t			bst;
+	bus_space_handle_t		bsh;
+	device_t			dev;
+	void				*ih;
+	struct sdma_channel_control	*ccb;
+	struct sdma_buffer_descriptor	*bd0;
+	struct sdma_context_data	*context;
+	struct sdma_channel		channel[SDMA_N_CHANNELS];
+	uint32_t			num_bd;
+	uint32_t			ccb_phys;
+	uint32_t			context_phys;
+	struct sdma_firmware_header	*fw_header;
+	struct sdma_script_start_addrs	*fw_scripts;
+};
+
+struct sdma_conf {
+	bus_addr_t	saddr;
+	bus_addr_t	daddr;
+	uint32_t	word_length;
+	uint32_t	nbits;
+	uint32_t	command;
+	uint32_t	num_bd;
+	uint32_t	event;
+	uint32_t	period;
+	uint32_t	(*ih)(void *, int);
+	void		*ih_user;
+};
+
+int sdma_configure(int, struct sdma_conf *);
+int sdma_start(int);
+int sdma_stop(int);
+int sdma_alloc(void);
+int sdma_free(int);
diff --git a/sys/arm/freescale/imx/imx6_ssi.c b/sys/arm/freescale/imx/imx6_ssi.c
new file mode 100644
index 00000000000..af8a9e8f279
--- /dev/null
+++ b/sys/arm/freescale/imx/imx6_ssi.c
@@ -0,0 +1,855 @@
+/*-
+ * Copyright (c) 2015 Ruslan Bukin <br@bsdpad.com>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * i.MX6 Synchronous Serial Interface (SSI)
+ *
+ * Chapter 61, i.MX 6Dual/6Quad Applications Processor Reference Manual,
+ * Rev. 1, 04/2013
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/bus.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+#include <sys/malloc.h>
+#include <sys/rman.h>
+#include <sys/timeet.h>
+#include <sys/timetc.h>
+
+#include <dev/sound/pcm/sound.h>
+#include <dev/sound/chip.h>
+#include <mixer_if.h>
+
+#include <dev/fdt/fdt_common.h>
+#include <dev/ofw/openfirm.h>
+#include <dev/ofw/ofw_bus.h>
+#include <dev/ofw/ofw_bus_subr.h>
+
+#include <machine/bus.h>
+#include <machine/fdt.h>
+#include <machine/cpu.h>
+#include <machine/intr.h>
+
+#include <arm/freescale/imx/imx6_sdma.h>
+#include <arm/freescale/imx/imx6_anatopvar.h>
+#include <arm/freescale/imx/imx_ccmvar.h>
+
+#define	READ4(_sc, _reg)	\
+	bus_space_read_4(_sc->bst, _sc->bsh, _reg)
+#define	WRITE4(_sc, _reg, _val)	\
+	bus_space_write_4(_sc->bst, _sc->bsh, _reg, _val)
+
+#define	SSI_NCHANNELS	1
+
+/* i.MX6 SSI registers */
+
+#define	SSI_STX0	0x00 /* Transmit Data Register n */
+#define	SSI_STX1	0x04 /* Transmit Data Register n */
+#define	SSI_SRX0	0x08 /* Receive Data Register n */
+#define	SSI_SRX1	0x0C /* Receive Data Register n */
+#define	SSI_SCR		0x10 /* Control Register */
+#define	 SCR_I2S_MODE_S	5    /* I2S Mode Select. */
+#define	 SCR_I2S_MODE_M	0x3
+#define	 SCR_SYN	(1 << 4)
+#define	 SCR_NET       	(1 << 3)  /* Network mode */
+#define	 SCR_RE		(1 << 2)  /* Receive Enable. */
+#define	 SCR_TE		(1 << 1)  /* Transmit Enable. */
+#define	 SCR_SSIEN	(1 << 0)  /* SSI Enable */
+#define	SSI_SISR	0x14      /* Interrupt Status Register */
+#define	SSI_SIER	0x18      /* Interrupt Enable Register */
+#define	 SIER_RDMAE	(1 << 22) /* Receive DMA Enable. */
+#define	 SIER_RIE	(1 << 21) /* Receive Interrupt Enable. */
+#define	 SIER_TDMAE	(1 << 20) /* Transmit DMA Enable. */
+#define	 SIER_TIE	(1 << 19) /* Transmit Interrupt Enable. */
+#define	 SIER_TDE0IE	(1 << 12) /* Transmit Data Register Empty 0. */
+#define	 SIER_TUE0IE	(1 << 8)  /* Transmitter Underrun Error 0. */
+#define	 SIER_TFE0IE	(1 << 0)  /* Transmit FIFO Empty 0 IE. */
+#define	SSI_STCR	0x1C	  /* Transmit Configuration Register */
+#define	 STCR_TXBIT0	(1 << 9)  /* Transmit Bit 0 shift MSB/LSB */
+#define	 STCR_TFEN1	(1 << 8)  /* Transmit FIFO Enable 1. */
+#define	 STCR_TFEN0	(1 << 7)  /* Transmit FIFO Enable 0. */
+#define	 STCR_TFDIR	(1 << 6)  /* Transmit Frame Direction. */
+#define	 STCR_TXDIR	(1 << 5)  /* Transmit Clock Direction. */
+#define	 STCR_TSHFD	(1 << 4)  /* Transmit Shift Direction. */
+#define	 STCR_TSCKP	(1 << 3)  /* Transmit Clock Polarity. */
+#define	 STCR_TFSI	(1 << 2)  /* Transmit Frame Sync Invert. */
+#define	 STCR_TFSL	(1 << 1)  /* Transmit Frame Sync Length. */
+#define	 STCR_TEFS	(1 << 0)  /* Transmit Early Frame Sync. */
+#define	SSI_SRCR	0x20      /* Receive Configuration Register */
+#define	SSI_STCCR	0x24      /* Transmit Clock Control Register */
+#define	 STCCR_DIV2	(1 << 18) /* Divide By 2. */
+#define	 STCCR_PSR	(1 << 17) /* Divide clock by 8. */
+#define	 WL3_WL0_S	13
+#define	 WL3_WL0_M	0xf
+#define	 DC4_DC0_S	8
+#define	 DC4_DC0_M	0x1f
+#define	 PM7_PM0_S	0
+#define	 PM7_PM0_M	0xff
+#define	SSI_SRCCR	0x28	/* Receive Clock Control Register */
+#define	SSI_SFCSR	0x2C	/* FIFO Control/Status Register */
+#define	 SFCSR_RFWM1_S	20	/* Receive FIFO Empty WaterMark 1 */
+#define	 SFCSR_RFWM1_M	0xf
+#define	 SFCSR_TFWM1_S	16	/* Transmit FIFO Empty WaterMark 1 */
+#define	 SFCSR_TFWM1_M	0xf
+#define	 SFCSR_RFWM0_S	4	/* Receive FIFO Empty WaterMark 0 */
+#define	 SFCSR_RFWM0_M	0xf
+#define	 SFCSR_TFWM0_S	0	/* Transmit FIFO Empty WaterMark 0 */
+#define	 SFCSR_TFWM0_M	0xf
+#define	SSI_SACNT	0x38	/* AC97 Control Register */
+#define	SSI_SACADD	0x3C	/* AC97 Command Address Register */
+#define	SSI_SACDAT	0x40	/* AC97 Command Data Register */
+#define	SSI_SATAG	0x44	/* AC97 Tag Register */
+#define	SSI_STMSK	0x48	/* Transmit Time Slot Mask Register */
+#define	SSI_SRMSK	0x4C	/* Receive Time Slot Mask Register */
+#define	SSI_SACCST	0x50	/* AC97 Channel Status Register */
+#define	SSI_SACCEN	0x54	/* AC97 Channel Enable Register */
+#define	SSI_SACCDIS	0x58	/* AC97 Channel Disable Register */
+
+static MALLOC_DEFINE(M_SSI, "ssi", "ssi audio");
+
+uint32_t ssi_dma_intr(void *arg, int chn);
+
+struct ssi_rate {
+	uint32_t speed;
+	uint32_t mfi; /* PLL4 Multiplication Factor Integer */
+	uint32_t mfn; /* PLL4 Multiplication Factor Numerator */
+	uint32_t mfd; /* PLL4 Multiplication Factor Denominator */
+	/* More dividers to configure can be added here */
+};
+
+static struct ssi_rate rate_map[] = {
+	{ 192000, 49, 152, 1000 }, /* PLL4 49.152 Mhz */
+	/* TODO: add more frequences */
+	{ 0, 0 },
+};
+
+/*
+ *  i.MX6 example bit clock formula
+ *
+ *  BCLK = 2 channels * 192000 hz * 24 bit = 9216000 hz = 
+ *     (24000000 * (49 + 152/1000.0) / 4 / 4 / 2 / 2 / 2 / 1 / 1)
+ *             ^     ^     ^      ^    ^   ^   ^   ^   ^   ^   ^
+ *             |     |     |      |    |   |   |   |   |   |   |
+ *  Fref ------/     |     |      |    |   |   |   |   |   |   |
+ *  PLL4 div select -/     |      |    |   |   |   |   |   |   |
+ *  PLL4 num --------------/      |    |   |   |   |   |   |   |
+ *  PLL4 denom -------------------/    |   |   |   |   |   |   |
+ *  PLL4 post div ---------------------/   |   |   |   |   |   |
+ *  CCM ssi pre div (CCM_CS1CDR) ----------/   |   |   |   |   |
+ *  CCM ssi post div (CCM_CS1CDR) -------------/   |   |   |   |
+ *  SSI PM7_PM0_S ---------------------------------/   |   |   |
+ *  SSI Fixed divider ---------------------------------/   |   |
+ *  SSI DIV2 ----------------------------------------------/   |
+ *  SSI PSR (prescaler /1 or /8) ------------------------------/
+ *
+ *  MCLK (Master clock) depends on DAC, usually BCLK * 4
+ */
+
+struct sc_info {
+	struct resource		*res[2];
+	bus_space_tag_t		bst;
+	bus_space_handle_t	bsh;
+	device_t		dev;
+	struct mtx		*lock;
+	void			*ih;
+	int			pos;
+	int			dma_size;
+	bus_dma_tag_t		dma_tag;
+	bus_dmamap_t		dma_map;
+	bus_addr_t		buf_base_phys;
+	uint32_t		*buf_base;
+	struct sdma_conf	*conf;
+	struct ssi_rate		*sr;
+	struct sdma_softc	*sdma_sc;
+	int			sdma_ev_rx;
+	int			sdma_ev_tx;
+	int			sdma_channel;
+};
+
+/* Channel registers */
+struct sc_chinfo {
+	struct snd_dbuf		*buffer;
+	struct pcm_channel	*channel;
+	struct sc_pcminfo	*parent;
+
+	/* Channel information */
+	uint32_t	dir;
+	uint32_t	format;
+
+	/* Flags */
+	uint32_t	run;
+};
+
+/* PCM device private data */
+struct sc_pcminfo {
+	device_t		dev;
+	uint32_t		(*ih)(struct sc_pcminfo *scp);
+	uint32_t		chnum;
+	struct sc_chinfo	chan[SSI_NCHANNELS];
+	struct sc_info		*sc;
+};
+
+static struct resource_spec ssi_spec[] = {
+	{ SYS_RES_MEMORY,	0,	RF_ACTIVE },
+	{ SYS_RES_IRQ,		0,	RF_ACTIVE },
+	{ -1, 0 }
+};
+
+static int setup_dma(struct sc_pcminfo *scp);
+static void setup_ssi(struct sc_info *);
+static void ssi_configure_clock(struct sc_info *);
+
+/*
+ * Mixer interface.
+ */
+
+static int
+ssimixer_init(struct snd_mixer *m)
+{
+	struct sc_pcminfo *scp;
+	struct sc_info *sc;
+	int mask;
+
+	scp = mix_getdevinfo(m);
+	sc = scp->sc;
+
+	if (sc == NULL)
+		return -1;
+
+	mask = SOUND_MASK_PCM;
+	mask |= SOUND_MASK_VOLUME;
+
+	snd_mtxlock(sc->lock);
+	pcm_setflags(scp->dev, pcm_getflags(scp->dev) | SD_F_SOFTPCMVOL);
+	mix_setdevs(m, mask);
+	snd_mtxunlock(sc->lock);
+
+	return (0);
+}
+
+static int
+ssimixer_set(struct snd_mixer *m, unsigned dev,
+    unsigned left, unsigned right)
+{
+	struct sc_pcminfo *scp;
+
+	scp = mix_getdevinfo(m);
+
+	/* Here we can configure hardware volume on our DAC */
+
+#if 1
+	device_printf(scp->dev, "ssimixer_set() %d %d\n",
+	    left, right);
+#endif
+
+	return (0);
+}
+
+static kobj_method_t ssimixer_methods[] = {
+	KOBJMETHOD(mixer_init,      ssimixer_init),
+	KOBJMETHOD(mixer_set,       ssimixer_set),
+	KOBJMETHOD_END
+};
+MIXER_DECLARE(ssimixer);
+
+
+/*
+ * Channel interface.
+ */
+
+static void *
+ssichan_init(kobj_t obj, void *devinfo, struct snd_dbuf *b,
+    struct pcm_channel *c, int dir)
+{
+	struct sc_pcminfo *scp;
+	struct sc_chinfo *ch;
+	struct sc_info *sc;
+
+	scp = (struct sc_pcminfo *)devinfo;
+	sc = scp->sc;
+
+	snd_mtxlock(sc->lock);
+	ch = &scp->chan[0];
+	ch->dir = dir;
+	ch->run = 0;
+	ch->buffer = b;
+	ch->channel = c;
+	ch->parent = scp;
+	snd_mtxunlock(sc->lock);
+
+	if (sndbuf_setup(ch->buffer, sc->buf_base, sc->dma_size) != 0) {
+		device_printf(scp->dev, "Can't setup sndbuf.\n");
+		return NULL;
+	}
+
+	return ch;
+}
+
+static int
+ssichan_free(kobj_t obj, void *data)
+{
+	struct sc_chinfo *ch = data;
+	struct sc_pcminfo *scp = ch->parent;
+	struct sc_info *sc = scp->sc;
+
+#if 0
+	device_printf(scp->dev, "ssichan_free()\n");
+#endif
+
+	snd_mtxlock(sc->lock);
+	/* TODO: free channel buffer */
+	snd_mtxunlock(sc->lock);
+
+	return (0);
+}
+
+static int
+ssichan_setformat(kobj_t obj, void *data, uint32_t format)
+{
+	struct sc_chinfo *ch = data;
+
+	ch->format = format;
+
+	return (0);
+}
+
+static uint32_t
+ssichan_setspeed(kobj_t obj, void *data, uint32_t speed)
+{
+	struct sc_pcminfo *scp;
+	struct sc_chinfo *ch;
+	struct ssi_rate *sr;
+	struct sc_info *sc;
+	int threshold;
+	int i;
+
+	ch = data;
+	scp = ch->parent;
+	sc = scp->sc;
+
+	sr = NULL;
+
+	/* First look for equal frequency. */
+	for (i = 0; rate_map[i].speed != 0; i++) {
+		if (rate_map[i].speed == speed)
+			sr = &rate_map[i];
+	}
+
+	/* If no match, just find nearest. */
+	if (sr == NULL) {
+		for (i = 0; rate_map[i].speed != 0; i++) {
+			sr = &rate_map[i];
+			threshold = sr->speed + ((rate_map[i + 1].speed != 0) ?
+			    ((rate_map[i + 1].speed - sr->speed) >> 1) : 0);
+			if (speed < threshold)
+				break;
+		}
+	}
+
+	sc->sr = sr;
+
+	ssi_configure_clock(sc);
+
+	return (sr->speed);
+}
+
+static void
+ssi_configure_clock(struct sc_info *sc)
+{
+	struct ssi_rate *sr;
+
+	sr = sc->sr;
+
+	pll4_configure_output(sr->mfi, sr->mfn, sr->mfd);
+
+	/* Configure other dividers here, if any */
+}
+
+static uint32_t
+ssichan_setblocksize(kobj_t obj, void *data, uint32_t blocksize)
+{
+	struct sc_chinfo *ch = data;
+	struct sc_pcminfo *scp = ch->parent;
+	struct sc_info *sc = scp->sc;
+
+	sndbuf_resize(ch->buffer, sc->dma_size / blocksize, blocksize);
+
+	setup_dma(scp);
+
+	return (sndbuf_getblksz(ch->buffer));
+}
+
+uint32_t
+ssi_dma_intr(void *arg, int chn)
+{
+	struct sc_pcminfo *scp;
+	struct sdma_conf *conf;
+	struct sc_chinfo *ch;
+	struct sc_info *sc;
+	int bufsize;
+
+	scp = arg;
+	ch = &scp->chan[0];
+	sc = scp->sc;
+	conf = sc->conf;
+
+	bufsize = sndbuf_getsize(ch->buffer);
+
+	sc->pos += conf->period;
+	if (sc->pos >= bufsize)
+		sc->pos -= bufsize;
+
+	if (ch->run)
+		chn_intr(ch->channel);
+
+	return (0);
+}
+
+static int
+find_sdma_controller(struct sc_info *sc)
+{
+	struct sdma_softc *sdma_sc;
+	phandle_t node, sdma_node;
+	device_t sdma_dev;
+	int dts_value[8];
+	int len;
+
+	if ((node = ofw_bus_get_node(sc->dev)) == -1)
+		return (ENXIO);
+
+	if ((len = OF_getproplen(node, "dmas")) <= 0)
+		return (ENXIO);
+
+	OF_getprop(node, "dmas", &dts_value, len);
+
+	sc->sdma_ev_rx = fdt32_to_cpu(dts_value[1]);
+	sc->sdma_ev_tx = fdt32_to_cpu(dts_value[5]);
+
+	sdma_node = OF_node_from_xref(fdt32_to_cpu(dts_value[0]));
+
+	sdma_sc = NULL;
+
+	sdma_dev = devclass_get_device(devclass_find("sdma"), 0);
+	if (sdma_dev)
+		sdma_sc = device_get_softc(sdma_dev);
+
+	if (sdma_sc == NULL) {
+		device_printf(sc->dev, "No sDMA found. Can't operate\n");
+		return (ENXIO);
+	};
+
+	sc->sdma_sc = sdma_sc;
+
+	return (0);
+};
+
+static int
+setup_dma(struct sc_pcminfo *scp)
+{
+	struct sdma_conf *conf;
+	struct sc_chinfo *ch;
+	struct sc_info *sc;
+	int fmt;
+
+	ch = &scp->chan[0];
+	sc = scp->sc;
+	conf = sc->conf;
+
+	conf->ih = ssi_dma_intr;
+	conf->ih_user = scp;
+	conf->saddr = sc->buf_base_phys;
+	conf->daddr = rman_get_start(sc->res[0]) + SSI_STX0;
+	conf->event = sc->sdma_ev_tx; /* SDMA TX event */
+	conf->period = sndbuf_getblksz(ch->buffer);
+	conf->num_bd = sndbuf_getblkcnt(ch->buffer);
+
+	/*
+	 * Word Length
+	 * Can be 32, 24, 16 or 8 for sDMA.
+	 *
+	 * SSI supports 24 at max.
+	 */
+
+	fmt = sndbuf_getfmt(ch->buffer);
+
+	if (fmt & AFMT_16BIT) {
+		conf->word_length = 16;
+		conf->command = CMD_2BYTES;
+	} else if (fmt & AFMT_24BIT) {
+		conf->word_length = 24;
+		conf->command = CMD_3BYTES;
+	} else {
+		device_printf(sc->dev, "Unknown format\n");
+		return (-1);
+	}
+
+	return (0);
+}
+
+static int
+ssi_start(struct sc_pcminfo *scp)
+{
+	struct sc_info *sc;
+	int reg;
+
+	sc = scp->sc;
+
+	if (sdma_configure(sc->sdma_channel, sc->conf) != 0) {
+		device_printf(sc->dev, "Can't configure sDMA\n");
+		return (-1);
+	}
+
+	/* Enable DMA interrupt */
+	reg = (SIER_TDMAE);
+	WRITE4(sc, SSI_SIER, reg);
+
+	sdma_start(sc->sdma_channel);
+
+	return (0);
+}
+
+static int
+ssi_stop(struct sc_pcminfo *scp)
+{
+	struct sc_info *sc;
+	int reg;
+
+	sc = scp->sc;
+
+	reg = READ4(sc, SSI_SIER);
+	reg &= ~(SIER_TDMAE);
+	WRITE4(sc, SSI_SIER, reg);
+
+	sdma_stop(sc->sdma_channel);
+
+	bzero(sc->buf_base, sc->dma_size);
+
+	return (0);
+}
+
+static int
+ssichan_trigger(kobj_t obj, void *data, int go)
+{
+	struct sc_pcminfo *scp;
+	struct sc_chinfo *ch;
+	struct sc_info *sc;
+
+	ch = data;
+	scp = ch->parent;
+	sc = scp->sc;
+
+	snd_mtxlock(sc->lock);
+
+	switch (go) {
+	case PCMTRIG_START:
+#if 0
+		device_printf(scp->dev, "trigger start\n");
+#endif
+		ch->run = 1;
+
+		ssi_start(scp);
+
+		break;
+
+	case PCMTRIG_STOP:
+	case PCMTRIG_ABORT:
+#if 0
+		device_printf(scp->dev, "trigger stop or abort\n");
+#endif
+		ch->run = 0;
+
+		ssi_stop(scp);
+
+		break;
+	}
+
+	snd_mtxunlock(sc->lock);
+
+	return (0);
+}
+
+static uint32_t
+ssichan_getptr(kobj_t obj, void *data)
+{
+	struct sc_pcminfo *scp;
+	struct sc_chinfo *ch;
+	struct sc_info *sc;
+
+	ch = data;
+	scp = ch->parent;
+	sc = scp->sc;
+
+	return (sc->pos);
+}
+
+static uint32_t ssi_pfmt[] = {
+	SND_FORMAT(AFMT_S24_LE, 2, 0),
+	0
+};
+
+static struct pcmchan_caps ssi_pcaps = {44100, 192000, ssi_pfmt, 0};
+
+static struct pcmchan_caps *
+ssichan_getcaps(kobj_t obj, void *data)
+{
+
+	return (&ssi_pcaps);
+}
+
+static kobj_method_t ssichan_methods[] = {
+	KOBJMETHOD(channel_init,         ssichan_init),
+	KOBJMETHOD(channel_free,         ssichan_free),
+	KOBJMETHOD(channel_setformat,    ssichan_setformat),
+	KOBJMETHOD(channel_setspeed,     ssichan_setspeed),
+	KOBJMETHOD(channel_setblocksize, ssichan_setblocksize),
+	KOBJMETHOD(channel_trigger,      ssichan_trigger),
+	KOBJMETHOD(channel_getptr,       ssichan_getptr),
+	KOBJMETHOD(channel_getcaps,      ssichan_getcaps),
+	KOBJMETHOD_END
+};
+CHANNEL_DECLARE(ssichan);
+
+static int
+ssi_probe(device_t dev)
+{
+
+	if (!ofw_bus_status_okay(dev))
+		return (ENXIO);
+
+	if (!ofw_bus_is_compatible(dev, "fsl,imx6q-ssi"))
+		return (ENXIO);
+
+	device_set_desc(dev, "i.MX6 Synchronous Serial Interface (SSI)");
+	return (BUS_PROBE_DEFAULT);
+}
+
+static void
+ssi_intr(void *arg)
+{
+	struct sc_pcminfo *scp;
+	struct sc_chinfo *ch;
+	struct sc_info *sc;
+
+	scp = arg;
+	sc = scp->sc;
+	ch = &scp->chan[0];
+
+	/* We don't use SSI interrupt */
+#if 0
+	device_printf(sc->dev, "SSI Intr 0x%08x\n",
+	    READ4(sc, SSI_SISR));
+#endif
+}
+
+static void
+setup_ssi(struct sc_info *sc)
+{
+	int reg;
+
+	reg = READ4(sc, SSI_STCCR);
+	reg &= ~(WL3_WL0_M << WL3_WL0_S);
+	reg |= (0xb << WL3_WL0_S); /* 24 bit */
+	reg &= ~(DC4_DC0_M << DC4_DC0_S);
+	reg |= (1 << DC4_DC0_S); /* 2 words per frame */
+	reg &= ~(STCCR_DIV2); /* Divide by 1 */
+	reg &= ~(STCCR_PSR); /* Divide by 1 */
+	reg &= ~(PM7_PM0_M << PM7_PM0_S);
+	reg |= (1 << PM7_PM0_S); /* Divide by 2 */
+	WRITE4(sc, SSI_STCCR, reg);
+
+	reg = READ4(sc, SSI_SFCSR);
+	reg &= ~(SFCSR_TFWM0_M << SFCSR_TFWM0_S);
+	reg |= (8 << SFCSR_TFWM0_S); /* empty slots */
+	WRITE4(sc, SSI_SFCSR, reg);
+
+	reg = READ4(sc, SSI_STCR);
+	reg |= (STCR_TFEN0);
+	reg &= ~(STCR_TFEN1);
+	reg &= ~(STCR_TSHFD); /* MSB */
+	reg |= (STCR_TXBIT0);
+	reg |= (STCR_TXDIR | STCR_TFDIR);
+	reg |= (STCR_TSCKP); /* falling edge */
+	reg |= (STCR_TFSI);
+	reg &= ~(STCR_TFSI); /* active high frame sync */
+	reg &= ~(STCR_TFSL);
+	reg |= STCR_TEFS;
+	WRITE4(sc, SSI_STCR, reg);
+
+	reg = READ4(sc, SSI_SCR);
+	reg &= ~(SCR_I2S_MODE_M << SCR_I2S_MODE_S); /* Not master */
+	reg |= (SCR_SSIEN | SCR_TE);
+	reg |= (SCR_NET);
+	reg |= (SCR_SYN);
+	WRITE4(sc, SSI_SCR, reg);
+}
+
+static void
+ssi_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int err)
+{
+	bus_addr_t *addr;
+
+	if (err)
+		return;
+
+	addr = (bus_addr_t*)arg;
+	*addr = segs[0].ds_addr;
+}
+
+static int
+ssi_attach(device_t dev)
+{
+	char status[SND_STATUSLEN];
+	struct sc_pcminfo *scp;
+	struct sc_info *sc;
+	int err;
+
+	sc = malloc(sizeof(*sc), M_DEVBUF, M_WAITOK | M_ZERO);
+	sc->dev = dev;
+	sc->sr = &rate_map[0];
+	sc->pos = 0;
+	sc->conf = malloc(sizeof(struct sdma_conf), M_DEVBUF, M_WAITOK | M_ZERO);
+
+	sc->lock = snd_mtxcreate(device_get_nameunit(dev), "ssi softc");
+	if (sc->lock == NULL) {
+		device_printf(dev, "Cant create mtx\n");
+		return (ENXIO);
+	}
+
+	if (bus_alloc_resources(dev, ssi_spec, sc->res)) {
+		device_printf(dev, "could not allocate resources\n");
+		return (ENXIO);
+	}
+
+	/* Memory interface */
+	sc->bst = rman_get_bustag(sc->res[0]);
+	sc->bsh = rman_get_bushandle(sc->res[0]);
+
+	/* SDMA */
+	if (find_sdma_controller(sc)) {
+		device_printf(dev, "could not find active SDMA\n");
+		return (ENXIO);
+	}
+
+	/* Setup PCM */
+	scp = malloc(sizeof(struct sc_pcminfo), M_DEVBUF, M_NOWAIT | M_ZERO);
+	scp->sc = sc;
+	scp->dev = dev;
+
+	/*
+	 * Maximum possible DMA buffer.
+	 * Will be used partialy to match 24 bit word.
+	 */
+	sc->dma_size = 131072;
+
+	/*
+	 * Must use dma_size boundary as modulo feature required.
+	 * Modulo feature allows setup circular buffer.
+	 */
+
+	err = bus_dma_tag_create(
+	    bus_get_dma_tag(sc->dev),
+	    4, sc->dma_size,		/* alignment, boundary */
+	    BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
+	    BUS_SPACE_MAXADDR,		/* highaddr */
+	    NULL, NULL,			/* filter, filterarg */
+	    sc->dma_size, 1,		/* maxsize, nsegments */
+	    sc->dma_size, 0,		/* maxsegsize, flags */
+	    NULL, NULL,			/* lockfunc, lockarg */
+	    &sc->dma_tag);
+
+	err = bus_dmamem_alloc(sc->dma_tag, (void **)&sc->buf_base,
+	    BUS_DMA_NOWAIT | BUS_DMA_COHERENT, &sc->dma_map);
+	if (err) {
+		device_printf(dev, "cannot allocate framebuffer\n");
+		return (ENXIO);
+	}
+
+	err = bus_dmamap_load(sc->dma_tag, sc->dma_map, sc->buf_base,
+	    sc->dma_size, ssi_dmamap_cb, &sc->buf_base_phys, BUS_DMA_NOWAIT);
+	if (err) {
+		device_printf(dev, "cannot load DMA map\n");
+		return (ENXIO);
+	}
+
+	bzero(sc->buf_base, sc->dma_size);
+
+	/* Setup interrupt handler */
+	err = bus_setup_intr(dev, sc->res[1], INTR_MPSAFE | INTR_TYPE_AV,
+	    NULL, ssi_intr, scp, &sc->ih);
+	if (err) {
+		device_printf(dev, "Unable to alloc interrupt resource.\n");
+		return (ENXIO);
+	}
+
+	pcm_setflags(dev, pcm_getflags(dev) | SD_F_MPSAFE);
+
+	err = pcm_register(dev, scp, 1, 0);
+	if (err) {
+		device_printf(dev, "Can't register pcm.\n");
+		return (ENXIO);
+	}
+
+	scp->chnum = 0;
+	pcm_addchan(dev, PCMDIR_PLAY, &ssichan_class, scp);
+	scp->chnum++;
+
+	snprintf(status, SND_STATUSLEN, "at simplebus");
+	pcm_setstatus(dev, status);
+
+	mixer_init(dev, &ssimixer_class, scp);
+	setup_ssi(sc);
+
+	imx_ccm_ssi_configure(dev);
+
+	sc->sdma_channel = sdma_alloc();
+	if (sc->sdma_channel < 0) {
+		device_printf(sc->dev, "Can't get sDMA channel\n");
+		return (1);
+	}
+
+	return (0);
+}
+
+static device_method_t ssi_pcm_methods[] = {
+	DEVMETHOD(device_probe,		ssi_probe),
+	DEVMETHOD(device_attach,	ssi_attach),
+	{ 0, 0 }
+};
+
+static driver_t ssi_pcm_driver = {
+	"pcm",
+	ssi_pcm_methods,
+	PCM_SOFTC_SIZE,
+};
+
+DRIVER_MODULE(ssi, simplebus, ssi_pcm_driver, pcm_devclass, 0, 0);
+MODULE_DEPEND(ssi, sound, SOUND_MINVER, SOUND_PREFVER, SOUND_MAXVER);
+MODULE_VERSION(ssi, 1);
diff --git a/sys/arm/freescale/imx/imx_ccmvar.h b/sys/arm/freescale/imx/imx_ccmvar.h
index 354e6163b45..f15943791c6 100644
--- a/sys/arm/freescale/imx/imx_ccmvar.h
+++ b/sys/arm/freescale/imx/imx_ccmvar.h
@@ -51,5 +51,6 @@ uint32_t imx_ccm_ahb_hz(void);
 
 void imx_ccm_usb_enable(device_t _usbdev);
 void imx_ccm_usbphy_enable(device_t _phydev);
+void imx_ccm_ssi_configure(device_t _ssidev);
 
 #endif
diff --git a/sys/boot/fdt/dts/arm/apalis-imx6.dts b/sys/boot/fdt/dts/arm/apalis-imx6.dts
index 0806f4e3981..b6440603c58 100644
--- a/sys/boot/fdt/dts/arm/apalis-imx6.dts
+++ b/sys/boot/fdt/dts/arm/apalis-imx6.dts
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014 Ruslan Bukin <br@bsdpad.com>
+ * Copyright (c) 2014-2015 Ruslan Bukin <br@bsdpad.com>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -26,6 +26,8 @@
  * $FreeBSD$
  */
 
+#include "imx6q-pinfunc.h"
+
 /dts-v1/;
 /include/ "imx6.dtsi"
 
@@ -42,7 +44,20 @@
 
 	SOC: soc@00000000 {
 		aips@02000000 { /* AIPS1 */
-			iomux@020e0000		{ status = "okay"; };
+			iomux@020e0000 {
+				status = "okay";
+				pinctrl-names = "default";
+				pinctrl-0 = <&pins_ssi>;
+				pins_ssi: ssi {
+					fsl,pins = <
+					MX6QDL_PAD_DISP0_DAT16__AUD5_TXC 0x130b0
+					MX6QDL_PAD_DISP0_DAT17__AUD5_TXD 0x110b0
+					MX6QDL_PAD_DISP0_DAT18__AUD5_TXFS 0x130b0
+					MX6QDL_PAD_DISP0_DAT19__AUD5_RXD 0x130b0
+					MX6QDL_PAD_GPIO_19__CCM_CLKO1 0x130b0
+					>;
+				};
+			};
 			gpio@0209c000		{ status = "okay"; };
 			gpio@020a0000		{ status = "okay"; };
 			gpio@020a4000		{ status = "okay"; };
@@ -76,6 +91,7 @@
 			usdhc@02198000		{ status = "disabled"; };
 			usdhc@0219c000		{ status = "disabled"; };
 			audmux@021d8000		{ status = "okay"; };
+			i2c@021a0000		{ status = "okay"; };
 		};
 	};
 
diff --git a/sys/boot/fdt/dts/arm/imx6.dtsi b/sys/boot/fdt/dts/arm/imx6.dtsi
index c24bb81a004..db8b44e53b2 100644
--- a/sys/boot/fdt/dts/arm/imx6.dtsi
+++ b/sys/boot/fdt/dts/arm/imx6.dtsi
@@ -126,11 +126,12 @@
 			gpt: timer@02098000 {
 				compatible = "fsl,imx6q-gpt", "fsl,imx51-gpt";
 				reg = <0x02098000 0x4000>;
-				interrupt-parent = <&gic>; interrupts = <87>;
+				interrupt-parent = <&gic>;
+				interrupts = <87>;
 			};
 
 			iomux@020e0000 {
-				compatible = "fsl,imx6q-iomux";
+				compatible = "fsl,imx6q-iomuxc";
 				reg = <0x020e0000 0x4000>;
 				interrupt-parent = <&gic>;
 				interrupts = <32>;
@@ -311,6 +312,9 @@
 				compatible = "fsl,imx6q-ssi";
 				reg = <0x02028000 0x4000>;
 				interrupts = < 78 >;
+				dmas = <&sdma 37 1 0>,
+				       <&sdma 38 1 0>;
+				dma-names = "rx", "tx";
 				status = "disabled";
 			};
 

From 5faf2ae1601299ad5561e80c9a4647c224aae336 Mon Sep 17 00:00:00 2001
From: Jilles Tjoelker <jilles@FreeBSD.org>
Date: Sat, 24 Jan 2015 13:50:13 +0000
Subject: [PATCH 236/279] cp,mv,touch: Set timestamps with nanosecond
 precision.

This uses utimensat().
---
 bin/cp/utils.c        | 11 +++---
 bin/mv/mv.c           |  9 +++--
 usr.bin/touch/touch.c | 79 +++++++++++++++++++++----------------------
 3 files changed, 48 insertions(+), 51 deletions(-)

diff --git a/bin/cp/utils.c b/bin/cp/utils.c
index ad9695c08a1..e77a6c13a37 100644
--- a/bin/cp/utils.c
+++ b/bin/cp/utils.c
@@ -330,7 +330,7 @@ copy_special(struct stat *from_stat, int exists)
 int
 setfile(struct stat *fs, int fd)
 {
-	static struct timeval tv[2];
+	static struct timespec tspec[2];
 	struct stat ts;
 	int rval, gotstat, islink, fdval;
 
@@ -340,10 +340,11 @@ setfile(struct stat *fs, int fd)
 	fs->st_mode &= S_ISUID | S_ISGID | S_ISVTX |
 		       S_IRWXU | S_IRWXG | S_IRWXO;
 
-	TIMESPEC_TO_TIMEVAL(&tv[0], &fs->st_atim);
-	TIMESPEC_TO_TIMEVAL(&tv[1], &fs->st_mtim);
-	if (islink ? lutimes(to.p_path, tv) : utimes(to.p_path, tv)) {
-		warn("%sutimes: %s", islink ? "l" : "", to.p_path);
+	tspec[0] = fs->st_atim;
+	tspec[1] = fs->st_mtim;
+	if (utimensat(AT_FDCWD, to.p_path, tspec,
+	    islink ? AT_SYMLINK_NOFOLLOW : 0)) {
+		warn("utimensat: %s", to.p_path);
 		rval = 1;
 	}
 	if (fdval ? fstat(fd, &ts) :
diff --git a/bin/mv/mv.c b/bin/mv/mv.c
index 3e832acbfdf..999c8fc8af4 100644
--- a/bin/mv/mv.c
+++ b/bin/mv/mv.c
@@ -273,7 +273,7 @@ do_move(const char *from, const char *to)
 static int
 fastcopy(const char *from, const char *to, struct stat *sbp)
 {
-	struct timeval tval[2];
+	struct timespec ts[2];
 	static u_int blen = MAXPHYS;
 	static char *bp = NULL;
 	mode_t oldmode;
@@ -350,10 +350,9 @@ err:		if (unlink(to))
 	} else
 		warn("%s: cannot stat", to);
 
-	tval[0].tv_sec = sbp->st_atime;
-	tval[1].tv_sec = sbp->st_mtime;
-	tval[0].tv_usec = tval[1].tv_usec = 0;
-	if (utimes(to, tval))
+	ts[0] = sbp->st_atim;
+	ts[1] = sbp->st_mtim;
+	if (utimensat(AT_FDCWD, to, ts, 0))
 		warn("%s: set times", to);
 
 	if (close(to_fd)) {
diff --git a/usr.bin/touch/touch.c b/usr.bin/touch/touch.c
index 4439c07eb43..c8bfd28d692 100644
--- a/usr.bin/touch/touch.c
+++ b/usr.bin/touch/touch.c
@@ -56,10 +56,10 @@ static const char sccsid[] = "@(#)touch.c	8.1 (Berkeley) 6/6/93";
 #include <time.h>
 #include <unistd.h>
 
-static void	stime_arg1(const char *, struct timeval *);
-static void	stime_arg2(const char *, int, struct timeval *);
-static void	stime_darg(const char *, struct timeval *);
-static void	stime_file(const char *, struct timeval *);
+static void	stime_arg1(const char *, struct timespec *);
+static void	stime_arg2(const char *, int, struct timespec *);
+static void	stime_darg(const char *, struct timespec *);
+static void	stime_file(const char *, struct timespec *);
 static int	timeoffset(const char *);
 static void	usage(const char *);
 
@@ -67,19 +67,17 @@ int
 main(int argc, char *argv[])
 {
 	struct stat sb;
-	struct timeval tv[2];
-	int (*stat_f)(const char *, struct stat *);
-	int (*utimes_f)(const char *, const struct timeval *);
+	struct timespec ts[2];
+	int atflag;
 	int Aflag, aflag, cflag, mflag, ch, fd, len, rval, timeset;
 	char *p;
 	char *myname;
 
 	myname = basename(argv[0]);
 	Aflag = aflag = cflag = mflag = timeset = 0;
-	stat_f = stat;
-	utimes_f = utimes;
-	if (gettimeofday(&tv[0], NULL) == -1)
-		err(1, "gettimeofday");
+	atflag = 0;
+	if (clock_gettime(CLOCK_REALTIME, &ts[0]) == -1)
+		err(1, "clock_gettime(CLOCK_REALTIME)");
 
 	while ((ch = getopt(argc, argv, "A:acd:fhmr:t:")) != -1)
 		switch(ch) {
@@ -94,26 +92,25 @@ main(int argc, char *argv[])
 			break;
 		case 'd':
 			timeset = 1;
-			stime_darg(optarg, tv);
+			stime_darg(optarg, ts);
 			break;
 		case 'f':
 			/* No-op for compatibility. */
 			break;
 		case 'h':
 			cflag = 1;
-			stat_f = lstat;
-			utimes_f = lutimes;
+			atflag = AT_SYMLINK_NOFOLLOW;
 			break;
 		case 'm':
 			mflag = 1;
 			break;
 		case 'r':
 			timeset = 1;
-			stime_file(optarg, tv);
+			stime_file(optarg, ts);
 			break;
 		case 't':
 			timeset = 1;
-			stime_arg1(optarg, tv);
+			stime_arg1(optarg, ts);
 			break;
 		default:
 			usage(myname);
@@ -132,9 +129,9 @@ main(int argc, char *argv[])
 			 * that time once and for all here.
 			 */
 			if (aflag)
-				tv[0].tv_sec += Aflag;
+				ts[0].tv_sec += Aflag;
 			if (mflag)
-				tv[1].tv_sec += Aflag;
+				ts[1].tv_sec += Aflag;
 			Aflag = 0;		/* done our job */
 		}
 	} else {
@@ -148,11 +145,11 @@ main(int argc, char *argv[])
 			len = p - argv[0];
 			if (*p == '\0' && (len == 8 || len == 10)) {
 				timeset = 1;
-				stime_arg2(*argv++, len == 10, tv);
+				stime_arg2(*argv++, len == 10, ts);
 			}
 		}
 		/* Both times default to the same. */
-		tv[1] = tv[0];
+		ts[1] = ts[0];
 	}
 
 	if (*argv == NULL)
@@ -163,7 +160,7 @@ main(int argc, char *argv[])
 
 	for (rval = 0; *argv; ++argv) {
 		/* See if the file exists. */
-		if (stat_f(*argv, &sb) != 0) {
+		if (fstatat(AT_FDCWD, *argv, &sb, atflag) != 0) {
 			if (errno != ENOENT) {
 				rval = 1;
 				warn("%s", *argv);
@@ -187,9 +184,9 @@ main(int argc, char *argv[])
 		}
 
 		if (!aflag)
-			TIMESPEC_TO_TIMEVAL(&tv[0], &sb.st_atim);
+			ts[0] = sb.st_atim;
 		if (!mflag)
-			TIMESPEC_TO_TIMEVAL(&tv[1], &sb.st_mtim);
+			ts[1] = sb.st_mtim;
 
 		/*
 		 * We're adjusting the times based on the file times, not a
@@ -197,17 +194,17 @@ main(int argc, char *argv[])
 		 */
 		if (Aflag) {
 			if (aflag) {
-				TIMESPEC_TO_TIMEVAL(&tv[0], &sb.st_atim);
-				tv[0].tv_sec += Aflag;
+				ts[0] = sb.st_atim;
+				ts[0].tv_sec += Aflag;
 			}
 			if (mflag) {
-				TIMESPEC_TO_TIMEVAL(&tv[1], &sb.st_mtim);
-				tv[1].tv_sec += Aflag;
+				ts[1] = sb.st_mtim;
+				ts[1].tv_sec += Aflag;
 			}
 		}
 
-		/* Try utimes(2). */
-		if (!utimes_f(*argv, tv))
+		/* Try utimensat(2). */
+		if (!utimensat(AT_FDCWD, *argv, ts, atflag))
 			continue;
 
 		/* If the user specified a time, nothing else we can do. */
@@ -223,7 +220,7 @@ main(int argc, char *argv[])
 		 * The permission checks are different, too, in that the
 		 * ability to write the file is sufficient.  Take a shot.
 		 */
-		 if (!utimes_f(*argv, NULL))
+		 if (!utimensat(AT_FDCWD, *argv, NULL, atflag))
 			continue;
 
 		rval = 1;
@@ -235,7 +232,7 @@ main(int argc, char *argv[])
 #define	ATOI2(ar)	((ar)[0] - '0') * 10 + ((ar)[1] - '0'); (ar) += 2;
 
 static void
-stime_arg1(const char *arg, struct timeval *tvp)
+stime_arg1(const char *arg, struct timespec *tvp)
 {
 	time_t now;
 	struct tm *t;
@@ -291,7 +288,7 @@ stime_arg1(const char *arg, struct timeval *tvp)
 	if (tvp[0].tv_sec == -1)
 		goto terr;
 
-	tvp[0].tv_usec = tvp[1].tv_usec = 0;
+	tvp[0].tv_nsec = tvp[1].tv_nsec = 0;
 	return;
 
 terr:
@@ -299,7 +296,7 @@ terr:
 }
 
 static void
-stime_arg2(const char *arg, int year, struct timeval *tvp)
+stime_arg2(const char *arg, int year, struct timespec *tvp)
 {
 	time_t now;
 	struct tm *t;
@@ -325,18 +322,18 @@ stime_arg2(const char *arg, int year, struct timeval *tvp)
 		errx(1,
 	"out of range or illegal time specification: MMDDhhmm[yy]");
 
-	tvp[0].tv_usec = tvp[1].tv_usec = 0;
+	tvp[0].tv_nsec = tvp[1].tv_nsec = 0;
 }
 
 static void
-stime_darg(const char *arg, struct timeval *tvp)
+stime_darg(const char *arg, struct timespec *tvp)
 {
 	struct tm t = { .tm_sec = 0 };
 	const char *fmt, *colon;
 	char *p;
 	int val, isutc = 0;
 
-	tvp[0].tv_usec = 0;
+	tvp[0].tv_nsec = 0;
 	t.tm_isdst = -1;
 	colon = strchr(arg, ':');
 	if (colon == NULL || strchr(colon + 1, ':') == NULL)
@@ -349,9 +346,9 @@ stime_darg(const char *arg, struct timeval *tvp)
 	/* POSIX: must have at least one digit after dot */
 	if ((*p == '.' || *p == ',') && isdigit((unsigned char)p[1])) {
 		p++;
-		val = 100000;
+		val = 100000000;
 		while (isdigit((unsigned char)*p)) {
-			tvp[0].tv_usec += val * (*p - '0');
+			tvp[0].tv_nsec += val * (*p - '0');
 			p++;
 			val /= 10;
 		}
@@ -403,14 +400,14 @@ timeoffset(const char *arg)
 }
 
 static void
-stime_file(const char *fname, struct timeval *tvp)
+stime_file(const char *fname, struct timespec *tsp)
 {
 	struct stat sb;
 
 	if (stat(fname, &sb))
 		err(1, "%s", fname);
-	TIMESPEC_TO_TIMEVAL(tvp, &sb.st_atim);
-	TIMESPEC_TO_TIMEVAL(tvp + 1, &sb.st_mtim);
+	tsp[0] = sb.st_atim;
+	tsp[1] = sb.st_mtim;
 }
 
 static void

From 71943c3d359adc2450d40abffbcda2ec748c4c86 Mon Sep 17 00:00:00 2001
From: Konstantin Belousov <kib@FreeBSD.org>
Date: Sat, 24 Jan 2015 15:33:42 +0000
Subject: [PATCH 237/279] Avoid calling vmspace_free() while owning the process
 lock.  Freeing of an vm space may require obtaining sleepable locks.  Hold
 the process to keep the pointer valid, and change trylock to lock, since
 there is no longer two process locks owned simultaneously in
 vm_pageout_oom().

Note that after the process lock is dropped, process might exec, and
no longer qualify as the owner of biggest vm space.

In collaboration with:	rstone
Sponsored by:	The FreeBSD Foundation
MFC after:	1 week
---
 sys/vm/vm_pageout.c | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index ca9d7f9a3b5..c81290ca097 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -1516,15 +1516,15 @@ vm_pageout_oom(int shortage)
 	FOREACH_PROC_IN_SYSTEM(p) {
 		int breakout;
 
-		if (PROC_TRYLOCK(p) == 0)
-			continue;
+		PROC_LOCK(p);
+
 		/*
 		 * If this is a system, protected or killed process, skip it.
 		 */
-		if (p->p_state != PRS_NORMAL ||
-		    (p->p_flag & (P_INEXEC | P_PROTECTED | P_SYSTEM)) ||
-		    (p->p_pid == 1) || P_KILLED(p) ||
-		    ((p->p_pid < 48) && (swap_pager_avail != 0))) {
+		if (p->p_state != PRS_NORMAL || (p->p_flag & (P_INEXEC |
+		    P_PROTECTED | P_SYSTEM | P_WEXIT)) != 0 ||
+		    p->p_pid == 1 || P_KILLED(p) ||
+		    (p->p_pid < 48 && swap_pager_avail != 0)) {
 			PROC_UNLOCK(p);
 			continue;
 		}
@@ -1557,11 +1557,14 @@ vm_pageout_oom(int shortage)
 			PROC_UNLOCK(p);
 			continue;
 		}
+		_PHOLD(p);
 		if (!vm_map_trylock_read(&vm->vm_map)) {
-			vmspace_free(vm);
+			_PRELE(p);
 			PROC_UNLOCK(p);
+			vmspace_free(vm);
 			continue;
 		}
+		PROC_UNLOCK(p);
 		size = vmspace_swap_count(vm);
 		vm_map_unlock_read(&vm->vm_map);
 		if (shortage == VM_OOM_MEM)
@@ -1573,16 +1576,19 @@ vm_pageout_oom(int shortage)
 		 */
 		if (size > bigsize) {
 			if (bigproc != NULL)
-				PROC_UNLOCK(bigproc);
+				PRELE(bigproc);
 			bigproc = p;
 			bigsize = size;
-		} else
-			PROC_UNLOCK(p);
+		} else {
+			PRELE(p);
+		}
 	}
 	sx_sunlock(&allproc_lock);
 	if (bigproc != NULL) {
+		PROC_LOCK(bigproc);
 		killproc(bigproc, "out of swap space");
 		sched_nice(bigproc, PRIO_MIN);
+		_PRELE(bigproc);
 		PROC_UNLOCK(bigproc);
 		wakeup(&vm_cnt.v_free_count);
 	}

From 117f1bc17f8c9dcee4c7256a2dddceab1954c219 Mon Sep 17 00:00:00 2001
From: Alexander Motin <mav@FreeBSD.org>
Date: Sat, 24 Jan 2015 15:40:52 +0000
Subject: [PATCH 238/279] Fix wrong LUN reference in XCOPY block-to-block
 operation.

This could cause data corruption due to accessing wrong LUN in case of
retries on write errors.  Failed writes were retried to read LUN.

MFC after:	3 days
---
 sys/cam/ctl/ctl_tpc.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/cam/ctl/ctl_tpc.c b/sys/cam/ctl/ctl_tpc.c
index 4e67ce14d32..63aee278bb9 100644
--- a/sys/cam/ctl/ctl_tpc.c
+++ b/sys/cam/ctl/ctl_tpc.c
@@ -916,7 +916,7 @@ tpc_process_b2b(struct tpc_list *list)
 				    /*control*/ 0);
 		tiow->io->io_hdr.retries = 3;
 		tiow->lun = dl;
-		tiow->io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = tior;
+		tiow->io->io_hdr.ctl_private[CTL_PRIV_FRONTEND].ptr = tiow;
 
 		TAILQ_INSERT_TAIL(&tior->run, tiow, rlinks);
 		TAILQ_INSERT_TAIL(prun, tior, rlinks);

From 8dd985befefa1d755af67820b7b32461899d9852 Mon Sep 17 00:00:00 2001
From: Jilles Tjoelker <jilles@FreeBSD.org>
Date: Sat, 24 Jan 2015 15:49:40 +0000
Subject: [PATCH 239/279] Enable utimensat tests from NetBSD.

As with other tests from c063, a required #include <sys/stat.h> was missing.
---
 contrib/netbsd-tests/lib/libc/c063/t_utimensat.c | 3 +++
 lib/libc/tests/c063/Makefile                     | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/contrib/netbsd-tests/lib/libc/c063/t_utimensat.c b/contrib/netbsd-tests/lib/libc/c063/t_utimensat.c
index 9f21fd6f126..bbfa28bd01d 100644
--- a/contrib/netbsd-tests/lib/libc/c063/t_utimensat.c
+++ b/contrib/netbsd-tests/lib/libc/c063/t_utimensat.c
@@ -40,6 +40,9 @@ __RCSID("$NetBSD: t_utimensat.c,v 1.5 2013/03/17 04:46:06 jmmv Exp $");
 #include <string.h>
 #include <unistd.h>
 #include <sys/param.h>
+#ifdef __FreeBSD__
+#include <sys/stat.h>
+#endif
 #include <sys/time.h>
 
 #define DIR "dir"
diff --git a/lib/libc/tests/c063/Makefile b/lib/libc/tests/c063/Makefile
index 0b8f3b0974f..5f4e1ca9c31 100644
--- a/lib/libc/tests/c063/Makefile
+++ b/lib/libc/tests/c063/Makefile
@@ -2,7 +2,7 @@
 
 TESTSDIR=	${TESTSBASE}/lib/libc/c063
 
-#TODO: t_o_search, t_utimensat
+#TODO: t_o_search
 
 NETBSD_ATF_TESTS_C=	faccessat
 NETBSD_ATF_TESTS_C+=	fchmodat
@@ -18,6 +18,7 @@ NETBSD_ATF_TESTS_C+=	readlinkat
 NETBSD_ATF_TESTS_C+=	renameat
 NETBSD_ATF_TESTS_C+=	symlinkat
 NETBSD_ATF_TESTS_C+=	unlinkat
+NETBSD_ATF_TESTS_C+=	utimensat
 
 CFLAGS+= -D_INCOMPLETE_XOPEN_C063
 

From 423521aa33a2ca4d2846956aed7a0e0d5a0b78a6 Mon Sep 17 00:00:00 2001
From: Ryan Stone <rstone@FreeBSD.org>
Date: Sat, 24 Jan 2015 16:59:38 +0000
Subject: [PATCH 240/279] vmspace_release() may sleep if the last reference is
 being released, so add a WITNESS_WARN() to catch cases where it is called
 with a non-sleepable lock held.

MFC after:	1 month
Sponsored by:	Sandvine Inc.
---
 sys/vm/vm_map.c | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c
index b8d67bd30a3..b7e668b928b 100644
--- a/sys/vm/vm_map.c
+++ b/sys/vm/vm_map.c
@@ -344,6 +344,9 @@ void
 vmspace_free(struct vmspace *vm)
 {
 
+	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL,
+	    "vmspace_free() called with non-sleepable lock held");
+
 	if (vm->vm_refcnt == 0)
 		panic("vmspace_free: attempt to free already freed vmspace");
 

From ce7e32798f0a5b337b63929c4c5dd237206862f1 Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Sat, 24 Jan 2015 17:11:11 +0000
Subject: [PATCH 241/279] Add tests/etc/rc.d to mtree.

Submitted by:	stefanf
MFC after:	1 week
MFC with:	277627
---
 etc/mtree/BSD.tests.dist | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/etc/mtree/BSD.tests.dist b/etc/mtree/BSD.tests.dist
index c43f1aea48d..7ea2155d372 100644
--- a/etc/mtree/BSD.tests.dist
+++ b/etc/mtree/BSD.tests.dist
@@ -52,6 +52,8 @@
         ..
     ..
     etc
+        rc.d
+        ..
     ..
     games
     ..

From 8311a2b8a4ee641d7ef0c46331378da0a328467b Mon Sep 17 00:00:00 2001
From: Will Andrews <will@FreeBSD.org>
Date: Sat, 24 Jan 2015 17:32:45 +0000
Subject: [PATCH 242/279] Add vm.panic_on_oom sysctl, which enables those who
 would rather panic than kill a process, when the system runs out of memory. 
 Defaults to off.

Usually, this is most useful when the OOM condition is due to mismanagement
of memory, on a system where the applications in question don't respond well
to being killed.

In theory, if the system is properly managed, it shouldn't be possible to
hit this condition. If it does, the panic can be more desirable for some
users (since it can be a good means of finding the root cause) rather than
killing the largest process and continuing on its merry way.

As kib@ mentions in the differential, there is also protect(1), which uses
procctl(PROC_SPROTECT) to ensure that some processes are immune.  However,
a panic approach is still useful in some environments.  This is primarily
intended as a development/debugging tool.

Differential Revision:	D1627
Reviewed by:		kib
MFC after:		1 week
---
 sys/vm/vm_pageout.c | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index c81290ca097..ecc3dabfe70 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -180,6 +180,12 @@ static int vm_swap_enabled = 1;
 static int vm_swap_idle_enabled = 0;
 #endif
 
+static int vm_panic_on_oom = 0;
+
+SYSCTL_INT(_vm, OID_AUTO, panic_on_oom,
+	CTLFLAG_RWTUN, &vm_panic_on_oom, 0,
+	"panic on out of memory instead of killing the largest process");
+
 SYSCTL_INT(_vm, OID_AUTO, pageout_wakeup_thresh,
 	CTLFLAG_RW, &vm_pageout_wakeup_thresh, 0,
 	"free page threshold for waking up the pageout daemon");
@@ -1585,6 +1591,8 @@ vm_pageout_oom(int shortage)
 	}
 	sx_sunlock(&allproc_lock);
 	if (bigproc != NULL) {
+		if (vm_panic_on_oom != 0)
+			panic("out of swap space");
 		PROC_LOCK(bigproc);
 		killproc(bigproc, "out of swap space");
 		sched_nice(bigproc, PRIO_MIN);

From 70a89b8588dae5847fbdfcfc5732a246ec630463 Mon Sep 17 00:00:00 2001
From: Baptiste Daroussin <bapt@FreeBSD.org>
Date: Sat, 24 Jan 2015 19:13:03 +0000
Subject: [PATCH 243/279] Allow negative numbers in -u and -g options

PR:		196514
MFC after:	1 week
---
 usr.sbin/pw/pw_group.c            |  6 +++++-
 usr.sbin/pw/pw_user.c             |  5 ++++-
 usr.sbin/pw/tests/Makefile        |  4 +++-
 usr.sbin/pw/tests/pw_groupshow.sh | 19 +++++++++++++++++++
 usr.sbin/pw/tests/pw_usershow.sh  | 19 +++++++++++++++++++
 5 files changed, 50 insertions(+), 3 deletions(-)
 create mode 100755 usr.sbin/pw/tests/pw_groupshow.sh
 create mode 100755 usr.sbin/pw/tests/pw_usershow.sh

diff --git a/usr.sbin/pw/pw_group.c b/usr.sbin/pw/pw_group.c
index b20ce88fb30..51166cdebca 100644
--- a/usr.sbin/pw/pw_group.c
+++ b/usr.sbin/pw/pw_group.c
@@ -68,7 +68,11 @@ pw_group(struct userconf * cnf, int mode, struct cargs * args)
 	};
 
 	if (a_gid != NULL) {
-		if (strspn(a_gid->val, "0123456789") != strlen(a_gid->val))
+		const char *teststr;
+		teststr = a_gid->val;
+		if (*teststr == '-')
+			teststr++;
+		if (strspn(teststr, "0123456789") != strlen(teststr))
 			errx(EX_USAGE, "-g expects a number");
 	}
 
diff --git a/usr.sbin/pw/pw_user.c b/usr.sbin/pw/pw_user.c
index 483148af490..f146b464adb 100644
--- a/usr.sbin/pw/pw_user.c
+++ b/usr.sbin/pw/pw_user.c
@@ -322,7 +322,10 @@ pw_user(struct userconf * cnf, int mode, struct cargs * args)
 			a_name = NULL;
 		}
 	} else {
-		if (strspn(a_uid->val, "0123456789") != strlen(a_uid->val))
+		const char *teststr = a_uid->val;
+		if (*teststr == '-')
+			teststr++;
+		if (strspn(teststr, "0123456789") != strlen(teststr))
 			errx(EX_USAGE, "-u expects a number");
 	}
 
diff --git a/usr.sbin/pw/tests/Makefile b/usr.sbin/pw/tests/Makefile
index 1283ff2a7d2..c6098841822 100644
--- a/usr.sbin/pw/tests/Makefile
+++ b/usr.sbin/pw/tests/Makefile
@@ -9,9 +9,11 @@ ATF_TESTS_SH=	pw_etcdir \
 		pw_lock \
 		pw_groupdel \
 		pw_groupmod \
+		pw_groupshow \
 		pw_useradd \
 		pw_userdel \
-		pw_usermod
+		pw_usermod \
+		pw_usershow
 
 .for tp in ${ATF_TESTS_SH}
 TEST_METADATA.${tp}+=	required_user="root"
diff --git a/usr.sbin/pw/tests/pw_groupshow.sh b/usr.sbin/pw/tests/pw_groupshow.sh
new file mode 100755
index 00000000000..2ba53d6ff19
--- /dev/null
+++ b/usr.sbin/pw/tests/pw_groupshow.sh
@@ -0,0 +1,19 @@
+# $FreeBSD$
+
+# Import helper functions
+. $(atf_get_srcdir)/helper_functions.shin
+
+
+# Test negative uid are still valid
+# PR: 196514
+atf_test_case show_group_with_negative_number
+show_group_with_negative_number_body() {
+	populate_etc_skel
+	atf_check -s exit:0 \
+		-o inline:"wheel:*:0:root\n" \
+		${PW} groupshow -n wheel -g -1
+}
+
+atf_init_test_cases() {
+	atf_add_test_case show_group_with_negative_number
+}
diff --git a/usr.sbin/pw/tests/pw_usershow.sh b/usr.sbin/pw/tests/pw_usershow.sh
new file mode 100755
index 00000000000..4703644ca36
--- /dev/null
+++ b/usr.sbin/pw/tests/pw_usershow.sh
@@ -0,0 +1,19 @@
+# $FreeBSD$
+
+# Import helper functions
+. $(atf_get_srcdir)/helper_functions.shin
+
+
+# Test negative uid are still valid
+# PR: 196514
+atf_test_case show_user_with_negative_number
+show_user_with_negative_number_body() {
+	populate_etc_skel
+	atf_check -s exit:0 \
+		-o inline:"root:*:0:0::0:0:Charlie &:/root:/bin/csh\n" \
+		${PW} usershow -n root -u -1
+}
+
+atf_init_test_cases() {
+	atf_add_test_case show_user_with_negative_number
+}

From 11c0b69c08a46625f1264b5fde253b55b152398c Mon Sep 17 00:00:00 2001
From: Adrian Chadd <adrian@FreeBSD.org>
Date: Sat, 24 Jan 2015 19:49:27 +0000
Subject: [PATCH 244/279] Change the permissions from 0660 to 0600.

Otherwise people in wheel can do things with netmap, including
but not limited to promisc transmit/receive.

Approved by:	luigi
MFC after:	1 week
---
 sys/dev/netmap/netmap.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/sys/dev/netmap/netmap.c b/sys/dev/netmap/netmap.c
index 9369f6cf2a2..959b2703fc8 100644
--- a/sys/dev/netmap/netmap.c
+++ b/sys/dev/netmap/netmap.c
@@ -3075,10 +3075,10 @@ netmap_init(void)
 #ifdef __FreeBSD__
 	/* support for the 'eternal' flag */
 	netmap_dev = make_dev_credf(MAKEDEV_ETERNAL_KLD,
-		&netmap_cdevsw, 0, NULL, UID_ROOT, GID_WHEEL, 0660,
+		&netmap_cdevsw, 0, NULL, UID_ROOT, GID_WHEEL, 0600,
 			      "netmap");
 #else
-	netmap_dev = make_dev(&netmap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0660,
+	netmap_dev = make_dev(&netmap_cdevsw, 0, UID_ROOT, GID_WHEEL, 0600,
 			      "netmap");
 #endif
 	if (!netmap_dev)

From 8f228789016acf40c01a73284a7d541ba58985e5 Mon Sep 17 00:00:00 2001
From: "Pedro F. Giffuni" <pfg@FreeBSD.org>
Date: Sat, 24 Jan 2015 20:16:34 +0000
Subject: [PATCH 245/279] gdb: Add missing break statements

2004-05-21  Jim Blandy  <jimb@redhat.com>

       * dwarf2expr.c (execute_stack_op): Add 'break' statements after
       cases for DW_OP_div and DW_OP_shr.  (Thanks to Reva Cuthbertson.)

Sourceware commit	99c87dab95747d380392a3698740507a21ad3236
---
 gdb/dwarf2expr.c | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/gdb/dwarf2expr.c b/gdb/dwarf2expr.c
index 50baced5f2e..294afa06c63 100644
--- a/gdb/dwarf2expr.c
+++ b/gdb/dwarf2expr.c
@@ -575,6 +575,7 @@ execute_stack_op (struct dwarf_expr_context *ctx, unsigned char *op_ptr,
 		break;
 	      case DW_OP_div:
 		binop = BINOP_DIV;
+                break;
 	      case DW_OP_minus:
 		binop = BINOP_SUB;
 		break;
@@ -595,6 +596,7 @@ execute_stack_op (struct dwarf_expr_context *ctx, unsigned char *op_ptr,
 		break;
 	      case DW_OP_shr:
 		binop = BINOP_RSH;
+                break;
 	      case DW_OP_shra:
 		binop = BINOP_RSH;
 		val1 = value_from_longest (signed_address_type (), first);

From 50878a7bbdbbd7b42b2101e4ca1606b974b8da20 Mon Sep 17 00:00:00 2001
From: Ian Lepore <ian@FreeBSD.org>
Date: Sat, 24 Jan 2015 20:18:37 +0000
Subject: [PATCH 246/279] Reimplement fdt_clock_register_provider() correctly. 
 It turns out you can't use OF_xref_from_device() to implement the function
 that registers the xref association with the device.

Pointy hat:	     ian
Submitted by:	     loos
---
 sys/dev/fdt/fdt_clock.c | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/sys/dev/fdt/fdt_clock.c b/sys/dev/fdt/fdt_clock.c
index e471523b8a2..88c2ac4c63d 100644
--- a/sys/dev/fdt/fdt_clock.c
+++ b/sys/dev/fdt/fdt_clock.c
@@ -149,7 +149,8 @@ void
 fdt_clock_register_provider(device_t provider)
 {
 
-	OF_device_register_xref(OF_xref_from_device(provider), provider);
+	OF_device_register_xref(
+        OF_xref_from_node(ofw_bus_get_node(provider)), provider);
 }
 
 void

From 8d2edb694b0b40cfefe7372c9116efe267b1f034 Mon Sep 17 00:00:00 2001
From: "Pedro F. Giffuni" <pfg@FreeBSD.org>
Date: Sat, 24 Jan 2015 21:00:51 +0000
Subject: [PATCH 247/279] GDB: Replace use of sprintf.

2005-03-17  Mark Kettenis  <kettenis@gnu.org>

      * corelow.c (get_core_register_section): Replace usage of sprintf
      and strcpy with xstrprintf and xstrdup.

Sourceware commit:	3ecda4574edb38ad12fb491ccaf6d9b0caa3a07a
---
 gdb/corelow.c | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/gdb/corelow.c b/gdb/corelow.c
index 403cfa31338..c06e6220dc8 100644
--- a/gdb/corelow.c
+++ b/gdb/corelow.c
@@ -432,15 +432,16 @@ get_core_register_section (char *name,
 			   char *human_name,
 			   int required)
 {
-  char section_name[100];
+  static char *section_name = NULL;
   struct bfd_section *section;
   bfd_size_type size;
   char *contents;
 
+  xfree (section_name);
   if (PIDGET (inferior_ptid))
-    sprintf (section_name, "%s/%d", name, PIDGET (inferior_ptid));
+    section_name = xstrprintf ("%s/%d", name, PIDGET (inferior_ptid));
   else
-    strcpy (section_name, name);
+    section_name = xstrdup (name);
 
   section = bfd_get_section_by_name (core_bfd, section_name);
   if (! section)

From 79a86dafb7ed506794a640c2806474692793560b Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 00:03:44 +0000
Subject: [PATCH 248/279] Add MK_EE knob to control installing edit, ee, etc

MFC after: 2 weeks
Sponsored by: EMC / Isilon Storage Division
---
 share/mk/src.opts.mk                     |  1 +
 tools/build/mk/OptionalObsoleteFiles.inc | 17 +++++++++++++++++
 tools/build/options/WITHOUT_EE           |  5 +++++
 usr.bin/Makefile                         |  5 ++++-
 4 files changed, 27 insertions(+), 1 deletion(-)
 create mode 100644 tools/build/options/WITHOUT_EE

diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk
index f8adc30c963..b196e388059 100644
--- a/share/mk/src.opts.mk
+++ b/share/mk/src.opts.mk
@@ -73,6 +73,7 @@ __DEFAULT_YES_OPTIONS = \
     DMAGENT \
     DYNAMICROOT \
     ED_CRYPTO \
+    EE \
     ELFTOOLCHAIN_TOOLS \
     EXAMPLES \
     FDT \
diff --git a/tools/build/mk/OptionalObsoleteFiles.inc b/tools/build/mk/OptionalObsoleteFiles.inc
index 4f3b6e11d5a..fe2897ef721 100644
--- a/tools/build/mk/OptionalObsoleteFiles.inc
+++ b/tools/build/mk/OptionalObsoleteFiles.inc
@@ -1216,6 +1216,23 @@ OLD_FILES+=usr/share/dict/words
 OLD_DIRS+=usr/share/dict
 .endif
 
+.if ${MK_EE} == no
+OLD_FILES+=usr/bin/edit
+OLD_FILES+=usr/bin/ee
+OLD_FILES+=usr/bin/ree
+OLD_FILES+=usr/share/man/man1/edit.1.gz
+OLD_FILES+=usr/share/man/man1/ee.1.gz
+OLD_FILES+=usr/share/man/man1/ree.1.gz
+OLD_FILES+=usr/share/nls/C/ee.cat
+OLD_FILES+=usr/share/nls/de_DE.ISO8859-1/ee.cat
+OLD_FILES+=usr/share/nls/fr_FR.ISO8859-1/ee.cat
+OLD_FILES+=usr/share/nls/hu_HU.ISO8859-2/ee.cat
+OLD_FILES+=usr/share/nls/pl_PL.ISO8859-2/ee.cat
+OLD_FILES+=usr/share/nls/pt_BR.ISO8859-1/ee.cat
+OLD_FILES+=usr/share/nls/ru_RU.KOI8-R/ee.cat
+OLD_FILES+=usr/share/nls/uk_UA.KOI8-U/ee.cat
+.endif
+
 .if ${MK_ELFTOOLCHAIN_TOOLS} == no
 OLD_FILES+=usr/bin/elfcopy
 OLD_FILES+=usr/share/man/man1/elfcopy.1.gz
diff --git a/tools/build/options/WITHOUT_EE b/tools/build/options/WITHOUT_EE
new file mode 100644
index 00000000000..721ddaab9df
--- /dev/null
+++ b/tools/build/options/WITHOUT_EE
@@ -0,0 +1,5 @@
+.\" $FreeBSD$
+Set to not build and install
+.Xr edit 1 ,
+.Xr ee 1 ,
+and related programs.
diff --git a/usr.bin/Makefile b/usr.bin/Makefile
index 2e54cdfaba0..7106db75e50 100644
--- a/usr.bin/Makefile
+++ b/usr.bin/Makefile
@@ -40,7 +40,6 @@ SUBDIR=	${_addr2line} \
 	dirname \
 	dpv \
 	du \
-	ee \
 	elf2aout \
 	${_elfcopy} \
 	elfdump \
@@ -240,6 +239,10 @@ SUBDIR+=	calendar
 _clang=		clang
 .endif
 
+.if ${MK_EE} != "no"
+SUBDIR+=	ee
+.endif
+
 .if ${MK_ELFTOOLCHAIN_TOOLS} != "no"
 _addr2line=	addr2line
 _elfcopy=	elfcopy

From c50bf0f33728af061b9479f374eaa8c9bd8298ce Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 00:30:22 +0000
Subject: [PATCH 249/279] Only build vi support into rescue if MK_VI != no

Sponsored by: EMC / Isilon Storage Division
---
 rescue/rescue/Makefile | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/rescue/rescue/Makefile b/rescue/rescue/Makefile
index e08ec2953f1..cb4422d4528 100644
--- a/rescue/rescue/Makefile
+++ b/rescue/rescue/Makefile
@@ -199,8 +199,10 @@ CRUNCH_LIBS+= -lcrypto
 .endif
 CRUNCH_LIBS+= -lmd
 
+.if ${MK_VI} != "no"
 CRUNCH_PROGS_usr.bin+= vi
 CRUNCH_ALIAS_vi= ex
+.endif
 
 CRUNCH_PROGS_usr.bin+= id
 CRUNCH_ALIAS_id= groups whoami

From 57d9cab5080bc72a10506395120a54f3fa35d3f2 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 00:32:17 +0000
Subject: [PATCH 250/279] Only build share/dtrace if MK_CDDL != no

MFC after: 1 week
Sponsored by: EMC / Isilon Storage Division
---
 share/Makefile | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/share/Makefile b/share/Makefile
index 63c18bb6464..97d1dac540c 100644
--- a/share/Makefile
+++ b/share/Makefile
@@ -8,7 +8,7 @@
 SUBDIR=	${_colldef} \
 	${_dict} \
 	${_doc} \
-	dtrace \
+	${_dtrace} \
 	${_examples} \
 	${_i18n} \
 	keys \
@@ -37,6 +37,10 @@ SUBDIR=	${_colldef} \
 _snmp=		snmp
 .endif
 
+.if ${MK_CDDL} != "no"
+_dtrace=	dtrace
+.endif
+
 .if ${MK_DICT} != "no"
 _dict=		dict
 .endif

From cd9c9939501f157bda77d9d3d8c7c7a37b23776a Mon Sep 17 00:00:00 2001
From: Mark Johnston <markj@FreeBSD.org>
Date: Sun, 25 Jan 2015 00:34:43 +0000
Subject: [PATCH 251/279] Ensure that we don't try to demangle a symbol name if
 we failed to look up the symbol. Add a test to exercise this code path.

Reviewed by:	adrian
---
 lib/libproc/proc_sym.c        |  4 ++--
 lib/libproc/tests/proc_test.c | 40 ++++++++++++++++++++++++++++++++++-
 2 files changed, 41 insertions(+), 3 deletions(-)

diff --git a/lib/libproc/proc_sym.c b/lib/libproc/proc_sym.c
index e52f8ff563e..90d40a899c3 100644
--- a/lib/libproc/proc_sym.c
+++ b/lib/libproc/proc_sym.c
@@ -335,8 +335,8 @@ proc_addr2sym(struct proc_handle *p, uintptr_t addr, char *name,
 		goto out;
 
 	error = lookup_addr(e, symtabscn, symtabstridx, off, addr, &s, symcopy);
-	if (error == 0)
-		goto out;
+	if (error != 0)
+		goto err2;
 
 out:
 	demangle(s, name, namesz);
diff --git a/lib/libproc/tests/proc_test.c b/lib/libproc/tests/proc_test.c
index 0242b5b5ee9..dbaace84f5a 100644
--- a/lib/libproc/tests/proc_test.c
+++ b/lib/libproc/tests/proc_test.c
@@ -1,5 +1,5 @@
 /*-
- * Copyright (c) 2014 Mark Johnston <markj@FreeBSD.org>
+ * Copyright (c) 2014, 2015 Mark Johnston <markj@FreeBSD.org>
  * All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -295,6 +295,43 @@ ATF_TC_BODY(symbol_lookup, tc)
 	proc_free(phdl);
 }
 
+ATF_TC(symbol_lookup_fail);
+ATF_TC_HEAD(symbol_lookup_fail, tc)
+{
+	atf_tc_set_md_var(tc, "descr",
+	    "Verify that proc_addr2sym() returns an error when given an offset "
+	    "that it cannot resolve.");
+}
+ATF_TC_BODY(symbol_lookup_fail, tc)
+{
+	char symname[32];
+	GElf_Sym sym;
+	struct proc_handle *phdl;
+	prmap_t *map;
+	int error;
+
+	phdl = start_prog(tc, false);
+
+	/* Initialize the rtld_db handle. */
+	(void)proc_rdagent(phdl);
+
+	map = proc_obj2map(phdl, target_prog_file);
+	ATF_REQUIRE_MSG(map != NULL, "failed to look up map for '%s'",
+	    target_prog_file);
+
+	/*
+	 * We shouldn't be able to find symbols at the beginning of a mapped
+	 * file.
+	 */
+	error = proc_addr2sym(phdl, map->pr_vaddr, symname, sizeof(symname),
+	    &sym);
+	ATF_REQUIRE_MSG(error != 0, "unexpectedly found a symbol");
+
+	ATF_CHECK_EQ_MSG(proc_continue(phdl), 0, "failed to resume execution");
+
+	proc_free(phdl);
+}
+
 ATF_TC(signal_forward);
 ATF_TC_HEAD(signal_forward, tc)
 {
@@ -343,6 +380,7 @@ ATF_TP_ADD_TCS(tp)
 	ATF_TP_ADD_TC(tp, map_alias_name2map);
 	ATF_TP_ADD_TC(tp, map_alias_name2sym);
 	ATF_TP_ADD_TC(tp, symbol_lookup);
+	ATF_TP_ADD_TC(tp, symbol_lookup_fail);
 	ATF_TP_ADD_TC(tp, signal_forward);
 
 	return (atf_no_error());

From b8797aec6fea844a9fd0fa6d0d5cf022961ef6c2 Mon Sep 17 00:00:00 2001
From: Mark Johnston <markj@FreeBSD.org>
Date: Sun, 25 Jan 2015 00:36:42 +0000
Subject: [PATCH 252/279] Document the fact that modules declared with
 SYSCALL_MODULE(9) have their names prefixed with "sys/".

MFC after:	3 days
---
 share/man/man9/SYSCALL_MODULE.9 | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/share/man/man9/SYSCALL_MODULE.9 b/share/man/man9/SYSCALL_MODULE.9
index b803bcec11e..eb90aeb0832 100644
--- a/share/man/man9/SYSCALL_MODULE.9
+++ b/share/man/man9/SYSCALL_MODULE.9
@@ -28,7 +28,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd January 7, 2005
+.Dd January 24, 2015
 .Dt SYSCALL_MODULE 9
 .Os
 .Sh NAME
@@ -46,8 +46,8 @@ The
 .Fn SYSCALL_MODULE
 macro declares a new syscall.
 .Fn SYSCALL_MODULE
-expands into a kernel module declaration named as
-.Fa name .
+expands into a kernel module declaration with name
+.Ql sys/ Ns Fa name .
 .Pp
 The rest of the arguments expected by this macro are:
 .Bl -tag -width ".Fa new_sysent"

From f8c8839091395fffb3f4abb143b39ac97ff3aee2 Mon Sep 17 00:00:00 2001
From: Mark Johnston <markj@FreeBSD.org>
Date: Sun, 25 Jan 2015 00:47:06 +0000
Subject: [PATCH 253/279] gr_equal(): Fix a crash that could occur if the first
 group's member list was longer than the second's. There is no need to compute
 and compare the member list lengths in a separate pass, since we now just
 return false when comparing member names if the list lengths are not equal.

MFC after:	2 weeks
---
 lib/libutil/gr_util.c | 25 ++++++++++---------------
 1 file changed, 10 insertions(+), 15 deletions(-)

diff --git a/lib/libutil/gr_util.c b/lib/libutil/gr_util.c
index 465efd98242..b0b0b36f081 100644
--- a/lib/libutil/gr_util.c
+++ b/lib/libutil/gr_util.c
@@ -351,8 +351,6 @@ gr_fini(void)
 int
 gr_equal(const struct group *gr1, const struct group *gr2)
 {
-	int gr1_ndx;
-	int gr2_ndx;
 
 	/* Check that the non-member information is the same. */
 	if (gr1->gr_name == NULL || gr2->gr_name == NULL) {
@@ -368,7 +366,8 @@ gr_equal(const struct group *gr1, const struct group *gr2)
 	if (gr1->gr_gid != gr2->gr_gid)
 		return (false);
 
-	/* Check all members in both groups.
+	/*
+	 * Check all members in both groups.
 	 * getgrnam can return gr_mem with a pointer to NULL.
 	 * gr_dup and gr_add strip out this superfluous NULL, setting
 	 * gr_mem to NULL for no members.
@@ -376,22 +375,18 @@ gr_equal(const struct group *gr1, const struct group *gr2)
 	if (gr1->gr_mem != NULL && gr2->gr_mem != NULL) {
 		int i;
 
-		for (i = 0; gr1->gr_mem[i] != NULL; i++) {
+		for (i = 0;
+		    gr1->gr_mem[i] != NULL && gr2->gr_mem[i] != NULL; i++) {
 			if (strcmp(gr1->gr_mem[i], gr2->gr_mem[i]) != 0)
 				return (false);
 		}
-	}
-	/* Count number of members in both structs */
-	gr2_ndx = 0;
-	if (gr2->gr_mem != NULL)
-		for(; gr2->gr_mem[gr2_ndx] != NULL; gr2_ndx++)
-			/* empty */;
-	gr1_ndx = 0;
-	if (gr1->gr_mem != NULL)
-		for(; gr1->gr_mem[gr1_ndx] != NULL; gr1_ndx++)
-			/* empty */;
-	if (gr1_ndx != gr2_ndx)
+		if (gr1->gr_mem[i] != NULL || gr2->gr_mem[i] != NULL)
+			return (false);
+	} else if (gr1->gr_mem != NULL && gr1->gr_mem[0] != NULL) {
 		return (false);
+	} else if (gr2->gr_mem != NULL && gr2->gr_mem[0] != NULL) {
+		return (false);
+	}
 
 	return (true);
 }

From 6251a2629c2004292366779ce83a3a8d0ea29c37 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 02:30:44 +0000
Subject: [PATCH 254/279] Sort #include order

---
 sbin/rcorder/rcorder.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sbin/rcorder/rcorder.c b/sbin/rcorder/rcorder.c
index 8c46b4fd1de..1371e37504c 100644
--- a/sbin/rcorder/rcorder.c
+++ b/sbin/rcorder/rcorder.c
@@ -41,8 +41,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/stat.h>
 
 #include <err.h>
-#include <stdio.h>
 #include <libutil.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>

From 9a1f476703f5ad42eb7bb8ef2a7a07c0034b0f74 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 03:02:29 +0000
Subject: [PATCH 255/279] - Revert r277670 (#include order is required for
 fparseln) - Remove unnecessary CFLAGS+= -I now that make is no longer
 required to make   rcorder build

---
 sbin/rcorder/Makefile  | 3 ---
 sbin/rcorder/rcorder.c | 2 +-
 2 files changed, 1 insertion(+), 4 deletions(-)

diff --git a/sbin/rcorder/Makefile b/sbin/rcorder/Makefile
index 038bc35acee..2f1bbac1fa8 100644
--- a/sbin/rcorder/Makefile
+++ b/sbin/rcorder/Makefile
@@ -10,8 +10,5 @@ LIBADD=	util
 CFLAGS+= -DORDER
 
 #CFLAGS+= -DDEBUG
-.if ${CFLAGS:M-DDEBUG}
-CFLAGS+= -I${.CURDIR}/../../usr.bin/make
-.endif
 
 .include <bsd.prog.mk>
diff --git a/sbin/rcorder/rcorder.c b/sbin/rcorder/rcorder.c
index 1371e37504c..8c46b4fd1de 100644
--- a/sbin/rcorder/rcorder.c
+++ b/sbin/rcorder/rcorder.c
@@ -41,8 +41,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/stat.h>
 
 #include <err.h>
-#include <libutil.h>
 #include <stdio.h>
+#include <libutil.h>
 #include <stdlib.h>
 #include <string.h>
 #include <unistd.h>

From b29d6977f31adc2560a78f75c7b43d73fc10c93d Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 04:20:11 +0000
Subject: [PATCH 256/279] Add MK_ISCSI knob for building the iscsi initiator,
 iscsi daemon, kernel modules, etc

MFC after: 2 weeks
Sponsored by: EMC / Isilon Storage Division
---
 etc/rc.d/Makefile                        |  7 +++++--
 sbin/Makefile                            |  5 ++++-
 share/man/man4/Makefile                  |  8 ++++++--
 share/mk/src.opts.mk                     |  1 +
 sys/conf/kern.opts.mk                    |  1 +
 sys/modules/Makefile                     |  7 +++++--
 tools/build/mk/OptionalObsoleteFiles.inc | 14 ++++++++++++++
 tools/build/options/WITHOUT_ISCSI        |  4 ++++
 usr.bin/Makefile                         |  5 ++++-
 usr.sbin/Makefile                        |  5 ++++-
 10 files changed, 48 insertions(+), 9 deletions(-)
 create mode 100644 tools/build/options/WITHOUT_ISCSI

diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile
index 4994e9da8aa..bb10868ad4f 100644
--- a/etc/rc.d/Makefile
+++ b/etc/rc.d/Makefile
@@ -66,8 +66,6 @@ FILES=	DAEMON \
 	ipropd_master \
 	ipropd_slave \
 	ipsec \
-	iscsictl \
-	iscsid \
 	jail \
 	${_kadmind} \
 	${_kdc} \
@@ -174,6 +172,11 @@ _ubthidhci=	ubthidhci
 _casperd=	casperd
 .endif
 
+.if ${MK_ISCSI} != "no"
+FILES+=		iscsictl
+FILES+=		iscsid
+.endif
+
 .if ${MK_NS_CACHING} != "no"
 _nscd=		nscd
 .endif
diff --git a/sbin/Makefile b/sbin/Makefile
index 3725270c4c3..ae0db296e48 100644
--- a/sbin/Makefile
+++ b/sbin/Makefile
@@ -35,7 +35,6 @@ SUBDIR=adjkerntz \
 	hastd \
 	ifconfig \
 	init \
-	iscontrol \
 	kldconfig \
 	kldload \
 	kldstat \
@@ -109,6 +108,10 @@ SUBDIR+=	pflogd
 SUBDIR+=	ping6
 SUBDIR+=	rtsol
 .endif
+	
+.if ${MK_ISCSI} != "no"
+SUBDIR+=	iscontrol
+.endif
 
 .if ${MK_QUOTAS} != "no"
 SUBDIR+=	quotacheck
diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile
index ffedc0fca45..5c07303670c 100644
--- a/share/man/man4/Makefile
+++ b/share/man/man4/Makefile
@@ -206,8 +206,6 @@ MAN=	aac.4 \
 	ipw.4 \
 	ipwfw.4 \
 	isci.4 \
-	iscsi.4 \
-	iscsi_initiator.4 \
 	ismt.4 \
 	isp.4 \
 	ispfw.4 \
@@ -860,6 +858,12 @@ _nvram2env.4=	nvram2env.4
 SUBDIR=	man4.${MACHINE_CPUARCH}
 .endif
 
+.if ${MK_ISCSI} != "no"
+MAN+=		iscsi.4
+MAN+=		iscsi_initiator.4
+
+.endif
+
 .if ${MK_TESTS} != "no"
 ATF=            ${.CURDIR}/../../../contrib/atf
 .PATH:          ${ATF}/doc
diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk
index b196e388059..fc29acee171 100644
--- a/share/mk/src.opts.mk
+++ b/share/mk/src.opts.mk
@@ -97,6 +97,7 @@ __DEFAULT_YES_OPTIONS = \
     INET6 \
     IPFILTER \
     IPFW \
+    ISCSI \
     JAIL \
     KDUMP \
     KVM \
diff --git a/sys/conf/kern.opts.mk b/sys/conf/kern.opts.mk
index 16bb13447a7..63e8510e4fa 100644
--- a/sys/conf/kern.opts.mk
+++ b/sys/conf/kern.opts.mk
@@ -30,6 +30,7 @@ __DEFAULT_YES_OPTIONS = \
     INET \
     INET6 \
     IPFILTER \
+    ISCSI \
     KERNEL_SYMBOLS \
     NETGRAPH \
     PF \
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index 211c8e7508c..aedb6d09404 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -169,8 +169,6 @@ SUBDIR=	\
 	${_ipw} \
 	${_ipwfw} \
 	${_isci} \
-	iscsi \
-	iscsi_initiator \
 	isp \
 	${_ispfw} \
 	${_iwi} \
@@ -413,6 +411,11 @@ _ipfw=		ipfw
 _ipfilter=	ipfilter
 .endif
 
+.if ${MK_ISCSI} != "no" || defined(ALL_MODULES)
+SUBDIR+=	iscsi
+SUBDIR+=	iscsi_initiator
+.endif
+
 .if ${MK_NAND} != "no" || defined(ALL_MODULES)
 _nandfs=	nandfs
 _nandsim=	nandsim
diff --git a/tools/build/mk/OptionalObsoleteFiles.inc b/tools/build/mk/OptionalObsoleteFiles.inc
index fe2897ef721..ceb81c9b34f 100644
--- a/tools/build/mk/OptionalObsoleteFiles.inc
+++ b/tools/build/mk/OptionalObsoleteFiles.inc
@@ -2094,6 +2094,20 @@ OLD_FILES+=usr/share/man/man8/ipfwpcap.8.gz
 OLD_FILES+=usr/share/man/man8/natd.8.gz
 .endif
 
+.if ${MK_ISCSI} == no
+OLD_FILES+=etc/rc.d/iscsictl
+OLD_FILES+=etc/rc.d/iscsid
+OLD_FILES+=sbin/iscontrol
+OLD_FILES+=usr/bin/iscsictl
+OLD_FILES+=usr/sbin/iscsid
+OLD_FILES+=usr/share/man/man4/iscsi.4.gz
+OLD_FILES+=usr/share/man/man4/iscsi_initiator.4.gz
+OLD_FILES+=usr/share/man/man5/iscsi.conf.5.gz
+OLD_FILES+=usr/share/man/man8/iscontrol.8.gz
+OLD_FILES+=usr/share/man/man8/iscsictl.8.gz
+OLD_FILES+=usr/share/man/man8/iscsid.8.gz
+.endif
+
 .if ${MK_JAIL} == no
 OLD_FILES+=usr/sbin/jail
 OLD_FILES+=usr/sbin/jexec
diff --git a/tools/build/options/WITHOUT_ISCSI b/tools/build/options/WITHOUT_ISCSI
new file mode 100644
index 00000000000..83f3d7467ba
--- /dev/null
+++ b/tools/build/options/WITHOUT_ISCSI
@@ -0,0 +1,4 @@
+.\" $FreeBSD$
+Set to not build
+.Xr iscid 8
+and related utilities.
diff --git a/usr.bin/Makefile b/usr.bin/Makefile
index 7106db75e50..fb12ef8f0dc 100644
--- a/usr.bin/Makefile
+++ b/usr.bin/Makefile
@@ -69,7 +69,6 @@ SUBDIR=	${_addr2line} \
 	id \
 	ipcrm \
 	ipcs \
-	iscsictl \
 	join \
 	jot \
 	${_kdump} \
@@ -273,6 +272,10 @@ _mkcsmapper=	mkcsmapper
 _mkesdb=	mkesdb
 .endif
 
+.if ${MK_ISCSI} != "no"
+SUBDIR+=	iscsictl
+.endif
+
 .if ${MK_KDUMP} != "no"
 SUBDIR+=        kdump
 SUBDIR+=        truss
diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile
index 09e53adc981..e1e20d44dc4 100644
--- a/usr.sbin/Makefile
+++ b/usr.sbin/Makefile
@@ -38,7 +38,6 @@ SUBDIR=	adduser \
 	ifmcstat \
 	inetd \
 	iostat \
-	iscsid \
 	kldxref \
 	mailwrapper \
 	makefs \
@@ -178,6 +177,10 @@ SUBDIR+=	traceroute6
 SUBDIR+=	ipfwpcap
 .endif
 
+.if ${MK_ISCSI} != "no"
+SUBDIR+=	iscsid
+.endif
+
 .if ${MK_JAIL} != "no"
 SUBDIR+=	jail
 SUBDIR+=	jexec

From 3f802165ba9f979a71ec77199965193ca352d751 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 04:37:44 +0000
Subject: [PATCH 257/279] Add MK_TALK knob for building the talk and talkd

MFC after: 2 weeks
Sponsored by: EMC / Isilon Storage Division
---
 libexec/Makefile                         | 5 ++++-
 share/mk/src.opts.mk                     | 1 +
 tools/build/mk/OptionalObsoleteFiles.inc | 7 +++++++
 tools/build/options/WITHOUT_TALK         | 5 +++++
 usr.bin/Makefile                         | 5 ++++-
 5 files changed, 21 insertions(+), 2 deletions(-)
 create mode 100644 tools/build/options/WITHOUT_TALK

diff --git a/libexec/Makefile b/libexec/Makefile
index 7d1c1f8ddae..c8b5386678c 100644
--- a/libexec/Makefile
+++ b/libexec/Makefile
@@ -28,7 +28,6 @@ SUBDIR=	${_atf} \
 	${_rtld-elf} \
 	save-entropy \
 	${_smrsh} \
-	talkd \
 	tcpd \
 	${_telnetd} \
 	${_tests} \
@@ -81,6 +80,10 @@ _mail.local=	mail.local
 _smrsh=		smrsh
 .endif
 
+.if ${MK_TALK} != "no"
+SUBDIR+=	talkd
+.endif
+
 .if ${MK_TELNET} != "no"
 _telnetd=	telnetd
 .endif
diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk
index fc29acee171..126e7868c6f 100644
--- a/share/mk/src.opts.mk
+++ b/share/mk/src.opts.mk
@@ -144,6 +144,7 @@ __DEFAULT_YES_OPTIONS = \
     SYSCALL_COMPAT \
     SYSCONS \
     SYSINSTALL \
+    TALK \
     TCSH \
     TELNET \
     TESTS \
diff --git a/tools/build/mk/OptionalObsoleteFiles.inc b/tools/build/mk/OptionalObsoleteFiles.inc
index ceb81c9b34f..1f326b4c829 100644
--- a/tools/build/mk/OptionalObsoleteFiles.inc
+++ b/tools/build/mk/OptionalObsoleteFiles.inc
@@ -4073,6 +4073,13 @@ OLD_DIRS+=usr/share/doc/pjdfstest
 # to be filled in
 #.endif
 
+.if ${MK_TALK} == no
+OLD_FILES+=usr/bin/talk
+OLD_FILES+=usr/libexec/ntalkd
+OLD_FILES+=usr/share/man/man1/talk.1.gz
+OLD_FILES+=usr/share/man/man8/talkd.8.gz
+.endif
+
 .if ${MK_TCSH} == no
 OLD_FILES+=bin/csh
 OLD_FILES+=bin/tcsh
diff --git a/tools/build/options/WITHOUT_TALK b/tools/build/options/WITHOUT_TALK
new file mode 100644
index 00000000000..33d41675000
--- /dev/null
+++ b/tools/build/options/WITHOUT_TALK
@@ -0,0 +1,5 @@
+.\" $FreeBSD$
+Set to not build or install
+.Xr talk 1
+and
+.Xr talkd 8 .
diff --git a/usr.bin/Makefile b/usr.bin/Makefile
index fb12ef8f0dc..fbe87bd296d 100644
--- a/usr.bin/Makefile
+++ b/usr.bin/Makefile
@@ -162,7 +162,6 @@ SUBDIR=	${_addr2line} \
 	systat \
 	tabs \
 	tail \
-	talk \
 	tar \
 	tcopy \
 	tee \
@@ -348,6 +347,10 @@ SUBDIR+=	rwho
 SUBDIR+=	vacation
 .endif
 
+.if ${MK_TALK} != "no"
+SUBDIR+=	talk
+.endif
+
 .if ${MK_TELNET} != "no"
 SUBDIR+=	telnet
 .endif

From e45b569c9592bc86a54469f6fd0461c293f36211 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 04:43:13 +0000
Subject: [PATCH 258/279] Add MK_BSDINSTALL knob for building and installing
 bsdinstall

MFC after: 2 weeks
Sponsored by: EMC / Isilon Storage Division
---
 share/mk/src.opts.mk                     |  1 +
 tools/build/mk/OptionalObsoleteFiles.inc | 33 ++++++++++++++++++++++++
 tools/build/options/WITHOUT_BSDINSTALL   |  5 ++++
 usr.sbin/Makefile                        |  5 +++-
 4 files changed, 43 insertions(+), 1 deletion(-)
 create mode 100644 tools/build/options/WITHOUT_BSDINSTALL

diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk
index 126e7868c6f..0fcf0cc8a5b 100644
--- a/share/mk/src.opts.mk
+++ b/share/mk/src.opts.mk
@@ -57,6 +57,7 @@ __DEFAULT_YES_OPTIONS = \
     BLUETOOTH \
     BOOT \
     BSD_CPIO \
+    BSDINSTALL \
     BSNMP \
     BZIP2 \
     CALENDAR \
diff --git a/tools/build/mk/OptionalObsoleteFiles.inc b/tools/build/mk/OptionalObsoleteFiles.inc
index 1f326b4c829..110275fd577 100644
--- a/tools/build/mk/OptionalObsoleteFiles.inc
+++ b/tools/build/mk/OptionalObsoleteFiles.inc
@@ -274,6 +274,39 @@ OLD_FILES+=usr/share/man/man8/sdpd.8.gz
 # to be filled in
 #.endif
 
+.if ${MK_BSDINSTALL} == no
+OLD_FILES+=usr/libexec/bsdinstall/adduser
+OLD_FILES+=usr/libexec/bsdinstall/auto
+OLD_FILES+=usr/libexec/bsdinstall/autopart
+OLD_FILES+=usr/libexec/bsdinstall/checksum
+OLD_FILES+=usr/libexec/bsdinstall/config
+OLD_FILES+=usr/libexec/bsdinstall/distextract
+OLD_FILES+=usr/libexec/bsdinstall/distfetch
+OLD_FILES+=usr/libexec/bsdinstall/docsinstall
+OLD_FILES+=usr/libexec/bsdinstall/entropy
+OLD_FILES+=usr/libexec/bsdinstall/hostname
+OLD_FILES+=usr/libexec/bsdinstall/jail
+OLD_FILES+=usr/libexec/bsdinstall/keymap
+OLD_FILES+=usr/libexec/bsdinstall/mirrorselect
+OLD_FILES+=usr/libexec/bsdinstall/mount
+OLD_FILES+=usr/libexec/bsdinstall/netconfig
+OLD_FILES+=usr/libexec/bsdinstall/netconfig_ipv4
+OLD_FILES+=usr/libexec/bsdinstall/netconfig_ipv6
+OLD_FILES+=usr/libexec/bsdinstall/partedit
+OLD_FILES+=usr/libexec/bsdinstall/rootpass
+OLD_FILES+=usr/libexec/bsdinstall/script
+OLD_FILES+=usr/libexec/bsdinstall/scriptedpart
+OLD_FILES+=usr/libexec/bsdinstall/services
+OLD_FILES+=usr/libexec/bsdinstall/time
+OLD_FILES+=usr/libexec/bsdinstall/umount
+OLD_FILES+=usr/libexec/bsdinstall/wlanconfig
+OLD_FILES+=usr/libexec/bsdinstall/zfsboot
+OLD_FILES+=usr/sbin/bsdinstall
+OLD_FILES+=usr/share/man/man8/bsdinstall.8.gz
+OLD_FILES+=usr/share/man/man8/sade.8.gz
+OLD_DIRS+=usr/libexec/bsdinstall
+.endif
+
 .if ${MK_CALENDAR} == no
 OLD_FILES+=etc/periodic/daily/300.calendar
 OLD_FILES+=usr/bin/calendar
diff --git a/tools/build/options/WITHOUT_BSDINSTALL b/tools/build/options/WITHOUT_BSDINSTALL
new file mode 100644
index 00000000000..8aaf2a6544e
--- /dev/null
+++ b/tools/build/options/WITHOUT_BSDINSTALL
@@ -0,0 +1,5 @@
+.\" $FreeBSD$
+Set to not build
+.Xr bsdinstall 8 ,
+.Xr sade 8 ,
+and related programs.
diff --git a/usr.sbin/Makefile b/usr.sbin/Makefile
index e1e20d44dc4..1db40466657 100644
--- a/usr.sbin/Makefile
+++ b/usr.sbin/Makefile
@@ -9,7 +9,6 @@ SUBDIR=	adduser \
 	binmiscctl \
 	bootparamd \
 	bsdconfig \
-	bsdinstall \
 	cdcontrol \
 	chkgrp \
 	chown \
@@ -129,6 +128,10 @@ SUBDIR+=	authpf
 SUBDIR+=	bluetooth
 .endif
 
+.if ${MK_BSDINSTALL} != "no"
+SUBDIR+=	bsdinstall
+.endif
+
 .if ${MK_BSNMP} != "no"
 SUBDIR+=	bsnmpd
 .endif

From 18cc3172331cdea16e89083e735b7c2730d5c71e Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 04:52:48 +0000
Subject: [PATCH 259/279] Add MK_CCD knob for building and installing ccd(4),
 ccdconfig, etc

MFC after: 2 weeks
Sponsored by: EMC / Isilon Storage Division
---
 etc/rc.d/Makefile                        | 5 ++++-
 sbin/Makefile                            | 5 ++++-
 share/man/man4/Makefile                  | 6 +++++-
 share/mk/src.opts.mk                     | 1 +
 sys/conf/kern.opts.mk                    | 1 +
 sys/modules/geom/Makefile                | 8 +++++++-
 tools/build/mk/OptionalObsoleteFiles.inc | 7 +++++++
 tools/build/options/WITHOUT_CCD          | 4 ++++
 8 files changed, 33 insertions(+), 4 deletions(-)
 create mode 100644 tools/build/options/WITHOUT_CCD

diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile
index bb10868ad4f..465f3412d95 100644
--- a/etc/rc.d/Makefile
+++ b/etc/rc.d/Makefile
@@ -30,7 +30,6 @@ FILES=	DAEMON \
 	bsnmpd \
 	${_bthidd} \
 	${_casperd} \
-	ccd \
 	cleanvar \
 	cleartmp \
 	cron \
@@ -172,6 +171,10 @@ _ubthidhci=	ubthidhci
 _casperd=	casperd
 .endif
 
+.if ${MK_CCD} != "no"
+FILES+=		ccd
+.endif
+
 .if ${MK_ISCSI} != "no"
 FILES+=		iscsictl
 FILES+=		iscsid
diff --git a/sbin/Makefile b/sbin/Makefile
index ae0db296e48..1c34e0cc50f 100644
--- a/sbin/Makefile
+++ b/sbin/Makefile
@@ -8,7 +8,6 @@
 SUBDIR=adjkerntz \
 	badsect \
 	camcontrol \
-	ccdconfig \
 	clri \
 	comcontrol \
 	conscontrol \
@@ -81,6 +80,10 @@ SUBDIR+=	atm
 SUBDIR+=	casperd
 .endif
 
+.if ${MK_CCD} != "no"
+SUBDIR+=	ccdconfig
+.endif
+
 .if ${MK_CXX} != "no"
 SUBDIR+=	devd
 .endif
diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile
index 5c07303670c..8bfd87572fe 100644
--- a/share/man/man4/Makefile
+++ b/share/man/man4/Makefile
@@ -91,7 +91,7 @@ MAN=	aac.4 \
 	cc_htcp.4 \
 	cc_newreno.4 \
 	cc_vegas.4 \
-	ccd.4 \
+	${_ccd.4} \
 	cd.4 \
 	cdce.4 \
 	ch.4 \
@@ -858,6 +858,10 @@ _nvram2env.4=	nvram2env.4
 SUBDIR=	man4.${MACHINE_CPUARCH}
 .endif
 
+.if ${MK_CCD} != "no"
+_ccd.4=		ccd.4
+.endif
+
 .if ${MK_ISCSI} != "no"
 MAN+=		iscsi.4
 MAN+=		iscsi_initiator.4
diff --git a/share/mk/src.opts.mk b/share/mk/src.opts.mk
index 0fcf0cc8a5b..8082b141527 100644
--- a/share/mk/src.opts.mk
+++ b/share/mk/src.opts.mk
@@ -63,6 +63,7 @@ __DEFAULT_YES_OPTIONS = \
     CALENDAR \
     CAPSICUM \
     CASPER \
+    CCD \
     CDDL \
     CPP \
     CROSS_COMPILER \
diff --git a/sys/conf/kern.opts.mk b/sys/conf/kern.opts.mk
index 63e8510e4fa..8b3567928b8 100644
--- a/sys/conf/kern.opts.mk
+++ b/sys/conf/kern.opts.mk
@@ -24,6 +24,7 @@
 
 __DEFAULT_YES_OPTIONS = \
     BLUETOOTH \
+    CCD \
     CDDL \
     CRYPT \
     FORMAT_EXTENSIONS \
diff --git a/sys/modules/geom/Makefile b/sys/modules/geom/Makefile
index a94f7600505..f54a3e83c8a 100644
--- a/sys/modules/geom/Makefile
+++ b/sys/modules/geom/Makefile
@@ -1,8 +1,10 @@
 # $FreeBSD$
 
+SYSDIR?=${.CURDIR}/../..
+.include "${SYSDIR}/conf/kern.opts.mk"
+
 SUBDIR=	geom_bde \
 	geom_cache \
-	geom_ccd \
 	geom_concat \
 	geom_eli \
 	geom_gate \
@@ -25,4 +27,8 @@ SUBDIR=	geom_bde \
 	geom_virstor \
 	geom_zero
 
+.if ${MK_CCD} != "no" || defined(ALL_MODULES)
+SUBDIR+=	geom_ccd
+.endif
+
 .include <bsd.subdir.mk>
diff --git a/tools/build/mk/OptionalObsoleteFiles.inc b/tools/build/mk/OptionalObsoleteFiles.inc
index 110275fd577..66c331b2158 100644
--- a/tools/build/mk/OptionalObsoleteFiles.inc
+++ b/tools/build/mk/OptionalObsoleteFiles.inc
@@ -398,6 +398,13 @@ OLD_FILES+=sbin/casper
 OLD_FILES+=usr/lib/libcasper.a
 .endif
 
+.if ${MK_CCD} == no
+OLD_FILES+=etc/rc.d/ccd
+OLD_FILES+=sbin/ccdconfig
+OLD_FILES+=usr/share/man/man4/ccd.4.gz
+OLD_FILES+=usr/share/man/man8/ccdconfig.8.gz
+.endif
+
 .if ${MK_CDDL} == no
 OLD_LIBS+=lib/libavl.so.2
 OLD_LIBS+=lib/libctf.so.2
diff --git a/tools/build/options/WITHOUT_CCD b/tools/build/options/WITHOUT_CCD
new file mode 100644
index 00000000000..b4d044ecaaa
--- /dev/null
+++ b/tools/build/options/WITHOUT_CCD
@@ -0,0 +1,4 @@
+.\" $FreeBSD$
+Set to not build
+.Xr geom_ccd 4
+and related utilities.

From 93da83668688c3b53d3271d84e2ba87bea4fcd5c Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 04:56:43 +0000
Subject: [PATCH 260/279] Regen src.conf(5)

---
 share/man/man5/src.conf.5 | 34 +++++++++++++++++++++++++++++++---
 1 file changed, 31 insertions(+), 3 deletions(-)

diff --git a/share/man/man5/src.conf.5 b/share/man/man5/src.conf.5
index f0910cb1de7..ed806a40385 100644
--- a/share/man/man5/src.conf.5
+++ b/share/man/man5/src.conf.5
@@ -1,7 +1,7 @@
 .\" DO NOT EDIT-- this file is automatically generated.
 .\" from FreeBSD: head/tools/build/options/makeman 255964 2013-10-01 07:22:04Z des
 .\" $FreeBSD$
-.Dd January 7, 2015
+.Dd January 24, 2015
 .Dt SRC.CONF 5
 .Os
 .Sh NAME
@@ -142,6 +142,12 @@ Set to not build Bluetooth related kernel modules, programs and libraries.
 .It Va WITHOUT_BOOT
 .\" from FreeBSD: head/tools/build/options/WITHOUT_BOOT 156932 2006-03-21 07:50:50Z ru
 Set to not build the boot blocks and loader.
+.It Va WITHOUT_BSDINSTALL
+.\" from FreeBSD: head/tools/build/options/WITHOUT_BSDINSTALL 277677 2015-01-25 04:43:13Z ngie
+Set to not build
+.Xr bsdinstall 8 ,
+.Xr sade 8 ,
+and related programs.
 .It Va WITHOUT_BSD_CPIO
 .\" from FreeBSD: head/tools/build/options/WITHOUT_BSD_CPIO 179813 2008-06-16 05:48:15Z dougb
 Set to not build the BSD licensed version of cpio based on
@@ -179,6 +185,11 @@ Set to not build Capsicum support into system programs.
 .It Va WITHOUT_CASPER
 .\" from FreeBSD: head/tools/build/options/WITHOUT_CASPER 258838 2013-12-02 08:21:28Z pjd
 Set to not build Casper program and related libraries.
+.It Va WITHOUT_CCD
+.\" from FreeBSD: head/tools/build/options/WITHOUT_CCD 277678 2015-01-25 04:52:48Z ngie
+Set to not build
+.Xr geom_ccd 4
+and related utilities.
 .It Va WITHOUT_CDDL
 .\" from FreeBSD: head/tools/build/options/WITHOUT_CDDL 163861 2006-11-01 09:02:11Z jb
 Set to not build code licensed under Sun's CDDL.
@@ -218,13 +229,13 @@ unless an alternative compiler is provided via
 XCC.
 .Pp
 It is a default setting on
-arm/armeb, mips/mipsel, mips/mips, mips/mips64el, mips/mips64, mips/mipsn32 and sparc64/sparc64.
+arm/armeb, mips/mipsel, mips/mips, mips/mips64el, mips/mips64, mips/mipsn32, powerpc/powerpc, powerpc/powerpc64 and sparc64/sparc64.
 .It Va WITH_CLANG_BOOTSTRAP
 .\" from FreeBSD: head/tools/build/options/WITH_CLANG_BOOTSTRAP 264660 2014-04-18 17:03:58Z imp
 Set to build the Clang C/C++ compiler during the bootstrap phase of the build.
 .Pp
 It is a default setting on
-amd64/amd64, arm/arm, arm/armv6, arm/armv6hf, i386/i386, pc98/i386, powerpc/powerpc and powerpc/powerpc64.
+amd64/amd64, arm/arm, arm/armv6, arm/armv6hf, i386/i386 and pc98/i386.
 .It Va WITH_CLANG_EXTRAS
 .\" from FreeBSD: head/tools/build/options/WITH_CLANG_EXTRAS 231057 2012-02-05 23:56:22Z dim
 Set to build additional clang and llvm tools, such as bugpoint.
@@ -380,6 +391,12 @@ dynamically.
 Set to build
 .Xr ed 1
 without support for encryption/decryption.
+.It Va WITHOUT_EE
+.\" from FreeBSD: head/tools/build/options/WITHOUT_EE 277663 2015-01-25 00:03:44Z ngie
+Set to not build and install
+.Xr edit 1 ,
+.Xr ee 1 ,
+and related programs.
 .It Va WITH_EISA
 .\" from FreeBSD: head/tools/build/options/WITH_EISA 264654 2014-04-18 16:53:06Z imp
 Set to build EISA kernel modules.
@@ -593,6 +610,11 @@ Set to not build IP Filter package.
 .It Va WITHOUT_IPFW
 .\" from FreeBSD: head/tools/build/options/WITHOUT_IPFW 183242 2008-09-21 22:02:26Z sam
 Set to not build IPFW tools.
+.It Va WITHOUT_ISCSI
+.\" from FreeBSD: head/tools/build/options/WITHOUT_ISCSI 277675 2015-01-25 04:20:11Z ngie
+Set to not build
+.Xr iscid 8
+and related utilities.
 .It Va WITHOUT_JAIL
 .\" from FreeBSD: head/tools/build/options/WITHOUT_JAIL 249966 2013-04-27 04:09:09Z eadler
 Set to not build tools for the support of jails; e.g.,
@@ -1035,6 +1057,12 @@ support files such as keyboard maps, fonts, and screen output maps.
 Set to not build
 .Xr sysinstall 8
 and related programs.
+.It Va WITHOUT_TALK
+.\" from FreeBSD: head/tools/build/options/WITHOUT_TALK 277676 2015-01-25 04:37:44Z ngie
+Set to not build or install
+.Xr talk 1
+and
+.Xr talkd 8 .
 .It Va WITHOUT_TCSH
 .\" from FreeBSD: head/tools/build/options/WITHOUT_TCSH 156932 2006-03-21 07:50:50Z ru
 Set to not build and install

From 5a63bdd39b1bc66bea9a6c5063b3814d764ea612 Mon Sep 17 00:00:00 2001
From: Warner Losh <imp@FreeBSD.org>
Date: Sun, 25 Jan 2015 04:58:41 +0000
Subject: [PATCH 261/279] spl man page hasn't been relevant for a while, retire
 it.

---
 ObsoleteFiles.inc       |  14 +++
 share/man/man9/Makefile |  13 ---
 share/man/man9/spl.9    | 228 ----------------------------------------
 3 files changed, 14 insertions(+), 241 deletions(-)
 delete mode 100644 share/man/man9/spl.9

diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc
index 271cbef2c53..036e5d4cff2 100644
--- a/ObsoleteFiles.inc
+++ b/ObsoleteFiles.inc
@@ -38,6 +38,20 @@
 #   xargs -n1 | sort | uniq -d;
 # done
 
+# 20150124: spl.9 and friends
+OLD_FILES+=usr/share/man/man9/spl.9
+OLD_FILES+=usr/share/man/man9/spl0.9
+OLD_FILES+=usr/share/man/man9/splbio.9
+OLD_FILES+=usr/share/man/man9/splclock.9
+OLD_FILES+=usr/share/man/man9/splhigh.9
+OLD_FILES+=usr/share/man/man9/splimp.9
+OLD_FILES+=usr/share/man/man9/splnet.9
+OLD_FILES+=usr/share/man/man9/splsoftclock.9
+OLD_FILES+=usr/share/man/man9/splsofttty.9
+OLD_FILES+=usr/share/man/man9/splstatclock.9
+OLD_FILES+=usr/share/man/man9/spltty.9
+OLD_FILES+=usr/share/man/man9/splvm.9
+OLD_FILES+=usr/share/man/man9/splx.9
 # 20150118: new clang import which bumps version from 3.5.0 to 3.5.1.
 OLD_FILES+=usr/include/clang/3.5.0/__wmmintrin_aes.h
 OLD_FILES+=usr/include/clang/3.5.0/__wmmintrin_pclmul.h
diff --git a/share/man/man9/Makefile b/share/man/man9/Makefile
index cf64df7c811..ddd46aaf3f1 100644
--- a/share/man/man9/Makefile
+++ b/share/man/man9/Makefile
@@ -252,7 +252,6 @@ MAN=	accept_filter.9 \
 	sleep.9 \
 	sleepqueue.9 \
 	socket.9 \
-	spl.9 \
 	stack.9 \
 	store.9 \
 	style.9 \
@@ -1447,18 +1446,6 @@ MLINKS+=socket.9 soabort.9 \
 	socket.9 soupcall_clear.9 \
 	socket.9 soupcall_set.9 \
 	socket.9 sowakeup.9
-MLINKS+=spl.9 spl0.9 \
-	spl.9 splbio.9 \
-	spl.9 splclock.9 \
-	spl.9 splhigh.9 \
-	spl.9 splimp.9 \
-	spl.9 splnet.9 \
-	spl.9 splsoftclock.9 \
-	spl.9 splsofttty.9 \
-	spl.9 splstatclock.9 \
-	spl.9 spltty.9 \
-	spl.9 splvm.9 \
-	spl.9 splx.9
 MLINKS+=stack.9 stack_copy.9 \
 	stack.9 stack_create.9 \
 	stack.9 stack_destroy.9 \
diff --git a/share/man/man9/spl.9 b/share/man/man9/spl.9
deleted file mode 100644
index 3bfa556d4b7..00000000000
--- a/share/man/man9/spl.9
+++ /dev/null
@@ -1,228 +0,0 @@
-.\"
-.\" Copyright (c) 1996 Joerg Wunsch
-.\"
-.\" All rights reserved.
-.\"
-.\" Redistribution and use in source and binary forms, with or without
-.\" modification, are permitted provided that the following conditions
-.\" are met:
-.\" 1. Redistributions of source code must retain the above copyright
-.\"    notice, this list of conditions and the following disclaimer.
-.\" 2. Redistributions in binary form must reproduce the above copyright
-.\"    notice, this list of conditions and the following disclaimer in the
-.\"    documentation and/or other materials provided with the distribution.
-.\"
-.\" THIS SOFTWARE IS PROVIDED BY THE DEVELOPERS ``AS IS'' AND ANY EXPRESS OR
-.\" IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
-.\" OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
-.\" IN NO EVENT SHALL THE DEVELOPERS BE LIABLE FOR ANY DIRECT, INDIRECT,
-.\" INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
-.\" NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-.\" DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-.\" THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-.\" (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
-.\" THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-.\"
-.\" $FreeBSD$
-.\"
-.Dd July 21, 1996
-.Dt SPL 9
-.Os
-.Sh NAME
-.Nm splbio ,
-.Nm splclock ,
-.Nm splhigh ,
-.Nm splimp ,
-.Nm splnet ,
-.Nm splsoftclock ,
-.Nm splsofttty ,
-.Nm splstatclock ,
-.Nm spltty ,
-.Nm splvm ,
-.Nm spl0 ,
-.Nm splx
-.Nd manipulate interrupt priorities
-.Sh SYNOPSIS
-.In sys/types.h
-.In sys/systm.h
-.Ft intrmask_t
-.Fn splbio "void"
-.Ft intrmask_t
-.Fn splclock "void"
-.Ft intrmask_t
-.Fn splhigh "void"
-.Ft intrmask_t
-.Fn splimp "void"
-.Ft intrmask_t
-.Fn splnet "void"
-.Ft intrmask_t
-.Fn splsoftclock "void"
-.Ft intrmask_t
-.Fn splsofttty "void"
-.Ft intrmask_t
-.Fn splstatclock "void"
-.Ft intrmask_t
-.Fn spltty "void"
-.Ft void
-.Fn spl0 "void"
-.Ft void
-.Fn splx "intrmask_t ipl"
-.Sh DESCRIPTION
-.Bf -symbolic
-This API is deprecated.
-Use mutexes to protect data structures instead.
-See
-.Xr mutex 9
-for more information.
-The API is now a complete NOP.
-This man page documents historical behavior so you can understand the
-code locking that the spl did when converting code from versions of the
-kernel prior to
-.Fx 5.0 .
-The examples in this man page are also obsolete and should not be viewed
-as documenting
-.Fx 5.0
-and newer.
-.Ef
-.Pp
-The
-.Fn spl
-function family sets the interrupt priority
-.Dq level
-of the CPU.
-This prevents interrupt handlers of the blocked priority level from
-being run.
-This is used in the
-.Dq synchronous
-part of a driver (the part that runs on behalf of the user process) to
-examine or modify data areas that might be examined or modified by
-interrupt handlers.
-.Pp
-Each driver that uses interrupts is normally assigned to an interrupt
-priority group by a keyword in its config line.
-For example:
-.Bd -literal -offset indent
-device foo0 at isa? port 0x0815 irq 12 tty
-.Ed
-.Pp
-assigns interrupt 12 to the
-.Dq tty
-priority group.
-The system automatically arranges for interrupts in
-the
-.Em xxx
-group to be called at a priority >=
-.Em spl Ns Fn xxx
-.Pp
-The function
-.Fn splx
-sets the interrupt priority to an absolute value.
-The intent is that
-the value returned by the other functions should be saved in a local
-variable, and later passed to
-.Fn splx
-in order to restore the previous priority.
-.Pp
-The function
-.Fn spl0
-lowers the priority to a value where all interrupt handlers are
-unblocked, but ASTs (asynchronous system traps) remain blocked until
-the system is about to return to user mode.
-.Pp
-The traditional assignment of the various device drivers to the
-interrupt priority groups can be roughly classified as:
-.Bl -tag -width Fn
-.It Fn splnet
-Software part of the network interface drivers.
-.It Fn splimp
-All network interface drivers.
-.It Fn splbio
-All
-.Em buffered IO
-(i.e., disk and the like) drivers.
-.It Fn spltty
-Basically, all non-network communications devices, but effectively
-used for all drivers that are neither network nor disks.
-.El
-.Sh RETURN VALUES
-All functions except
-.Fn splx
-and
-.Fn spl0
-return the previous priority value.
-.Sh EXAMPLES
-This is a typical example demonstrating the usage:
-.Bd -literal
-struct foo_softc {
-	...
-	int flags;
-#define	FOO_ASLEEP	1
-#define	FOO_READY	2
-
-} foo_softc[NFOO];
-
-int
-foowrite(...)
-{
-	struct foo_softc *sc;
-	int s, error;
-
-	...
-	s = spltty();
-	if (!(sc->flags & FOO_READY)) {
-		/* Not ready, must sleep on resource. */
-		sc->flags |= FOO_ASLEEP;
-		error = tsleep(sc, PZERO, "foordy", 0);
-		sc->flags &= ~FOO_ASLEEP;
-	}
-	sc->flags &= ~FOO_READY;
-	splx(s);
-
-	...
-}
-
-void
-foointr(...)
-{
-	struct foo_softc *sc;
-
-	...
-	sc->flags |= FOO_READY;
-	if (sc->flags & FOO_ASLEEP)
-		/* Somebody was waiting for us, awake him. */
-		wakeup(sc);
-	...
-}
-
-.Ed
-Note that the interrupt handler should
-.Em never
-reduce the priority level.
-It is automatically called as it had
-raised the interrupt priority to its own level, i.e., further interrupts
-of the same group are being blocked.
-.Sh HISTORY
-The interrupt priority levels appeared in a very early version of
-.Ux .
-They have been traditionally known by number instead of by
-names, and were inclusive up to higher priority levels (i.e., priority
-5 has been blocking everything up to level 5).
-This is no longer the case in
-.Fx .
-The traditional name
-.Ql level
-for them is still reflected in the letter
-.Ql l
-of the respective functions and variables, although they are not
-really levels anymore, but rather different (partially inclusive)
-sets of functions to be blocked during some periods of the life of
-the system.
-The historical number scheme can be considered as a
-simple linearly ordered set of interrupt priority groups.
-.Pp
-.Fx 5.0
-eliminated spl entirely in favor of locking primitives which scale
-to more than one processor.
-.Sh AUTHORS
-This manual page was written by
-.An J\(:org Wunsch .

From e3d1e0f6abd297e056f3e7f94658ef5d853cf6a2 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 05:13:15 +0000
Subject: [PATCH 262/279] Build cuse(4) if MK_CUSE != no

MFC after: 1 week
Sponsored by: EMC / Isilon Storage Division
---
 sys/conf/kern.opts.mk | 1 +
 sys/modules/Makefile  | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/sys/conf/kern.opts.mk b/sys/conf/kern.opts.mk
index 8b3567928b8..84e4e4f8b4e 100644
--- a/sys/conf/kern.opts.mk
+++ b/sys/conf/kern.opts.mk
@@ -27,6 +27,7 @@ __DEFAULT_YES_OPTIONS = \
     CCD \
     CDDL \
     CRYPT \
+    CUSE \
     FORMAT_EXTENSIONS \
     INET \
     INET6 \
diff --git a/sys/modules/Makefile b/sys/modules/Makefile
index aedb6d09404..5c7485684fd 100644
--- a/sys/modules/Makefile
+++ b/sys/modules/Makefile
@@ -86,7 +86,6 @@ SUBDIR=	\
 	${_ct} \
 	${_ctau} \
 	ctl \
-	cuse \
 	${_cxgb} \
 	${_cxgbe} \
 	dc \
@@ -393,6 +392,10 @@ _random=	random
 .endif
 .endif
 
+.if ${MK_CUSE} != "no" || defined(ALL_MODULES)
+SUBDIR+=	cuse
+.endif
+
 .if (${MK_INET_SUPPORT} != "no" || ${MK_INET6_SUPPORT} != "no") || \
 	defined(ALL_MODULES)
 _carp=		carp

From 8b77741ea72db395dae0267d7dc28d757dc8d061 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 05:15:06 +0000
Subject: [PATCH 263/279] Make install cuse headers if MK_CUSE != no

MFC after: 2 weeks
Sponsored by: EMC / Isilon Storage Division
---
 include/Makefile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/include/Makefile b/include/Makefile
index fa815272c2b..e7da89bdd92 100644
--- a/include/Makefile
+++ b/include/Makefile
@@ -45,7 +45,6 @@ LSUBDIRS=	cam/ata cam/scsi \
 	dev/ic dev/iicbus dev/io dev/lmc dev/mfi dev/nvme \
 	dev/ofw dev/pbio dev/pci ${_dev_powermac_nvram} dev/ppbus dev/smbus \
 	dev/speaker dev/usb dev/utopia dev/vkbd dev/wi \
-	fs/cuse \
 	fs/devfs fs/fdescfs fs/msdosfs fs/nandfs fs/nfs fs/nullfs \
 	fs/procfs fs/smbfs fs/udf fs/unionfs \
 	geom/cache geom/concat geom/eli geom/gate geom/journal geom/label \
@@ -59,6 +58,10 @@ LSUBDIRS=	cam/ata cam/scsi \
 
 LSUBSUBDIRS=	dev/mpt/mpilib
 
+.if ${MK_CUSE} != "no"
+LSUBDIRS+=	fs/cuse
+.endif
+
 .if ${MACHINE_ARCH} == "powerpc" || ${MACHINE_ARCH} == "powerpc64"
 _dev_powermac_nvram=	dev/powermac_nvram
 .endif

From d4cd3eb85b0cec660aa372ded4958bde3ce3dc8c Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 05:15:45 +0000
Subject: [PATCH 264/279] Fill in entries for MK_CUSE == no

MFC after: 1 week
Sponsored by: EMC / Isilon Storage Division
---
 tools/build/mk/OptionalObsoleteFiles.inc | 37 ++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/tools/build/mk/OptionalObsoleteFiles.inc b/tools/build/mk/OptionalObsoleteFiles.inc
index 66c331b2158..de4e8701480 100644
--- a/tools/build/mk/OptionalObsoleteFiles.inc
+++ b/tools/build/mk/OptionalObsoleteFiles.inc
@@ -649,6 +649,43 @@ OLD_FILES+=usr/share/man/man1/ctm_smail.1.gz
 OLD_FILES+=usr/share/man/man5/ctm.5.gz
 .endif
 
+.if ${MK_CUSE} == no
+OLD_FILES+=usr/include/fs/cuse/cuse_defs.h
+OLD_FILES+=usr/include/fs/cuse/cuse_ioctl.h
+OLD_FILES+=usr/include/cuse.h
+OLD_FILES+=usr/lib/libcuse.a
+OLD_LIBS+=usr/lib/libcuse.so.1
+OLD_FILES+=usr/lib/libcuse_p.a
+OLD_FILES+=usr/share/man/man3/cuse.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_alloc_unit_number.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_alloc_unit_number_by_id.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_copy_in.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_copy_out.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_dev_create.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_dev_destroy.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_dev_get_current.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_dev_get_per_file_handle.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_dev_get_priv0.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_dev_get_priv1.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_dev_set_per_file_handle.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_dev_set_priv0.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_dev_set_priv1.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_free_unit_number.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_free_unit_number_by_id.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_get_local.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_got_peer_signal.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_init.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_is_vmalloc_addr.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_poll_wakeup.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_set_local.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_uninit.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_vmalloc.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_vmfree.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_vmoffset.3.gz
+OLD_FILES+=usr/share/man/man3/cuse_wait_and_process.3.gz
+OLD_DIRS+=usr/include/fs/cuse
+.endif
+
 # devd(8) not listed here on purpose
 .if ${MK_CXX} == no
 OLD_FILES+=usr/bin/CC

From f1574bb04d8777e0bc18fe71283e10b22021f046 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 05:23:22 +0000
Subject: [PATCH 265/279] Fill in some dtrace entries when MK_CDDL == no

MFC after: 1 week
Sponsored by: EMC / Isilon Storage Division
---
 tools/build/mk/OptionalObsoleteFiles.inc | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/tools/build/mk/OptionalObsoleteFiles.inc b/tools/build/mk/OptionalObsoleteFiles.inc
index de4e8701480..3c4b53a9024 100644
--- a/tools/build/mk/OptionalObsoleteFiles.inc
+++ b/tools/build/mk/OptionalObsoleteFiles.inc
@@ -428,7 +428,6 @@ OLD_FILES+=usr/lib/dtrace/signal.d
 OLD_FILES+=usr/lib/dtrace/tcp.d
 OLD_FILES+=usr/lib/dtrace/udp.d
 OLD_FILES+=usr/lib/dtrace/unistd.d
-OLD_DIRS+=usr/lib/dtrace
 OLD_FILES+=usr/lib/libavl.a
 OLD_FILES+=usr/lib/libavl.so
 OLD_FILES+=usr/lib/libavl_p.a
@@ -449,7 +448,6 @@ OLD_FILES+=usr/lib/libuutil.so
 OLD_FILES+=usr/lib/libuutil_p.a
 .if ${TARGET_ARCH} == "amd64" || ${TARGET_ARCH} == "powerpc64"
 OLD_FILES+=usr/lib32/dtrace/drti.o
-OLD_DIRS+=usr/lib32/dtrace
 OLD_FILES+=usr/lib32/libavl.a
 OLD_FILES+=usr/lib32/libavl.so
 OLD_LIBS+=usr/lib32/libavl.so.2
@@ -475,9 +473,24 @@ OLD_FILES+=usr/lib32/libuutil.so
 OLD_LIBS+=usr/lib32/libuutil.so.2
 OLD_FILES+=usr/lib32/libuutil_p.a
 .endif
+OLD_LIBS+=lib/libdtrace.so.2
 OLD_FILES+=usr/sbin/dtrace
 OLD_FILES+=usr/sbin/lockstat
 OLD_FILES+=usr/share/man/man1/dtrace.1.gz
+OLD_FILES+=usr/share/dtrace/disklatency
+OLD_FILES+=usr/share/dtrace/disklatencycmd
+OLD_FILES+=usr/share/dtrace/hotopen
+OLD_FILES+=usr/share/dtrace/nfsclienttime
+OLD_FILES+=usr/share/dtrace/toolkit/execsnoop
+OLD_FILES+=usr/share/dtrace/toolkit/hotkernel
+OLD_FILES+=usr/share/dtrace/toolkit/hotuser
+OLD_FILES+=usr/share/dtrace/toolkit/opensnoop
+OLD_FILES+=usr/share/dtrace/toolkit/procsystime
+OLD_FILES+=usr/share/man/man1/dtrace.1.gz
+OLD_DIRS+=usr/lib/dtrace
+OLD_DIRS+=usr/lib32/dtrace
+OLD_DIRS+=usr/share/dtrace/toolkit
+OLD_DIRS+=usr/share/dtrace
 .endif
 
 .if ${MK_ZFS} == no

From 9cb82d2e00f3c2376b4a6b508f129ba919143ce4 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 05:30:03 +0000
Subject: [PATCH 266/279] Install bsnmp rc.d script if MK_BSNMP != no

MFC after: 1 week
Sponsored by: EMC / Isilon Storage Division
---
 etc/rc.d/Makefile | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/etc/rc.d/Makefile b/etc/rc.d/Makefile
index 465f3412d95..3cd07110323 100644
--- a/etc/rc.d/Makefile
+++ b/etc/rc.d/Makefile
@@ -27,7 +27,6 @@ FILES=	DAEMON \
 	${_bluetooth} \
 	bootparams \
 	bridge \
-	bsnmpd \
 	${_bthidd} \
 	${_casperd} \
 	cleanvar \
@@ -167,6 +166,10 @@ _hcsecd=	hcsecd
 _ubthidhci=	ubthidhci
 .endif
 
+.if ${MK_BSNMP} != "no"
+FILES+=		bsnmpd
+.endif
+
 .if ${MK_CASPER} != "no"
 _casperd=	casperd
 .endif

From 2186d09cc8f7d00c6bae25c45a234fd5a11f9e7a Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 05:30:45 +0000
Subject: [PATCH 267/279] Fill in entries for MK_BSNMP == no

MFC after: 1 week
Sponsored by: EMC / Isilon Storage Division
---
 tools/build/mk/OptionalObsoleteFiles.inc | 93 ++++++++++++++++++++++++
 1 file changed, 93 insertions(+)

diff --git a/tools/build/mk/OptionalObsoleteFiles.inc b/tools/build/mk/OptionalObsoleteFiles.inc
index 3c4b53a9024..77f4dd897ff 100644
--- a/tools/build/mk/OptionalObsoleteFiles.inc
+++ b/tools/build/mk/OptionalObsoleteFiles.inc
@@ -307,6 +307,99 @@ OLD_FILES+=usr/share/man/man8/sade.8.gz
 OLD_DIRS+=usr/libexec/bsdinstall
 .endif
 
+.if ${MK_BSNMP} == no
+OLD_FILES+=etc/snmpd.config
+OLD_FILES+=etc/rc.d/bsnmpd
+OLD_FILES+=usr/bin/bsnmpget
+OLD_FILES+=usr/bin/bsnmpset
+OLD_FILES+=usr/bin/bsnmpwalk
+OLD_FILES+=usr/include/bsnmp/asn1.h
+OLD_FILES+=usr/include/bsnmp/bridge_snmp.h
+OLD_FILES+=usr/include/bsnmp/snmp.h
+OLD_FILES+=usr/include/bsnmp/snmp_atm.h
+OLD_FILES+=usr/include/bsnmp/snmp_mibII.h
+OLD_FILES+=usr/include/bsnmp/snmp_netgraph.h
+OLD_FILES+=usr/include/bsnmp/snmpagent.h
+OLD_FILES+=usr/include/bsnmp/snmpclient.h
+OLD_FILES+=usr/include/bsnmp/snmpmod.h
+OLD_FILES+=usr/lib/libbsnmp.a
+OLD_LIBS+=usr/lib/libbsnmp.so.6
+OLD_FILES+=usr/lib/libbsnmp_p.a
+OLD_FILES+=usr/lib/libbsnmptools.a
+OLD_LIBS+=usr/lib/libbsnmptools.so.0
+OLD_FILES+=usr/lib/libbsnmptools_p.a
+OLD_LIBS+=usr/lib/snmp_atm.so.6
+OLD_LIBS+=usr/lib/snmp_bridge.so.6
+OLD_LIBS+=usr/lib/snmp_hast.so.6
+OLD_LIBS+=usr/lib/snmp_hostres.so.6
+OLD_LIBS+=usr/lib/snmp_lm75.so.6
+OLD_LIBS+=usr/lib/snmp_mibII.so.6
+OLD_LIBS+=usr/lib/snmp_netgraph.so.6
+OLD_LIBS+=usr/lib/snmp_pf.so.6
+OLD_LIBS+=usr/lib/snmp_target.so.6
+OLD_LIBS+=usr/lib/snmp_usm.so.6
+OLD_LIBS+=usr/lib/snmp_vacm.so.6
+OLD_LIBS+=usr/lib/snmp_wlan.so.6
+OLD_FILES+=usr/sbin/bsnmpd
+OLD_FILES+=usr/sbin/gensnmptree
+OLD_FILES+=usr/share/man/man1/bsnmpd.1.gz
+OLD_FILES+=usr/share/man/man1/bsnmpget.1.gz
+OLD_FILES+=usr/share/man/man1/bsnmpset.1.gz
+OLD_FILES+=usr/share/man/man1/bsnmpwalk.1.gz
+OLD_FILES+=usr/share/man/man1/gensnmptree.1.gz
+OLD_FILES+=usr/share/man/man3/asn1.3.gz
+OLD_FILES+=usr/share/man/man3/bsnmpagent.3.gz
+OLD_FILES+=usr/share/man/man3/bsnmpclient.3.gz
+OLD_FILES+=usr/share/man/man3/bsnmplib.3.gz
+OLD_FILES+=usr/share/man/man3/snmp_atm.3.gz
+OLD_FILES+=usr/share/man/man3/snmp_bridge.3.gz
+OLD_FILES+=usr/share/man/man3/snmp_hast.3.gz
+OLD_FILES+=usr/share/man/man3/snmp_hostres.3.gz
+OLD_FILES+=usr/share/man/man3/snmp_lm75.3.gz
+OLD_FILES+=usr/share/man/man3/snmp_mibII.3.gz
+OLD_FILES+=usr/share/man/man3/snmp_netgraph.3.gz
+OLD_FILES+=usr/share/man/man3/snmp_target.3.gz
+OLD_FILES+=usr/share/man/man3/snmp_usm.3.gz
+OLD_FILES+=usr/share/man/man3/snmp_vacm.3.gz
+OLD_FILES+=usr/share/man/man3/snmp_wlan.3.gz
+OLD_FILES+=usr/share/man/man3/snmpmod.3.gz
+OLD_FILES+=usr/share/snmp/defs/atm_freebsd.def
+OLD_FILES+=usr/share/snmp/defs/atm_tree.def
+OLD_FILES+=usr/share/snmp/defs/bridge_tree.def
+OLD_FILES+=usr/share/snmp/defs/hast_tree.def
+OLD_FILES+=usr/share/snmp/defs/hostres_tree.def
+OLD_FILES+=usr/share/snmp/defs/lm75_tree.def
+OLD_FILES+=usr/share/snmp/defs/mibII_tree.def
+OLD_FILES+=usr/share/snmp/defs/netgraph_tree.def
+OLD_FILES+=usr/share/snmp/defs/pf_tree.def
+OLD_FILES+=usr/share/snmp/defs/target_tree.def
+OLD_FILES+=usr/share/snmp/defs/tree.def
+OLD_FILES+=usr/share/snmp/defs/usm_tree.def
+OLD_FILES+=usr/share/snmp/defs/vacm_tree.def
+OLD_FILES+=usr/share/snmp/defs/wlan_tree.def
+OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-ATM-FREEBSD-MIB.txt
+OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-ATM.txt
+OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-BRIDGE-MIB.txt
+OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-HAST-MIB.txt
+OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-HOSTRES-MIB.txt
+OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-IP-MIB.txt
+OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-LM75-MIB.txt
+OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-MIB.txt
+OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-MIB2-MIB.txt
+OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-NETGRAPH.txt
+OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-PF-MIB.txt
+OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-SNMPD.txt
+OLD_FILES+=usr/share/snmp/mibs/BEGEMOT-WIRELESS-MIB.txt
+OLD_FILES+=usr/share/snmp/mibs/BRIDGE-MIB.txt
+OLD_FILES+=usr/share/snmp/mibs/FOKUS-MIB.txt
+OLD_FILES+=usr/share/snmp/mibs/FREEBSD-MIB.txt
+OLD_FILES+=usr/share/snmp/mibs/RSTP-MIB.txt
+OLD_DIRS+=usr/include/bsnmp
+OLD_DIRS+=usr/share/snmp
+OLD_DIRS+=usr/share/snmp/defs
+OLD_DIRS+=usr/share/snmp/mibs
+.endif
+
 .if ${MK_CALENDAR} == no
 OLD_FILES+=etc/periodic/daily/300.calendar
 OLD_FILES+=usr/bin/calendar

From 3b09bf5e55207926e3072b96ffca3b551a26b613 Mon Sep 17 00:00:00 2001
From: Enji Cooper <ngie@FreeBSD.org>
Date: Sun, 25 Jan 2015 05:37:06 +0000
Subject: [PATCH 268/279] Build lib/libgpio if MK_GPIO != no

Fill in corresponding entries for MK_GPIO == no in OptionalObsoleteFiles.inc

MFC after: 2 weeks
Sponsored by: EMC / Isilon Storage Division
---
 lib/Makefile                             |  6 +++++-
 tools/build/mk/OptionalObsoleteFiles.inc | 25 ++++++++++++++++++++++++
 2 files changed, 30 insertions(+), 1 deletion(-)

diff --git a/lib/Makefile b/lib/Makefile
index c51ae96b1c3..83d9d636ee9 100644
--- a/lib/Makefile
+++ b/lib/Makefile
@@ -53,7 +53,7 @@ SUBDIR=	${SUBDIR_ORDERED} \
 	libfetch \
 	libfigpar \
 	libgeom \
-	libgpio \
+	${_libgpio} \
 	${_libgssapi} \
 	${_librpcsec_gss} \
 	${_libiconv_modules} \
@@ -195,6 +195,10 @@ _cuse=		libcuse
 _libelftc=	libelftc
 .endif
 
+.if ${MK_GPIO} != "no"
+_libgpio=	libgpio
+.endif
+
 .if ${MK_GSSAPI} != "no"
 _libgssapi=	libgssapi
 _librpcsec_gss=	librpcsec_gss
diff --git a/tools/build/mk/OptionalObsoleteFiles.inc b/tools/build/mk/OptionalObsoleteFiles.inc
index 77f4dd897ff..ecd471199fd 100644
--- a/tools/build/mk/OptionalObsoleteFiles.inc
+++ b/tools/build/mk/OptionalObsoleteFiles.inc
@@ -1550,7 +1550,32 @@ OLD_FILES+=usr/share/man/man1/kgdb.1.gz
 .endif
 
 .if ${MK_GPIO} == no
+OLD_FILES+=usr/include/libgpio.h
+OLD_FILES+=usr/lib/libgpio.a
+OLD_LIBS+=usr/lib/libgpio.so.0
+OLD_FILES+=usr/lib/libgpio_p.a
 OLD_FILES+=usr/sbin/gpioctl
+OLD_FILES+=usr/share/man/man3/gpio.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_close.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_open.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_open_device.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_config.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_get.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_high.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_input.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_invin.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_invout.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_list.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_low.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_opendrain.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_output.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_pulldown.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_pullup.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_pulsate.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_pushpull.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_set.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_set_flags.3.gz
+OLD_FILES+=usr/share/man/man3/gpio_pin_tristate.3.gz
 OLD_FILES+=usr/share/man/man8/gpioctl.8.gz
 .endif
 

From 12877d0bbd5d88084442235ffa45b21a47c86874 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dag-Erling=20Sm=C3=B8rgrav?= <des@FreeBSD.org>
Date: Sun, 25 Jan 2015 11:57:18 +0000
Subject: [PATCH 269/279] Fix the font in the text version.  This has bothered
 me for a long time...

MFC after:	1 week
---
 sys/boot/forth/beastie.4th | 11 +++++------
 sys/boot/forth/brand.4th   |  3 +--
 2 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/sys/boot/forth/beastie.4th b/sys/boot/forth/beastie.4th
index 8d2424408bb..6512d739c27 100644
--- a/sys/boot/forth/beastie.4th
+++ b/sys/boot/forth/beastie.4th
@@ -89,7 +89,7 @@ variable logoY
 	0 25 at-xy
 ;
 
-: fbsdbw-logo ( x y -- ) \ "FreeBSD" logo in B/W (13 rows x 21 columns)
+: fbsdbw-logo ( x y -- ) \ "FreeBSD" logo in B/W (12 rows x 21 columns)
 
 	\ We used to use the beastie himself as our default... until the
 	\ eventual complaint derided his reign of the advanced boot-menu.
@@ -106,17 +106,16 @@ variable logoY
 	5 + swap 6 + swap
 
 	2dup at-xy ."  ______" 1+
-	2dup at-xy ." |  ____| __ ___  ___ " 1+
-	2dup at-xy ." | |__ | '__/ _ \/ _ \" 1+
-	2dup at-xy ." |  __|| | |  __/  __/" 1+
-	2dup at-xy ." | |   | | |    |    |" 1+
+	2dup at-xy ." |  ____|" 1+
+	2dup at-xy ." | |__  _ __ ___  ___ " 1+
+	2dup at-xy ." |  __|| '__/ _ \/ _ \" 1+
+	2dup at-xy ." | |   | | |  __/  __/" 1+
 	2dup at-xy ." |_|   |_|  \___|\___|" 1+
 	2dup at-xy ."  ____   _____ _____" 1+
 	2dup at-xy ." |  _ \ / ____|  __ \" 1+
 	2dup at-xy ." | |_) | (___ | |  | |" 1+
 	2dup at-xy ." |  _ < \___ \| |  | |" 1+
 	2dup at-xy ." | |_) |____) | |__| |" 1+
-	2dup at-xy ." |     |      |      |" 1+
 	     at-xy ." |____/|_____/|_____/"
 
 	\ Put the cursor back at the bottom
diff --git a/sys/boot/forth/brand.4th b/sys/boot/forth/brand.4th
index 28d3c5c1d26..3dda97cc3e2 100644
--- a/sys/boot/forth/brand.4th
+++ b/sys/boot/forth/brand.4th
@@ -33,14 +33,13 @@ variable brandY
 2 brandX !
 1 brandY !
 
-: fbsd-logo ( x y -- ) \ "FreeBSD" [wide] logo in B/W (7 rows x 42 columns)
+: fbsd-logo ( x y -- ) \ "FreeBSD" [wide] logo in B/W (6 rows x 42 columns)
 
 	2dup at-xy ."  ______               ____   _____ _____  " 1+
 	2dup at-xy ." |  ____|             |  _ \ / ____|  __ \ " 1+
 	2dup at-xy ." | |___ _ __ ___  ___ | |_) | (___ | |  | |" 1+
 	2dup at-xy ." |  ___| '__/ _ \/ _ \|  _ < \___ \| |  | |" 1+
 	2dup at-xy ." | |   | | |  __/  __/| |_) |____) | |__| |" 1+
-	2dup at-xy ." | |   | | |    |    ||     |      |      |" 1+
 	     at-xy ." |_|   |_|  \___|\___||____/|_____/|_____/ "
 
 	\ Put the cursor back at the bottom

From d4ff1726f97fcb2bd594313e599de96592d66614 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dag-Erling=20Sm=C3=B8rgrav?= <des@FreeBSD.org>
Date: Sun, 25 Jan 2015 12:02:38 +0000
Subject: [PATCH 270/279] Remove ISA NICs.  Anyone still using these on amd64
 can build their own kernel.

---
 sys/amd64/conf/GENERIC | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/sys/amd64/conf/GENERIC b/sys/amd64/conf/GENERIC
index 370ab10bba9..86891b7507f 100644
--- a/sys/amd64/conf/GENERIC
+++ b/sys/amd64/conf/GENERIC
@@ -260,16 +260,6 @@ device		vr			# VIA Rhine, Rhine II
 device		wb			# Winbond W89C840F
 device		xl			# 3Com 3c90x (``Boomerang'', ``Cyclone'')
 
-# ISA Ethernet NICs.  pccard NICs included.
-device		cs			# Crystal Semiconductor CS89x0 NIC
-# 'device ed' requires 'device miibus'
-device		ed			# NE[12]000, SMC Ultra, 3c503, DS8390 cards
-device		ex			# Intel EtherExpress Pro/10 and Pro/10+
-device		ep			# Etherlink III based cards
-device		fe			# Fujitsu MB8696x based cards
-device		sn			# SMC's 9000 series of Ethernet chips
-device		xe			# Xircom pccard Ethernet
-
 # Wireless NIC cards
 device		wlan			# 802.11 support
 options 	IEEE80211_DEBUG		# enable debug msgs

From 4ac1e0a9fcd00850e993635f146519ab766d2456 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dag-Erling=20Sm=C3=B8rgrav?= <des@FreeBSD.org>
Date: Sun, 25 Jan 2015 12:11:50 +0000
Subject: [PATCH 271/279] Allow tracing dlfunc() / dlsym() events.

MFC after:	1 week
---
 libexec/rtld-elf/rtld.c | 17 +++++++++++++----
 usr.bin/kdump/kdump.c   |  9 +++++++++
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/libexec/rtld-elf/rtld.c b/libexec/rtld-elf/rtld.c
index 5f449c138e1..7b19f5aa65c 100644
--- a/libexec/rtld-elf/rtld.c
+++ b/libexec/rtld-elf/rtld.c
@@ -266,6 +266,8 @@ bool ld_library_path_rpath = false;
 #define	UTRACE_PRELOAD_FINISHED		8
 #define	UTRACE_INIT_CALL		9
 #define	UTRACE_FINI_CALL		10
+#define	UTRACE_DLSYM_START		11
+#define	UTRACE_DLSYM_STOP		12
 
 struct utrace_rtld {
 	char sig[4];			/* 'RTLD' */
@@ -3099,6 +3101,7 @@ do_dlsym(void *handle, const char *name, void *retaddr, const Ver_Entry *ve,
     SymLook req;
     RtldLockState lockstate;
     tls_index ti;
+    void *sym;
     int res;
 
     def = NULL;
@@ -3108,6 +3111,7 @@ do_dlsym(void *handle, const char *name, void *retaddr, const Ver_Entry *ve,
     req.flags = flags | SYMLOOK_IN_PLT;
     req.lockstate = &lockstate;
 
+    LD_UTRACE(UTRACE_DLSYM_START, handle, NULL, 0, 0, name);
     rlock_acquire(rtld_bind_lock, &lockstate);
     if (sigsetjmp(lockstate.env, 0) != 0)
 	    lock_upgrade(rtld_bind_lock, &lockstate);
@@ -3117,6 +3121,7 @@ do_dlsym(void *handle, const char *name, void *retaddr, const Ver_Entry *ve,
 	if ((obj = obj_from_addr(retaddr)) == NULL) {
 	    _rtld_error("Cannot determine caller's shared object");
 	    lock_release(rtld_bind_lock, &lockstate);
+	    LD_UTRACE(UTRACE_DLSYM_STOP, handle, NULL, 0, 0, name);
 	    return NULL;
 	}
 	if (handle == NULL) {	/* Just the caller's shared object. */
@@ -3164,6 +3169,7 @@ do_dlsym(void *handle, const char *name, void *retaddr, const Ver_Entry *ve,
     } else {
 	if ((obj = dlcheck(handle)) == NULL) {
 	    lock_release(rtld_bind_lock, &lockstate);
+	    LD_UTRACE(UTRACE_DLSYM_STOP, handle, NULL, 0, 0, name);
 	    return NULL;
 	}
 
@@ -3207,19 +3213,22 @@ do_dlsym(void *handle, const char *name, void *retaddr, const Ver_Entry *ve,
 	 * symbol.
 	 */
 	if (ELF_ST_TYPE(def->st_info) == STT_FUNC)
-	    return (make_function_pointer(def, defobj));
+	    sym = make_function_pointer(def, defobj);
 	else if (ELF_ST_TYPE(def->st_info) == STT_GNU_IFUNC)
-	    return (rtld_resolve_ifunc(defobj, def));
+	    sym = rtld_resolve_ifunc(defobj, def);
 	else if (ELF_ST_TYPE(def->st_info) == STT_TLS) {
 	    ti.ti_module = defobj->tlsindex;
 	    ti.ti_offset = def->st_value;
-	    return (__tls_get_addr(&ti));
+	    sym = __tls_get_addr(&ti);
 	} else
-	    return (defobj->relocbase + def->st_value);
+	    sym = defobj->relocbase + def->st_value;
+	LD_UTRACE(UTRACE_DLSYM_STOP, handle, sym, 0, 0, name);
+	return (sym);
     }
 
     _rtld_error("Undefined symbol \"%s\"", name);
     lock_release(rtld_bind_lock, &lockstate);
+    LD_UTRACE(UTRACE_DLSYM_STOP, handle, NULL, 0, 0, name);
     return NULL;
 }
 
diff --git a/usr.bin/kdump/kdump.c b/usr.bin/kdump/kdump.c
index 0daf7373cd1..b8e5903a0c8 100644
--- a/usr.bin/kdump/kdump.c
+++ b/usr.bin/kdump/kdump.c
@@ -1531,6 +1531,8 @@ ktrcsw(struct ktr_csw *cs)
 #define	UTRACE_PRELOAD_FINISHED		8
 #define	UTRACE_INIT_CALL		9
 #define	UTRACE_FINI_CALL		10
+#define	UTRACE_DLSYM_START		11
+#define	UTRACE_DLSYM_STOP		12
 
 struct utrace_rtld {
 	char sig[4];				/* 'RTLD' */
@@ -1610,6 +1612,13 @@ ktruser_rtld(int len, void *p)
 		printf("RTLD: fini %p for %p (%s)\n", ut->mapbase, ut->handle,
 		    ut->name);
 		break;
+	case UTRACE_DLSYM_START:
+		printf("RTLD: dlsym(%p, %s)\n", ut->handle, ut->name);
+		break;
+	case UTRACE_DLSYM_STOP:
+		printf("RTLD: %p = dlsym(%p, %s)\n", ut->mapbase, ut->handle,
+		    ut->name);
+		break;
 	default:
 		cp = p;
 		cp += 4;

From 4fa545cb14af3c7af3839cbc2feb2fc4eb706c8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Dag-Erling=20Sm=C3=B8rgrav?= <des@FreeBSD.org>
Date: Sun, 25 Jan 2015 15:44:46 +0000
Subject: [PATCH 272/279] Allow the user to specify the location of
 control.conf.

---
 etc/rc.d/local_unbound | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/etc/rc.d/local_unbound b/etc/rc.d/local_unbound
index ed69c193d88..d8eef7dcba5 100755
--- a/etc/rc.d/local_unbound
+++ b/etc/rc.d/local_unbound
@@ -26,6 +26,7 @@ pidfile="/var/run/${name}.pid"
 : ${local_unbound_config:=${local_unbound_workdir}/unbound.conf}
 : ${local_unbound_flags:=-c${local_unbound_config}}
 : ${local_unbound_forwardconf:=${local_unbound_workdir}/forward.conf}
+: ${local_unbound_controlconf:=${local_unbound_workdir}/control.conf}
 : ${local_unbound_anchor:=${local_unbound_workdir}/root.key}
 : ${local_unbound_forwarders:=}
 
@@ -66,6 +67,7 @@ local_unbound_setup()
 	    -w ${local_unbound_workdir} \
 	    -c ${local_unbound_config} \
 	    -f ${local_unbound_forwardconf} \
+	    -o ${local_unbound_controlconf} \
 	    -a ${local_unbound_anchor} \
 	    ${local_unbound_forwarders}
 }

From 002d455873f5f94749ba3b60c47cdb0ecf5e48c6 Mon Sep 17 00:00:00 2001
From: John Baldwin <jhb@FreeBSD.org>
Date: Sun, 25 Jan 2015 19:45:44 +0000
Subject: [PATCH 273/279] Use an sbuf to generate the output of the
 net.inet.tcp.hostcache.list sysctl to avoid a possible buffer overflow if the
 cache grows while the text is being generated.

PR:		172675
MFC after:	2 weeks
---
 sys/netinet/tcp_hostcache.c | 24 +++++++++++-------------
 1 file changed, 11 insertions(+), 13 deletions(-)

diff --git a/sys/netinet/tcp_hostcache.c b/sys/netinet/tcp_hostcache.c
index cbe24985a70..b03556ff67e 100644
--- a/sys/netinet/tcp_hostcache.c
+++ b/sys/netinet/tcp_hostcache.c
@@ -73,6 +73,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/malloc.h>
+#include <sys/sbuf.h>
 #include <sys/socket.h>
 #include <sys/socketvar.h>
 #include <sys/sysctl.h>
@@ -595,30 +596,27 @@ tcp_hc_update(struct in_conninfo *inc, struct hc_metrics_lite *hcml)
 static int
 sysctl_tcp_hc_list(SYSCTL_HANDLER_ARGS)
 {
-	int bufsize;
 	int linesize = 128;
-	char *p, *buf;
-	int len, i, error;
+	struct sbuf sb;
+	int i, error;
 	struct hc_metrics *hc_entry;
 #ifdef INET6
 	char ip6buf[INET6_ADDRSTRLEN];
 #endif
 
-	bufsize = linesize * (V_tcp_hostcache.cache_count + 1);
+	sbuf_new(&sb, NULL, linesize * (V_tcp_hostcache.cache_count + 1),
+	    SBUF_FIXEDLEN);
 
-	p = buf = (char *)malloc(bufsize, M_TEMP, M_WAITOK|M_ZERO);
-
-	len = snprintf(p, linesize,
-		"\nIP address        MTU  SSTRESH      RTT   RTTVAR BANDWIDTH "
+	sbuf_printf(&sb,
+	        "\nIP address        MTU  SSTRESH      RTT   RTTVAR BANDWIDTH "
 		"    CWND SENDPIPE RECVPIPE HITS  UPD  EXP\n");
-	p += len;
 
 #define msec(u) (((u) + 500) / 1000)
 	for (i = 0; i < V_tcp_hostcache.hashsize; i++) {
 		THC_LOCK(&V_tcp_hostcache.hashbase[i].hch_mtx);
 		TAILQ_FOREACH(hc_entry, &V_tcp_hostcache.hashbase[i].hch_bucket,
 			      rmx_q) {
-			len = snprintf(p, linesize,
+			sbuf_printf(&sb,
 			    "%-15s %5lu %8lu %6lums %6lums %9lu %8lu %8lu %8lu "
 			    "%4lu %4lu %4i\n",
 			    hc_entry->ip4.s_addr ? inet_ntoa(hc_entry->ip4) :
@@ -640,13 +638,13 @@ sysctl_tcp_hc_list(SYSCTL_HANDLER_ARGS)
 			    hc_entry->rmx_hits,
 			    hc_entry->rmx_updates,
 			    hc_entry->rmx_expire);
-			p += len;
 		}
 		THC_UNLOCK(&V_tcp_hostcache.hashbase[i].hch_mtx);
 	}
 #undef msec
-	error = SYSCTL_OUT(req, buf, p - buf);
-	free(buf, M_TEMP);
+	sbuf_finish(&sb);
+	error = SYSCTL_OUT(req, sbuf_data(&sb), sbuf_len(&sb));
+	sbuf_delete(&sb);
 	return(error);
 }
 

From 0e521c9a260716057045c35e341ae4e2605fe95c Mon Sep 17 00:00:00 2001
From: John Baldwin <jhb@FreeBSD.org>
Date: Sun, 25 Jan 2015 19:53:09 +0000
Subject: [PATCH 274/279] Pass a valid Dx state variable to
 PCIB_POWER_FOR_SLEEP() in pcib_resume() instead of NULL.

Submitted by:	dchagin
MFC after:	2 weeks
---
 sys/dev/pci/pci_pci.c | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/sys/dev/pci/pci_pci.c b/sys/dev/pci/pci_pci.c
index f35b481a1ab..c7029f28f0f 100644
--- a/sys/dev/pci/pci_pci.c
+++ b/sys/dev/pci/pci_pci.c
@@ -1114,11 +1114,13 @@ int
 pcib_resume(device_t dev)
 {
 	device_t	pcib;
+	int dstate;
 
 	if (pci_do_power_resume) {
 		pcib = device_get_parent(device_get_parent(dev));
-		if (PCIB_POWER_FOR_SLEEP(pcib, dev, NULL) == 0)
-			pci_set_powerstate(dev, PCI_POWERSTATE_D0);
+		dstate = PCI_POWERSTATE_D0;
+		if (PCIB_POWER_FOR_SLEEP(pcib, dev, &dstate) == 0)
+			pci_set_powerstate(dev, dstate);
 	}
 	pcib_cfg_restore(device_get_softc(dev));
 	return (bus_generic_resume(dev));

From a77a12340a51cf904274fad32d72f1e3d079263d Mon Sep 17 00:00:00 2001
From: John Baldwin <jhb@FreeBSD.org>
Date: Sun, 25 Jan 2015 19:56:45 +0000
Subject: [PATCH 275/279] Change the default VFS timestamp precision from
 seconds to microseconds.

Discussed on:	arch@
MFC after:	2 weeks
---
 sys/kern/vfs_subr.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index 29205236852..912863e6ddf 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -632,7 +632,7 @@ vfs_getnewfsid(struct mount *mp)
  */
 enum { TSP_SEC, TSP_HZ, TSP_USEC, TSP_NSEC };
 
-static int timestamp_precision = TSP_SEC;
+static int timestamp_precision = TSP_USEC;
 SYSCTL_INT(_vfs, OID_AUTO, timestamp_precision, CTLFLAG_RW,
     &timestamp_precision, 0, "File timestamp precision (0: seconds, "
     "1: sec + ns accurate to 1/HZ, 2: sec + ns truncated to ms, "

From 8d44440ad95651c362a4122e5a4e00e3e515a461 Mon Sep 17 00:00:00 2001
From: John Baldwin <jhb@FreeBSD.org>
Date: Sun, 25 Jan 2015 20:16:45 +0000
Subject: [PATCH 276/279] If the boot-time memory test is enabled, output a dot
 ('.') for each GB of RAM tested so people watching the console can see that
 the machine is making progress and not hung.

PR:		196650
Submitted by:	Ravi Pokala <rpokala@panasas.com>
Suggestions from:	Eric van Gyzen <eric@vangyzen.net>
MFC after:	2 weeks
---
 sys/amd64/amd64/machdep.c | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c
index bcfdac143b4..07378fddce3 100644
--- a/sys/amd64/amd64/machdep.c
+++ b/sys/amd64/amd64/machdep.c
@@ -1557,6 +1557,8 @@ native_parse_memmap(caddr_t kmdp, vm_paddr_t *physmap, int *physmap_idx)
 	}
 }
 
+#define	PAGES_PER_GB	(1024 * 1024 * 1024 / PAGE_SIZE)
+
 /*
  * Populate the (physmap) array with base/bound pairs describing the
  * available physical memory in the system, then test this memory and
@@ -1575,6 +1577,7 @@ getmemsize(caddr_t kmdp, u_int64_t first)
 	u_long physmem_start, physmem_tunable, memtest;
 	pt_entry_t *pte;
 	quad_t dcons_addr, dcons_size;
+	int page_counter;
 
 	bzero(physmap, sizeof(physmap));
 	basemem = 0;
@@ -1678,6 +1681,9 @@ getmemsize(caddr_t kmdp, u_int64_t first)
 	 * physmap is in bytes, so when converting to page boundaries,
 	 * round up the start address and round down the end address.
 	 */
+	page_counter = 0;
+	if (memtest != 0)
+		printf("Testing system memory");
 	for (i = 0; i <= physmap_idx; i += 2) {
 		vm_paddr_t end;
 
@@ -1707,6 +1713,14 @@ getmemsize(caddr_t kmdp, u_int64_t first)
 			if (memtest == 0)
 				goto skip_memtest;
 
+			/*
+			 * Print a "." every GB to show we're making
+			 * progress.
+			 */
+			page_counter++;
+			if ((page_counter % PAGES_PER_GB) == 0)
+				printf(".");
+
 			/*
 			 * map page into kernel: valid, read/write,non-cacheable
 			 */
@@ -1794,6 +1808,8 @@ do_next:
 	}
 	*pte = 0;
 	invltlb();
+	if (memtest != 0)
+		printf("\n");
 
 	/*
 	 * XXX

From 2aa533b91d244a0d6175fa82ef165aff90efbee1 Mon Sep 17 00:00:00 2001
From: John Baldwin <jhb@FreeBSD.org>
Date: Sun, 25 Jan 2015 20:37:32 +0000
Subject: [PATCH 277/279] natd(8) will work with an unconfigured interface and
 effectively not do anything until the interface is assigned an address.  This
 fixes ipfw_nat to do the same by using an IP of INADDR_ANY instead of
 aborting the nat setup if the requested interface is not yet configured.

Differential Revision:	https://reviews.freebsd.org/D1539
Reviewed by:	melifaro, glebius, gnn
MFC after:	1 week
---
 sbin/ipfw/nat.c | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/sbin/ipfw/nat.c b/sbin/ipfw/nat.c
index 3bd0259ad5a..439177875e2 100644
--- a/sbin/ipfw/nat.c
+++ b/sbin/ipfw/nat.c
@@ -163,9 +163,9 @@ set_addr_dynamic(const char *ifn, struct nat44_cfg_nat *n)
 		}
 	}
 	if (sin == NULL)
-		errx(1, "%s: cannot get interface address", ifn);
-
-	n->ip = sin->sin_addr;
+		n->ip.s_addr = htonl(INADDR_ANY);
+	else
+		n->ip = sin->sin_addr;
 	strncpy(n->if_name, ifn, IF_NAMESIZE);
 
 	free(buf);

From 72584f4cbae1d31f29790032ef26862003b33ebb Mon Sep 17 00:00:00 2001
From: Oleksandr Tymoshenko <gonzo@FreeBSD.org>
Date: Sun, 25 Jan 2015 22:08:36 +0000
Subject: [PATCH 278/279] Add vt(4) support to AM335x LCDC driver

---
 sys/arm/ti/am335x/am335x_lcd.c | 51 ++++++++++++++++++++++++++++++++--
 sys/arm/ti/am335x/files.am335x |  2 +-
 2 files changed, 49 insertions(+), 4 deletions(-)

diff --git a/sys/arm/ti/am335x/am335x_lcd.c b/sys/arm/ti/am335x/am335x_lcd.c
index 4fd811cc2ea..053567ec308 100644
--- a/sys/arm/ti/am335x/am335x_lcd.c
+++ b/sys/arm/ti/am335x/am335x_lcd.c
@@ -27,6 +27,7 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
+#include "opt_syscons.h"
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
@@ -41,8 +42,6 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <vm/vm.h>
 #include <vm/pmap.h>
-
-/* syscons bits */
 #include <sys/fbio.h>
 #include <sys/consio.h>
 
@@ -54,7 +53,11 @@ __FBSDID("$FreeBSD$");
 #include <dev/ofw/ofw_bus_subr.h>
 
 #include <dev/fb/fbreg.h>
+#ifdef DEV_SC
 #include <dev/syscons/syscons.h>
+#else /* VT */
+#include <dev/vt/vt.h>
+#endif
 
 #include <arm/ti/ti_prcm.h>
 #include <arm/ti/ti_scm.h>
@@ -62,6 +65,8 @@ __FBSDID("$FreeBSD$");
 #include "am335x_lcd.h"
 #include "am335x_pwm.h"
 
+#include "fb_if.h"
+
 #define	LCD_PID			0x00
 #define	LCD_CTRL		0x04
 #define		CTRL_DIV_MASK		0xff
@@ -178,6 +183,7 @@ __FBSDID("$FreeBSD$");
 
 struct am335x_lcd_softc {
 	device_t		sc_dev;
+	struct fb_info		sc_fb_info;
 	struct resource		*sc_mem_res;
 	struct resource		*sc_irq_res;
 	void			*sc_intr_hl;
@@ -410,7 +416,9 @@ done:
 static int
 am335x_lcd_probe(device_t dev)
 {
+#ifdef DEV_SC
 	int err;
+#endif
 
 	if (!ofw_bus_status_okay(dev))
 		return (ENXIO);
@@ -420,10 +428,12 @@ am335x_lcd_probe(device_t dev)
 
 	device_set_desc(dev, "AM335x LCD controller");
 
+#ifdef DEV_SC
 	err = sc_probe_unit(device_get_unit(dev), 
 	    device_get_flags(dev) | SC_AUTODETECT_KBD);
 	if (err != 0)
 		return (err);
+#endif
 
 	return (BUS_PROBE_DEFAULT);
 }
@@ -670,6 +680,16 @@ am335x_lcd_attach(device_t dev)
 	    PWM_PERIOD, PWM_PERIOD) == 0)
 		sc->sc_backlight = 100;
 
+	sc->sc_fb_info.fb_name = device_get_nameunit(sc->sc_dev);
+	sc->sc_fb_info.fb_vbase = (intptr_t)sc->sc_fb_base;
+	sc->sc_fb_info.fb_pbase = sc->sc_fb_phys;
+	sc->sc_fb_info.fb_size = sc->sc_fb_size;
+	sc->sc_fb_info.fb_bpp = sc->sc_fb_info.fb_depth = panel.bpp;
+	sc->sc_fb_info.fb_stride = panel.panel_width*panel.bpp / 8;
+	sc->sc_fb_info.fb_width = panel.panel_width;
+	sc->sc_fb_info.fb_height = panel.panel_height;
+
+#ifdef	DEV_SC
 	err = (sc_attach_unit(device_get_unit(dev),
 	    device_get_flags(dev) | SC_AUTODETECT_KBD));
 
@@ -679,6 +699,18 @@ am335x_lcd_attach(device_t dev)
 	}
 
 	am335x_lcd_syscons_setup((vm_offset_t)sc->sc_fb_base, sc->sc_fb_phys, &panel);
+#else /* VT */
+	device_t fbd = device_add_child(dev, "fbd",
+	device_get_unit(dev));
+	if (fbd == NULL) {
+		device_printf(dev, "Failed to add fbd child\n");
+		goto fail;
+	}
+	if (device_probe_and_attach(fbd) != 0) {
+		device_printf(dev, "Failed to attach fbd device\n");
+		goto fail;
+	}
+#endif
 
 	return (0);
 
@@ -693,16 +725,29 @@ am335x_lcd_detach(device_t dev)
 	return (EBUSY);
 }
 
+static struct fb_info *
+am335x_lcd_fb_getinfo(device_t dev)
+{
+	struct am335x_lcd_softc *sc;
+
+	sc = device_get_softc(dev);
+
+	return (&sc->sc_fb_info);
+}
+
 static device_method_t am335x_lcd_methods[] = {
 	DEVMETHOD(device_probe,		am335x_lcd_probe),
 	DEVMETHOD(device_attach,	am335x_lcd_attach),
 	DEVMETHOD(device_detach,	am335x_lcd_detach),
 
+	/* Framebuffer service methods */
+	DEVMETHOD(fb_getinfo,		am335x_lcd_fb_getinfo),
+
 	DEVMETHOD_END
 };
 
 static driver_t am335x_lcd_driver = {
-	"am335x_lcd",
+	"fb",
 	am335x_lcd_methods,
 	sizeof(struct am335x_lcd_softc),
 };
diff --git a/sys/arm/ti/am335x/files.am335x b/sys/arm/ti/am335x/files.am335x
index 465a7fc9dea..aee414fc7b8 100644
--- a/sys/arm/ti/am335x/files.am335x
+++ b/sys/arm/ti/am335x/files.am335x
@@ -4,7 +4,7 @@ arm/ti/aintc.c				standard
 
 arm/ti/am335x/am335x_dmtimer.c		standard
 arm/ti/am335x/am335x_gpio.c		optional	gpio
-arm/ti/am335x/am335x_lcd.c		optional	sc
+arm/ti/am335x/am335x_lcd.c		optional	sc | vt
 arm/ti/am335x/am335x_lcd_syscons.c	optional	sc
 arm/ti/am335x/am335x_pmic.c		optional	am335x_pmic
 arm/ti/am335x/am335x_prcm.c		standard

From fcafcbcb6fd14c0707ca8c01079d992361474a53 Mon Sep 17 00:00:00 2001
From: Scott Long <scottl@FreeBSD.org>
Date: Sun, 25 Jan 2015 22:29:23 +0000
Subject: [PATCH 279/279] Fix the ioctl interface to properly support fetching
 the header of regular and extended config pages.

Obtained from:	Netflix, Inc.
MFC after:	3 days
---
 sys/dev/mps/mps.c      | 14 ++++++++++----
 sys/dev/mps/mps_user.c |  4 ++++
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/sys/dev/mps/mps.c b/sys/dev/mps/mps.c
index 20c7f75308d..0957602ccc1 100644
--- a/sys/dev/mps/mps.c
+++ b/sys/dev/mps/mps.c
@@ -2621,9 +2621,12 @@ mps_read_config_page(struct mps_softc *sc, struct mps_config_params *params)
 
 	cm->cm_data = params->buffer;
 	cm->cm_length = params->length;
-	cm->cm_sge = &req->PageBufferSGE;
-	cm->cm_sglsize = sizeof(MPI2_SGE_IO_UNION);
-	cm->cm_flags = MPS_CM_FLAGS_SGE_SIMPLE | MPS_CM_FLAGS_DATAIN;
+	if (cm->cm_data != NULL) {
+		cm->cm_sge = &req->PageBufferSGE;
+		cm->cm_sglsize = sizeof(MPI2_SGE_IO_UNION);
+		cm->cm_flags = MPS_CM_FLAGS_SGE_SIMPLE | MPS_CM_FLAGS_DATAIN;
+	} else
+		cm->cm_sge = NULL;
 	cm->cm_desc.Default.RequestFlags = MPI2_REQ_DESCRIPT_FLAGS_DEFAULT_TYPE;
 
 	cm->cm_complete_data = params;
@@ -2680,9 +2683,12 @@ mps_config_complete(struct mps_softc *sc, struct mps_command *cm)
 		goto done;
 	}
 	params->status = reply->IOCStatus;
-	if (params->hdr.Ext.ExtPageType != 0) {
+	if (params->hdr.Struct.PageType == MPI2_CONFIG_PAGETYPE_EXTENDED) {
 		params->hdr.Ext.ExtPageType = reply->ExtPageType;
 		params->hdr.Ext.ExtPageLength = reply->ExtPageLength;
+		params->hdr.Ext.PageType = reply->Header.PageType;
+		params->hdr.Ext.PageNumber = reply->Header.PageNumber;
+		params->hdr.Ext.PageVersion = reply->Header.PageVersion;
 	} else {
 		params->hdr.Struct.PageType = reply->Header.PageType;
 		params->hdr.Struct.PageNumber = reply->Header.PageNumber;
diff --git a/sys/dev/mps/mps_user.c b/sys/dev/mps/mps_user.c
index 16ec4e4e623..63f78b6ccd4 100644
--- a/sys/dev/mps/mps_user.c
+++ b/sys/dev/mps/mps_user.c
@@ -309,6 +309,10 @@ mps_user_read_extcfg_header(struct mps_softc *sc,
 	hdr->PageNumber = ext_page_req->header.PageNumber;
 	hdr->ExtPageType = ext_page_req->header.ExtPageType;
 	params.page_address = le32toh(ext_page_req->page_address);
+	params.buffer = NULL;
+	params.length = 0;
+	params.callback = NULL;
+
 	if ((error = mps_read_config_page(sc, &params)) != 0) {
 		/*
 		 * Leave the request. Without resetting the chip, it's