From 6e0efc6a3979a5a6f784ebe60bb67df21d337eed Mon Sep 17 00:00:00 2001 From: Randall Stewart Date: Tue, 23 Feb 2016 17:53:39 +0000 Subject: [PATCH 01/25] This fixes the fastpath code to have a better module initialization sequence when included in loader.conf. It also fixes it so that no matter if some one incorrectly specifies a load order, the lists and such will be initialized on demand at that time so no one can make that mistake. Reviewed by: hiren Differential Revision: D5189 --- sys/netinet/tcp_stacks/fastpath.c | 2 +- sys/netinet/tcp_subr.c | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/sys/netinet/tcp_stacks/fastpath.c b/sys/netinet/tcp_stacks/fastpath.c index 6af9212b22c..2cf7895d4e1 100644 --- a/sys/netinet/tcp_stacks/fastpath.c +++ b/sys/netinet/tcp_stacks/fastpath.c @@ -2453,4 +2453,4 @@ static moduledata_t new_tcp_fastpaths = { }; MODULE_VERSION(kern_tcpfastpaths, 1); -DECLARE_MODULE(kern_tcpfastpaths, new_tcp_fastpaths, SI_SUB_PSEUDO, SI_ORDER_ANY); +DECLARE_MODULE(kern_tcpfastpaths, new_tcp_fastpaths, SI_SUB_PROTO_DOMAIN, SI_ORDER_ANY); diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c index e6fdb7df6b6..2b0571cea4c 100644 --- a/sys/netinet/tcp_subr.c +++ b/sys/netinet/tcp_subr.c @@ -263,9 +263,20 @@ static struct tcp_function_block tcp_def_funcblk = { 0 }; +int t_functions_inited = 0; struct tcp_funchead t_functions; static struct tcp_function_block *tcp_func_set_ptr = &tcp_def_funcblk; +static void +init_tcp_functions() +{ + if (t_functions_inited == 0) { + TAILQ_INIT(&t_functions); + rw_init_flags(&tcp_function_lock, "tcp_func_lock" , 0); + t_functions_inited = 1; + } +} + static struct tcp_function_block * find_tcp_functions_locked(struct tcp_function_set *fs) { @@ -503,6 +514,9 @@ register_tcp_functions(struct tcp_function_block *blk, int wait) struct tcp_function *n; struct tcp_function_set fs; + if (t_functions_inited == 0) { + init_tcp_functions(); + } if ((blk->tfb_tcp_output == NULL) || (blk->tfb_tcp_do_segment == NULL) || (blk->tfb_tcp_ctloutput == NULL) || @@ -681,8 +695,7 @@ tcp_init(void) tcp_finwait2_timeout = TCPTV_FINWAIT2_TIMEOUT; tcp_tcbhashsize = hashsize; /* Setup the tcp function block list */ - TAILQ_INIT(&t_functions); - rw_init_flags(&tcp_function_lock, "tcp_func_lock" , 0); + init_tcp_functions(); register_tcp_functions(&tcp_def_funcblk, M_WAITOK); if (tcp_soreceive_stream) { From b265897c37449d3d7cd99f909b97ec44616c1722 Mon Sep 17 00:00:00 2001 From: Hans Petter Selasky Date: Tue, 23 Feb 2016 18:17:01 +0000 Subject: [PATCH 02/25] Configure the correct bMaxPacketSize for control endpoints before requesting the initial complete device descriptor and not as part of the subsequent babble error recovery. Babble means that the received USB packet was bigger than than configured maximum packet size. This only affects enumeration of FULL speed USB devices which use a bMaxPacketSize different from 8 bytes. This patch might help fix enumeration of USB devices which exhibit USB I/O errors in dmesg during boot. MFC after: 1 week --- sys/dev/usb/controller/xhci.c | 13 +++++++++++++ sys/dev/usb/controller/xhci.h | 1 + 2 files changed, 14 insertions(+) diff --git a/sys/dev/usb/controller/xhci.c b/sys/dev/usb/controller/xhci.c index 1c1d2b0ee3d..0739e2865e3 100644 --- a/sys/dev/usb/controller/xhci.c +++ b/sys/dev/usb/controller/xhci.c @@ -2358,6 +2358,8 @@ xhci_configure_endpoint(struct usb_device *udev, /* store endpoint mode */ pepext->trb_ep_mode = ep_mode; + /* store bMaxPacketSize for control endpoints */ + pepext->trb_ep_maxp = edesc->wMaxPacketSize[0]; usb_pc_cpu_flush(pepext->page_cache); if (ep_mode == USB_EP_MODE_STREAMS) { @@ -2904,6 +2906,17 @@ xhci_transfer_insert(struct usb_xfer *xfer) return (USB_ERR_NOMEM); } + /* check if bMaxPacketSize changed */ + if (xfer->flags_int.control_xfr != 0 && + pepext->trb_ep_maxp != xfer->endpoint->edesc->wMaxPacketSize[0]) { + + DPRINTFN(8, "Reconfigure control endpoint\n"); + + /* force driver to reconfigure endpoint */ + pepext->trb_halted = 1; + pepext->trb_running = 0; + } + /* check for stopped condition, after putting transfer on interrupt queue */ if (pepext->trb_running == 0) { struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); diff --git a/sys/dev/usb/controller/xhci.h b/sys/dev/usb/controller/xhci.h index 48574506548..306400a52fd 100644 --- a/sys/dev/usb/controller/xhci.h +++ b/sys/dev/usb/controller/xhci.h @@ -383,6 +383,7 @@ struct xhci_endpoint_ext { uint8_t trb_halted; uint8_t trb_running; uint8_t trb_ep_mode; + uint8_t trb_ep_maxp; }; enum { From 9a59fb36b2a735ff2870555037af8b7744248e0d Mon Sep 17 00:00:00 2001 From: Michael Tuexen Date: Tue, 23 Feb 2016 18:50:34 +0000 Subject: [PATCH 03/25] Don't leak an address in an error path. CID: 1351729 MFC after: 3 days --- sys/netinet6/sctp6_usrreq.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sys/netinet6/sctp6_usrreq.c b/sys/netinet6/sctp6_usrreq.c index 176fc97956a..daae99e1a57 100644 --- a/sys/netinet6/sctp6_usrreq.c +++ b/sys/netinet6/sctp6_usrreq.c @@ -1009,6 +1009,7 @@ sctp6_getaddr(struct socket *so, struct sockaddr **addr) stcb = LIST_FIRST(&inp->sctp_asoc_list); if (stcb == NULL) { SCTP_INP_RUNLOCK(inp); + SCTP_FREE_SONAME(sin6); SCTP_LTRACE_ERR_RET(inp, NULL, NULL, SCTP_FROM_SCTP6_USRREQ, ENOENT); return (ENOENT); } @@ -1028,6 +1029,7 @@ sctp6_getaddr(struct socket *so, struct sockaddr **addr) if ((!fnd) || (sin_a6 == NULL)) { /* punt */ SCTP_INP_RUNLOCK(inp); + SCTP_FREE_SONAME(sin6); SCTP_LTRACE_ERR_RET(inp, NULL, NULL, SCTP_FROM_SCTP6_USRREQ, ENOENT); return (ENOENT); } From d70876fd7e2fe46f9d1fb2212a2687f9684ccb7a Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Tue, 23 Feb 2016 19:56:29 +0000 Subject: [PATCH 04/25] Add support for displaying thread IDs to truss(1). - Consolidate duplicate code for printing the metadata at the start of each line into a shared function. - Add an -H option which will log the thread ID of the relevant thread for each event. While here, remove some extraneous calls to clock_gettime() in print_syscall() and print_syscall_ret(). The caller of print_syscall_ret() always updates the current thread's "after" time before it is called. Reviewed by: kib MFC after: 1 month Differential Revision: https://reviews.freebsd.org/D5363 --- usr.bin/truss/extern.h | 1 + usr.bin/truss/main.c | 5 ++- usr.bin/truss/setup.c | 96 ++++++++++++++++++++-------------------- usr.bin/truss/syscalls.c | 23 +--------- usr.bin/truss/truss.1 | 12 +++-- usr.bin/truss/truss.h | 1 + 6 files changed, 63 insertions(+), 75 deletions(-) diff --git a/usr.bin/truss/extern.h b/usr.bin/truss/extern.h index 708d055cf2c..5ce3d3be8ec 100644 --- a/usr.bin/truss/extern.h +++ b/usr.bin/truss/extern.h @@ -31,6 +31,7 @@ * $FreeBSD$ */ +extern int print_line_prefix(struct trussinfo *); extern void setup_and_wait(struct trussinfo *, char **); extern void start_tracing(struct trussinfo *, pid_t); extern void restore_proc(int); diff --git a/usr.bin/truss/main.c b/usr.bin/truss/main.c index 531d7dbb4c1..38626fd3acc 100644 --- a/usr.bin/truss/main.c +++ b/usr.bin/truss/main.c @@ -96,7 +96,7 @@ main(int ac, char **av) trussinfo->curthread = NULL; LIST_INIT(&trussinfo->proclist); init_syscalls(); - while ((c = getopt(ac, av, "p:o:facedDs:S")) != -1) { + while ((c = getopt(ac, av, "p:o:facedDs:SH")) != -1) { switch (c) { case 'p': /* specified pid */ pid = atoi(optarg); @@ -132,6 +132,9 @@ main(int ac, char **av) case 'S': /* Don't trace signals */ trussinfo->flags |= NOSIGS; break; + case 'H': + trussinfo->flags |= DISPLAYTIDS; + break; default: usage(); } diff --git a/usr.bin/truss/setup.c b/usr.bin/truss/setup.c index 24126adfd67..36dd571fae7 100644 --- a/usr.bin/truss/setup.c +++ b/usr.bin/truss/setup.c @@ -439,26 +439,46 @@ exit_syscall(struct trussinfo *info, struct ptrace_lwpinfo *pl) } } +int +print_line_prefix(struct trussinfo *info) +{ + struct timespec timediff; + struct threadinfo *t; + int len; + + len = 0; + t = info->curthread; + if (info->flags & (FOLLOWFORKS | DISPLAYTIDS)) { + if (info->flags & FOLLOWFORKS) + len += fprintf(info->outfile, "%5d", t->proc->pid); + if ((info->flags & (FOLLOWFORKS | DISPLAYTIDS)) == + (FOLLOWFORKS | DISPLAYTIDS)) + len += fprintf(info->outfile, " "); + if (info->flags & DISPLAYTIDS) + len += fprintf(info->outfile, "%6d", t->tid); + len += fprintf(info->outfile, ": "); + } + if (info->flags & ABSOLUTETIMESTAMPS) { + timespecsubt(&t->after, &info->start_time, &timediff); + len += fprintf(info->outfile, "%jd.%09ld ", + (intmax_t)timediff.tv_sec, timediff.tv_nsec); + } + if (info->flags & RELATIVETIMESTAMPS) { + timespecsubt(&t->after, &t->before, &timediff); + len += fprintf(info->outfile, "%jd.%09ld ", + (intmax_t)timediff.tv_sec, timediff.tv_nsec); + } + return (len); +} + static void report_exit(struct trussinfo *info, siginfo_t *si) { - struct timespec timediff; + struct threadinfo *t; - if (info->flags & FOLLOWFORKS) - fprintf(info->outfile, "%5d: ", si->si_pid); - clock_gettime(CLOCK_REALTIME, &info->curthread->after); - if (info->flags & ABSOLUTETIMESTAMPS) { - timespecsubt(&info->curthread->after, &info->start_time, - &timediff); - fprintf(info->outfile, "%jd.%09ld ", (intmax_t)timediff.tv_sec, - timediff.tv_nsec); - } - if (info->flags & RELATIVETIMESTAMPS) { - timespecsubt(&info->curthread->after, &info->curthread->before, - &timediff); - fprintf(info->outfile, "%jd.%09ld ", (intmax_t)timediff.tv_sec, - timediff.tv_nsec); - } + t = info->curthread; + clock_gettime(CLOCK_REALTIME, &t->after); + print_line_prefix(info); if (si->si_code == CLD_EXITED) fprintf(info->outfile, "process exit, rval = %u\n", si->si_status); @@ -469,48 +489,26 @@ report_exit(struct trussinfo *info, siginfo_t *si) } static void -report_new_child(struct trussinfo *info, pid_t pid) +report_new_child(struct trussinfo *info) { - struct timespec timediff; + struct threadinfo *t; - clock_gettime(CLOCK_REALTIME, &info->curthread->after); - assert(info->flags & FOLLOWFORKS); - fprintf(info->outfile, "%5d: ", pid); - if (info->flags & ABSOLUTETIMESTAMPS) { - timespecsubt(&info->curthread->after, &info->start_time, - &timediff); - fprintf(info->outfile, "%jd.%09ld ", (intmax_t)timediff.tv_sec, - timediff.tv_nsec); - } - if (info->flags & RELATIVETIMESTAMPS) { - timediff.tv_sec = 0; - timediff.tv_nsec = 0; - fprintf(info->outfile, "%jd.%09ld ", (intmax_t)timediff.tv_sec, - timediff.tv_nsec); - } + t = info->curthread; + clock_gettime(CLOCK_REALTIME, &t->after); + t->before = t->after; + print_line_prefix(info); fprintf(info->outfile, "\n"); } static void report_signal(struct trussinfo *info, siginfo_t *si) { - struct timespec timediff; + struct threadinfo *t; char *signame; - if (info->flags & FOLLOWFORKS) - fprintf(info->outfile, "%5d: ", si->si_pid); - if (info->flags & ABSOLUTETIMESTAMPS) { - timespecsubt(&info->curthread->after, &info->start_time, - &timediff); - fprintf(info->outfile, "%jd.%09ld ", (intmax_t)timediff.tv_sec, - timediff.tv_nsec); - } - if (info->flags & RELATIVETIMESTAMPS) { - timespecsubt(&info->curthread->after, &info->curthread->before, - &timediff); - fprintf(info->outfile, "%jd.%09ld ", (intmax_t)timediff.tv_sec, - timediff.tv_nsec); - } + t = info->curthread; + clock_gettime(CLOCK_REALTIME, &t->after); + print_line_prefix(info); signame = strsig(si->si_status); fprintf(info->outfile, "SIGNAL %u (%s)\n", si->si_status, signame == NULL ? "?" : signame); @@ -572,7 +570,7 @@ eventloop(struct trussinfo *info) pending_signal = 0; } else if (pl.pl_flags & PL_FLAG_CHILD) { if ((info->flags & COUNTONLY) == 0) - report_new_child(info, si.si_pid); + report_new_child(info); pending_signal = 0; } else { if ((info->flags & NOSIGS) == 0) diff --git a/usr.bin/truss/syscalls.c b/usr.bin/truss/syscalls.c index b0133905651..97723211c7a 100644 --- a/usr.bin/truss/syscalls.c +++ b/usr.bin/truss/syscalls.c @@ -2005,38 +2005,18 @@ print_arg(struct syscall_args *sc, unsigned long *args, long *retval, void print_syscall(struct trussinfo *trussinfo) { - struct timespec timediff; struct threadinfo *t; const char *name; char **s_args; int i, len, nargs; - len = 0; t = trussinfo->curthread; - if (trussinfo->flags & FOLLOWFORKS) - len += fprintf(trussinfo->outfile, "%5d: ", - t->proc->pid); name = t->cs.name; nargs = t->cs.nargs; s_args = t->cs.s_args; - if (name != NULL && (strcmp(name, "execve") == 0 || - strcmp(name, "exit") == 0)) { - clock_gettime(CLOCK_REALTIME, &t->after); - } - - if (trussinfo->flags & ABSOLUTETIMESTAMPS) { - timespecsubt(&t->after, &trussinfo->start_time, &timediff); - len += fprintf(trussinfo->outfile, "%jd.%09ld ", - (intmax_t)timediff.tv_sec, timediff.tv_nsec); - } - - if (trussinfo->flags & RELATIVETIMESTAMPS) { - timespecsubt(&t->after, &t->before, &timediff); - len += fprintf(trussinfo->outfile, "%jd.%09ld ", - (intmax_t)timediff.tv_sec, timediff.tv_nsec); - } + len = print_line_prefix(trussinfo); len += fprintf(trussinfo->outfile, "%s(", name); for (i = 0; i < nargs; i++) { @@ -2063,7 +2043,6 @@ print_syscall_ret(struct trussinfo *trussinfo, int errorp, long *retval) t = trussinfo->curthread; sc = t->cs.sc; if (trussinfo->flags & COUNTONLY) { - clock_gettime(CLOCK_REALTIME, &t->after); timespecsubt(&t->after, &t->before, &timediff); timespecadd(&sc->time, &timediff, &sc->time); sc->ncalls++; diff --git a/usr.bin/truss/truss.1 b/usr.bin/truss/truss.1 index fb02cb4e75a..0246fa93392 100644 --- a/usr.bin/truss/truss.1 +++ b/usr.bin/truss/truss.1 @@ -1,6 +1,6 @@ .\" $FreeBSD$ .\" -.Dd October 9, 2015 +.Dd February 23, 2016 .Dt TRUSS 1 .Os .Sh NAME @@ -8,12 +8,12 @@ .Nd trace system calls .Sh SYNOPSIS .Nm -.Op Fl facedDS +.Op Fl facedDHS .Op Fl o Ar file .Op Fl s Ar strsize .Fl p Ar pid .Nm -.Op Fl facedDS +.Op Fl facedDHS .Op Fl o Ar file .Op Fl s Ar strsize .Ar command Op Ar args @@ -32,6 +32,10 @@ Trace descendants of the original traced process created by .Xr fork 2 , .Xr vfork 2 , etc. +To distinguish events between processes, +the process ID +.Pq PID +of the process is included in the output of each event. .It Fl a Show the argument strings that are passed in each .Xr execve 2 @@ -52,6 +56,8 @@ since the trace was started. .It Fl D Include timestamps in the output showing the time elapsed since the last recorded event. +.It Fl H +Include the thread ID of in the output of each event. .It Fl S Do not display information about signals received by the process. (Normally, diff --git a/usr.bin/truss/truss.h b/usr.bin/truss/truss.h index 5ef24447c25..582ab0baa74 100644 --- a/usr.bin/truss/truss.h +++ b/usr.bin/truss/truss.h @@ -35,6 +35,7 @@ #define EXECVEARGS 0x00000010 #define EXECVEENVS 0x00000020 #define COUNTONLY 0x00000040 +#define DISPLAYTIDS 0x00000080 struct procinfo; struct trussinfo; From 287b96dd253d69d2a3ff2033fb2eb19aec44605f Mon Sep 17 00:00:00 2001 From: John Baldwin Date: Tue, 23 Feb 2016 20:00:55 +0000 Subject: [PATCH 05/25] Add handling for non-native error values to libsysdecode. Add two new functions, sysdecode_abi_to_freebsd_errno() and sysdecode_freebsd_to_abi_errno(), which convert errno values between the native FreeBSD ABI and other supported ABIs. Note that the mappings are not necessarily perfect meaning in some cases multiple errors in one ABI might map to a single error in another ABI. In that case, the reverse mapping will return one of the errors that maps, but which error is non-deterministic. Change truss to always report the raw error value to the user but use libsysdecode to map it to a native errno value that can be used with strerror() to generate a description. Previously truss reported the "converted" error value. Now the user will always see the exact error value that the application sees. Change kdump to report the truly raw error value to the user. Previously kdump would report the absolute value of the raw error value (so for Linux binaries it didn't output the FreeBSD error value, but the positive value of the Linux error). Now it reports the real (i.e. negative) error value for Linux binaries. Also, use libsysdecode to convert the native FreeBSD error reported in the ktrace record to the raw error used by the ABI. This means that the Linux ABI can now be handled directly in ktrsysret() and removes the need for linux_ktrsysret(). Reviewed by: bdrewery, kib Helpful notes: wblock (manpage) Differential Revision: https://reviews.freebsd.org/D5314 --- lib/libsysdecode/Makefile | 4 +- lib/libsysdecode/errno.c | 209 ++++++++++++++++++ lib/libsysdecode/sysdecode.3 | 1 + lib/libsysdecode/sysdecode.h | 2 + .../sysdecode_abi_to_freebsd_errno.3 | 94 ++++++++ lib/libsysdecode/sysdecode_syscallnames.3 | 5 +- usr.bin/kdump/kdump.c | 71 +----- usr.bin/truss/Makefile | 2 +- usr.bin/truss/aarch64-cloudabi64.c | 4 - usr.bin/truss/amd64-cloudabi64.c | 4 - usr.bin/truss/amd64-linux32.c | 27 --- usr.bin/truss/cloudabi.c | 122 ---------- usr.bin/truss/cloudabi.h | 28 --- usr.bin/truss/i386-linux.c | 27 --- usr.bin/truss/syscalls.c | 8 +- 15 files changed, 322 insertions(+), 286 deletions(-) create mode 100644 lib/libsysdecode/errno.c create mode 100644 lib/libsysdecode/sysdecode_abi_to_freebsd_errno.3 delete mode 100644 usr.bin/truss/cloudabi.c delete mode 100644 usr.bin/truss/cloudabi.h diff --git a/lib/libsysdecode/Makefile b/lib/libsysdecode/Makefile index e1e69fef64e..405cab8c784 100644 --- a/lib/libsysdecode/Makefile +++ b/lib/libsysdecode/Makefile @@ -4,15 +4,17 @@ LIB= sysdecode -SRCS= ioctl.c syscallnames.c utrace.c +SRCS= errno.c ioctl.c syscallnames.c utrace.c INCS= sysdecode.h CFLAGS+= -I${.CURDIR}/../../sys MAN+= sysdecode.3 \ + sysdecode_abi_to_freebsd_errno.3 \ sysdecode_ioctlname.3 \ sysdecode_syscallnames.3 \ sysdecode_utrace.3 +MLINKS+= sysdecode_abi_to_freebsd_errno.3 sysdecode_freebsd_to_abi_errno.3 CLEANFILES= ioctl.c diff --git a/lib/libsysdecode/errno.c b/lib/libsysdecode/errno.c new file mode 100644 index 00000000000..1e863bb69dc --- /dev/null +++ b/lib/libsysdecode/errno.c @@ -0,0 +1,209 @@ +/*- + * Copyright (c) 2015 John H. Baldwin + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include + +#if defined(__i386__) || defined(__amd64__) +/* + * Linux syscalls return negative errno's, we do positive and map them + * Reference: + * FreeBSD: src/sys/sys/errno.h + * Linux: linux-2.6.17.8/include/asm-generic/errno-base.h + * linux-2.6.17.8/include/asm-generic/errno.h + */ +static int bsd_to_linux_errno[ELAST + 1] = { + -0, -1, -2, -3, -4, -5, -6, -7, -8, -9, + -10, -35, -12, -13, -14, -15, -16, -17, -18, -19, + -20, -21, -22, -23, -24, -25, -26, -27, -28, -29, + -30, -31, -32, -33, -34, -11,-115,-114, -88, -89, + -90, -91, -92, -93, -94, -95, -96, -97, -98, -99, + -100,-101,-102,-103,-104,-105,-106,-107,-108,-109, + -110,-111, -40, -36,-112,-113, -39, -11, -87,-122, + -116, -66, -6, -6, -6, -6, -6, -37, -38, -9, + -6, -6, -43, -42, -75,-125, -84, -95, -16, -74, + -72, -67, -71 +}; +#endif + +#if defined(__aarch64__) || defined(__amd64__) +#include + +static const int cloudabi_errno_table[] = { + [CLOUDABI_E2BIG] = E2BIG, + [CLOUDABI_EACCES] = EACCES, + [CLOUDABI_EADDRINUSE] = EADDRINUSE, + [CLOUDABI_EADDRNOTAVAIL] = EADDRNOTAVAIL, + [CLOUDABI_EAFNOSUPPORT] = EAFNOSUPPORT, + [CLOUDABI_EAGAIN] = EAGAIN, + [CLOUDABI_EALREADY] = EALREADY, + [CLOUDABI_EBADF] = EBADF, + [CLOUDABI_EBADMSG] = EBADMSG, + [CLOUDABI_EBUSY] = EBUSY, + [CLOUDABI_ECANCELED] = ECANCELED, + [CLOUDABI_ECHILD] = ECHILD, + [CLOUDABI_ECONNABORTED] = ECONNABORTED, + [CLOUDABI_ECONNREFUSED] = ECONNREFUSED, + [CLOUDABI_ECONNRESET] = ECONNRESET, + [CLOUDABI_EDEADLK] = EDEADLK, + [CLOUDABI_EDESTADDRREQ] = EDESTADDRREQ, + [CLOUDABI_EDOM] = EDOM, + [CLOUDABI_EDQUOT] = EDQUOT, + [CLOUDABI_EEXIST] = EEXIST, + [CLOUDABI_EFAULT] = EFAULT, + [CLOUDABI_EFBIG] = EFBIG, + [CLOUDABI_EHOSTUNREACH] = EHOSTUNREACH, + [CLOUDABI_EIDRM] = EIDRM, + [CLOUDABI_EILSEQ] = EILSEQ, + [CLOUDABI_EINPROGRESS] = EINPROGRESS, + [CLOUDABI_EINTR] = EINTR, + [CLOUDABI_EINVAL] = EINVAL, + [CLOUDABI_EIO] = EIO, + [CLOUDABI_EISCONN] = EISCONN, + [CLOUDABI_EISDIR] = EISDIR, + [CLOUDABI_ELOOP] = ELOOP, + [CLOUDABI_EMFILE] = EMFILE, + [CLOUDABI_EMLINK] = EMLINK, + [CLOUDABI_EMSGSIZE] = EMSGSIZE, + [CLOUDABI_EMULTIHOP] = EMULTIHOP, + [CLOUDABI_ENAMETOOLONG] = ENAMETOOLONG, + [CLOUDABI_ENETDOWN] = ENETDOWN, + [CLOUDABI_ENETRESET] = ENETRESET, + [CLOUDABI_ENETUNREACH] = ENETUNREACH, + [CLOUDABI_ENFILE] = ENFILE, + [CLOUDABI_ENOBUFS] = ENOBUFS, + [CLOUDABI_ENODEV] = ENODEV, + [CLOUDABI_ENOENT] = ENOENT, + [CLOUDABI_ENOEXEC] = ENOEXEC, + [CLOUDABI_ENOLCK] = ENOLCK, + [CLOUDABI_ENOLINK] = ENOLINK, + [CLOUDABI_ENOMEM] = ENOMEM, + [CLOUDABI_ENOMSG] = ENOMSG, + [CLOUDABI_ENOPROTOOPT] = ENOPROTOOPT, + [CLOUDABI_ENOSPC] = ENOSPC, + [CLOUDABI_ENOSYS] = ENOSYS, + [CLOUDABI_ENOTCONN] = ENOTCONN, + [CLOUDABI_ENOTDIR] = ENOTDIR, + [CLOUDABI_ENOTEMPTY] = ENOTEMPTY, + [CLOUDABI_ENOTRECOVERABLE] = ENOTRECOVERABLE, + [CLOUDABI_ENOTSOCK] = ENOTSOCK, + [CLOUDABI_ENOTSUP] = ENOTSUP, + [CLOUDABI_ENOTTY] = ENOTTY, + [CLOUDABI_ENXIO] = ENXIO, + [CLOUDABI_EOVERFLOW] = EOVERFLOW, + [CLOUDABI_EOWNERDEAD] = EOWNERDEAD, + [CLOUDABI_EPERM] = EPERM, + [CLOUDABI_EPIPE] = EPIPE, + [CLOUDABI_EPROTO] = EPROTO, + [CLOUDABI_EPROTONOSUPPORT] = EPROTONOSUPPORT, + [CLOUDABI_EPROTOTYPE] = EPROTOTYPE, + [CLOUDABI_ERANGE] = ERANGE, + [CLOUDABI_EROFS] = EROFS, + [CLOUDABI_ESPIPE] = ESPIPE, + [CLOUDABI_ESRCH] = ESRCH, + [CLOUDABI_ESTALE] = ESTALE, + [CLOUDABI_ETIMEDOUT] = ETIMEDOUT, + [CLOUDABI_ETXTBSY] = ETXTBSY, + [CLOUDABI_EXDEV] = EXDEV, + [CLOUDABI_ENOTCAPABLE] = ENOTCAPABLE, +}; +#endif + +int +sysdecode_abi_to_freebsd_errno(enum sysdecode_abi abi, int error) +{ + + switch (abi) { + case SYSDECODE_ABI_FREEBSD: + case SYSDECODE_ABI_FREEBSD32: + return (error); +#if defined(__i386__) || defined(__amd64__) + case SYSDECODE_ABI_LINUX: + case SYSDECODE_ABI_LINUX32: { + unsigned int i; + + /* + * This is imprecise since it returns the first + * matching errno. + */ + for (i = 0; i < nitems(bsd_to_linux_errno); i++) { + if (error == bsd_to_linux_errno[i]) + return (i); + } + break; + } +#endif +#if defined(__aarch64__) || defined(__amd64__) + case SYSDECODE_ABI_CLOUDABI64: + if (error >= 0 && + (unsigned int)error < nitems(cloudabi_errno_table)) + return (cloudabi_errno_table[error]); + break; +#endif + default: + break; + } + return (INT_MAX); +} + +int +sysdecode_freebsd_to_abi_errno(enum sysdecode_abi abi, int error) +{ + + switch (abi) { + case SYSDECODE_ABI_FREEBSD: + case SYSDECODE_ABI_FREEBSD32: + return (error); +#if defined(__i386__) || defined(__amd64__) + case SYSDECODE_ABI_LINUX: + case SYSDECODE_ABI_LINUX32: + if (error >= 0 && error <= ELAST) + return (bsd_to_linux_errno[error]); + break; +#endif +#if defined(__aarch64__) || defined(__amd64__) + case SYSDECODE_ABI_CLOUDABI64: { + unsigned int i; + + for (i = 0; i < nitems(cloudabi_errno_table); i++) { + if (error == cloudabi_errno_table[i]) + return (i); + } + break; + } +#endif + default: + break; + } + return (INT_MAX); +} + diff --git a/lib/libsysdecode/sysdecode.3 b/lib/libsysdecode/sysdecode.3 index f058f59c60c..ab7e972b2f9 100644 --- a/lib/libsysdecode/sysdecode.3 +++ b/lib/libsysdecode/sysdecode.3 @@ -64,6 +64,7 @@ Supported on aarch64 and amd64. A placeholder for use when the ABI is not known. .El .Sh SEE ALSO +.Xr sysdecode_abi_to_freebsd_errno 3 , .Xr sysdecode_ioctlname 3 , .Xr sysdecode_syscallnames 3 , .Xr sysdecode_utrace 3 diff --git a/lib/libsysdecode/sysdecode.h b/lib/libsysdecode/sysdecode.h index fa00716826d..8a30206f66f 100644 --- a/lib/libsysdecode/sysdecode.h +++ b/lib/libsysdecode/sysdecode.h @@ -38,6 +38,8 @@ enum sysdecode_abi { SYSDECODE_ABI_CLOUDABI64 }; +int sysdecode_abi_to_freebsd_errno(enum sysdecode_abi _abi, int _error); +int sysdecode_freebsd_to_abi_errno(enum sysdecode_abi _abi, int _error); const char *sysdecode_ioctlname(unsigned long _val); const char *sysdecode_syscallname(enum sysdecode_abi _abi, unsigned int _code); int sysdecode_utrace(FILE *_fp, void *_buf, size_t _len); diff --git a/lib/libsysdecode/sysdecode_abi_to_freebsd_errno.3 b/lib/libsysdecode/sysdecode_abi_to_freebsd_errno.3 new file mode 100644 index 00000000000..74826f8e20c --- /dev/null +++ b/lib/libsysdecode/sysdecode_abi_to_freebsd_errno.3 @@ -0,0 +1,94 @@ +.\" +.\" Copyright (c) 2016 John Baldwin +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd February 23, 2016 +.Dt sysdecode_abi_to_freebsd_errno 3 +.Os +.Sh NAME +.Nm sysdecode_abi_to_freebsd_errno , +.Nm sysdecode_freebsd_to_abi_errno +.Nd translate error numbers between process ABIs +.Sh LIBRARY +.Lb libsysdecode +.Sh SYNOPSIS +.Ft int +.Fn sysdecode_abi_to_freebsd_errno "enum sysdecode_abi abi" "int error" +.Ft int +.Fn sysdecode_freebsd_to_abi_errno "enum sysdecode_abi abi" "int error" +.Sh DESCRIPTION +The +.Fn sysdecode_abi_to_freebsd_errno +function returns the native +.Xr errno 3 +value that corresponds to the error indicated by +.Fa error +for the process ABI +.Fa abi . +If +.Fa error +does not identify a valid error for +.Fa abi , +.Dv INT_MAX +is returned. +.Pp +The +.Fn sysdecode_freebsd_to_abi_errno +function the error value for the process ABI +.Fa abi +that corresponds to the native +.Xr errno 3 +value +.Fa error . +If +.Fa error +does not identify a valid +.Xr errno 3 +error, +.Dv INT_MAX +is returned. +.Pp +Note that the mappings between native +.Xr errno 3 +values and errors for other ABIs are not exhaustive. +If a mapping does not exist, +these functions return +.Dv INT_MAX . +In addition, multiple error values in one ABI may map to a single +error in another ABI. +.Sh RETURN VALUES +These functions return an error value on success or +.Dv INT_MAX +if +.Fa error +is not valid. +.Pp +For the list of supported ABIs, +see +.Xr sysdecode 3 . +.Sh SEE ALSO +.Xr sysdecode 3 , +.Xr sysdecode_syscallnames 3 diff --git a/lib/libsysdecode/sysdecode_syscallnames.3 b/lib/libsysdecode/sysdecode_syscallnames.3 index 1c75ab6e9d1..e920edb29c5 100644 --- a/lib/libsysdecode/sysdecode_syscallnames.3 +++ b/lib/libsysdecode/sysdecode_syscallnames.3 @@ -25,7 +25,7 @@ .\" .\" $FreeBSD$ .\" -.Dd January 24, 2016 +.Dd January 30, 2016 .Dt sysdecode_syscallnames 3 .Os .Sh NAME @@ -64,4 +64,5 @@ or .Fa ABI is invalid . .Sh SEE ALSO -.Xr sysdecode 3 +.Xr sysdecode 3 , +.Xr sysdecode_abi_to_freebsd_errno 3 diff --git a/usr.bin/kdump/kdump.c b/usr.bin/kdump/kdump.c index 8fe03ea2776..b4a1f696512 100644 --- a/usr.bin/kdump/kdump.c +++ b/usr.bin/kdump/kdump.c @@ -142,28 +142,6 @@ static struct ktr_header ktr_header; c = ','; \ } while (0) -#if defined(__amd64__) || defined(__i386__) - -void linux_ktrsysret(struct ktr_sysret *, u_int); - -/* - * from linux.h - * Linux syscalls return negative errno's, we do positive and map them - */ -static int bsd_to_linux_errno[ELAST + 1] = { - -0, -1, -2, -3, -4, -5, -6, -7, -8, -9, - -10, -35, -12, -13, -14, -15, -16, -17, -18, -19, - -20, -21, -22, -23, -24, -25, -26, -27, -28, -29, - -30, -31, -32, -33, -34, -11,-115,-114, -88, -89, - -90, -91, -92, -93, -94, -95, -96, -97, -98, -99, - -100,-101,-102,-103,-104,-105,-106,-107,-108,-109, - -110,-111, -40, -36,-112,-113, -39, -11, -87,-122, - -116, -66, -6, -6, -6, -6, -6, -37, -38, -9, - -6, -6, -43, -42, -75,-125, -84, -95, -16, -74, - -72, -67, -71 -}; -#endif - struct proc_info { TAILQ_ENTRY(proc_info) info; @@ -393,13 +371,7 @@ main(int argc, char *argv[]) ktrsyscall((struct ktr_syscall *)m, sv_flags); break; case KTR_SYSRET: -#if defined(__amd64__) || defined(__i386__) - if ((sv_flags & SV_ABI_MASK) == SV_ABI_LINUX) - linux_ktrsysret((struct ktr_sysret *)m, - sv_flags); - else -#endif - ktrsysret((struct ktr_sysret *)m, sv_flags); + ktrsysret((struct ktr_sysret *)m, sv_flags); break; case KTR_NAMEI: case KTR_SYSCTL: @@ -1366,7 +1338,8 @@ ktrsysret(struct ktr_sysret *ktr, u_int sv_flags) else if (error == EJUSTRETURN) printf("JUSTRETURN"); else { - printf("-1 errno %d", ktr->ktr_error); + printf("-1 errno %d", sysdecode_freebsd_to_abi_errno( + syscallabi(sv_flags), error)); if (fancy) printf(" %s", strerror(ktr->ktr_error)); } @@ -1852,44 +1825,6 @@ ktrfaultend(struct ktr_faultend *ktr) printf("\n"); } -#if defined(__amd64__) || defined(__i386__) -void -linux_ktrsysret(struct ktr_sysret *ktr, u_int sv_flags) -{ - register_t ret = ktr->ktr_retval; - int error = ktr->ktr_error; - - syscallname(ktr->ktr_code, sv_flags); - printf(" "); - - if (error == 0) { - if (fancy) { - printf("%ld", (long)ret); - if (ret < 0 || ret > 9) - printf("/%#lx", (unsigned long)ret); - } else { - if (decimal) - printf("%ld", (long)ret); - else - printf("%#lx", (unsigned long)ret); - } - } else if (error == ERESTART) - printf("RESTART"); - else if (error == EJUSTRETURN) - printf("JUSTRETURN"); - else { - if (ktr->ktr_error <= ELAST + 1) - error = abs(bsd_to_linux_errno[ktr->ktr_error]); - else - error = 999; - printf("-1 errno %d", error); - if (fancy) - printf(" %s", strerror(ktr->ktr_error)); - } - putchar('\n'); -} -#endif - void usage(void) { diff --git a/usr.bin/truss/Makefile b/usr.bin/truss/Makefile index b28098c802a..7ba439283ed 100644 --- a/usr.bin/truss/Makefile +++ b/usr.bin/truss/Makefile @@ -2,7 +2,7 @@ NO_WERROR= PROG= truss -SRCS= cloudabi.c main.c setup.c syscalls.c +SRCS= main.c setup.c syscalls.c LIBADD= sysdecode diff --git a/usr.bin/truss/aarch64-cloudabi64.c b/usr.bin/truss/aarch64-cloudabi64.c index f2891afe02a..27906ea9204 100644 --- a/usr.bin/truss/aarch64-cloudabi64.c +++ b/usr.bin/truss/aarch64-cloudabi64.c @@ -31,11 +31,9 @@ __FBSDID("$FreeBSD$"); #include -#include #include #include -#include "cloudabi.h" #include "truss.h" static int @@ -74,8 +72,6 @@ aarch64_cloudabi64_fetch_retval(struct trussinfo *trussinfo, long *retval, retval[0] = regs.x[0]; retval[1] = regs.x[1]; *errorp = (regs.spsr & PSR_C) != 0; - if (*errorp) - retval[0] = cloudabi_convert_errno(retval[0]); return (0); } diff --git a/usr.bin/truss/amd64-cloudabi64.c b/usr.bin/truss/amd64-cloudabi64.c index ce8b18918fd..f7d7c1aa144 100644 --- a/usr.bin/truss/amd64-cloudabi64.c +++ b/usr.bin/truss/amd64-cloudabi64.c @@ -31,11 +31,9 @@ __FBSDID("$FreeBSD$"); #include -#include #include #include -#include "cloudabi.h" #include "truss.h" static int @@ -83,8 +81,6 @@ amd64_cloudabi64_fetch_retval(struct trussinfo *trussinfo, long *retval, retval[0] = regs.r_rax; retval[1] = regs.r_rdx; *errorp = (regs.r_rflags & PSL_C) != 0; - if (*errorp) - retval[0] = cloudabi_convert_errno(retval[0]); return (0); } diff --git a/usr.bin/truss/amd64-linux32.c b/usr.bin/truss/amd64-linux32.c index 873ef01ee38..279d2285b3b 100644 --- a/usr.bin/truss/amd64-linux32.c +++ b/usr.bin/truss/amd64-linux32.c @@ -83,28 +83,12 @@ amd64_linux32_fetch_args(struct trussinfo *trussinfo, u_int narg) return (0); } -/* - * Linux syscalls return negative errno's, we do positive and map them - */ -static const int bsd_to_linux_errno[] = { - -0, -1, -2, -3, -4, -5, -6, -7, -8, -9, - -10, -35, -12, -13, -14, -15, -16, -17, -18, -19, - -20, -21, -22, -23, -24, -25, -26, -27, -28, -29, - -30, -31, -32, -33, -34, -11,-115,-114, -88, -89, - -90, -91, -92, -93, -94, -95, -96, -97, -98, -99, - -100,-101,-102,-103,-104,-105,-106,-107,-108,-109, - -110,-111, -40, -36,-112,-113, -39, -11, -87,-122, - -116, -66, -6, -6, -6, -6, -6, -37, -38, -9, - -6, -}; - static int amd64_linux32_fetch_retval(struct trussinfo *trussinfo, long *retval, int *errorp) { struct reg regs; lwpid_t tid; - size_t i; tid = trussinfo->curthread->tid; if (ptrace(PT_GETREGS, tid, (caddr_t)®s, 0) < 0) { @@ -117,17 +101,6 @@ amd64_linux32_fetch_retval(struct trussinfo *trussinfo, long *retval, *errorp = !!(regs.r_rflags & PSL_C); if (*errorp) retval[0] = (int)retval[0]; - - if (*errorp) { - for (i = 0; i < nitems(bsd_to_linux_errno); i++) { - if (retval[0] == bsd_to_linux_errno[i]) { - retval[0] = i; - return (0); - } - } - - /* XXX: How to handle unknown errors? */ - } return (0); } diff --git a/usr.bin/truss/cloudabi.c b/usr.bin/truss/cloudabi.c deleted file mode 100644 index d4265002739..00000000000 --- a/usr.bin/truss/cloudabi.c +++ /dev/null @@ -1,122 +0,0 @@ -/*- - * Copyright (c) 2015 Nuxi, https://nuxi.nl/ - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -#include -__FBSDID("$FreeBSD$"); - -#include - -#include - -#include - -#include "cloudabi.h" - -long -cloudabi_convert_errno(long error) -{ - static const int table[] = { - [CLOUDABI_E2BIG] = E2BIG, - [CLOUDABI_EACCES] = EACCES, - [CLOUDABI_EADDRINUSE] = EADDRINUSE, - [CLOUDABI_EADDRNOTAVAIL] = EADDRNOTAVAIL, - [CLOUDABI_EAFNOSUPPORT] = EAFNOSUPPORT, - [CLOUDABI_EAGAIN] = EAGAIN, - [CLOUDABI_EALREADY] = EALREADY, - [CLOUDABI_EBADF] = EBADF, - [CLOUDABI_EBADMSG] = EBADMSG, - [CLOUDABI_EBUSY] = EBUSY, - [CLOUDABI_ECANCELED] = ECANCELED, - [CLOUDABI_ECHILD] = ECHILD, - [CLOUDABI_ECONNABORTED] = ECONNABORTED, - [CLOUDABI_ECONNREFUSED] = ECONNREFUSED, - [CLOUDABI_ECONNRESET] = ECONNRESET, - [CLOUDABI_EDEADLK] = EDEADLK, - [CLOUDABI_EDESTADDRREQ] = EDESTADDRREQ, - [CLOUDABI_EDOM] = EDOM, - [CLOUDABI_EDQUOT] = EDQUOT, - [CLOUDABI_EEXIST] = EEXIST, - [CLOUDABI_EFAULT] = EFAULT, - [CLOUDABI_EFBIG] = EFBIG, - [CLOUDABI_EHOSTUNREACH] = EHOSTUNREACH, - [CLOUDABI_EIDRM] = EIDRM, - [CLOUDABI_EILSEQ] = EILSEQ, - [CLOUDABI_EINPROGRESS] = EINPROGRESS, - [CLOUDABI_EINTR] = EINTR, - [CLOUDABI_EINVAL] = EINVAL, - [CLOUDABI_EIO] = EIO, - [CLOUDABI_EISCONN] = EISCONN, - [CLOUDABI_EISDIR] = EISDIR, - [CLOUDABI_ELOOP] = ELOOP, - [CLOUDABI_EMFILE] = EMFILE, - [CLOUDABI_EMLINK] = EMLINK, - [CLOUDABI_EMSGSIZE] = EMSGSIZE, - [CLOUDABI_EMULTIHOP] = EMULTIHOP, - [CLOUDABI_ENAMETOOLONG] = ENAMETOOLONG, - [CLOUDABI_ENETDOWN] = ENETDOWN, - [CLOUDABI_ENETRESET] = ENETRESET, - [CLOUDABI_ENETUNREACH] = ENETUNREACH, - [CLOUDABI_ENFILE] = ENFILE, - [CLOUDABI_ENOBUFS] = ENOBUFS, - [CLOUDABI_ENODEV] = ENODEV, - [CLOUDABI_ENOENT] = ENOENT, - [CLOUDABI_ENOEXEC] = ENOEXEC, - [CLOUDABI_ENOLCK] = ENOLCK, - [CLOUDABI_ENOLINK] = ENOLINK, - [CLOUDABI_ENOMEM] = ENOMEM, - [CLOUDABI_ENOMSG] = ENOMSG, - [CLOUDABI_ENOPROTOOPT] = ENOPROTOOPT, - [CLOUDABI_ENOSPC] = ENOSPC, - [CLOUDABI_ENOSYS] = ENOSYS, - [CLOUDABI_ENOTCONN] = ENOTCONN, - [CLOUDABI_ENOTDIR] = ENOTDIR, - [CLOUDABI_ENOTEMPTY] = ENOTEMPTY, - [CLOUDABI_ENOTRECOVERABLE] = ENOTRECOVERABLE, - [CLOUDABI_ENOTSOCK] = ENOTSOCK, - [CLOUDABI_ENOTSUP] = ENOTSUP, - [CLOUDABI_ENOTTY] = ENOTTY, - [CLOUDABI_ENXIO] = ENXIO, - [CLOUDABI_EOVERFLOW] = EOVERFLOW, - [CLOUDABI_EOWNERDEAD] = EOWNERDEAD, - [CLOUDABI_EPERM] = EPERM, - [CLOUDABI_EPIPE] = EPIPE, - [CLOUDABI_EPROTO] = EPROTO, - [CLOUDABI_EPROTONOSUPPORT] = EPROTONOSUPPORT, - [CLOUDABI_EPROTOTYPE] = EPROTOTYPE, - [CLOUDABI_ERANGE] = ERANGE, - [CLOUDABI_EROFS] = EROFS, - [CLOUDABI_ESPIPE] = ESPIPE, - [CLOUDABI_ESRCH] = ESRCH, - [CLOUDABI_ESTALE] = ESTALE, - [CLOUDABI_ETIMEDOUT] = ETIMEDOUT, - [CLOUDABI_ETXTBSY] = ETXTBSY, - [CLOUDABI_EXDEV] = EXDEV, - [CLOUDABI_ENOTCAPABLE] = ENOTCAPABLE, - }; - - if (error < 0 || error >= nitems(table) || table[error] == 0) - return (error); - return (table[error]); -} diff --git a/usr.bin/truss/cloudabi.h b/usr.bin/truss/cloudabi.h deleted file mode 100644 index abc8c0cd65a..00000000000 --- a/usr.bin/truss/cloudabi.h +++ /dev/null @@ -1,28 +0,0 @@ -/*- - * Copyright (c) 2015 Nuxi, https://nuxi.nl/ - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ - */ - -long cloudabi_convert_errno(long); diff --git a/usr.bin/truss/i386-linux.c b/usr.bin/truss/i386-linux.c index c5f625d7303..52232b41d45 100644 --- a/usr.bin/truss/i386-linux.c +++ b/usr.bin/truss/i386-linux.c @@ -83,27 +83,11 @@ i386_linux_fetch_args(struct trussinfo *trussinfo, u_int narg) return (0); } -/* - * Linux syscalls return negative errno's, we do positive and map them - */ -static const int bsd_to_linux_errno[] = { - -0, -1, -2, -3, -4, -5, -6, -7, -8, -9, - -10, -35, -12, -13, -14, -15, -16, -17, -18, -19, - -20, -21, -22, -23, -24, -25, -26, -27, -28, -29, - -30, -31, -32, -33, -34, -11,-115,-114, -88, -89, - -90, -91, -92, -93, -94, -95, -96, -97, -98, -99, - -100,-101,-102,-103,-104,-105,-106,-107,-108,-109, - -110,-111, -40, -36,-112,-113, -39, -11, -87,-122, - -116, -66, -6, -6, -6, -6, -6, -37, -38, -9, - -6, -}; - static int i386_linux_fetch_retval(struct trussinfo *trussinfo, long *retval, int *errorp) { struct reg regs; lwpid_t tid; - size_t i; tid = trussinfo->curthread->tid; if (ptrace(PT_GETREGS, tid, (caddr_t)®s, 0) < 0) { @@ -114,17 +98,6 @@ i386_linux_fetch_retval(struct trussinfo *trussinfo, long *retval, int *errorp) retval[0] = regs.r_eax; retval[1] = regs.r_edx; *errorp = !!(regs.r_eflags & PSL_C); - - if (*errorp) { - for (i = 0; i < nitems(bsd_to_linux_errno); i++) { - if (retval[0] == bsd_to_linux_errno[i]) { - retval[0] = i; - return (0); - } - } - - /* XXX: How to handle unknown errors? */ - } return (0); } diff --git a/usr.bin/truss/syscalls.c b/usr.bin/truss/syscalls.c index 97723211c7a..b3a4c423f7b 100644 --- a/usr.bin/truss/syscalls.c +++ b/usr.bin/truss/syscalls.c @@ -2039,6 +2039,7 @@ print_syscall_ret(struct trussinfo *trussinfo, int errorp, long *retval) struct timespec timediff; struct threadinfo *t; struct syscall *sc; + int error; t = trussinfo->curthread; sc = t->cs.sc; @@ -2053,9 +2054,12 @@ print_syscall_ret(struct trussinfo *trussinfo, int errorp, long *retval) print_syscall(trussinfo); fflush(trussinfo->outfile); - if (errorp) + if (errorp) { + error = sysdecode_abi_to_freebsd_errno(t->proc->abi->abi, + retval[0]); fprintf(trussinfo->outfile, " ERR#%ld '%s'\n", retval[0], - strerror(retval[0])); + error == INT_MAX ? "Unknown error" : strerror(error)); + } #ifndef __LP64__ else if (sc->ret_type == 2) { off_t off; From ffadb4acda8f6b2b5dbdf4900f5c154e7a26a137 Mon Sep 17 00:00:00 2001 From: Andriy Voskoboinyk Date: Tue, 23 Feb 2016 21:11:42 +0000 Subject: [PATCH 06/25] net80211: fix TIM cleanup. Remove duplicate 'ni->ni_associd = 0' assignment from ieee80211_node_leave(), since it breaks iv_set_tim() in ic->ic_node_cleanup() (associd is cleared right after this call). Tested with RTL8188EU (HOSTAP mode) and WUSB54GC (STA mode, with powersaving enabled). Approved by: adrian (mentor) Differential Revision: https://reviews.freebsd.org/D5398 --- sys/net80211/ieee80211_node.c | 1 - 1 file changed, 1 deletion(-) diff --git a/sys/net80211/ieee80211_node.c b/sys/net80211/ieee80211_node.c index 82cfccfaee7..6ac36550d45 100644 --- a/sys/net80211/ieee80211_node.c +++ b/sys/net80211/ieee80211_node.c @@ -2683,7 +2683,6 @@ ieee80211_node_leave(struct ieee80211_node *ni) IEEE80211_LOCK(ic); IEEE80211_AID_CLR(vap, ni->ni_associd); - ni->ni_associd = 0; vap->iv_sta_assoc--; ic->ic_sta_assoc--; From 6971a6379553f399f2f8b4558331cfced256ac40 Mon Sep 17 00:00:00 2001 From: Bryan Drewery Date: Tue, 23 Feb 2016 21:21:47 +0000 Subject: [PATCH 07/25] Fix build after r29592. --- sys/netinet/tcp_subr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/netinet/tcp_subr.c b/sys/netinet/tcp_subr.c index 2b0571cea4c..ddbd12946ec 100644 --- a/sys/netinet/tcp_subr.c +++ b/sys/netinet/tcp_subr.c @@ -268,7 +268,7 @@ struct tcp_funchead t_functions; static struct tcp_function_block *tcp_func_set_ptr = &tcp_def_funcblk; static void -init_tcp_functions() +init_tcp_functions(void) { if (t_functions_inited == 0) { TAILQ_INIT(&t_functions); From 94e6fdd806ee040d08be27e8a06c3bf6ddb0f65b Mon Sep 17 00:00:00 2001 From: Mariusz Zaborski Date: Tue, 23 Feb 2016 22:02:25 +0000 Subject: [PATCH 08/25] According to the sys/kern/capabilities.conf, gethostid(3) should be allowed. Pointed out by: Milosz Kaniewski Approved by: pjd (mentor) MFC after: 3 days Sponsored by: Wheel Systems, http://wheelsystems.com --- sys/kern/kern_mib.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/kern/kern_mib.c b/sys/kern/kern_mib.c index 2ee260d8d5e..d02b5b1a20c 100644 --- a/sys/kern/kern_mib.c +++ b/sys/kern/kern_mib.c @@ -422,7 +422,7 @@ sysctl_hostid(SYSCTL_HANDLER_ARGS) } SYSCTL_PROC(_kern, KERN_HOSTID, hostid, - CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE, + CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE | CTFLAG_CAPRD, NULL, 0, sysctl_hostid, "LU", "Host ID"); /* From 8d4a1dbf7b6fdd37f376d75c9b681289dd32c215 Mon Sep 17 00:00:00 2001 From: Jung-uk Kim Date: Tue, 23 Feb 2016 22:22:15 +0000 Subject: [PATCH 09/25] Fix style(9) bugs. --- sys/dev/acpica/acpi_video.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/sys/dev/acpica/acpi_video.c b/sys/dev/acpica/acpi_video.c index 5b9e526e346..0bc264290b0 100644 --- a/sys/dev/acpica/acpi_video.c +++ b/sys/dev/acpica/acpi_video.c @@ -602,14 +602,16 @@ acpi_video_vo_bind(struct acpi_video_output *vo, ACPI_HANDLE handle) vo->handle = handle; vo->vo_numlevels = vo_get_brightness_levels(handle, &vo->vo_levels); if (vo->vo_numlevels >= 2) { - if (vo->vo_fullpower == -1 - || acpi_video_vo_check_level(vo, vo->vo_fullpower) != 0) + if (vo->vo_fullpower == -1 || + acpi_video_vo_check_level(vo, vo->vo_fullpower) != 0) { /* XXX - can't deal with rebinding... */ vo->vo_fullpower = vo->vo_levels[BCL_FULLPOWER]; - if (vo->vo_economy == -1 - || acpi_video_vo_check_level(vo, vo->vo_economy) != 0) + } + if (vo->vo_economy == -1 || + acpi_video_vo_check_level(vo, vo->vo_economy) != 0) { /* XXX - see above. */ vo->vo_economy = vo->vo_levels[BCL_ECONOMY]; + } } if (vo->vo_levels != NULL) AcpiInstallNotifyHandler(handle, ACPI_DEVICE_NOTIFY, From 3e4f6cc6c787bc960604f8e1daffa669aa2d7588 Mon Sep 17 00:00:00 2001 From: Jung-uk Kim Date: Tue, 23 Feb 2016 22:30:45 +0000 Subject: [PATCH 10/25] Fix white spaces. --- sys/dev/acpica/acpi_video.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/dev/acpica/acpi_video.c b/sys/dev/acpica/acpi_video.c index 0bc264290b0..dc8aa41ef77 100644 --- a/sys/dev/acpica/acpi_video.c +++ b/sys/dev/acpica/acpi_video.c @@ -608,7 +608,7 @@ acpi_video_vo_bind(struct acpi_video_output *vo, ACPI_HANDLE handle) vo->vo_fullpower = vo->vo_levels[BCL_FULLPOWER]; } if (vo->vo_economy == -1 || - acpi_video_vo_check_level(vo, vo->vo_economy) != 0) { + acpi_video_vo_check_level(vo, vo->vo_economy) != 0) { /* XXX - see above. */ vo->vo_economy = vo->vo_levels[BCL_ECONOMY]; } From 84f1891033161f057ede75e5102d4b28c616acee Mon Sep 17 00:00:00 2001 From: Jilles Tjoelker Date: Tue, 23 Feb 2016 22:44:01 +0000 Subject: [PATCH 11/25] sh: Fix set -v and multiline history after r295825. r295825 erroneously dropped the newline from the input data for 'set -v' output and multiline history. Reported by: vangyzen --- bin/sh/input.c | 3 ++- bin/sh/tests/parser/Makefile | 1 + bin/sh/tests/parser/set-v1.0 | 8 ++++++++ bin/sh/tests/parser/set-v1.0.stderr | 5 +++++ 4 files changed, 16 insertions(+), 1 deletion(-) create mode 100644 bin/sh/tests/parser/set-v1.0 create mode 100644 bin/sh/tests/parser/set-v1.0.stderr diff --git a/bin/sh/input.c b/bin/sh/input.c index 27b835a55b8..3292f9905f7 100644 --- a/bin/sh/input.c +++ b/bin/sh/input.c @@ -242,7 +242,8 @@ again: parsenleft = parselleft; parselleft = 0; } else /* *q == '\n' */ { - parsenleft = q - parsenextc + 1; + q++; + parsenleft = q - parsenextc; parselleft -= parsenleft; } parsenleft--; diff --git a/bin/sh/tests/parser/Makefile b/bin/sh/tests/parser/Makefile index eb1e6f567b5..dd126ee7ea9 100644 --- a/bin/sh/tests/parser/Makefile +++ b/bin/sh/tests/parser/Makefile @@ -79,6 +79,7 @@ FILES+= only-redir2.0 FILES+= only-redir3.0 FILES+= only-redir4.0 FILES+= pipe-not1.0 +FILES+= set-v1.0 set-v1.0.stderr FILES+= var-assign1.0 .include diff --git a/bin/sh/tests/parser/set-v1.0 b/bin/sh/tests/parser/set-v1.0 new file mode 100644 index 00000000000..687cb831755 --- /dev/null +++ b/bin/sh/tests/parser/set-v1.0 @@ -0,0 +1,8 @@ +# $FreeBSD$ + +${SH} <<\EOF +echo one >&2 +set -v +echo two >&2 +echo three >&2 +EOF diff --git a/bin/sh/tests/parser/set-v1.0.stderr b/bin/sh/tests/parser/set-v1.0.stderr new file mode 100644 index 00000000000..d904fa5ffdb --- /dev/null +++ b/bin/sh/tests/parser/set-v1.0.stderr @@ -0,0 +1,5 @@ +one +echo two >&2 +two +echo three >&2 +three From 7054df7f5b9be8a90dcd1a3c4994100e1d1df544 Mon Sep 17 00:00:00 2001 From: Jung-uk Kim Date: Tue, 23 Feb 2016 22:50:45 +0000 Subject: [PATCH 12/25] Remove brightness notify handler before reinstalling new one. --- sys/dev/acpica/acpi_video.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sys/dev/acpica/acpi_video.c b/sys/dev/acpica/acpi_video.c index dc8aa41ef77..135f5bc5ad6 100644 --- a/sys/dev/acpica/acpi_video.c +++ b/sys/dev/acpica/acpi_video.c @@ -597,8 +597,12 @@ acpi_video_vo_bind(struct acpi_video_output *vo, ACPI_HANDLE handle) { ACPI_SERIAL_BEGIN(video_output); - if (vo->vo_levels != NULL) + if (vo->vo_levels != NULL) { + AcpiRemoveNotifyHandler(vo->handle, ACPI_DEVICE_NOTIFY, + acpi_video_vo_notify_handler); AcpiOsFree(vo->vo_levels); + vo->vo_levels = NULL; + } vo->handle = handle; vo->vo_numlevels = vo_get_brightness_levels(handle, &vo->vo_levels); if (vo->vo_numlevels >= 2) { From 8444599e1e03c22265bb1b45d34ad4716194e075 Mon Sep 17 00:00:00 2001 From: Jung-uk Kim Date: Tue, 23 Feb 2016 22:55:44 +0000 Subject: [PATCH 13/25] Silence PVS-Studio warning (V595). --- sys/dev/acpica/acpi_video.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sys/dev/acpica/acpi_video.c b/sys/dev/acpica/acpi_video.c index 135f5bc5ad6..6360071b50a 100644 --- a/sys/dev/acpica/acpi_video.c +++ b/sys/dev/acpica/acpi_video.c @@ -616,10 +616,9 @@ acpi_video_vo_bind(struct acpi_video_output *vo, ACPI_HANDLE handle) /* XXX - see above. */ vo->vo_economy = vo->vo_levels[BCL_ECONOMY]; } - } - if (vo->vo_levels != NULL) AcpiInstallNotifyHandler(handle, ACPI_DEVICE_NOTIFY, acpi_video_vo_notify_handler, vo); + } ACPI_SERIAL_END(video_output); } From 2f977ab449f480dbdcdc169da17dc19a18ad91c0 Mon Sep 17 00:00:00 2001 From: Jung-uk Kim Date: Tue, 23 Feb 2016 23:09:45 +0000 Subject: [PATCH 14/25] Silence PVS-Studio warning (V595). --- sys/dev/acpica/acpi_ec.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sys/dev/acpica/acpi_ec.c b/sys/dev/acpica/acpi_ec.c index 8a8cb6ea4f3..6482b3f1b24 100644 --- a/sys/dev/acpica/acpi_ec.c +++ b/sys/dev/acpica/acpi_ec.c @@ -434,9 +434,7 @@ out: params->gpe_bit, (params->glk) ? ", GLK" : "", ecdt ? ", ECDT" : ""); device_set_desc_copy(dev, desc); - } - - if (ret > 0 && params) + } else free(params, M_TEMP); if (buf.Pointer) AcpiOsFree(buf.Pointer); From 200241b504fb22d357f587bb9bb3e9467a0fab50 Mon Sep 17 00:00:00 2001 From: Bryan Drewery Date: Tue, 23 Feb 2016 23:37:10 +0000 Subject: [PATCH 15/25] Fix build after r295934. --- sys/kern/kern_mib.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/kern/kern_mib.c b/sys/kern/kern_mib.c index d02b5b1a20c..60c8b7e3f38 100644 --- a/sys/kern/kern_mib.c +++ b/sys/kern/kern_mib.c @@ -422,7 +422,7 @@ sysctl_hostid(SYSCTL_HANDLER_ARGS) } SYSCTL_PROC(_kern, KERN_HOSTID, hostid, - CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE | CTFLAG_CAPRD, + CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_PRISON | CTLFLAG_MPSAFE | CTLFLAG_CAPRD, NULL, 0, sysctl_hostid, "LU", "Host ID"); /* From 0eda5b3f23f8867823fc1587b77b547b3a738a69 Mon Sep 17 00:00:00 2001 From: Jung-uk Kim Date: Tue, 23 Feb 2016 23:57:24 +0000 Subject: [PATCH 16/25] Silence PVS-Studio warning (V595). It can never be NULL here. --- sys/x86/isa/clock.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/x86/isa/clock.c b/sys/x86/isa/clock.c index 8778c0dc627..cced5e34afb 100644 --- a/sys/x86/isa/clock.c +++ b/sys/x86/isa/clock.c @@ -166,7 +166,7 @@ clkintr(void *arg) mtx_unlock_spin(&clock_lock); } - if (sc && sc->et.et_active && sc->mode != MODE_STOP) + if (sc->et.et_active && sc->mode != MODE_STOP) sc->et.et_event_cb(&sc->et, sc->et.et_arg); #ifdef DEV_MCA From 8f8cb840b0d64bb227909d4866b7734d3a0d200d Mon Sep 17 00:00:00 2001 From: Maxim Sobolev Date: Tue, 23 Feb 2016 23:59:08 +0000 Subject: [PATCH 17/25] Improve mkuzip(8) and geom_uzip(4), merge in LZMA support from mkulzma(8) and geom_uncompress(4): 1. mkuzip(8): - Proper support for eliminating all-zero blocks when compressing an image. This feature is already supported by the geom_uzip(4) module and CLOOP format in general, so it's just a matter of making mkuzip(8) match. It should be noted, however that this feature while it sounds great, results in very slight improvement in the overall compression ratio, since compressing default 16k all-zero block produces only 39 bytes compressed output block, which is 99.8% compression ratio. With typical average compression ratio of amd64 binaries and data being around 60-70% the difference between 99.8% and 100.0% is not that great further diluted by the ratio of number of zero blocks in the uncompressed image to the overall number of blocks being less than 0.5 (typically). However, this may be important from performance standpoint, so that kernel are not spinning its wheels decompressing those empty blocks every time this zero region is read. It could also be important when you create huge image mostly filled with zero blocks for testing purposes. - New feature allowing to de-duplicate output image. It turns out that if you twist CLOOP format a bit you can do that as well. And unlike zero-blocks elimination, this gives a noticeable improvement in the overall compression ratio, reducing output image by something like 3-4% on my test UFS2 3GB image consisting of full FreeBSD base system plus some of the packages (openjdk, apache etc), about 2.3GB worth of file data (800+MB compressed). The only caveat is that images created with this feature "on" would not work on older versions of FeeBSDxi kernel, hence it's turned off by default. - provide options to control both features and document them in manual page. - merge in all relevant LZMA compression support from the mkulzma(8), add new option to select between both. - switch license from ad-hoc beerware into standard 2-clause BSD. 2. geom_uzip(4): - implement support for de-duplicated images; - optimize some code paths to handle "all-zero" blocks without reading any compressed data; - beef up manual page to explain that geom_uzip(4) is not limited only to md(4) images. The compressed data can be written to the block device and accessed directly via magic of GEOM(4) and devfs(4), including to mount root fs from a compressed drive. - convert debug log code from being compiled in conditionally into being present all the time and provide two sysctls to turn it on or off. Due to intended use of the module, it can be used in environments where there may not be a luxury to put new kernel with debug code enabled. Having those options handy allows debug issues without as much problem by just having access to serial console or network shell access to a box/appliance. The resulting additional CPU cycles are just few int comparisons and branches, and those are minuscule when compared to data decompression which is the main feature of the module. - hopefully improve robustness and resiliency of the geom_uzip(4) by performing some of the data validation / range checking on the TOC entries and rejecting to attach to an image if those checks fail. - merge in all relevant LZMA decompression support from the geom_uncompress(4), enable automatically when appropriate format is indicated in the header. - move compilation work into its own worker thread so that it does not clog g_up. This allows multiple instances work in parallel utilizing smp cores. - document new knobs in the manual page. Reviewed by: adrian MFC after: 1 month Differential Revision: https://reviews.freebsd.org/D5333 --- share/man/man4/geom_uzip.4 | 90 ++++- sys/conf/files | 13 +- sys/geom/uzip/g_uzip.c | 535 ++++++++++++++++++++-------- sys/geom/uzip/g_uzip.h | 35 ++ sys/geom/uzip/g_uzip_cloop.h | 55 +++ sys/geom/uzip/g_uzip_dapi.h | 41 +++ sys/geom/uzip/g_uzip_lzma.c | 111 ++++++ sys/geom/uzip/g_uzip_lzma.h | 32 ++ sys/geom/uzip/g_uzip_softc.h | 56 +++ sys/geom/uzip/g_uzip_wrkthr.c | 71 ++++ sys/geom/uzip/g_uzip_wrkthr.h | 30 ++ sys/geom/uzip/g_uzip_zlib.c | 137 +++++++ sys/geom/uzip/g_uzip_zlib.h | 33 ++ sys/modules/geom/geom_uzip/Makefile | 15 +- usr.bin/mkuzip/Makefile | 3 +- usr.bin/mkuzip/mkuz_blockcache.c | 97 +++++ usr.bin/mkuzip/mkuz_blockcache.h | 36 ++ usr.bin/mkuzip/mkuz_cloop.h | 50 +++ usr.bin/mkuzip/mkuz_lzma.c | 110 ++++++ usr.bin/mkuzip/mkuz_lzma.h | 42 +++ usr.bin/mkuzip/mkuz_zlib.c | 81 +++++ usr.bin/mkuzip/mkuz_zlib.h | 36 ++ usr.bin/mkuzip/mkuzip.8 | 82 ++++- usr.bin/mkuzip/mkuzip.c | 221 ++++++++---- usr.bin/mkuzip/mkuzip.h | 29 ++ 25 files changed, 1810 insertions(+), 231 deletions(-) create mode 100644 sys/geom/uzip/g_uzip.h create mode 100644 sys/geom/uzip/g_uzip_cloop.h create mode 100644 sys/geom/uzip/g_uzip_dapi.h create mode 100644 sys/geom/uzip/g_uzip_lzma.c create mode 100644 sys/geom/uzip/g_uzip_lzma.h create mode 100644 sys/geom/uzip/g_uzip_softc.h create mode 100644 sys/geom/uzip/g_uzip_wrkthr.c create mode 100644 sys/geom/uzip/g_uzip_wrkthr.h create mode 100644 sys/geom/uzip/g_uzip_zlib.c create mode 100644 sys/geom/uzip/g_uzip_zlib.h create mode 100644 usr.bin/mkuzip/mkuz_blockcache.c create mode 100644 usr.bin/mkuzip/mkuz_blockcache.h create mode 100644 usr.bin/mkuzip/mkuz_cloop.h create mode 100644 usr.bin/mkuzip/mkuz_lzma.c create mode 100644 usr.bin/mkuzip/mkuz_lzma.h create mode 100644 usr.bin/mkuzip/mkuz_zlib.c create mode 100644 usr.bin/mkuzip/mkuz_zlib.h create mode 100644 usr.bin/mkuzip/mkuzip.h diff --git a/share/man/man4/geom_uzip.4 b/share/man/man4/geom_uzip.4 index 0d36322121e..4dbab524cca 100644 --- a/share/man/man4/geom_uzip.4 +++ b/share/man/man4/geom_uzip.4 @@ -30,7 +30,7 @@ .Os .Sh NAME .Nm geom_uzip -.Nd "GEOM based compressed disk images" +.Nd "GEOM based compressed disk images and partitions" .Sh SYNOPSIS To compile this driver into the kernel, place the following line in your @@ -51,7 +51,7 @@ The framework provides support for compressed read-only disk images. This allows significant storage savings at the expense of -a little CPU time on each read. +a some CPU time on each read. Data written in the GEOM label area allows .Nm to detect compressed images which have been created with @@ -63,17 +63,53 @@ creates a unique .Pa md#.uzip device for each image. .Pp +.Nm +is not limited to supporting only +.Xr md 4 +images. +The image can also reside on a block device. +.Pq For example, a disk, USB flash drive, DVD-ROM, etc. +The appropriate device node will appear with the +.Pa .uzip +suffix. +.Bd -literal -offset indent +# gpart show da0 +=> 0 7833600 da0 BSD (3.7G) + 0 2097152 1 freebsd-ufs (1.0G) + 2097152 5736448 - free - (2.7G) +# gpart add -t freebsd-ufs -s 1G da0 +da0b added +# dd if=/tmp/20160217_dcomp_zcomp.uzip bs=256k of=/dev/da0b +3190+1 records in +3190+1 records out +836331008 bytes transferred in 111.021489 secs (7533055 bytes/sec) +# fsck -t ffs /dev/da0b.uzip +** /dev/da0b.uzip (NO WRITE) +** Last Mounted on /mnt +** Phase 1 - Check Blocks and Sizes +** Phase 2 - Check Pathnames +** Phase 3 - Check Connectivity +** Phase 4 - Check Reference Counts +** Phase 5 - Check Cyl groups +97455 files, 604242 used, 184741 free (2349 frags, 22799 blocks, + 0.3% fragmentation) +# mount -o ro /dev/da0b.uzip /mnt +# df /dev/da0b.uzip +Filesystem 1K-blocks Used Avail Capacity Mounted on +/dev/da0b.uzip 3155932 2416968 738964 77% /mnt +.Ed +.Pp The .Nm -device is subsequently used by the +device is subsequently used by .Fx -kernel to access the disk images. +kernel to access the uncompressed data. The .Nm driver does not allow write operations to the underlying disk image. To check which -.Xr md 4 -devices match a given +.Dq providers +match a given .Nm device: .Bd -literal -offset indent @@ -83,13 +119,44 @@ Providers: 1. Name: md1.uzip Mediasize: 22003712 (21M) Sectorsize: 512 - Mode: r1w0e1 Consumers: 1. Name: md1 Mediasize: 9563648 (9.1M) Sectorsize: 512 - Mode: r1w0e1 + +Geom name: da0b.uzip +Providers: +1. Name: da0b.uzip + Mediasize: 3355443200 (3.1G) + Sectorsize: 512 +Consumers: +1. Name: da0b + Mediasize: 1073741824 (1.0G) + Sectorsize: 512 .Ed +.Pp +.Nm +allows mounting the root file system from a compressed disk partition by +setting the +.Dv vfs.root.mountfrom +tunable. +See +.Xr loader.conf 5 +for details. +.Sh DIAGNOSTICS +Several flags are provided for tracing +.Nm +I/O operations and TOC parsing via the following sysctls. +.Bl -tag -width indent +.It Va kern.geom.uzip.debug +Log level. +Zero disables logging. +Higher values enable more verbose debug logging for +.Nm . +Supported levels are from 0 (no logging) to 4 (maximum amount of logging). +.It Va kern.geom.uzip.debug_block +Log operations involving compressed cluster number. +.El .Sh SEE ALSO .Xr GEOM 4 , .Xr md 4 , @@ -101,5 +168,12 @@ The .Nm driver was written by .An Max Khon Aq Mt fjoe@FreeBSD.org . +The block de-duplication code as well as some +.Nm +driver optimizations have been contributed by +.An Maxim Sobolev Aq Mt sobomax@FreeBSD.org . +The LZMA decompression support and CLOOP 3.0 support have been added by +.An Aleksandr Rybalko Aq Mt ray@FreeBSD.org . +.Pp This manual page was written by .An Ceri Davies Aq Mt ceri@FreeBSD.org . diff --git a/sys/conf/files b/sys/conf/files index bbf971319e0..7102c03f8a7 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -3083,21 +3083,24 @@ geom/shsec/g_shsec.c optional geom_shsec geom/stripe/g_stripe.c optional geom_stripe geom/uncompress/g_uncompress.c optional geom_uncompress contrib/xz-embedded/freebsd/xz_malloc.c \ - optional xz_embedded | geom_uncompress \ + optional xz_embedded | geom_uncompress | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_crc32.c \ - optional xz_embedded | geom_uncompress \ + optional xz_embedded | geom_uncompress | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_bcj.c \ - optional xz_embedded | geom_uncompress \ + optional xz_embedded | geom_uncompress | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_lzma2.c \ - optional xz_embedded | geom_uncompress \ + optional xz_embedded | geom_uncompress | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_stream.c \ - optional xz_embedded | geom_uncompress \ + optional xz_embedded | geom_uncompress | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" geom/uzip/g_uzip.c optional geom_uzip +geom/uzip/g_uzip_lzma.c optional geom_uzip +geom/uzip/g_uzip_wrkthr.c optional geom_uzip +geom/uzip/g_uzip_zlib.c optional geom_uzip geom/vinum/geom_vinum.c optional geom_vinum geom/vinum/geom_vinum_create.c optional geom_vinum geom/vinum/geom_vinum_drive.c optional geom_vinum diff --git a/sys/geom/uzip/g_uzip.c b/sys/geom/uzip/g_uzip.c index 8363d06ebae..17d179bc115 100644 --- a/sys/geom/uzip/g_uzip.c +++ b/sys/geom/uzip/g_uzip.c @@ -1,6 +1,7 @@ /*- * Copyright (c) 2004 Max Khon * Copyright (c) 2014 Juniper Networks, Inc. + * Copyright (c) 2006-2016 Maxim Sobolev * All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -38,91 +39,116 @@ __FBSDID("$FreeBSD$"); #include #include #include -#include +#include #include -FEATURE(geom_uzip, "GEOM uzip read-only compressed disks support"); +MALLOC_DEFINE(M_GEOM_UZIP, "geom_uzip", "GEOM UZIP data structures"); -#undef GEOM_UZIP_DEBUG -#ifdef GEOM_UZIP_DEBUG -#define DPRINTF(a) printf a -#else -#define DPRINTF(a) +#include +#include +#include +#include +#include +#include +#include + +FEATURE(geom_uzip, "GEOM read-only compressed disks support"); + +struct g_uzip_blk { + uint64_t offset; + uint32_t blen; +#define BLEN_UNDEF UINT32_MAX +}; + +#ifndef ABS +#define ABS(a) ((a) < 0 ? -(a) : (a)) #endif -static MALLOC_DEFINE(M_GEOM_UZIP, "geom_uzip", "GEOM UZIP data structures"); +#define BLK_IN_RANGE(mcn, bcn, ilen) \ + (((bcn) != BLEN_UNDEF) && ( \ + ((ilen) >= 0 && (mcn >= bcn) && (mcn <= ((intmax_t)(bcn) + (ilen)))) || \ + ((ilen) < 0 && (mcn <= bcn) && (mcn >= ((intmax_t)(bcn) + (ilen)))) \ + )) + +#ifdef GEOM_UZIP_DEBUG +# define GEOM_UZIP_DBG_DEFAULT 3 +#else +# define GEOM_UZIP_DBG_DEFAULT 0 +#endif + +#define GUZ_DBG_ERR 1 +#define GUZ_DBG_INFO 2 +#define GUZ_DBG_IO 3 +#define GUZ_DBG_TOC 4 + +SYSCTL_DECL(_kern_geom); +SYSCTL_NODE(_kern_geom, OID_AUTO, uzip, CTLFLAG_RW, 0, "GEOM_UZIP stuff"); +static u_int g_uzip_debug = GEOM_UZIP_DBG_DEFAULT; +SYSCTL_UINT(_kern_geom_uzip, OID_AUTO, debug, CTLFLAG_RWTUN, &g_uzip_debug, 0, + "Debug level (0-4)"); +static u_int g_uzip_debug_block = BLEN_UNDEF; +SYSCTL_UINT(_kern_geom_uzip, OID_AUTO, debug_block, CTLFLAG_RWTUN, + &g_uzip_debug_block, 0, "Debug operations around specific cluster#"); + +#define DPRINTF(lvl, a) \ + if ((lvl) <= g_uzip_debug) { \ + printf a; \ + } +#define DPRINTF_BLK(lvl, cn, a) \ + if ((lvl) <= g_uzip_debug || \ + BLK_IN_RANGE(cn, g_uzip_debug_block, 8) || \ + BLK_IN_RANGE(cn, g_uzip_debug_block, -8)) { \ + printf a; \ + } +#define DPRINTF_BRNG(lvl, bcn, ecn, a) \ + if (bcn >= ecn) { \ + printf("DPRINTF_BRNG: invalid range (%ju, %ju), BUG BUG " \ + "BUG!\n", (uintmax_t)bcn, (uintmax_t)ecn); \ + } else if (((lvl) <= g_uzip_debug) || \ + BLK_IN_RANGE(g_uzip_debug_block, bcn, \ + (intmax_t)ecn - (intmax_t)bcn)) { \ + printf a; \ + } #define UZIP_CLASS_NAME "UZIP" /* * Maximum allowed valid block size (to prevent foot-shooting) */ -#define MAX_BLKSZ (MAXPHYS - MAXPHYS / 1000 - 12) +#define MAX_BLKSZ (MAXPHYS) -/* - * Integer values (block size, number of blocks, offsets) - * are stored in big-endian (network) order on disk and struct cloop_header - * and in native order in struct g_uzip_softc - */ - -#define CLOOP_MAGIC_LEN 128 static char CLOOP_MAGIC_START[] = "#!/bin/sh\n"; -struct cloop_header { - char magic[CLOOP_MAGIC_LEN]; /* cloop magic */ - uint32_t blksz; /* block size */ - uint32_t nblocks; /* number of blocks */ -}; - -struct g_uzip_softc { - uint32_t blksz; /* block size */ - uint32_t nblocks; /* number of blocks */ - uint64_t *offsets; - - struct mtx last_mtx; - uint32_t last_blk; /* last blk no */ - char *last_buf; /* last blk data */ - int req_total; /* total requests */ - int req_cached; /* cached requests */ -}; - -static void g_uzip_done(struct bio *bp); +static void g_uzip_read_done(struct bio *bp); +static void g_uzip_do(struct g_uzip_softc *, struct bio *bp); static void g_uzip_softc_free(struct g_uzip_softc *sc, struct g_geom *gp) { if (gp != NULL) { - DPRINTF(("%s: %d requests, %d cached\n", + DPRINTF(GUZ_DBG_INFO, ("%s: %d requests, %d cached\n", gp->name, sc->req_total, sc->req_cached)); } - if (sc->offsets != NULL) { - free(sc->offsets, M_GEOM_UZIP); - sc->offsets = NULL; + + mtx_lock(&sc->queue_mtx); + sc->wrkthr_flags |= GUZ_SHUTDOWN; + wakeup(sc); + while (!(sc->wrkthr_flags & GUZ_EXITING)) { + msleep(sc->procp, &sc->queue_mtx, PRIBIO, "guzfree", + hz / 10); } + mtx_unlock(&sc->queue_mtx); + + sc->dcp->free(sc->dcp); + free(sc->toc, M_GEOM_UZIP); + mtx_destroy(&sc->queue_mtx); mtx_destroy(&sc->last_mtx); free(sc->last_buf, M_GEOM_UZIP); free(sc, M_GEOM_UZIP); } -static void * -z_alloc(void *nil, u_int type, u_int size) -{ - void *ptr; - - ptr = malloc(type * size, M_GEOM_UZIP, M_NOWAIT); - - return (ptr); -} - -static void -z_free(void *nil, void *ptr) -{ - - free(ptr, M_GEOM_UZIP); -} - static int g_uzip_cached(struct g_geom *gp, struct bio *bp) { @@ -144,8 +170,9 @@ g_uzip_cached(struct g_geom *gp, struct bio *bp) sc->req_cached++; mtx_unlock(&sc->last_mtx); - DPRINTF(("%s/%s: %p: offset=%jd: got %jd bytes from cache\n", - __func__, gp->name, bp, (intmax_t)ofs, (intmax_t)usz)); + DPRINTF(GUZ_DBG_IO, ("%s/%s: %p: offset=%jd: got %jd bytes " + "from cache\n", __func__, gp->name, bp, (intmax_t)ofs, + (intmax_t)usz)); bp->bio_completed += usz; bp->bio_resid -= usz; @@ -160,6 +187,19 @@ g_uzip_cached(struct g_geom *gp, struct bio *bp) return (0); } +#define BLK_ENDS(sc, bi) ((sc)->toc[(bi)].offset + \ + (sc)->toc[(bi)].blen) + +#define BLK_IS_CONT(sc, bi) (BLK_ENDS((sc), (bi) - 1) == \ + (sc)->toc[(bi)].offset) +#define BLK_IS_NIL(sc, bi) ((sc)->toc[(bi)].blen == 0) + +#define TOFF_2_BOFF(sc, pp, bi) ((sc)->toc[(bi)].offset - \ + (sc)->toc[(bi)].offset % (pp)->sectorsize) +#define TLEN_2_BLEN(sc, pp, bp, ei) ((BLK_ENDS((sc), (ei)) - \ + (bp)->bio_offset + (pp)->sectorsize - 1) / \ + (pp)->sectorsize * (pp)->sectorsize) + static int g_uzip_request(struct g_geom *gp, struct bio *bp) { @@ -167,21 +207,14 @@ g_uzip_request(struct g_geom *gp, struct bio *bp) struct bio *bp2; struct g_consumer *cp; struct g_provider *pp; - off_t ofs; - size_t start_blk, end_blk; + off_t ofs, start_blk_ofs; + size_t i, start_blk, end_blk, zsize; if (g_uzip_cached(gp, bp) != 0) return (1); sc = gp->softc; - bp2 = g_clone_bio(bp); - if (bp2 == NULL) { - g_io_deliver(bp, ENOMEM); - return (1); - } - bp2->bio_done = g_uzip_done; - cp = LIST_FIRST(&gp->consumer); pp = cp->provider; @@ -191,17 +224,54 @@ g_uzip_request(struct g_geom *gp, struct bio *bp) end_blk = (ofs + bp->bio_resid + sc->blksz - 1) / sc->blksz; KASSERT(end_blk <= sc->nblocks, ("end_blk out of range")); - DPRINTF(("%s/%s: %p: start=%u (%jd), end=%u (%jd)\n", - __func__, gp->name, bp, - (u_int)start_blk, (intmax_t)sc->offsets[start_blk], - (u_int)end_blk, (intmax_t)sc->offsets[end_blk])); + for (; BLK_IS_NIL(sc, start_blk) && start_blk < end_blk; start_blk++) { + /* Fill in any leading Nil blocks */ + start_blk_ofs = ofs % sc->blksz; + zsize = MIN(sc->blksz - start_blk_ofs, bp->bio_resid); + DPRINTF_BLK(GUZ_DBG_IO, start_blk, ("%s/%s: %p/%ju: " + "filling %ju zero bytes\n", __func__, gp->name, gp, + (uintmax_t)bp->bio_completed, (uintmax_t)zsize)); + bzero(bp->bio_data + bp->bio_completed, zsize); + bp->bio_completed += zsize; + bp->bio_resid -= zsize; + ofs += zsize; + } - bp2->bio_offset = sc->offsets[start_blk] - - sc->offsets[start_blk] % pp->sectorsize; + if (start_blk == end_blk) { + KASSERT(bp->bio_resid == 0, ("bp->bio_resid is invalid")); + /* + * No non-Nil data is left, complete request immediately. + */ + DPRINTF(GUZ_DBG_IO, ("%s/%s: %p: all done returning %ju " + "bytes\n", __func__, gp->name, gp, + (uintmax_t)bp->bio_completed)); + g_io_deliver(bp, 0); + return (1); + } + + for (i = start_blk + 1; i < end_blk; i++) { + /* Trim discontinuous areas if any */ + if (!BLK_IS_CONT(sc, i)) { + end_blk = i; + break; + } + } + + DPRINTF_BRNG(GUZ_DBG_IO, start_blk, end_blk, ("%s/%s: %p: " + "start=%u (%ju), end=%u (%ju)\n", __func__, gp->name, bp, + (u_int)start_blk, (uintmax_t)sc->toc[start_blk].offset, + (u_int)end_blk, (uintmax_t)BLK_ENDS(sc, end_blk - 1))); + + bp2 = g_clone_bio(bp); + if (bp2 == NULL) { + g_io_deliver(bp, ENOMEM); + return (1); + } + bp2->bio_done = g_uzip_read_done; + + bp2->bio_offset = TOFF_2_BOFF(sc, pp, start_blk); while (1) { - bp2->bio_length = sc->offsets[end_blk] - bp2->bio_offset; - bp2->bio_length = (bp2->bio_length + pp->sectorsize - 1) / - pp->sectorsize * pp->sectorsize; + bp2->bio_length = TLEN_2_BLEN(sc, pp, bp2, end_blk - 1); if (bp2->bio_length <= MAXPHYS) break; @@ -215,8 +285,8 @@ g_uzip_request(struct g_geom *gp, struct bio *bp) return (1); } - DPRINTF(("%s/%s: %p: reading %jd bytes from offset %jd\n", - __func__, gp->name, bp, + DPRINTF_BRNG(GUZ_DBG_IO, start_blk, end_blk, ("%s/%s: %p: " + "reading %jd bytes from offset %jd\n", __func__, gp->name, bp, (intmax_t)bp2->bio_length, (intmax_t)bp2->bio_offset)); g_io_request(bp2, cp); @@ -224,22 +294,37 @@ g_uzip_request(struct g_geom *gp, struct bio *bp) } static void -g_uzip_done(struct bio *bp) +g_uzip_read_done(struct bio *bp) { - z_stream zs; struct bio *bp2; - struct g_provider *pp; - struct g_consumer *cp; struct g_geom *gp; struct g_uzip_softc *sc; - char *data, *data2; - off_t ofs; - size_t blk, blkofs, len, ulen; bp2 = bp->bio_parent; gp = bp2->bio_to->geom; sc = gp->softc; + mtx_lock(&sc->queue_mtx); + bioq_disksort(&sc->bio_queue, bp); + mtx_unlock(&sc->queue_mtx); + wakeup(sc); +} + +static void +g_uzip_do(struct g_uzip_softc *sc, struct bio *bp) +{ + struct bio *bp2; + struct g_provider *pp; + struct g_consumer *cp; + struct g_geom *gp; + char *data, *data2; + off_t ofs; + size_t blk, blkofs, len, ulen, firstblk; + int err; + + bp2 = bp->bio_parent; + gp = bp2->bio_to->geom; + cp = LIST_FIRST(&gp->consumer); pp = cp->provider; @@ -253,46 +338,47 @@ g_uzip_done(struct bio *bp) goto done; } - zs.zalloc = z_alloc; - zs.zfree = z_free; - if (inflateInit(&zs) != Z_OK) { - bp2->bio_error = EILSEQ; - goto done; - } - ofs = bp2->bio_offset + bp2->bio_completed; - blk = ofs / sc->blksz; + firstblk = blk = ofs / sc->blksz; blkofs = ofs % sc->blksz; - data = bp->bio_data + sc->offsets[blk] % pp->sectorsize; + data = bp->bio_data + sc->toc[blk].offset % pp->sectorsize; data2 = bp2->bio_data + bp2->bio_completed; while (bp->bio_completed && bp2->bio_resid) { + if (blk > firstblk && !BLK_IS_CONT(sc, blk)) { + DPRINTF_BLK(GUZ_DBG_IO, blk, ("%s/%s: %p: backref'ed " + "cluster #%u requested, looping around\n", + __func__, gp->name, bp2, (u_int)blk)); + goto done; + } ulen = MIN(sc->blksz - blkofs, bp2->bio_resid); - len = sc->offsets[blk + 1] - sc->offsets[blk]; - DPRINTF(("%s/%s: %p/%ju: data2=%p, ulen=%u, data=%p, len=%u\n", - __func__, gp->name, gp, bp->bio_completed, - data2, (u_int)ulen, data, (u_int)len)); + len = sc->toc[blk].blen; + DPRINTF(GUZ_DBG_IO, ("%s/%s: %p/%ju: data2=%p, ulen=%u, " + "data=%p, len=%u\n", __func__, gp->name, gp, + bp->bio_completed, data2, (u_int)ulen, data, (u_int)len)); if (len == 0) { /* All zero block: no cache update */ bzero(data2, ulen); } else if (len <= bp->bio_completed) { - zs.next_in = data; - zs.avail_in = len; - zs.next_out = sc->last_buf; - zs.avail_out = sc->blksz; mtx_lock(&sc->last_mtx); - if (inflate(&zs, Z_FINISH) != Z_STREAM_END) { + err = sc->dcp->decompress(sc->dcp, gp->name, data, + len, sc->last_buf); + if (err != 0) { sc->last_blk = -1; mtx_unlock(&sc->last_mtx); - inflateEnd(&zs); bp2->bio_error = EILSEQ; + DPRINTF(GUZ_DBG_ERR, ("%s/%s: decompress" + "(%p) failed\n", __func__, gp->name, + sc->dcp)); goto done; } sc->last_blk = blk; memcpy(data2, sc->last_buf + blkofs, ulen); mtx_unlock(&sc->last_mtx); - if (inflateReset(&zs) != Z_OK) { - inflateEnd(&zs); + err = sc->dcp->rewind(sc->dcp, gp->name); + if (err != 0) { bp2->bio_error = EILSEQ; + DPRINTF(GUZ_DBG_ERR, ("%s/%s: rewind(%p) " + "failed\n", __func__, gp->name, sc->dcp)); goto done; } data += len; @@ -307,9 +393,6 @@ g_uzip_done(struct bio *bp) blk++; } - if (inflateEnd(&zs) != Z_OK) - bp2->bio_error = EILSEQ; - done: /* Finish processing the request. */ free(bp->bio_data, M_GEOM_UZIP); @@ -330,9 +413,9 @@ g_uzip_start(struct bio *bp) pp = bp->bio_to; gp = pp->geom; - DPRINTF(("%s/%s: %p: cmd=%d, offset=%jd, length=%jd, buffer=%p\n", - __func__, gp->name, bp, bp->bio_cmd, (intmax_t)bp->bio_offset, - (intmax_t)bp->bio_length, bp->bio_data)); + DPRINTF(GUZ_DBG_IO, ("%s/%s: %p: cmd=%d, offset=%jd, length=%jd, " + "buffer=%p\n", __func__, gp->name, bp, bp->bio_cmd, + (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length, bp->bio_data)); sc = gp->softc; sc->req_total++; @@ -392,6 +475,92 @@ g_uzip_spoiled(struct g_consumer *cp) g_wither_geom(gp, ENXIO); } +static int +g_uzip_parse_toc(struct g_uzip_softc *sc, struct g_provider *pp, + struct g_geom *gp) +{ + uint32_t i, j, backref_to; + uint64_t max_offset, min_offset; + + min_offset = sizeof(struct cloop_header) + + (sc->nblocks + 1) * sizeof(uint64_t); + max_offset = sc->toc[0].offset - 1; + for (i = 0; i < sc->nblocks; i++) { + /* First do some bounds checking */ + if ((sc->toc[i].offset < min_offset) || + (sc->toc[i].offset >= pp->mediasize)) { + goto error_offset; + } + DPRINTF_BLK(GUZ_DBG_IO, i, ("%s: cluster #%u " + "sc->toc[i].offset=%ju max_offset=%ju\n", gp->name, + (u_int)i, (uintmax_t)sc->toc[i].offset, + (uintmax_t)max_offset)); + backref_to = BLEN_UNDEF; + if (sc->toc[i].offset < max_offset) { + /* + * For the backref'ed blocks search already parsed + * TOC entries for the matching offset and copy the + * size from matched entry. + */ + for (j = 0; j <= i; j++) { + if (sc->toc[j].offset == sc->toc[i].offset && + !BLK_IS_NIL(sc, j)) { + break; + } + if (j != i) { + continue; + } + DPRINTF(GUZ_DBG_ERR, ("%s: cannot match " + "backref'ed offset at cluster #%u\n", + gp->name, i)); + return (-1); + } + sc->toc[i].blen = sc->toc[j].blen; + backref_to = j; + } else { + /* + * For the "normal blocks" seek forward until we hit + * block whose offset is larger than ours and assume + * it's going to be the next one. + */ + for (j = i + 1; j < sc->nblocks; j++) { + if (sc->toc[j].offset > max_offset) { + break; + } + } + sc->toc[i].blen = sc->toc[j].offset - + sc->toc[i].offset; + if (BLK_ENDS(sc, i) > pp->mediasize) { + DPRINTF(GUZ_DBG_ERR, ("%s: cluster #%u " + "extends past media boundary (%ju > %ju)\n", + gp->name, (u_int)i, + (uintmax_t)BLK_ENDS(sc, i), + (intmax_t)pp->mediasize)); + return (-1); + } + KASSERT(max_offset <= sc->toc[i].offset, ( + "%s: max_offset is incorrect: %ju", + gp->name, (uintmax_t)max_offset)); + max_offset = BLK_ENDS(sc, i) - 1; + } + DPRINTF_BLK(GUZ_DBG_TOC, i, ("%s: cluster #%u, original %u " + "bytes, in %u bytes", gp->name, i, sc->blksz, + sc->toc[i].blen)); + if (backref_to != BLEN_UNDEF) { + DPRINTF_BLK(GUZ_DBG_TOC, i, (" (->#%u)", + (u_int)backref_to)); + } + DPRINTF_BLK(GUZ_DBG_TOC, i, ("\n")); + } + return (0); + +error_offset: + DPRINTF(GUZ_DBG_ERR, ("%s: cluster #%u: invalid offset %ju, " + "min_offset=%ju mediasize=%jd\n", gp->name, (u_int)i, + sc->toc[i].offset, min_offset, pp->mediasize)); + return (-1); +} + static struct g_geom * g_uzip_taste(struct g_class *mp, struct g_provider *pp, int flags) { @@ -403,6 +572,10 @@ g_uzip_taste(struct g_class *mp, struct g_provider *pp, int flags) struct g_geom *gp; struct g_provider *pp2; struct g_uzip_softc *sc; + enum { + GEOM_UZIP = 1, + GEOM_ULZMA + } type; g_trace(G_T_TOPOLOGY, "%s(%s,%s)", __func__, mp->name, pp->name); g_topology_assert(); @@ -422,10 +595,7 @@ g_uzip_taste(struct g_class *mp, struct g_provider *pp, int flags) if (error == 0) error = g_access(cp, 1, 0, 0); if (error) { - g_detach(cp); - g_destroy_consumer(cp); - g_destroy_geom(gp); - return (NULL); + goto e1; } g_topology_unlock(); @@ -433,22 +603,47 @@ g_uzip_taste(struct g_class *mp, struct g_provider *pp, int flags) * Read cloop header, look for CLOOP magic, perform * other validity checks. */ - DPRINTF(("%s: media sectorsize %u, mediasize %jd\n", + DPRINTF(GUZ_DBG_INFO, ("%s: media sectorsize %u, mediasize %jd\n", gp->name, pp->sectorsize, (intmax_t)pp->mediasize)); buf = g_read_data(cp, 0, pp->sectorsize, NULL); if (buf == NULL) - goto err; + goto e2; header = (struct cloop_header *) buf; if (strncmp(header->magic, CLOOP_MAGIC_START, sizeof(CLOOP_MAGIC_START) - 1) != 0) { - DPRINTF(("%s: no CLOOP magic\n", gp->name)); - goto err; - } - if (header->magic[0x0b] != 'V' || header->magic[0x0c] < '2') { - DPRINTF(("%s: image version too old\n", gp->name)); - goto err; + DPRINTF(GUZ_DBG_ERR, ("%s: no CLOOP magic\n", gp->name)); + goto e3; } + switch (header->magic[CLOOP_OFS_COMPR]) { + case CLOOP_COMP_LZMA: + case CLOOP_COMP_LZMA_DDP: + type = GEOM_ULZMA; + if (header->magic[CLOOP_OFS_VERSN] < CLOOP_MINVER_LZMA) { + DPRINTF(GUZ_DBG_ERR, ("%s: image version too old\n", + gp->name)); + goto e3; + } + DPRINTF(GUZ_DBG_INFO, ("%s: GEOM_UZIP_LZMA image found\n", + gp->name)); + break; + case CLOOP_COMP_LIBZ: + case CLOOP_COMP_LIBZ_DDP: + type = GEOM_UZIP; + if (header->magic[CLOOP_OFS_VERSN] < CLOOP_MINVER_ZLIB) { + DPRINTF(GUZ_DBG_ERR, ("%s: image version too old\n", + gp->name)); + goto e3; + } + DPRINTF(GUZ_DBG_INFO, ("%s: GEOM_UZIP_ZLIB image found\n", + gp->name)); + break; + default: + DPRINTF(GUZ_DBG_ERR, ("%s: unsupported image type\n", + gp->name)); + goto e3; + } + /* * Initialize softc and read offsets. */ @@ -459,7 +654,7 @@ g_uzip_taste(struct g_class *mp, struct g_provider *pp, int flags) if (sc->blksz % 512 != 0) { printf("%s: block size (%u) should be multiple of 512.\n", gp->name, sc->blksz); - goto err; + goto e4; } if (sc->blksz > MAX_BLKSZ) { printf("%s: block size (%u) should not be larger than %d.\n", @@ -470,15 +665,17 @@ g_uzip_taste(struct g_class *mp, struct g_provider *pp, int flags) total_offsets * sizeof(uint64_t) > pp->mediasize) { printf("%s: media too small for %u blocks\n", gp->name, sc->nblocks); - goto err; + goto e4; } - sc->offsets = malloc( - total_offsets * sizeof(uint64_t), M_GEOM_UZIP, M_WAITOK); + sc->toc = malloc(total_offsets * sizeof(struct g_uzip_blk), + M_GEOM_UZIP, M_WAITOK | M_ZERO); offsets_read = MIN(total_offsets, (pp->sectorsize - sizeof(*header)) / sizeof(uint64_t)); - for (i = 0; i < offsets_read; i++) - sc->offsets[i] = be64toh(((uint64_t *) (header + 1))[i]); - DPRINTF(("%s: %u offsets in the first sector\n", + for (i = 0; i < offsets_read; i++) { + sc->toc[i].offset = be64toh(((uint64_t *) (header + 1))[i]); + sc->toc[i].blen = BLEN_UNDEF; + } + DPRINTF(GUZ_DBG_INFO, ("%s: %u offsets in the first sector\n", gp->name, offsets_read)); for (blk = 1; offsets_read < total_offsets; blk++) { uint32_t nread; @@ -487,25 +684,59 @@ g_uzip_taste(struct g_class *mp, struct g_provider *pp, int flags) buf = g_read_data( cp, blk * pp->sectorsize, pp->sectorsize, NULL); if (buf == NULL) - goto err; + goto e5; nread = MIN(total_offsets - offsets_read, pp->sectorsize / sizeof(uint64_t)); - DPRINTF(("%s: %u offsets read from sector %d\n", + DPRINTF(GUZ_DBG_TOC, ("%s: %u offsets read from sector %d\n", gp->name, nread, blk)); for (i = 0; i < nread; i++) { - sc->offsets[offsets_read + i] = + sc->toc[offsets_read + i].offset = be64toh(((uint64_t *) buf)[i]); + sc->toc[offsets_read + i].blen = BLEN_UNDEF; } offsets_read += nread; } free(buf, M_GEOM); - DPRINTF(("%s: done reading offsets\n", gp->name)); + buf = NULL; + offsets_read -= 1; + DPRINTF(GUZ_DBG_INFO, ("%s: done reading %u block offsets from %u " + "sectors\n", gp->name, offsets_read, blk)); + if (sc->nblocks != offsets_read) { + DPRINTF(GUZ_DBG_ERR, ("%s: read %s offsets than expected " + "blocks\n", gp->name, + sc->nblocks < offsets_read ? "more" : "less")); + goto e5; + } + /* Massage TOC (table of contents), make sure it is sound */ + if (g_uzip_parse_toc(sc, pp, gp) != 0) { + DPRINTF(GUZ_DBG_ERR, ("%s: TOC error\n", gp->name)); + goto e5; + } mtx_init(&sc->last_mtx, "geom_uzip cache", NULL, MTX_DEF); + mtx_init(&sc->queue_mtx, "geom_uzip wrkthread", NULL, MTX_DEF); + bioq_init(&sc->bio_queue); sc->last_blk = -1; sc->last_buf = malloc(sc->blksz, M_GEOM_UZIP, M_WAITOK); sc->req_total = 0; sc->req_cached = 0; + if (type == GEOM_UZIP) { + sc->dcp = g_uzip_zlib_ctor(sc->blksz); + } else { + sc->dcp = g_uzip_lzma_ctor(sc->blksz); + } + if (sc->dcp == NULL) { + goto e6; + } + + sc->uzip_do = &g_uzip_do; + + error = kproc_create(g_uzip_wrkthr, sc, &sc->procp, 0, 0, "%s", + gp->name); + if (error != 0) { + goto e7; + } + g_topology_lock(); pp2 = g_new_providerf(gp, "%s", gp->name); pp2->sectorsize = 512; @@ -515,22 +746,31 @@ g_uzip_taste(struct g_class *mp, struct g_provider *pp, int flags) g_error_provider(pp2, 0); g_access(cp, -1, 0, 0); - DPRINTF(("%s: taste ok (%d, %jd), (%d, %d), %x\n", - gp->name, - pp2->sectorsize, (intmax_t)pp2->mediasize, + DPRINTF(GUZ_DBG_INFO, ("%s: taste ok (%d, %jd), (%d, %d), %x\n", + gp->name, pp2->sectorsize, (intmax_t)pp2->mediasize, pp2->stripeoffset, pp2->stripesize, pp2->flags)); - DPRINTF(("%s: %u x %u blocks\n", gp->name, sc->nblocks, sc->blksz)); + DPRINTF(GUZ_DBG_INFO, ("%s: %u x %u blocks\n", gp->name, sc->nblocks, + sc->blksz)); return (gp); -err: +e7: + sc->dcp->free(sc->dcp); +e6: + free(sc->last_buf, M_GEOM); + mtx_destroy(&sc->queue_mtx); + mtx_destroy(&sc->last_mtx); +e5: + free(sc->toc, M_GEOM); +e4: + free(gp->softc, M_GEOM_UZIP); +e3: + if (buf != NULL) { + free(buf, M_GEOM); + } +e2: g_topology_lock(); g_access(cp, -1, 0, 0); - if (buf != NULL) - free(buf, M_GEOM); - if (gp->softc != NULL) { - g_uzip_softc_free(gp->softc, NULL); - gp->softc = NULL; - } +e1: g_detach(cp); g_destroy_consumer(cp); g_destroy_geom(gp); @@ -547,7 +787,8 @@ g_uzip_destroy_geom(struct gctl_req *req, struct g_class *mp, struct g_geom *gp) g_topology_assert(); if (gp->softc == NULL) { - DPRINTF(("%s(%s): gp->softc == NULL\n", __func__, gp->name)); + DPRINTF(GUZ_DBG_ERR, ("%s(%s): gp->softc == NULL\n", __func__, + gp->name)); return (ENXIO); } diff --git a/sys/geom/uzip/g_uzip.h b/sys/geom/uzip/g_uzip.h new file mode 100644 index 00000000000..7a16827e7eb --- /dev/null +++ b/sys/geom/uzip/g_uzip.h @@ -0,0 +1,35 @@ +/*- + * Copyright (c) 2004 Max Khon + * Copyright (c) 2014 Juniper Networks, Inc. + * Copyright (c) 2006-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#if !defined(M_GEOM_UZIP) +MALLOC_DECLARE(M_GEOM_UZIP); +#endif + +#define DEFINE_RAW_METHOD(func, rval, args...) typedef rval (*func##_t)(args) diff --git a/sys/geom/uzip/g_uzip_cloop.h b/sys/geom/uzip/g_uzip_cloop.h new file mode 100644 index 00000000000..219a7b67d6e --- /dev/null +++ b/sys/geom/uzip/g_uzip_cloop.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +/* CLOOP format and related constants */ + +/* + * Integer values (block size, number of blocks, offsets) + * are stored in big-endian (network) order on disk. + */ + +#define CLOOP_MAGIC_LEN 128 +#define CLOOP_OFS_COMPR 0x0b +#define CLOOP_OFS_VERSN (CLOOP_OFS_COMPR + 1) + +#define CLOOP_MAJVER_2 '2' +#define CLOOP_MAJVER_3 '3' + +#define CLOOP_COMP_LIBZ 'V' +#define CLOOP_COMP_LIBZ_DDP 'v' +#define CLOOP_COMP_LZMA 'L' +#define CLOOP_COMP_LZMA_DDP 'l' + +#define CLOOP_MINVER_LZMA CLOOP_MAJVER_3 +#define CLOOP_MINVER_ZLIB CLOOP_MAJVER_2 + +struct cloop_header { + char magic[CLOOP_MAGIC_LEN]; /* cloop magic */ + uint32_t blksz; /* block size */ + uint32_t nblocks; /* number of blocks */ +}; diff --git a/sys/geom/uzip/g_uzip_dapi.h b/sys/geom/uzip/g_uzip_dapi.h new file mode 100644 index 00000000000..0689c323491 --- /dev/null +++ b/sys/geom/uzip/g_uzip_dapi.h @@ -0,0 +1,41 @@ +/*- + * Copyright (c) 2006-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +struct g_uzip_dapi; + +DEFINE_RAW_METHOD(g_uzip_dapi_decompress, int, struct g_uzip_dapi *, + const char *, void *, size_t, void *); +DEFINE_RAW_METHOD(g_uzip_dapi_free, void, struct g_uzip_dapi *); +DEFINE_RAW_METHOD(g_uzip_dapi_rewind, int, struct g_uzip_dapi *, const char *); + +struct g_uzip_dapi { + g_uzip_dapi_decompress_t decompress; + g_uzip_dapi_free_t free; + g_uzip_dapi_rewind_t rewind; + void *pvt; +}; diff --git a/sys/geom/uzip/g_uzip_lzma.c b/sys/geom/uzip/g_uzip_lzma.c new file mode 100644 index 00000000000..0a981ba8320 --- /dev/null +++ b/sys/geom/uzip/g_uzip_lzma.c @@ -0,0 +1,111 @@ +/*- + * Copyright (c) 2004 Max Khon + * Copyright (c) 2014 Juniper Networks, Inc. + * Copyright (c) 2006-2016 Maxim Sobolev + * Copyright (c) 2010-2012 Aleksandr Rybalko + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include + +#include + +#include +#include +#include + +struct g_uzip_lzma { + struct g_uzip_dapi pub; + /* XZ decoder structs */ + struct xz_buf b; + struct xz_dec *s; +}; + +static int g_uzip_lzma_nop(struct g_uzip_dapi *, const char *); + +static void +g_uzip_lzma_free(struct g_uzip_dapi *lzpp) +{ + struct g_uzip_lzma *lzp; + + lzp = (struct g_uzip_lzma *)lzpp->pvt; + if (lzp->s != NULL) { + xz_dec_end(lzp->s); + lzp->s = NULL; + } + + free(lzp, M_GEOM_UZIP); +} + +static int +g_uzip_lzma_decompress(struct g_uzip_dapi *lzpp, const char *gp_name, + void *ibp, size_t ilen, void *obp) +{ + struct g_uzip_lzma *lzp; + int err; + + lzp = (struct g_uzip_lzma *)lzpp->pvt; + + lzp->b.in = ibp; + lzp->b.out = obp; + lzp->b.in_pos = lzp->b.out_pos = 0; + lzp->b.in_size = ilen; + lzp->b.out_size = (size_t)-1; + err = (xz_dec_run(lzp->s, &lzp->b) != XZ_STREAM_END) ? 1 : 0; + /* TODO decoder recovery, if needed */ + + return (err); +} + +struct g_uzip_dapi * +g_uzip_lzma_ctor(uint32_t blksz) +{ + struct g_uzip_lzma *lzp; + + lzp = malloc(sizeof(struct g_uzip_lzma), M_GEOM_UZIP, M_WAITOK); + xz_crc32_init(); + lzp->s = xz_dec_init(XZ_SINGLE, 0); + if (lzp->s == NULL) { + goto e1; + } + lzp->pub.decompress = &g_uzip_lzma_decompress; + lzp->pub.free = &g_uzip_lzma_free; + lzp->pub.rewind = &g_uzip_lzma_nop; + lzp->pub.pvt = lzp; + return (&lzp->pub); +e1: + free(lzp, M_GEOM_UZIP); + return (NULL); +} + +static int +g_uzip_lzma_nop(struct g_uzip_dapi *zpp, const char *gp_name) +{ + + return (0); +} diff --git a/sys/geom/uzip/g_uzip_lzma.h b/sys/geom/uzip/g_uzip_lzma.h new file mode 100644 index 00000000000..f9f8a27a8a3 --- /dev/null +++ b/sys/geom/uzip/g_uzip_lzma.h @@ -0,0 +1,32 @@ +/*- + * Copyright (c) 2004 Max Khon + * Copyright (c) 2014 Juniper Networks, Inc. + * Copyright (c) 2006-2016 Maxim Sobolev + * Copyright (c) 2010-2012 Aleksandr Rybalko + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +struct g_uzip_dapi *g_uzip_lzma_ctor(uint32_t); diff --git a/sys/geom/uzip/g_uzip_softc.h b/sys/geom/uzip/g_uzip_softc.h new file mode 100644 index 00000000000..1d12ac097fb --- /dev/null +++ b/sys/geom/uzip/g_uzip_softc.h @@ -0,0 +1,56 @@ +/*- + * Copyright (c) 2004 Max Khon + * Copyright (c) 2014 Juniper Networks, Inc. + * Copyright (c) 2006-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +struct g_uzip_softc; +struct bio; + +DEFINE_RAW_METHOD(g_uzip_do, void, struct g_uzip_softc *, struct bio *); + +struct g_uzip_softc { + uint32_t blksz; /* block size */ + uint32_t nblocks; /* number of blocks */ + struct g_uzip_blk *toc; /* table of contents */ + + struct mtx last_mtx; + uint32_t last_blk; /* last blk no */ + char *last_buf; /* last blk data */ + int req_total; /* total requests */ + int req_cached; /* cached requests */ + struct g_uzip_dapi *dcp; /* decompressor instance */ + + g_uzip_do_t uzip_do; + + struct proc *procp; + struct bio_queue_head bio_queue; + struct mtx queue_mtx; + unsigned wrkthr_flags; +#define GUZ_SHUTDOWN (0x1 << 0) +#define GUZ_EXITING (0x1 << 1) +}; diff --git a/sys/geom/uzip/g_uzip_wrkthr.c b/sys/geom/uzip/g_uzip_wrkthr.c new file mode 100644 index 00000000000..ad24cb808bf --- /dev/null +++ b/sys/geom/uzip/g_uzip_wrkthr.c @@ -0,0 +1,71 @@ +/*- + * Copyright (c) 2006-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +void +g_uzip_wrkthr(void *arg) +{ + struct g_uzip_softc *sc; + struct bio *bp; + + sc = (struct g_uzip_softc *)arg; + thread_lock(curthread); + sched_prio(curthread, PRIBIO); + thread_unlock(curthread); + + for (;;) { + mtx_lock(&sc->queue_mtx); + if (sc->wrkthr_flags & GUZ_SHUTDOWN) { + sc->wrkthr_flags |= GUZ_EXITING; + mtx_unlock(&sc->queue_mtx); + kproc_exit(0); + } + bp = bioq_takefirst(&sc->bio_queue); + if (!bp) { + msleep(sc, &sc->queue_mtx, PRIBIO | PDROP, + "wrkwait", 0); + continue; + } + mtx_unlock(&sc->queue_mtx); + sc->uzip_do(sc, bp); + } +} diff --git a/sys/geom/uzip/g_uzip_wrkthr.h b/sys/geom/uzip/g_uzip_wrkthr.h new file mode 100644 index 00000000000..8492378b4aa --- /dev/null +++ b/sys/geom/uzip/g_uzip_wrkthr.h @@ -0,0 +1,30 @@ +/*- + * Copyright (c) 2006-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +void g_uzip_wrkthr(void *); + diff --git a/sys/geom/uzip/g_uzip_zlib.c b/sys/geom/uzip/g_uzip_zlib.c new file mode 100644 index 00000000000..cecac6e006c --- /dev/null +++ b/sys/geom/uzip/g_uzip_zlib.c @@ -0,0 +1,137 @@ +/*- + * Copyright (c) 2004 Max Khon + * Copyright (c) 2014 Juniper Networks, Inc. + * Copyright (c) 2006-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include + +#include + +#include +#include +#include + +struct g_uzip_zlib { + uint32_t blksz; + struct g_uzip_dapi pub; + /* Zlib decoder structs */ + z_stream zs; +}; + +static void *z_alloc(void *, u_int, u_int); +static void z_free(void *, void *); +static int g_uzip_zlib_rewind(struct g_uzip_dapi *, const char *); + +static void +g_uzip_zlib_free(struct g_uzip_dapi *zpp) +{ + struct g_uzip_zlib *zp; + + zp = (struct g_uzip_zlib *)zpp->pvt; + inflateEnd(&zp->zs); + free(zp, M_GEOM_UZIP); +} + +static int +g_uzip_zlib_decompress(struct g_uzip_dapi *zpp, const char *gp_name, void *ibp, + size_t ilen, void *obp) +{ + int err; + struct g_uzip_zlib *zp; + + zp = (struct g_uzip_zlib *)zpp->pvt; + + zp->zs.next_in = ibp; + zp->zs.avail_in = ilen; + zp->zs.next_out = obp; + zp->zs.avail_out = zp->blksz; + + err = (inflate(&zp->zs, Z_FINISH) != Z_STREAM_END) ? 1 : 0; + if (err != 0) { + printf("%s: UZIP(zlib) inflate() failed\n", gp_name); + } + return (err); +} + +static int +g_uzip_zlib_rewind(struct g_uzip_dapi *zpp, const char *gp_name) +{ + int err; + struct g_uzip_zlib *zp; + + zp = (struct g_uzip_zlib *)zpp->pvt; + + err = 0; + if (inflateReset(&zp->zs) != Z_OK) { + printf("%s: UZIP(zlib) decoder reset failed\n", gp_name); + err = 1; + } + return (err); +} + +struct g_uzip_dapi * +g_uzip_zlib_ctor(uint32_t blksz) +{ + struct g_uzip_zlib *zp; + + zp = malloc(sizeof(struct g_uzip_zlib), M_GEOM_UZIP, M_WAITOK); + zp->zs.zalloc = z_alloc; + zp->zs.zfree = z_free; + if (inflateInit(&zp->zs) != Z_OK) { + goto e1; + } + zp->blksz = blksz; + zp->pub.decompress = &g_uzip_zlib_decompress; + zp->pub.free = &g_uzip_zlib_free; + zp->pub.rewind = &g_uzip_zlib_rewind; + zp->pub.pvt = (void *)zp; + return (&zp->pub); +e1: + free(zp, M_GEOM_UZIP); + return (NULL); +} + +static void * +z_alloc(void *nil, u_int type, u_int size) +{ + void *ptr; + + ptr = malloc(type * size, M_GEOM_UZIP, M_NOWAIT); + + return (ptr); +} + +static void +z_free(void *nil, void *ptr) +{ + + free(ptr, M_GEOM_UZIP); +} diff --git a/sys/geom/uzip/g_uzip_zlib.h b/sys/geom/uzip/g_uzip_zlib.h new file mode 100644 index 00000000000..7a767b84598 --- /dev/null +++ b/sys/geom/uzip/g_uzip_zlib.h @@ -0,0 +1,33 @@ +/*- + * Copyright (c) 2004 Max Khon + * Copyright (c) 2014 Juniper Networks, Inc. + * Copyright (c) 2006-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +struct g_geom; + +struct g_uzip_dapi *g_uzip_zlib_ctor(uint32_t); diff --git a/sys/modules/geom/geom_uzip/Makefile b/sys/modules/geom/geom_uzip/Makefile index 3efeb809527..a336f63c224 100644 --- a/sys/modules/geom/geom_uzip/Makefile +++ b/sys/modules/geom/geom_uzip/Makefile @@ -3,7 +3,20 @@ .PATH: ${.CURDIR}/../../../geom/uzip ${.CURDIR}/../../../net KMOD= geom_uzip -SRCS= g_uzip.c +SRCS= g_uzip.c g_uzip_zlib.c g_uzip_lzma.c g_uzip_wrkthr.c +SRCS+= g_uzip.h g_uzip_dapi.h g_uzip_lzma.h g_uzip_zlib.h g_uzip_softc.h \ + g_uzip_wrkthr.h #CFLAGS= -g -DINVARIANT_SUPPORT -DINVARIANTS +.PATH: ${.CURDIR}/../../../contrib/xz-embedded/freebsd/ \ + ${.CURDIR}/../../../contrib/xz-embedded/linux/lib/xz/ \ + ${.CURDIR}/../../../contrib/xz-embedded/linux/include/linux/ \ + ${.CURDIR}/../../../net + +CFLAGS+= -I${.CURDIR}/../../../contrib/xz-embedded/freebsd \ + -I${.CURDIR}/../../../contrib/xz-embedded/linux/lib/xz/ +SRCS+= xz_crc32.c xz_dec_bcj.c xz_dec_lzma2.c xz_dec_stream.c \ + xz_malloc.c +SRCS+= xz.h xz_config.h xz_lzma2.h xz_malloc.h xz_private.h xz_stream.h + .include diff --git a/usr.bin/mkuzip/Makefile b/usr.bin/mkuzip/Makefile index 92ab43e897c..4050ad06814 100644 --- a/usr.bin/mkuzip/Makefile +++ b/usr.bin/mkuzip/Makefile @@ -2,7 +2,8 @@ PROG= mkuzip MAN= mkuzip.8 +SRCS= mkuzip.c mkuz_blockcache.c mkuz_lzma.c mkuz_zlib.c -LIBADD= z +LIBADD= z md lzma .include diff --git a/usr.bin/mkuzip/mkuz_blockcache.c b/usr.bin/mkuzip/mkuz_blockcache.c new file mode 100644 index 00000000000..a3ac564388f --- /dev/null +++ b/usr.bin/mkuzip/mkuz_blockcache.c @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include +#include +#include + +#if defined(MKUZ_DEBUG) +# include +#endif + +#include "mkuz_blockcache.h" + +struct mkuz_blkcache { + struct mkuz_blkcache_hit hit; + off_t data_offset; + unsigned char digest[16]; + struct mkuz_blkcache *next; +}; + +static struct mkuz_blkcache blkcache; + +struct mkuz_blkcache_hit * +mkuz_blkcache_regblock(int fd, uint32_t blkno, off_t offset, ssize_t len, + void *data) +{ + struct mkuz_blkcache *bcep; + MD5_CTX mcontext; + off_t data_offset; + unsigned char mdigest[16]; + + data_offset = lseek(fd, 0, SEEK_CUR); + if (data_offset < 0) { + return (NULL); + } + MD5Init(&mcontext); + MD5Update(&mcontext, data, len); + MD5Final(mdigest, &mcontext); + if (blkcache.hit.len == 0) { + bcep = &blkcache; + } else { + for (bcep = &blkcache; bcep != NULL; bcep = bcep->next) { + if (bcep->hit.len != len) + continue; + if (memcmp(mdigest, bcep->digest, sizeof(mdigest)) == 0) { + break; + } + } + if (bcep != NULL) { +#if defined(MKUZ_DEBUG) + printf("cache hit %d, %d, %d\n", (int)bcep->hit.offset, (int)data_offset, (int)len); +#endif + return (&bcep->hit); + } + bcep = malloc(sizeof(struct mkuz_blkcache)); + if (bcep == NULL) + return (NULL); + memset(bcep, '\0', sizeof(struct mkuz_blkcache)); + bcep->next = blkcache.next; + blkcache.next = bcep; + } + memcpy(bcep->digest, mdigest, sizeof(mdigest)); + bcep->data_offset = data_offset; + bcep->hit.offset = offset; + bcep->hit.len = len; + bcep->hit.blkno = blkno; + return (NULL); +} diff --git a/usr.bin/mkuzip/mkuz_blockcache.h b/usr.bin/mkuzip/mkuz_blockcache.h new file mode 100644 index 00000000000..2154ae83180 --- /dev/null +++ b/usr.bin/mkuzip/mkuz_blockcache.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +struct mkuz_blkcache_hit { + uint64_t offset; + ssize_t len; + uint32_t blkno; +}; + +struct mkuz_blkcache_hit *mkuz_blkcache_regblock(int, uint32_t, off_t, ssize_t, + void *); diff --git a/usr.bin/mkuzip/mkuz_cloop.h b/usr.bin/mkuzip/mkuz_cloop.h new file mode 100644 index 00000000000..4ed7c502639 --- /dev/null +++ b/usr.bin/mkuzip/mkuz_cloop.h @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +/* CLOOP format and related constants */ + +/* + * Integer values (block size, number of blocks, offsets) + * are stored in big-endian (network) order on disk. + */ + +#define CLOOP_MAGIC_LEN 128 +#define CLOOP_OFS_COMPR 0x0b +#define CLOOP_OFS_VERSN (CLOOP_OFS_COMPR + 1) + +#define CLOOP_MAJVER_2 '2' +#define CLOOP_MAJVER_3 '3' + +#define CLOOP_COMP_LIBZ 'V' +#define CLOOP_COMP_LZMA 'L' + +struct cloop_header { + char magic[CLOOP_MAGIC_LEN]; /* cloop magic */ + uint32_t blksz; /* block size */ + uint32_t nblocks; /* number of blocks */ +}; diff --git a/usr.bin/mkuzip/mkuz_lzma.c b/usr.bin/mkuzip/mkuz_lzma.c new file mode 100644 index 00000000000..d62376e3435 --- /dev/null +++ b/usr.bin/mkuzip/mkuz_lzma.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev + * Copyright (c) 2011 Aleksandr Rybalko + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include + +#include + +#include "mkuzip.h" +#include "mkuz_lzma.h" + +#define USED_BLOCKSIZE DEV_BSIZE + +struct mkuz_lzma { + lzma_filter filters[2]; + lzma_options_lzma opt_lzma; + lzma_stream strm; + char *obuf; + uint32_t blksz; +}; + +static struct mkuz_lzma ulzma = {.strm = LZMA_STREAM_INIT}; + +void * +mkuz_lzma_init(uint32_t blksz) +{ + if (blksz % USED_BLOCKSIZE != 0) { + errx(1, "cluster size should be multiple of %d", + USED_BLOCKSIZE); + /* Not reached */ + } + if (blksz > MAXPHYS) { + errx(1, "cluster size is too large"); + /* Not reached */ + } + ulzma.obuf = mkuz_safe_malloc(blksz * 2); + + /* Init lzma encoder */ + if (lzma_lzma_preset(&ulzma.opt_lzma, LZMA_PRESET_DEFAULT)) + errx(1, "Error loading LZMA preset"); + + ulzma.filters[0].id = LZMA_FILTER_LZMA2; + ulzma.filters[0].options = &ulzma.opt_lzma; + ulzma.filters[1].id = LZMA_VLI_UNKNOWN; + + ulzma.blksz = blksz; + + return (ulzma.obuf); +} + +void +mkuz_lzma_compress(const char *ibuf, uint32_t *destlen) +{ + lzma_ret ret; + + ret = lzma_stream_encoder(&ulzma.strm, ulzma.filters, LZMA_CHECK_CRC32); + if (ret != LZMA_OK) { + if (ret == LZMA_MEMLIMIT_ERROR) + errx(1, "can't compress data: LZMA_MEMLIMIT_ERROR"); + + errx(1, "can't compress data: LZMA compressor ERROR"); + } + + ulzma.strm.next_in = ibuf; + ulzma.strm.avail_in = ulzma.blksz; + ulzma.strm.next_out = ulzma.obuf; + ulzma.strm.avail_out = ulzma.blksz * 2; + + ret = lzma_code(&ulzma.strm, LZMA_FINISH); + + if (ret != LZMA_STREAM_END) { + /* Error */ + errx(1, "lzma_code FINISH failed, code=%d, pos(in=%zd, " + "out=%zd)", ret, (ulzma.blksz - ulzma.strm.avail_in), + (ulzma.blksz * 2 - ulzma.strm.avail_out)); + } + + lzma_end(&ulzma.strm); + + *destlen = (ulzma.blksz * 2) - ulzma.strm.avail_out; +} diff --git a/usr.bin/mkuzip/mkuz_lzma.h b/usr.bin/mkuzip/mkuz_lzma.h new file mode 100644 index 00000000000..426f8bc4975 --- /dev/null +++ b/usr.bin/mkuzip/mkuz_lzma.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev + * Copyright (c) 2011 Aleksandr Rybalko + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +/* Format L3.0, since we move to XZ API */ +#define CLOOP_MAGIC_LZMA \ + "#!/bin/sh\n" \ + "#L3.0\n" \ + "n=uncompress\n" \ + "m=geom_$n\n" \ + "(kldstat -m $m 2>&-||kldload $m)>&-&&" \ + "mount_cd9660 /dev/`mdconfig -af $0`.$n $1\n" \ + "exit $?\n" +#define DEFAULT_SUFX_LZMA ".ulzma" + +void *mkuz_lzma_init(uint32_t); +void mkuz_lzma_compress(const char *, uint32_t *); diff --git a/usr.bin/mkuzip/mkuz_zlib.c b/usr.bin/mkuzip/mkuz_zlib.c new file mode 100644 index 00000000000..8828891ab6e --- /dev/null +++ b/usr.bin/mkuzip/mkuz_zlib.c @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + */ + +#include +__FBSDID("$FreeBSD$"); + +#include +#include +#include + +#include + +#include "mkuzip.h" +#include "mkuz_zlib.h" + +struct mkuz_zlib { + char *obuf; + uLongf oblen; + uint32_t blksz; +}; + +static struct mkuz_zlib uzip; + +void * +mkuz_zlib_init(uint32_t blksz) +{ + if (blksz % DEV_BSIZE != 0) { + errx(1, "cluster size should be multiple of %d", + DEV_BSIZE); + /* Not reached */ + } + if (compressBound(blksz) > MAXPHYS) { + errx(1, "cluster size is too large"); + /* Not reached */ + } + uzip.oblen = compressBound(blksz); + uzip.obuf = mkuz_safe_malloc(uzip.oblen); + uzip.blksz = blksz; + + return (uzip.obuf); +} + +void +mkuz_zlib_compress(const char *ibuf, uint32_t *destlen) +{ + uLongf destlen_z; + + destlen_z = uzip.oblen; + if (compress2(uzip.obuf, &destlen_z, ibuf, uzip.blksz, + Z_BEST_COMPRESSION) != Z_OK) { + errx(1, "can't compress data: compress2() " + "failed"); + /* Not reached */ + } + + *destlen = (uint32_t)destlen_z; +} diff --git a/usr.bin/mkuzip/mkuz_zlib.h b/usr.bin/mkuzip/mkuz_zlib.h new file mode 100644 index 00000000000..f50bba2410b --- /dev/null +++ b/usr.bin/mkuzip/mkuz_zlib.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#define DEFAULT_SUFX_ZLIB ".uzip" + +#define CLOOP_MAGIC_ZLIB "#!/bin/sh\n#V2.0 Format\n" \ + "(kldstat -qm g_uzip||kldload geom_uzip)>&-&&" \ + "mount_cd9660 /dev/`mdconfig -af $0`.uzip $1\nexit $?\n" + +void *mkuz_zlib_init(uint32_t); +void mkuz_zlib_compress(const char *, uint32_t *); diff --git a/usr.bin/mkuzip/mkuzip.8 b/usr.bin/mkuzip/mkuzip.8 index c0b8315d891..c8395512d34 100644 --- a/usr.bin/mkuzip/mkuzip.8 +++ b/usr.bin/mkuzip/mkuzip.8 @@ -1,9 +1,27 @@ -.\" ---------------------------------------------------------------------------- -.\" "THE BEER-WARE LICENSE" (Revision 42): -.\" wrote this file. As long as you retain this notice you -.\" can do whatever you want with this stuff. If we meet some day, and you think -.\" this stuff is worth it, you can buy me a beer in return. Maxim Sobolev -.\" ---------------------------------------------------------------------------- +.\"- +.\" Copyright (c) 2004-2016 Maxim Sobolev +.\" All rights reserved. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. .\" .\" $FreeBSD$ .\" @@ -39,7 +57,9 @@ works in two phases: An .Ar infile image is split into clusters; each cluster is compressed using -.Xr zlib 3 . +.Xr zlib 3 +or +.Xr lzma 3 . .It The resulting set of compressed clusters along with headers that allow locating each individual cluster is written to the output file. @@ -51,7 +71,23 @@ The options are: Name of the output file .Ar outfile . The default is to use the input name with the suffix -.Pa .uzip . +.Pa .uzip +for the +.Xr zlib 3 +compression or +.Pa .ulzma +for the +.Xr lzma 3 . +.It Fl L +Use +.Xr lzma 3 +compression algorithm instead of the default +.Xr zlib 3 . +The +.Xr lzma 3 +provides noticeable better compression levels on the same data set +at the expense of much slower compression speed (10-20x) and somewhat slower +decompression (2-3x). .It Fl s Ar cluster_size Split the image into clusters of .Ar cluster_size @@ -61,6 +97,27 @@ The should be a multiple of 512 bytes. .It Fl v Display verbose messages. +.It Fl Z +Disable zero-blocks detection and elimination. +When this option is set, the +.Nm +would compress empty blocks (i.e. clusters that consist of only zero bytes) +just as it would any other block. +When the option is not set, the +.Nm +detects such blocks and skips them from the output. +Setting +.Fl Z +results is slight increase of compressed image size, typically less than 0.1% +of a final size of the compressed image. +.It Fl d +Enable de-duplication. +When the option is enabled the +.Nm +detects identical blocks in the input and replaces each subsequent occurence +of such block with pointer to the very first one in the output. +Setting this option results is moderate decrease of compressed image size, +typically around 3-5% of a final size of the compressed image. .El .Sh NOTES The compression ratio largely depends on the cluster size used. @@ -92,11 +149,20 @@ disk device using and automatically mount it using .Xr mount_cd9660 8 on the mount point provided as the first argument to the script. +.Pp +The de-duplication is a +.Fx +specific feature and while it does not require any changes to on-disk +compressed image format, however it did require some matching changes to the +.Xr geom_uzip 4 +to handle resulting images correctly. .Sh EXIT STATUS .Ex -std .Sh SEE ALSO .Xr gzip 1 , +.Xr xz 1 , .Xr zlib 3 , +.Xr lzma 3 , .Xr geom 4 , .Xr geom_uzip 4 , .Xr md 4 , diff --git a/usr.bin/mkuzip/mkuzip.c b/usr.bin/mkuzip/mkuzip.c index 08c8ed0beab..1e3a204ebcd 100644 --- a/usr.bin/mkuzip/mkuzip.c +++ b/usr.bin/mkuzip/mkuzip.c @@ -1,15 +1,33 @@ /* - * ---------------------------------------------------------------------------- - * "THE BEER-WARE LICENSE" (Revision 42): - * wrote this file. As long as you retain this notice you - * can do whatever you want with this stuff. If we meet some day, and you think - * this stuff is worth it, you can buy me a beer in return. Maxim Sobolev - * ---------------------------------------------------------------------------- + * Copyright (c) 2004-2016 Maxim Sobolev + * All rights reserved. * - * $FreeBSD$ + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. * */ +#include +__FBSDID("$FreeBSD$"); + #include #include #include @@ -17,7 +35,7 @@ #include #include #include -#include +#include #include #include #include @@ -26,18 +44,44 @@ #include #include -#define CLSTSIZE 16384 -#define DEFAULT_SUFX ".uzip" +#include "mkuzip.h" +#include "mkuz_cloop.h" +#include "mkuz_blockcache.h" +#include "mkuz_zlib.h" +#include "mkuz_lzma.h" -#define CLOOP_MAGIC_LEN 128 -static char CLOOP_MAGIC_START[] = "#!/bin/sh\n#V2.0 Format\n" - "(kldstat -qm g_uzip||kldload geom_uzip)>&-&&" - "mount_cd9660 /dev/`mdconfig -af $0`.uzip $1\nexit $?\n"; +#define DEFINE_RAW_METHOD(func, rval, args...) typedef rval (*func##_t)(args) + +#define DEFAULT_CLSTSIZE 16384 + +DEFINE_RAW_METHOD(f_init, void *, uint32_t); +DEFINE_RAW_METHOD(f_compress, void, const char *, uint32_t *); + +struct mkuz_format { + const char *magic; + const char *default_sufx; + f_init_t f_init; + f_compress_t f_compress; +}; + +static struct mkuz_format uzip_fmt = { + .magic = CLOOP_MAGIC_ZLIB, + .default_sufx = DEFAULT_SUFX_ZLIB, + .f_init = &mkuz_zlib_init, + .f_compress = &mkuz_zlib_compress +}; + +static struct mkuz_format ulzma_fmt = { + .magic = CLOOP_MAGIC_LZMA, + .default_sufx = DEFAULT_SUFX_LZMA, + .f_init = &mkuz_lzma_init, + .f_compress = &mkuz_lzma_compress +}; static char *readblock(int, char *, u_int32_t); static void usage(void); -static void *safe_malloc(size_t); static void cleanup(void); +static int memvcmp(const void *, unsigned char, size_t); static char *cleanfile = NULL; @@ -45,24 +89,24 @@ int main(int argc, char **argv) { char *iname, *oname, *obuf, *ibuf; uint64_t *toc; - int fdr, fdw, i, opt, verbose, tmp; + int fdr, fdw, i, opt, verbose, no_zcomp, tmp, en_dedup; struct iovec iov[2]; struct stat sb; - uLongf destlen; - uint64_t offset; - struct cloop_header { - char magic[CLOOP_MAGIC_LEN]; /* cloop magic */ - uint32_t blksz; /* block size */ - uint32_t nblocks; /* number of blocks */ - } hdr; + uint32_t destlen; + uint64_t offset, last_offset; + struct cloop_header hdr; + struct mkuz_blkcache_hit *chit; + const struct mkuz_format *handler; memset(&hdr, 0, sizeof(hdr)); - hdr.blksz = CLSTSIZE; - strcpy(hdr.magic, CLOOP_MAGIC_START); + hdr.blksz = DEFAULT_CLSTSIZE; oname = NULL; verbose = 0; + no_zcomp = 0; + en_dedup = 0; + handler = &uzip_fmt; - while((opt = getopt(argc, argv, "o:s:v")) != -1) { + while((opt = getopt(argc, argv, "o:s:vZdL")) != -1) { switch(opt) { case 'o': oname = optarg; @@ -75,15 +119,6 @@ int main(int argc, char **argv) optarg); /* Not reached */ } - if (tmp % DEV_BSIZE != 0) { - errx(1, "cluster size should be multiple of %d", - DEV_BSIZE); - /* Not reached */ - } - if (compressBound(tmp) > MAXPHYS) { - errx(1, "cluster size is too large"); - /* Not reached */ - } hdr.blksz = tmp; break; @@ -91,6 +126,18 @@ int main(int argc, char **argv) verbose = 1; break; + case 'Z': + no_zcomp = 1; + break; + + case 'd': + en_dedup = 1; + break; + + case 'L': + handler = &ulzma_fmt; + break; + default: usage(); /* Not reached */ @@ -104,17 +151,26 @@ int main(int argc, char **argv) /* Not reached */ } + strcpy(hdr.magic, handler->magic); + + if (en_dedup != 0) { + hdr.magic[CLOOP_OFS_VERSN] = CLOOP_MAJVER_3; + hdr.magic[CLOOP_OFS_COMPR] = + tolower(hdr.magic[CLOOP_OFS_COMPR]); + } + + obuf = handler->f_init(hdr.blksz); + iname = argv[0]; if (oname == NULL) { - asprintf(&oname, "%s%s", iname, DEFAULT_SUFX); + asprintf(&oname, "%s%s", iname, handler->default_sufx); if (oname == NULL) { err(1, "can't allocate memory"); /* Not reached */ } } - obuf = safe_malloc(compressBound(hdr.blksz)); - ibuf = safe_malloc(hdr.blksz); + ibuf = mkuz_safe_malloc(hdr.blksz); signal(SIGHUP, exit); signal(SIGINT, exit); @@ -152,7 +208,7 @@ int main(int argc, char **argv) "of %d, padding data\n", hdr.blksz); hdr.nblocks++; } - toc = safe_malloc((hdr.nblocks + 1) * sizeof(*toc)); + toc = mkuz_safe_malloc((hdr.nblocks + 1) * sizeof(*toc)); fdw = open(oname, O_WRONLY | O_TRUNC | O_CREAT, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); @@ -177,40 +233,73 @@ int main(int argc, char **argv) "%u, index length %zu bytes\n", sb.st_size, hdr.nblocks, iov[1].iov_len); + last_offset = 0; for(i = 0; i == 0 || ibuf != NULL; i++) { ibuf = readblock(fdr, ibuf, hdr.blksz); if (ibuf != NULL) { - destlen = compressBound(hdr.blksz); - if (compress2(obuf, &destlen, ibuf, hdr.blksz, - Z_BEST_COMPRESSION) != Z_OK) { - errx(1, "can't compress data: compress2() " - "failed"); - /* Not reached */ + if (no_zcomp == 0 && \ + memvcmp(ibuf, '\0', hdr.blksz) != 0) { + /* All zeroes block */ + destlen = 0; + } else { + handler->f_compress(ibuf, &destlen); } - if (verbose != 0) - fprintf(stderr, "cluster #%d, in %u bytes, " - "out %lu bytes\n", i, hdr.blksz, destlen); } else { destlen = DEV_BSIZE - (offset % DEV_BSIZE); memset(obuf, 0, destlen); if (verbose != 0) - fprintf(stderr, "padding data with %lu bytes so " - "that file size is multiple of %d\n", destlen, - DEV_BSIZE); + fprintf(stderr, "padding data with %lu bytes " + "so that file size is multiple of %d\n", + (u_long)destlen, DEV_BSIZE); } - if (write(fdw, obuf, destlen) < 0) { - err(1, "write(%s)", oname); - /* Not reached */ + if (destlen > 0 && en_dedup != 0) { + chit = mkuz_blkcache_regblock(fdw, i, offset, destlen, + obuf); + /* + * There should be at least one non-empty block + * between us and the backref'ed offset, otherwise + * we won't be able to parse that sequence correctly + * as it would be indistinguishible from another + * empty block. + */ + if (chit != NULL && chit->offset == last_offset) { + chit = NULL; + } + } else { + chit = NULL; + } + if (chit != NULL) { + toc[i] = htobe64(chit->offset); + } else { + if (destlen > 0 && write(fdw, obuf, destlen) < 0) { + err(1, "write(%s)", oname); + /* Not reached */ + } + toc[i] = htobe64(offset); + last_offset = offset; + offset += destlen; + } + if (ibuf != NULL && verbose != 0) { + fprintf(stderr, "cluster #%d, in %u bytes, " + "out len=%lu offset=%lu", i, hdr.blksz, + chit == NULL ? (u_long)destlen : 0, + (u_long)be64toh(toc[i])); + if (chit != NULL) { + fprintf(stderr, " (backref'ed to #%d)", + chit->blkno); + } + fprintf(stderr, "\n"); + } - toc[i] = htobe64(offset); - offset += destlen; } close(fdr); if (verbose != 0) fprintf(stderr, "compressed data to %ju bytes, saved %lld " - "bytes, %.2f%% decrease.\n", offset, (long long)(sb.st_size - offset), - 100.0 * (long long)(sb.st_size - offset) / (float)sb.st_size); + "bytes, %.2f%% decrease.\n", offset, + (long long)(sb.st_size - offset), + 100.0 * (long long)(sb.st_size - offset) / + (float)sb.st_size); /* Convert to big endian */ hdr.blksz = htonl(hdr.blksz); @@ -248,12 +337,13 @@ static void usage(void) { - fprintf(stderr, "usage: mkuzip [-v] [-o outfile] [-s cluster_size] infile\n"); + fprintf(stderr, "usage: mkuzip [-vZdL] [-o outfile] [-s cluster_size] " + "infile\n"); exit(1); } -static void * -safe_malloc(size_t size) +void * +mkuz_safe_malloc(size_t size) { void *retval; @@ -272,3 +362,12 @@ cleanup(void) if (cleanfile != NULL) unlink(cleanfile); } + +static int +memvcmp(const void *memory, unsigned char val, size_t size) +{ + const u_char *mm; + + mm = (const u_char *)memory; + return (*mm == val) && memcmp(mm, mm + 1, size - 1) == 0; +} diff --git a/usr.bin/mkuzip/mkuzip.h b/usr.bin/mkuzip/mkuzip.h new file mode 100644 index 00000000000..4028b3a5f0b --- /dev/null +++ b/usr.bin/mkuzip/mkuzip.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2004-2016 Maxim Sobolev + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +void *mkuz_safe_malloc(size_t); From 361e1fb408f94d4496e077a63d4fc180cae7733c Mon Sep 17 00:00:00 2001 From: Jim Harris Date: Wed, 24 Feb 2016 00:01:10 +0000 Subject: [PATCH 18/25] nvme: fix intx handler to not dereference ioq during initialization This was a regression from r293328, which deferred allocation of the controller's ioq array until after interrupts are enabled during boot. PR: 207432 Reported and tested by: Andy Carrel MFC after: 3 days Sponsored by: Intel --- sys/dev/nvme/nvme_ctrlr.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c index 2c8a5649ad1..00277f08263 100644 --- a/sys/dev/nvme/nvme_ctrlr.c +++ b/sys/dev/nvme/nvme_ctrlr.c @@ -810,7 +810,7 @@ nvme_ctrlr_intx_handler(void *arg) nvme_qpair_process_completions(&ctrlr->adminq); - if (ctrlr->ioq[0].cpl) + if (ctrlr->ioq && ctrlr->ioq[0].cpl) nvme_qpair_process_completions(&ctrlr->ioq[0]); nvme_mmio_write_4(ctrlr, intmc, 1); From 5497acc527bc4cb3360da4699afa0872599e35b7 Mon Sep 17 00:00:00 2001 From: Maxim Sobolev Date: Wed, 24 Feb 2016 00:39:36 +0000 Subject: [PATCH 19/25] Obsolete mkulzma(8) and geom_uncompress(4), their functionality is now provided by mkuzip(8) and geom_uzip(4) respectively. MFC after: 1 month --- ObsoleteFiles.inc | 2 + share/man/man4/Makefile | 1 - share/man/man4/geom_map.4 | 2 +- share/man/man4/geom_uncompress.4 | 118 ---- sys/boot/forth/loader.conf | 1 - sys/conf/files | 13 +- sys/geom/uncompress/g_uncompress.c | 684 ---------------------- sys/mips/conf/ALFA_HORNET_UB | 6 +- sys/mips/conf/AP121 | 6 +- sys/mips/conf/AP135 | 6 +- sys/mips/conf/AP143 | 6 +- sys/mips/conf/AP91 | 6 +- sys/mips/conf/AP93 | 6 +- sys/mips/conf/AP96 | 6 +- sys/mips/conf/CARAMBOLA2 | 6 +- sys/mips/conf/DB120 | 6 +- sys/mips/conf/DIR-655A1 | 6 +- sys/mips/conf/DIR-825B1 | 6 +- sys/mips/conf/DIR-825C1 | 4 +- sys/mips/conf/ENH200 | 6 +- sys/mips/conf/MT7620 | 2 +- sys/mips/conf/ONIONOMEGA | 6 +- sys/mips/conf/PB47 | 6 +- sys/mips/conf/PB92 | 6 +- sys/mips/conf/PICOSTATION_M2HP | 6 +- sys/mips/conf/ROUTERSTATION | 6 +- sys/mips/conf/RSPRO | 6 +- sys/mips/conf/RT305X | 2 +- sys/mips/conf/RT5350 | 6 +- sys/mips/conf/TL-ARCHERC7V2 | 6 +- sys/mips/conf/TL-WDR4300 | 6 +- sys/mips/conf/TL-WR1043NDv2 | 6 +- sys/mips/conf/TL-WR740Nv4 | 6 +- sys/mips/conf/TP-WN1043ND | 6 +- sys/mips/conf/WZR-300HP | 4 +- sys/mips/conf/WZR-HPAG300H | 4 +- sys/modules/geom/Makefile | 1 - sys/modules/geom/geom_uncompress/Makefile | 17 - targets/pseudo/userland/Makefile.depend | 1 - usr.bin/Makefile | 1 - usr.bin/mkulzma/Makefile | 8 - usr.bin/mkulzma/Makefile.depend | 20 - usr.bin/mkulzma/mkulzma.8 | 107 ---- usr.bin/mkulzma/mkulzma.c | 330 ----------- 44 files changed, 89 insertions(+), 1377 deletions(-) delete mode 100644 share/man/man4/geom_uncompress.4 delete mode 100644 sys/geom/uncompress/g_uncompress.c delete mode 100644 sys/modules/geom/geom_uncompress/Makefile delete mode 100644 usr.bin/mkulzma/Makefile delete mode 100644 usr.bin/mkulzma/Makefile.depend delete mode 100644 usr.bin/mkulzma/mkulzma.8 delete mode 100644 usr.bin/mkulzma/mkulzma.c diff --git a/ObsoleteFiles.inc b/ObsoleteFiles.inc index a5a7e3c8f84..f49544ea1a5 100644 --- a/ObsoleteFiles.inc +++ b/ObsoleteFiles.inc @@ -38,6 +38,8 @@ # xargs -n1 | sort | uniq -d; # done +# 20160223: functionality from mkulzma(1) merged into mkuzip(1) +OLD_FILES+=usr/bin/mkulzma # 20160211: Remove obsolete unbound-control-setup OLD_FILES+=usr/sbin/unbound-control-setup # 20160116: Update mandoc to cvs snapshot 20160116 diff --git a/share/man/man4/Makefile b/share/man/man4/Makefile index 2c04dd3071c..2508da085d2 100644 --- a/share/man/man4/Makefile +++ b/share/man/man4/Makefile @@ -165,7 +165,6 @@ MAN= aac.4 \ geom_fox.4 \ geom_linux_lvm.4 \ geom_map.4 \ - geom_uncompress.4 \ geom_uzip.4 \ gif.4 \ gpio.4 \ diff --git a/share/man/man4/geom_map.4 b/share/man/man4/geom_map.4 index 47b2aa878f8..b84f90a7172 100644 --- a/share/man/man4/geom_map.4 +++ b/share/man/man4/geom_map.4 @@ -78,7 +78,7 @@ or by using 0 DISK cfid0 8388608 4 hd 0 sc 0 1 MAP map/config 131072 4 i 5 o 8257536 entry 0 dsize 131072 1 MAP map/rootfs 6881280 4 i 4 o 1376256 entry 0 dsize 6881280 -2 UNCOMPRESS map/rootfs.uncompress 18677760 512 +2 UZIP map/rootfs.uzip 18677760 512 1 MAP map/kernel 1114112 4 i 3 o 262144 entry 0 dsize 1114112 1 MAP map/upgrade 7995392 4 i 2 o 262144 entry 0 dsize 7995392 1 MAP map/factory 65536 4 i 1 o 196608 entry 0 dsize 65536 diff --git a/share/man/man4/geom_uncompress.4 b/share/man/man4/geom_uncompress.4 deleted file mode 100644 index af12d270578..00000000000 --- a/share/man/man4/geom_uncompress.4 +++ /dev/null @@ -1,118 +0,0 @@ -.\" Copyright (c) 2006, Ceri Davies -.\" Copyright (c) 2014, Luiz Otavio O Souza -.\" All rights reserved. -.\" -.\" Redistribution and use in source and binary forms, with or without -.\" modification, are permitted provided that the following conditions -.\" are met: -.\" 1. Redistributions of source code must retain the above copyright -.\" notice, this list of conditions and the following disclaimer. -.\" 2. Redistributions in binary form must reproduce the above copyright -.\" notice, this list of conditions and the following disclaimer in the -.\" documentation and/or other materials provided with the distribution. -.\" -.\" THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE -.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS -.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) -.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -.\" SUCH DAMAGE. -.\" -.\" $FreeBSD$ -.\" -.Dd January 9, 2014 -.Dt GEOM_UNCOMPRESS 4 -.Os -.Sh NAME -.Nm geom_uncompress -.Nd "GEOM based compressed disk images" -.Sh SYNOPSIS -To compile this driver into the kernel, place the following line in your -kernel configuration file: -.Bd -ragged -offset indent -.Cd "options GEOM_UNCOMPRESS" -.Ed -.Pp -Alternatively, to load the driver as a module at boot time, place the -following line in -.Xr loader.conf 5 : -.Bd -literal -offset indent -geom_uncompress_load="YES" -.Ed -.Sh DESCRIPTION -The -.Nm -framework provides support for compressed read-only disk images. -This allows significant storage savings at the expense of a little CPU -time on each read. -Data written in the GEOM label area allows -.Nm -to detect compressed images which have been created with -.Xr mkulzma 8 -or -.Xr mkuzip 8 -and presented to the kernel as a logical disk device via -.Xr md 4 . -.Nm -creates a unique -.Pa md#.uncompress -device for each image. -.Pp -The -.Nm -device is subsequently used by the -.Fx -kernel to access the disk images. -The -.Nm -driver does not allow write operations to the underlying disk image. -To check which -.Xr md 4 -devices match a given -.Nm -device: -.Bd -literal -offset indent -# geom uncompress list -Geom name: md0.uncompress -Providers: -1. Name: md0.uncompress - Mediasize: 52428800 (50M) - Sectorsize: 512 - Mode: r1w0e0 -Consumers: -1. Name: md0 - Mediasize: 20864000 (20M) - Sectorsize: 512 - Mode: r1w0e0 -.Ed -.Sh SEE ALSO -.Xr GEOM 4 , -.Xr md 4 , -.Xr geom 8 , -.Xr mkulzma 8 , -.Xr mkuzip 8 -.Sh AUTHORS -.An -nosplit -The -.Nm -driver was written by -.An Max Khon Aq Mt fjoe@FreeBSD.org -as -.Xr geom_uzip 4 . -.An Aleksandr Rybalko Aq Mt ray@FreeBSD.org -copied it over as -.Nm -and added LZMA functionality . -This manual page was written by -.An Ceri Davies Aq Mt ceri@FreeBSD.org -for the -.Xr geom_uzip 4 , -and modified by -.An Luiz Otavio O Souza Aq Mt loos@FreeBSD.org -to match -.Nm . diff --git a/sys/boot/forth/loader.conf b/sys/boot/forth/loader.conf index c13e90ec7d4..916205570b1 100644 --- a/sys/boot/forth/loader.conf +++ b/sys/boot/forth/loader.conf @@ -238,7 +238,6 @@ geom_nop_load="NO" # Transparent disk driver (see gnop(8)) geom_raid3_load="NO" # RAID3 disk driver (see graid3(8)) geom_shsec_load="NO" # Shared secret disk driver (see gshsec(8)) geom_stripe_load="NO" # RAID0 disk driver (see gstripe(8)) -geom_uncompress_load="NO" # Compressed disk images driver (see mkulzma(8)) geom_uzip_load="NO" # Compressed disk images driver (see mkuzip(8)) geom_vinum_load="NO" # Concatenated/mirror/raid driver (see vinum(4)) diff --git a/sys/conf/files b/sys/conf/files index 7102c03f8a7..ba5e04bcad6 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -3081,21 +3081,20 @@ geom/raid3/g_raid3.c optional geom_raid3 geom/raid3/g_raid3_ctl.c optional geom_raid3 geom/shsec/g_shsec.c optional geom_shsec geom/stripe/g_stripe.c optional geom_stripe -geom/uncompress/g_uncompress.c optional geom_uncompress contrib/xz-embedded/freebsd/xz_malloc.c \ - optional xz_embedded | geom_uncompress | geom_uzip \ + optional xz_embedded | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_crc32.c \ - optional xz_embedded | geom_uncompress | geom_uzip \ + optional xz_embedded | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_bcj.c \ - optional xz_embedded | geom_uncompress | geom_uzip \ + optional xz_embedded | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_lzma2.c \ - optional xz_embedded | geom_uncompress | geom_uzip \ + optional xz_embedded | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" contrib/xz-embedded/linux/lib/xz/xz_dec_stream.c \ - optional xz_embedded | geom_uncompress | geom_uzip \ + optional xz_embedded | geom_uzip \ compile-with "${NORMAL_C} -I$S/contrib/xz-embedded/freebsd/ -I$S/contrib/xz-embedded/linux/lib/xz/ -I$S/contrib/xz-embedded/linux/include/linux/" geom/uzip/g_uzip.c optional geom_uzip geom/uzip/g_uzip_lzma.c optional geom_uzip @@ -3458,7 +3457,7 @@ libkern/strvalid.c standard libkern/timingsafe_bcmp.c standard libkern/zlib.c optional crypto | geom_uzip | ipsec | \ mxge | netgraph_deflate | \ - ddb_ctf | gzio | geom_uncompress + ddb_ctf | gzio net/altq/altq_cbq.c optional altq net/altq/altq_cdnr.c optional altq net/altq/altq_codel.c optional altq diff --git a/sys/geom/uncompress/g_uncompress.c b/sys/geom/uncompress/g_uncompress.c deleted file mode 100644 index 1a83cda0153..00000000000 --- a/sys/geom/uncompress/g_uncompress.c +++ /dev/null @@ -1,684 +0,0 @@ -/*- - * Copyright (c) 2010-2012 Aleksandr Rybalko - * Copyright (c) 2004 Max Khon - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - */ - -/* - * GEOM UNCOMPRESS module - simple decompression module for use with read-only - * copressed images maked by mkuzip(8) or mkulzma(8) utilities. - * - * To enable module in kernel config, put this line: - * device geom_uncompress - */ - -#include -__FBSDID("$FreeBSD$"); - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include - -#ifdef GEOM_UNCOMPRESS_DEBUG -#define DPRINTF(a) printf a -extern int g_debugflags; -#else -#define DPRINTF(a) -#endif - -static MALLOC_DEFINE(M_GEOM_UNCOMPRESS, "geom_uncompress", - "GEOM UNCOMPRESS data structures"); - -#define UNCOMPRESS_CLASS_NAME "UNCOMPRESS" -#define GEOM_UZIP_MAJVER '2' -#define GEOM_ULZMA_MAJVER '3' - -/* - * Maximum allowed valid block size (to prevent foot-shooting) - */ -#define MAX_BLKSZ (MAXPHYS) - -/* - * Integer values (block size, number of blocks, offsets) - * are stored in big-endian (network) order on disk and struct cloop_header - * and in native order in struct g_uncompress_softc - */ - -#define CLOOP_MAGIC_LEN 128 -static char CLOOP_MAGIC_START[] = "#!/bin/sh\n"; - -struct cloop_header { - char magic[CLOOP_MAGIC_LEN]; /* cloop magic */ - uint32_t blksz; /* block size */ - uint32_t nblocks; /* number of blocks */ -}; - -struct g_uncompress_softc { - uint32_t blksz; /* block size */ - uint32_t nblocks; /* number of blocks */ - uint64_t *offsets; - enum { - GEOM_UZIP = 1, - GEOM_ULZMA - } type; - - struct mtx last_mtx; - uint32_t last_blk; /* last blk no */ - char *last_buf; /* last blk data */ - int req_total; /* total requests */ - int req_cached; /* cached requests */ - - /* XZ decoder structs */ - struct xz_buf *b; - struct xz_dec *s; - z_stream *zs; -}; - -static void -g_uncompress_softc_free(struct g_uncompress_softc *sc, struct g_geom *gp) -{ - - if (gp != NULL) { - DPRINTF(("%s: %d requests, %d cached\n", - gp->name, sc->req_total, sc->req_cached)); - } - if (sc->offsets != NULL) { - free(sc->offsets, M_GEOM_UNCOMPRESS); - sc->offsets = NULL; - } - - switch (sc->type) { - case GEOM_ULZMA: - if (sc->b) { - free(sc->b, M_GEOM_UNCOMPRESS); - sc->b = 0; - } - if (sc->s) { - xz_dec_end(sc->s); - sc->s = 0; - } - break; - case GEOM_UZIP: - if (sc->zs) { - inflateEnd(sc->zs); - free(sc->zs, M_GEOM_UNCOMPRESS); - sc->zs = 0; - } - break; - } - - mtx_destroy(&sc->last_mtx); - free(sc->last_buf, M_GEOM_UNCOMPRESS); - free(sc, M_GEOM_UNCOMPRESS); -} - -static void * -z_alloc(void *nil, u_int type, u_int size) -{ - void *ptr; - - ptr = malloc(type * size, M_GEOM_UNCOMPRESS, M_NOWAIT); - - return (ptr); -} - -static void -z_free(void *nil, void *ptr) -{ - - free(ptr, M_GEOM_UNCOMPRESS); -} - -static void -g_uncompress_done(struct bio *bp) -{ - struct g_uncompress_softc *sc; - struct g_provider *pp, *pp2; - struct g_consumer *cp; - struct g_geom *gp; - struct bio *bp2; - uint32_t start_blk, i; - off_t iolen, pos, upos; - size_t bsize; - int err; - - err = 0; - bp2 = bp->bio_parent; - pp = bp2->bio_to; - gp = pp->geom; - cp = LIST_FIRST(&gp->consumer); - pp2 = cp->provider; - sc = gp->softc; - DPRINTF(("%s: done\n", gp->name)); - - bp2->bio_error = bp->bio_error; - if (bp2->bio_error != 0) - goto done; - - /* - * Example: - * Uncompressed block size = 65536 - * User request: 65540-261632 - * (4 uncompressed blocks -4B at start, -512B at end) - * - * We have 512(secsize)*63(nsec) = 32256B at offset 1024 - * From: 1024 bp->bio_offset = 1024 - * To: 33280 bp->bio_length = 33280 - 1024 = 32256 - * Compressed blocks: 0-1020, 1020-1080, 1080-4845, 4845-12444, - * 12444-33210, 33210-44100, ... - * - * Get start_blk from user request: - * start_blk = bp2->bio_offset / 65536 = 65540/65536 = 1 - * bsize (block size of parent) = pp2->sectorsize (Now is 4B) - * pos(in stream from parent) = sc->offsets[start_blk] % bsize = - * = sc->offsets[1] % 4 = 1020 % 4 = 0 - */ - - /* - * Uncompress data. - */ - start_blk = bp2->bio_offset / sc->blksz; - bsize = pp2->sectorsize; - iolen = bp->bio_completed; - pos = sc->offsets[start_blk] % bsize; - upos = 0; - - DPRINTF(("%s: done: bio_length %jd bio_completed %jd start_blk %d, " - "pos %jd, upos %jd (%jd, %d, %zu)\n", - gp->name, (intmax_t)bp->bio_length, (intmax_t)bp->bio_completed, - start_blk, (intmax_t)pos, (intmax_t)upos, - (intmax_t)bp2->bio_offset, sc->blksz, bsize)); - - for (i = start_blk; upos < bp2->bio_length; i++) { - off_t len, ulen, uoff; - - uoff = i == start_blk ? bp2->bio_offset % sc->blksz : 0; - ulen = MIN(sc->blksz - uoff, bp2->bio_length - upos); - len = sc->offsets[i + 1] - sc->offsets[i]; - - DPRINTF(( - "%s: done: inflate block %d, start %ju, end %ju len %jd\n", - gp->name, i, (uintmax_t)sc->offsets[i], - (uintmax_t)sc->offsets[i + 1], (intmax_t)len)); - - if (len == 0) { - /* All zero block: no cache update */ - bzero(bp2->bio_data + upos, ulen); - upos += ulen; - bp2->bio_completed += ulen; - continue; - } - if (len > iolen) { - DPRINTF(("%s: done: early termination: len (%jd) > " - "iolen (%jd)\n", - gp->name, (intmax_t)len, (intmax_t)iolen)); - break; - } - mtx_lock(&sc->last_mtx); - -#ifdef GEOM_UNCOMPRESS_DEBUG - if (g_debugflags & 32) - hexdump(bp->bio_data + pos, len, 0, 0); -#endif - - switch (sc->type) { - case GEOM_ULZMA: - sc->b->in = bp->bio_data + pos; - sc->b->out = sc->last_buf; - sc->b->in_pos = sc->b->out_pos = 0; - sc->b->in_size = len; - sc->b->out_size = (size_t)-1; - - err = (xz_dec_run(sc->s, sc->b) != XZ_STREAM_END) ? - 1 : 0; - /* TODO decoder recovery, if needed */ - break; - case GEOM_UZIP: - sc->zs->next_in = bp->bio_data + pos; - sc->zs->avail_in = len; - sc->zs->next_out = sc->last_buf; - sc->zs->avail_out = sc->blksz; - - err = (inflate(sc->zs, Z_FINISH) != Z_STREAM_END) ? - 1 : 0; - if ((err) || (inflateReset(sc->zs) != Z_OK)) - printf("%s: UZIP decoder reset failed\n", - gp->name); - break; - } - - if (err) { - sc->last_blk = -1; - mtx_unlock(&sc->last_mtx); - DPRINTF(("%s: done: inflate failed, code=%d\n", - gp->name, err)); - bp2->bio_error = EIO; - goto done; - } - -#ifdef GEOM_UNCOMPRESS_DEBUG - if (g_debugflags & 32) - hexdump(sc->last_buf, sc->b->out_size, 0, 0); -#endif - - sc->last_blk = i; - DPRINTF(("%s: done: inflated \n", gp->name)); - memcpy(bp2->bio_data + upos, sc->last_buf + uoff, ulen); - mtx_unlock(&sc->last_mtx); - - pos += len; - iolen -= len; - upos += ulen; - bp2->bio_completed += ulen; - } - -done: - /* - * Finish processing the request. - */ - DPRINTF(("%s: done: (%d, %jd, %ld)\n", - gp->name, bp2->bio_error, (intmax_t)bp2->bio_completed, - bp2->bio_resid)); - free(bp->bio_data, M_GEOM_UNCOMPRESS); - g_destroy_bio(bp); - g_io_deliver(bp2, bp2->bio_error); -} - -static void -g_uncompress_start(struct bio *bp) -{ - struct g_uncompress_softc *sc; - struct g_provider *pp, *pp2; - struct g_consumer *cp; - struct g_geom *gp; - struct bio *bp2; - uint32_t start_blk, end_blk; - size_t bsize; - - pp = bp->bio_to; - gp = pp->geom; - DPRINTF(("%s: start (%d:%s) to %s off=%jd len=%jd\n", - gp->name, bp->bio_cmd, - (bp->bio_cmd == BIO_READ) ? "BIO_READ" : "NOTSUPPORTED", - pp->name, (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length)); - - if (bp->bio_cmd != BIO_READ) { - g_io_deliver(bp, EOPNOTSUPP); - return; - } - - cp = LIST_FIRST(&gp->consumer); - pp2 = cp->provider; - sc = gp->softc; - - start_blk = bp->bio_offset / sc->blksz; - end_blk = howmany(bp->bio_offset + bp->bio_length, sc->blksz); - KASSERT(start_blk < sc->nblocks, ("start_blk out of range")); - KASSERT(end_blk <= sc->nblocks, ("end_blk out of range")); - - sc->req_total++; - if (start_blk + 1 == end_blk) { - mtx_lock(&sc->last_mtx); - if (start_blk == sc->last_blk) { - off_t uoff; - - uoff = bp->bio_offset % sc->blksz; - KASSERT(bp->bio_length <= sc->blksz - uoff, - ("cached data error")); - memcpy(bp->bio_data, sc->last_buf + uoff, - bp->bio_length); - sc->req_cached++; - mtx_unlock(&sc->last_mtx); - - DPRINTF(("%s: start: cached 0 + %jd, %jd + 0 + %jd\n", - gp->name, (intmax_t)bp->bio_length, (intmax_t)uoff, - (intmax_t)bp->bio_length)); - bp->bio_completed = bp->bio_length; - g_io_deliver(bp, 0); - return; - } - mtx_unlock(&sc->last_mtx); - } - - bp2 = g_clone_bio(bp); - if (bp2 == NULL) { - g_io_deliver(bp, ENOMEM); - return; - } - DPRINTF(("%s: start (%d..%d), %s: %d + %jd, %s: %d + %jd\n", - gp->name, start_blk, end_blk, - pp->name, pp->sectorsize, (intmax_t)pp->mediasize, - pp2->name, pp2->sectorsize, (intmax_t)pp2->mediasize)); - bsize = pp2->sectorsize; - bp2->bio_done = g_uncompress_done; - bp2->bio_offset = rounddown(sc->offsets[start_blk], bsize); - while (1) { - bp2->bio_length = roundup(sc->offsets[end_blk], bsize) - - bp2->bio_offset; - if (bp2->bio_length < MAXPHYS) - break; - - end_blk--; - DPRINTF(( - "%s: bio_length (%jd) > MAXPHYS: lowering end_blk to %u\n", - gp->name, (intmax_t)bp2->bio_length, end_blk)); - } - DPRINTF(("%s: start %jd + %jd -> %ju + %ju -> %jd + %jd\n", - gp->name, - (intmax_t)bp->bio_offset, (intmax_t)bp->bio_length, - (uintmax_t)sc->offsets[start_blk], - (uintmax_t)sc->offsets[end_blk] - sc->offsets[start_blk], - (intmax_t)bp2->bio_offset, (intmax_t)bp2->bio_length)); - bp2->bio_data = malloc(bp2->bio_length, M_GEOM_UNCOMPRESS, M_NOWAIT); - if (bp2->bio_data == NULL) { - g_destroy_bio(bp2); - g_io_deliver(bp, ENOMEM); - return; - } - - g_io_request(bp2, cp); - DPRINTF(("%s: start ok\n", gp->name)); -} - -static void -g_uncompress_orphan(struct g_consumer *cp) -{ - struct g_geom *gp; - - g_trace(G_T_TOPOLOGY, "%s(%p/%s)", __func__, cp, cp->provider->name); - g_topology_assert(); - - gp = cp->geom; - g_uncompress_softc_free(gp->softc, gp); - gp->softc = NULL; - g_wither_geom(gp, ENXIO); -} - -static int -g_uncompress_access(struct g_provider *pp, int dr, int dw, int de) -{ - struct g_consumer *cp; - struct g_geom *gp; - - gp = pp->geom; - cp = LIST_FIRST(&gp->consumer); - KASSERT (cp != NULL, ("g_uncompress_access but no consumer")); - - if (cp->acw + dw > 0) - return (EROFS); - - return (g_access(cp, dr, dw, de)); -} - -static void -g_uncompress_spoiled(struct g_consumer *cp) -{ - struct g_geom *gp; - - gp = cp->geom; - g_trace(G_T_TOPOLOGY, "%s(%p/%s)", __func__, cp, gp->name); - g_topology_assert(); - - g_uncompress_softc_free(gp->softc, gp); - gp->softc = NULL; - g_wither_geom(gp, ENXIO); -} - -static struct g_geom * -g_uncompress_taste(struct g_class *mp, struct g_provider *pp, int flags) -{ - struct cloop_header *header; - struct g_uncompress_softc *sc; - struct g_provider *pp2; - struct g_consumer *cp; - struct g_geom *gp; - uint64_t *offsets; - uint32_t i, r, total, total_offsets, type; - uint8_t *buf; - int error; - - g_trace(G_T_TOPOLOGY, "%s(%s,%s)", __func__, mp->name, pp->name); - g_topology_assert(); - - /* Skip providers that are already open for writing. */ - if (pp->acw > 0) - return (NULL); - - buf = NULL; - - /* - * Create geom instance. - */ - gp = g_new_geomf(mp, "%s.uncompress", pp->name); - cp = g_new_consumer(gp); - error = g_attach(cp, pp); - if (error == 0) - error = g_access(cp, 1, 0, 0); - if (error) { - g_detach(cp); - g_destroy_consumer(cp); - g_destroy_geom(gp); - return (NULL); - } - g_topology_unlock(); - - /* - * Read cloop header, look for CLOOP magic, perform - * other validity checks. - */ - DPRINTF(("%s: media sectorsize %u, mediasize %jd\n", - gp->name, pp->sectorsize, (intmax_t)pp->mediasize)); - total = roundup(sizeof(struct cloop_header), pp->sectorsize); - buf = g_read_data(cp, 0, total, NULL); - if (buf == NULL) - goto err; - header = (struct cloop_header *) buf; - if (strncmp(header->magic, CLOOP_MAGIC_START, - sizeof(CLOOP_MAGIC_START) - 1) != 0) { - DPRINTF(("%s: no CLOOP magic\n", gp->name)); - goto err; - } - - switch (header->magic[0x0b]) { - case 'L': - type = GEOM_ULZMA; - if (header->magic[0x0c] < GEOM_ULZMA_MAJVER) { - DPRINTF(("%s: image version too old\n", gp->name)); - goto err; - } - DPRINTF(("%s: GEOM_ULZMA image found\n", gp->name)); - break; - case 'V': - type = GEOM_UZIP; - if (header->magic[0x0c] < GEOM_UZIP_MAJVER) { - DPRINTF(("%s: image version too old\n", gp->name)); - goto err; - } - DPRINTF(("%s: GEOM_UZIP image found\n", gp->name)); - break; - default: - DPRINTF(("%s: unsupported image type\n", gp->name)); - goto err; - } - - DPRINTF(("%s: found CLOOP magic\n", gp->name)); - /* - * Initialize softc and read offsets. - */ - sc = malloc(sizeof(*sc), M_GEOM_UNCOMPRESS, M_WAITOK | M_ZERO); - gp->softc = sc; - sc->type = type; - sc->blksz = ntohl(header->blksz); - sc->nblocks = ntohl(header->nblocks); - if (sc->blksz % 4 != 0) { - printf("%s: block size (%u) should be multiple of 4.\n", - gp->name, sc->blksz); - goto err; - } - if (sc->blksz > MAX_BLKSZ) { - printf("%s: block size (%u) should not be larger than %d.\n", - gp->name, sc->blksz, MAX_BLKSZ); - } - total_offsets = sc->nblocks + 1; - if (sizeof(struct cloop_header) + - total_offsets * sizeof(uint64_t) > pp->mediasize) { - printf("%s: media too small for %u blocks\n", - gp->name, sc->nblocks); - goto err; - } - g_free(buf); - - sc->offsets = malloc(total_offsets * sizeof(uint64_t), - M_GEOM_UNCOMPRESS, M_WAITOK | M_ZERO); - total = roundup((sizeof(struct cloop_header) + - total_offsets * sizeof(uint64_t)), pp->sectorsize); -#define RSZ ((total - r) > MAXPHYS ? MAXPHYS: (total - r)) - for (r = 0, i = 0; r < total; r += MAXPHYS) { - buf = g_read_data(cp, r, RSZ, &error); - if (buf == NULL) { - free(sc->offsets, M_GEOM_UNCOMPRESS); - goto err; - } - offsets = (uint64_t *)buf; - if (r == 0) - offsets += - sizeof(struct cloop_header) / sizeof(uint64_t); - for (; i < total_offsets && offsets < (uint64_t *)(buf + RSZ); - i++, offsets++) - sc->offsets[i] = be64toh(*offsets); - g_free(buf); - } -#undef RSZ - buf = NULL; - DPRINTF(("%s: done reading offsets\n", gp->name)); - mtx_init(&sc->last_mtx, "geom_uncompress cache", NULL, MTX_DEF); - sc->last_blk = -1; - sc->last_buf = malloc(sc->blksz, M_GEOM_UNCOMPRESS, M_WAITOK); - sc->req_total = 0; - sc->req_cached = 0; - - switch (sc->type) { - case GEOM_ULZMA: - xz_crc32_init(); - sc->s = xz_dec_init(XZ_SINGLE, 0); - sc->b = (struct xz_buf*)malloc(sizeof(struct xz_buf), - M_GEOM_UNCOMPRESS, M_WAITOK); - break; - case GEOM_UZIP: - sc->zs = (z_stream *)malloc(sizeof(z_stream), - M_GEOM_UNCOMPRESS, M_WAITOK); - sc->zs->zalloc = z_alloc; - sc->zs->zfree = z_free; - if (inflateInit(sc->zs) != Z_OK) { - goto err; - } - break; - } - - g_topology_lock(); - pp2 = g_new_providerf(gp, "%s", gp->name); - pp2->sectorsize = 512; - pp2->mediasize = (off_t)sc->nblocks * sc->blksz; - if (pp->stripesize > 0) { - pp2->stripesize = pp->stripesize; - pp2->stripeoffset = pp->stripeoffset; - } - g_error_provider(pp2, 0); - g_access(cp, -1, 0, 0); - - DPRINTF(("%s: taste ok (%d, %jd), (%d, %d), %x\n", - gp->name, - pp2->sectorsize, (intmax_t)pp2->mediasize, - pp2->stripeoffset, pp2->stripesize, pp2->flags)); - DPRINTF(("%s: %u x %u blocks\n", gp->name, sc->nblocks, sc->blksz)); - return (gp); - -err: - g_topology_lock(); - g_access(cp, -1, 0, 0); - if (buf != NULL) - g_free(buf); - if (gp->softc != NULL) { - g_uncompress_softc_free(gp->softc, NULL); - gp->softc = NULL; - } - g_detach(cp); - g_destroy_consumer(cp); - g_destroy_geom(gp); - - return (NULL); -} - -static int -g_uncompress_destroy_geom(struct gctl_req *req, struct g_class *mp, - struct g_geom *gp) -{ - struct g_provider *pp; - - g_trace(G_T_TOPOLOGY, "%s(%s, %s)", __func__, mp->name, gp->name); - g_topology_assert(); - - if (gp->softc == NULL) { - DPRINTF(("%s(%s): gp->softc == NULL\n", __func__, gp->name)); - return (ENXIO); - } - - KASSERT(gp != NULL, ("NULL geom")); - pp = LIST_FIRST(&gp->provider); - KASSERT(pp != NULL, ("NULL provider")); - if (pp->acr > 0 || pp->acw > 0 || pp->ace > 0) - return (EBUSY); - - g_uncompress_softc_free(gp->softc, gp); - gp->softc = NULL; - g_wither_geom(gp, ENXIO); - - return (0); -} - -static struct g_class g_uncompress_class = { - .name = UNCOMPRESS_CLASS_NAME, - .version = G_VERSION, - .taste = g_uncompress_taste, - .destroy_geom = g_uncompress_destroy_geom, - - .start = g_uncompress_start, - .orphan = g_uncompress_orphan, - .access = g_uncompress_access, - .spoiled = g_uncompress_spoiled, -}; - -DECLARE_GEOM_CLASS(g_uncompress_class, g_uncompress); -MODULE_DEPEND(g_uncompress, zlib, 1, 1, 1); diff --git a/sys/mips/conf/ALFA_HORNET_UB b/sys/mips/conf/ALFA_HORNET_UB index 1986ec5ed69..99e8c8182b5 100644 --- a/sys/mips/conf/ALFA_HORNET_UB +++ b/sys/mips/conf/ALFA_HORNET_UB @@ -49,11 +49,11 @@ device arswitch options AR71XX_ENV_UBOOT # uzip - to boot natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS +device geom_uzip +options GEOM_UZIP # Used for the static uboot partition map device geom_map # Boot off of the rootfs, as defined in the geom_map setup. -options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" diff --git a/sys/mips/conf/AP121 b/sys/mips/conf/AP121 index cc49be06a4c..3ffbd79f6d8 100644 --- a/sys/mips/conf/AP121 +++ b/sys/mips/conf/AP121 @@ -43,11 +43,11 @@ device arswitch options AR71XX_ENV_UBOOT # uzip - to boot natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS +device geom_uzip +options GEOM_UZIP # Used for the static uboot partition map device geom_map # Boot off of the rootfs, as defined in the geom_map setup. -options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" diff --git a/sys/mips/conf/AP135 b/sys/mips/conf/AP135 index 2807528cc77..11c73eab8dc 100644 --- a/sys/mips/conf/AP135 +++ b/sys/mips/conf/AP135 @@ -46,8 +46,8 @@ device arswitch options AR71XX_ENV_UBOOT # uzip - to boot natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS +device geom_uzip +options GEOM_UZIP # Used for the static uboot partition map device geom_map @@ -64,7 +64,7 @@ device firmware # Used by the above options ATH_EEPROM_FIRMWARE # Boot off of the rootfs, as defined in the geom_map setup. -options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" # Default to accept options IPFIREWALL_DEFAULT_TO_ACCEPT diff --git a/sys/mips/conf/AP143 b/sys/mips/conf/AP143 index 7cf72b86419..021d9722c89 100644 --- a/sys/mips/conf/AP143 +++ b/sys/mips/conf/AP143 @@ -43,11 +43,11 @@ device arswitch options AR71XX_ENV_UBOOT # uzip - to boot natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS +device geom_uzip +options GEOM_UZIP # Used for the static uboot partition map device geom_map # Boot off of the rootfs, as defined in the geom_map setup. -options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" diff --git a/sys/mips/conf/AP91 b/sys/mips/conf/AP91 index 58e70732b90..c5e6c566aaa 100644 --- a/sys/mips/conf/AP91 +++ b/sys/mips/conf/AP91 @@ -38,9 +38,9 @@ options NO_SWAPPING # options MSDOSFS # uncompress - to boot read-only lzma natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS -options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uncompress\" +device geom_uzip +options GEOM_UZIP +options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uzip\" # Not enough space for these.. nooptions INVARIANTS diff --git a/sys/mips/conf/AP93 b/sys/mips/conf/AP93 index b53bb747808..927bf7fb887 100644 --- a/sys/mips/conf/AP93 +++ b/sys/mips/conf/AP93 @@ -26,9 +26,9 @@ options AR71XX_ENV_UBOOT options MSDOSFS # uncompress - to boot read-only lzma natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS -options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uncompress\" +device geom_uzip +options GEOM_UZIP +options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uzip\" # Used for the static uboot partition map device geom_map diff --git a/sys/mips/conf/AP96 b/sys/mips/conf/AP96 index ae7f9125d75..30d1ff8338d 100644 --- a/sys/mips/conf/AP96 +++ b/sys/mips/conf/AP96 @@ -27,9 +27,9 @@ options AR71XX_ENV_UBOOT options MSDOSFS # uncompress - to boot read-only lzma natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS -options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uncompress\" +device geom_uzip +options GEOM_UZIP +options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uzip\" # Used for the static uboot partition map device geom_map diff --git a/sys/mips/conf/CARAMBOLA2 b/sys/mips/conf/CARAMBOLA2 index b5b419ae7d1..033f37bafe7 100644 --- a/sys/mips/conf/CARAMBOLA2 +++ b/sys/mips/conf/CARAMBOLA2 @@ -45,11 +45,11 @@ device arswitch options AR71XX_ENV_UBOOT # uzip - to boot natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS +device geom_uzip +options GEOM_UZIP # Used for the static uboot partition map device geom_map # Boot off of the rootfs, as defined in the geom_map setup. -options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" diff --git a/sys/mips/conf/DB120 b/sys/mips/conf/DB120 index 0924d25cbcb..a491a22235d 100644 --- a/sys/mips/conf/DB120 +++ b/sys/mips/conf/DB120 @@ -38,8 +38,8 @@ device arswitch options AR71XX_ENV_UBOOT # uzip - to boot natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS +device geom_uzip +options GEOM_UZIP # Used for the static uboot partition map device geom_map @@ -51,4 +51,4 @@ device firmware # Used by the above options ATH_EEPROM_FIRMWARE # Boot off of the rootfs, as defined in the geom_map setup. -options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" diff --git a/sys/mips/conf/DIR-655A1 b/sys/mips/conf/DIR-655A1 index 22004f63b6d..4b66b91282e 100644 --- a/sys/mips/conf/DIR-655A1 +++ b/sys/mips/conf/DIR-655A1 @@ -41,8 +41,8 @@ device arswitch options AR71XX_ENV_UBOOT # uzip - to boot natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS +device geom_uzip +options GEOM_UZIP # Used for the static uboot partition map device geom_map @@ -52,7 +52,7 @@ device firmware # Used by the above options ATH_EEPROM_FIRMWARE # Boot off of the rootfs, as defined in the geom_map setup. -options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" # Default to accept options IPFIREWALL_DEFAULT_TO_ACCEPT diff --git a/sys/mips/conf/DIR-825B1 b/sys/mips/conf/DIR-825B1 index 40dcbf5e50e..3385c6bd1d4 100644 --- a/sys/mips/conf/DIR-825B1 +++ b/sys/mips/conf/DIR-825B1 @@ -54,11 +54,11 @@ options NO_SYSCTL_DESCR # GEOM modules device geom_map # to get access to the SPI flash partitions -device geom_uncompress # compressed in-memory filesystem hackery! -options GEOM_UNCOMPRESS +device geom_uzip # compressed in-memory filesystem hackery! +options GEOM_UZIP options GEOM_PART_GPT -options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uzip\" options AR71XX_REALMEM=64*1024*1024 options AR71XX_ENV_UBOOT diff --git a/sys/mips/conf/DIR-825C1 b/sys/mips/conf/DIR-825C1 index 2721962ef8c..4c37e972926 100644 --- a/sys/mips/conf/DIR-825C1 +++ b/sys/mips/conf/DIR-825C1 @@ -58,11 +58,11 @@ device firmware # Used by the above options ATH_EEPROM_FIRMWARE # Boot off of the rootfs, as defined in the geom_map setup. -options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" # In order to netboot, you have to build the mfsroot into the kernel # 19443712 or 19M is the biggest rootfs via netboot this thing supports #options MD_ROOT # md device usable as a potential root device #options MD_ROOT_SIZE=19444 #makeoptions MFS_IMAGE=/tftpboot/mfsroot-dir825c1.img.ulzma -#options ROOTDEVNAME=\"ufs:md0.uncompress\" +#options ROOTDEVNAME=\"ufs:md0.uzip\" diff --git a/sys/mips/conf/ENH200 b/sys/mips/conf/ENH200 index bece380d2a9..f9d8c0d2d94 100644 --- a/sys/mips/conf/ENH200 +++ b/sys/mips/conf/ENH200 @@ -26,9 +26,9 @@ options AR71XX_ENV_UBOOT options MSDOSFS # uncompress - to boot read-only lzma natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS -options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uncompress\" +device geom_uzip +options GEOM_UZIP +options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uzip\" # Used for the static uboot partition map device geom_map diff --git a/sys/mips/conf/MT7620 b/sys/mips/conf/MT7620 index 4c4c18ad100..3ac1469e4d4 100644 --- a/sys/mips/conf/MT7620 +++ b/sys/mips/conf/MT7620 @@ -65,7 +65,7 @@ options PSEUDOFS #Pseudo-filesystem framework #options BOOTP_WIRED_TO=rt0 #options BOOTP_COMPAT #options CD9660 # ISO 9660 Filesystem -#options ROOTDEVNAME=\"cd9660:/dev/map/rootfs.uncompress\" +#options ROOTDEVNAME=\"cd9660:/dev/map/rootfs.uzip\" #options TMPFS # TMP Memory Filesystem options FFS #Berkeley Fast Filesystem diff --git a/sys/mips/conf/ONIONOMEGA b/sys/mips/conf/ONIONOMEGA index 8b3ec4662f1..cdc5c1fb657 100644 --- a/sys/mips/conf/ONIONOMEGA +++ b/sys/mips/conf/ONIONOMEGA @@ -45,11 +45,11 @@ device arswitch options AR71XX_ENV_UBOOT # uzip - to boot natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS +device geom_uzip +options GEOM_UZIP # Used for the static uboot partition map device geom_map # Boot off of the rootfs, as defined in the geom_map setup. -options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" diff --git a/sys/mips/conf/PB47 b/sys/mips/conf/PB47 index 8386365438c..0fe8c82c9ce 100644 --- a/sys/mips/conf/PB47 +++ b/sys/mips/conf/PB47 @@ -32,11 +32,11 @@ options AR71XX_REALMEM=64*1024*1024 options MSDOSFS # uncompress - to boot read-only lzma natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS +device geom_uzip +options GEOM_UZIP # Used for the static uboot partition map device geom_map # Boot off of the rootfs, as defined in the geom_map setup. -options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" diff --git a/sys/mips/conf/PB92 b/sys/mips/conf/PB92 index 9ed14e36dfc..6176668fb12 100644 --- a/sys/mips/conf/PB92 +++ b/sys/mips/conf/PB92 @@ -54,9 +54,9 @@ options FFS #Berkeley Fast Filesystem #options UFS_DIRHASH #Improve performance on big directories # Support uncompress lzma rootfs -device geom_uncompress -options GEOM_UNCOMPRESS -options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uncompress\" +device geom_uzip +options GEOM_UZIP +options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uzip\" # PCI bus device pci diff --git a/sys/mips/conf/PICOSTATION_M2HP b/sys/mips/conf/PICOSTATION_M2HP index 3e738e6ca5f..aa821ad5d2b 100644 --- a/sys/mips/conf/PICOSTATION_M2HP +++ b/sys/mips/conf/PICOSTATION_M2HP @@ -38,9 +38,9 @@ options NO_SWAPPING # options MSDOSFS # uncompress - to boot read-only lzma natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS -options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uncompress\" +device geom_uzip +options GEOM_UZIP +options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uzip\" # Not enough space for these.. nooptions INVARIANTS diff --git a/sys/mips/conf/ROUTERSTATION b/sys/mips/conf/ROUTERSTATION index 41adccd2065..3dceace74f4 100644 --- a/sys/mips/conf/ROUTERSTATION +++ b/sys/mips/conf/ROUTERSTATION @@ -14,8 +14,8 @@ hints "ROUTERSTATION.hints" # GEOM modules device geom_redboot # to get access to the SPI flash partitions -device geom_uncompress # compressed in-memory filesystem support -options GEOM_UNCOMPRESS +device geom_uzip # compressed in-memory filesystem support +options GEOM_UZIP # For DOS options MSDOSFS @@ -27,4 +27,4 @@ device etherswitch device ukswitch # Boot path - redboot MFS -options ROOTDEVNAME=\"ufs:redboot/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:redboot/rootfs.uzip\" diff --git a/sys/mips/conf/RSPRO b/sys/mips/conf/RSPRO index 43d87b69b26..fa28ceeab14 100644 --- a/sys/mips/conf/RSPRO +++ b/sys/mips/conf/RSPRO @@ -15,8 +15,8 @@ device pcf2123_rtc # GEOM modules device geom_redboot # to get access to the SPI flash partitions -device geom_uncompress # compressed in-memory filesystem support -options GEOM_UNCOMPRESS +device geom_uzip # compressed in-memory filesystem support +options GEOM_UZIP # For DOS options MSDOSFS @@ -28,5 +28,5 @@ device etherswitch device arswitch # Boot off of flash -options ROOTDEVNAME=\"ufs:redboot/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:redboot/rootfs.uzip\" diff --git a/sys/mips/conf/RT305X b/sys/mips/conf/RT305X index e5e26f513a2..ca38c72c427 100644 --- a/sys/mips/conf/RT305X +++ b/sys/mips/conf/RT305X @@ -64,7 +64,7 @@ options BOOTP_NFSV3 options BOOTP_WIRED_TO=rt0 options BOOTP_COMPAT options CD9660 # ISO 9660 Filesystem -options ROOTDEVNAME=\"cd9660:/dev/map/rootfs.uncompress\" +options ROOTDEVNAME=\"cd9660:/dev/map/rootfs.uzip\" options TMPFS # TMP Memory Filesystem #options FFS #Berkeley Fast Filesystem diff --git a/sys/mips/conf/RT5350 b/sys/mips/conf/RT5350 index b6ec20bc2ab..f897178b607 100644 --- a/sys/mips/conf/RT5350 +++ b/sys/mips/conf/RT5350 @@ -67,10 +67,10 @@ options TMPFS # TMP Memory Filesystem options FFS #Berkeley Fast Filesystem #options ROOTDEVNAME=\"nfs:193.178.153.200:/bsdmips\" -#device geom_uncompress -#options GEOM_UNCOMPRESS +#device geom_uzip +#options GEOM_UZIP #options MD_ROOT -#options ROOTDEVNAME=\"ufs:md0.uncompress\" +#options ROOTDEVNAME=\"ufs:md0.uzip\" # Options for making kernel less hangry makeoptions INLINE_LIMIT=1024 diff --git a/sys/mips/conf/TL-ARCHERC7V2 b/sys/mips/conf/TL-ARCHERC7V2 index 21f26a4de51..cedd823f2ec 100644 --- a/sys/mips/conf/TL-ARCHERC7V2 +++ b/sys/mips/conf/TL-ARCHERC7V2 @@ -46,8 +46,8 @@ options MSDOSFS options AR71XX_ENV_UBOOT # uzip - to boot natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS +device geom_uzip +options GEOM_UZIP # Used for the static uboot partition map device geom_map @@ -64,7 +64,7 @@ device firmware # Used by the above options ATH_EEPROM_FIRMWARE # Boot off of the rootfs, as defined in the geom_map setup. -options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" # Default to accept options IPFIREWALL_DEFAULT_TO_ACCEPT diff --git a/sys/mips/conf/TL-WDR4300 b/sys/mips/conf/TL-WDR4300 index 3c797441132..e0915267be5 100644 --- a/sys/mips/conf/TL-WDR4300 +++ b/sys/mips/conf/TL-WDR4300 @@ -38,8 +38,8 @@ options MSDOSFS options AR71XX_ENV_UBOOT # uzip - to boot natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS +device geom_uzip +options GEOM_UZIP # Used for the static uboot partition map device geom_map @@ -51,4 +51,4 @@ device firmware # Used by the above options ATH_EEPROM_FIRMWARE # Boot off of the rootfs, as defined in the geom_map setup. -options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" diff --git a/sys/mips/conf/TL-WR1043NDv2 b/sys/mips/conf/TL-WR1043NDv2 index 2bcde0cd4c4..53a5847480f 100644 --- a/sys/mips/conf/TL-WR1043NDv2 +++ b/sys/mips/conf/TL-WR1043NDv2 @@ -34,8 +34,8 @@ options MSDOSFS options AR71XX_ENV_UBOOT # uzip - to boot natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS +device geom_uzip +options GEOM_UZIP # Used for the static uboot partition map device geom_map @@ -47,7 +47,7 @@ device geom_map #options ATH_EEPROM_FIRMWARE # Boot off of the rootfs, as defined in the geom_map setup. -options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" # Default to accept options IPFIREWALL_DEFAULT_TO_ACCEPT diff --git a/sys/mips/conf/TL-WR740Nv4 b/sys/mips/conf/TL-WR740Nv4 index 5257dd9b70e..59bf73eab62 100644 --- a/sys/mips/conf/TL-WR740Nv4 +++ b/sys/mips/conf/TL-WR740Nv4 @@ -43,13 +43,13 @@ device arswitch options AR71XX_ENV_UBOOT # uzip - to boot natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS +device geom_uzip +options GEOM_UZIP # Used for the static uboot partition map device geom_map # Boot off of the rootfs, as defined in the geom_map setup. -# options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +# options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" # Note: we don't fit in 4MB flash, so the rootfs must be on USB for now options ROOTDEVNAME=\"ufs:da0\" diff --git a/sys/mips/conf/TP-WN1043ND b/sys/mips/conf/TP-WN1043ND index c5832a2151b..7866de3be4b 100644 --- a/sys/mips/conf/TP-WN1043ND +++ b/sys/mips/conf/TP-WN1043ND @@ -37,14 +37,14 @@ options MSDOSFS options AR71XX_ENV_UBOOT # uncompress - to boot natively from flash -device geom_uncompress -options GEOM_UNCOMPRESS +device geom_uzip +options GEOM_UZIP # Used for the static uboot partition map device geom_map # Boot off of the rootfs, as defined in the geom_map setup. -options ROOTDEVNAME=\"ufs:map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:map/rootfs.uzip\" # We bite the performance overhead for now; the kernel won't # fit if the mutexes are inlined. diff --git a/sys/mips/conf/WZR-300HP b/sys/mips/conf/WZR-300HP index 90c5dc652f5..c059dbb34e8 100644 --- a/sys/mips/conf/WZR-300HP +++ b/sys/mips/conf/WZR-300HP @@ -26,9 +26,9 @@ options BOOTVERBOSE # GEOM modules device geom_map # to get access to the SPI flash partitions -device geom_uncompress # compressed in-memory filesystem hackery! +device geom_uzip # compressed in-memory filesystem hackery! -options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uzip\" options AR71XX_ATH_EEPROM # Fetch EEPROM/PCI config from flash options ATH_EEPROM_FIRMWARE # Use EEPROM from flash diff --git a/sys/mips/conf/WZR-HPAG300H b/sys/mips/conf/WZR-HPAG300H index 7d01b6d1971..dcb36f5281b 100644 --- a/sys/mips/conf/WZR-HPAG300H +++ b/sys/mips/conf/WZR-HPAG300H @@ -25,10 +25,10 @@ options AR71XX_ENV_UBOOT options BOOTVERBOSE # GEOM modules -device geom_uncompress # compressed in-memory filesystem hackery! +device geom_uzip # compressed in-memory filesystem hackery! device geom_map # to get access to the SPI flash partitions -options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uncompress\" +options ROOTDEVNAME=\"ufs:/dev/map/rootfs.uzip\" options AR71XX_ATH_EEPROM # Fetch EEPROM/PCI config from flash options ATH_EEPROM_FIRMWARE # Use EEPROM from flash diff --git a/sys/modules/geom/Makefile b/sys/modules/geom/Makefile index f54a3e83c8a..8d7e3c6deb3 100644 --- a/sys/modules/geom/Makefile +++ b/sys/modules/geom/Makefile @@ -21,7 +21,6 @@ SUBDIR= geom_bde \ geom_sched \ geom_shsec \ geom_stripe \ - geom_uncompress \ geom_uzip \ geom_vinum \ geom_virstor \ diff --git a/sys/modules/geom/geom_uncompress/Makefile b/sys/modules/geom/geom_uncompress/Makefile deleted file mode 100644 index 1bebeb5f012..00000000000 --- a/sys/modules/geom/geom_uncompress/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -# $FreeBSD$ - -.PATH: ${.CURDIR}/../../../geom/uncompress \ - ${.CURDIR}/../../../contrib/xz-embedded/freebsd/ \ - ${.CURDIR}/../../../contrib/xz-embedded/linux/lib/xz/ \ - ${.CURDIR}/../../../contrib/xz-embedded/linux/include/linux/ \ - ${.CURDIR}/../../../net - -KMOD= geom_uncompress -CFLAGS+= -I${.CURDIR}/../../../geom/uncompress/ \ - -I${.CURDIR}/../../../contrib/xz-embedded/freebsd \ - -I${.CURDIR}/../../../contrib/xz-embedded/linux/lib/xz/ -SRCS= g_uncompress.c xz_crc32.c xz_dec_bcj.c xz_dec_lzma2.c xz_dec_stream.c \ - xz_malloc.c -SRCS+= xz.h xz_config.h xz_lzma2.h xz_malloc.h xz_private.h xz_stream.h - -.include diff --git a/targets/pseudo/userland/Makefile.depend b/targets/pseudo/userland/Makefile.depend index a7130ddbb62..cebce7cdc24 100644 --- a/targets/pseudo/userland/Makefile.depend +++ b/targets/pseudo/userland/Makefile.depend @@ -297,7 +297,6 @@ DIRDEPS+= \ usr.bin/mklocale \ usr.bin/mkstr \ usr.bin/mktemp \ - usr.bin/mkulzma \ usr.bin/mkuzip \ usr.bin/msgs \ usr.bin/mt \ diff --git a/usr.bin/Makefile b/usr.bin/Makefile index 474bee1091d..a30476f94ee 100644 --- a/usr.bin/Makefile +++ b/usr.bin/Makefile @@ -99,7 +99,6 @@ SUBDIR= alias \ mkimg \ mklocale \ mktemp \ - mkulzma \ mkuzip \ mt \ ncal \ diff --git a/usr.bin/mkulzma/Makefile b/usr.bin/mkulzma/Makefile deleted file mode 100644 index 15596bfc22f..00000000000 --- a/usr.bin/mkulzma/Makefile +++ /dev/null @@ -1,8 +0,0 @@ -# $FreeBSD$ - -PROG= mkulzma -MAN= mkulzma.8 - -LIBADD= lzma - -.include diff --git a/usr.bin/mkulzma/Makefile.depend b/usr.bin/mkulzma/Makefile.depend deleted file mode 100644 index 987c48738e8..00000000000 --- a/usr.bin/mkulzma/Makefile.depend +++ /dev/null @@ -1,20 +0,0 @@ -# $FreeBSD$ -# Autogenerated - do NOT edit! - -DIRDEPS = \ - gnu/lib/csu \ - gnu/lib/libgcc \ - include \ - include/xlocale \ - lib/${CSU_DIR} \ - lib/libc \ - lib/libcompiler_rt \ - lib/liblzma \ - lib/libthr \ - - -.include - -.if ${DEP_RELDIR} == ${_DEP_RELDIR} -# local dependencies - needed for -jN in clean tree -.endif diff --git a/usr.bin/mkulzma/mkulzma.8 b/usr.bin/mkulzma/mkulzma.8 deleted file mode 100644 index b4ac33fe64b..00000000000 --- a/usr.bin/mkulzma/mkulzma.8 +++ /dev/null @@ -1,107 +0,0 @@ -.\" ---------------------------------------------------------------------------- -.\" Derived from mkuzip.8 by Aleksandr Rybalko -.\" ---------------------------------------------------------------------------- -.\" "THE BEER-WARE LICENSE" (Revision 42): -.\" wrote this file. As long as you retain this notice you -.\" can do whatever you want with this stuff. If we meet some day, and you think -.\" this stuff is worth it, you can buy me a beer in return. Maxim Sobolev -.\" ---------------------------------------------------------------------------- -.\" -.\" $FreeBSD$ -.\" -.Dd March 17, 2006 -.Dt MKULZMA 8 -.Os -.Sh NAME -.Nm mkulzma -.Nd compress disk image for use with -.Xr geom_uncompress 4 -class -.Sh SYNOPSIS -.Nm -.Op Fl v -.Op Fl o Ar outfile -.Op Fl s Ar cluster_size -.Ar infile -.Sh DESCRIPTION -The -.Nm -utility compresses a disk image file so that the -.Xr geom_uncompress 4 -class will be able to decompress the resulting image at run-time. -This allows for a significant reduction of size of disk image at -the expense of some CPU time required to decompress the data each -time it is read. -The -.Nm -utility -works in two phases: -.Bl -enum -.It -An -.Ar infile -image is split into clusters; each cluster is compressed using liblzma. -.It -The resulting set of compressed clusters along with headers that allow -locating each individual cluster is written to the output file. -.El -.Pp -The options are: -.Bl -tag -width indent -.It Fl o Ar outfile -Name of the output file -.Ar outfile . -The default is to use the input name with the suffix -.Pa .ulzma . -.It Fl s Ar cluster_size -Split the image into clusters of -.Ar cluster_size -bytes, 16384 bytes by default. -The -.Ar cluster_size -should be a multiple of 512 bytes. -.It Fl v -Display verbose messages. -.El -.Sh NOTES -The compression ratio largely depends on the cluster size used. -.\" The following two sentences are unclear: how can xz(1) be -.\" used in a comparable fashion, and wouldn't a lzma-compressed -.\" image suffer from larger cluster sizes as well? -For large cluster sizes (16K and higher), typical compression ratios -are only 1-2% less than those achieved with -.Xr lzma 1 . -However, it should be kept in mind that larger cluster -sizes lead to higher overhead in the -.Xr geom_uncompress 4 -class, as the class has to decompress the whole cluster even if -only a few bytes from that cluster have to be read. -.Pp -The -.Nm -utility -inserts a short shell script at the beginning of the generated image, -which makes it possible to -.Dq run -the image just like any other shell script. -The script tries to load the -.Xr geom_uncompress 4 -class if it is not loaded, configure the image as an -.Xr md 4 -disk device using -.Xr mdconfig 8 , -and automatically mount it using -.Xr mount_cd9660 8 -on the mount point provided as the first argument to the script. -.Sh EXIT STATUS -.Ex -std -.Sh SEE ALSO -.Xr lzma 1 , -.Xr geom 4 , -.Xr geom_uncompress 4 , -.Xr md 4 , -.Xr mdconfig 8 , -.Xr mount_cd9660 8 -.Sh AUTHORS -.An Maxim Sobolev Aq Mt sobomax@FreeBSD.org -.An Aleksandr Rybalko Aq Mt ray@ddteam.net diff --git a/usr.bin/mkulzma/mkulzma.c b/usr.bin/mkulzma/mkulzma.c deleted file mode 100644 index b046c1e4c77..00000000000 --- a/usr.bin/mkulzma/mkulzma.c +++ /dev/null @@ -1,330 +0,0 @@ -/* - * ---------------------------------------------------------------------------- - * Derived from mkuzip.c by Aleksandr Rybalko - * ---------------------------------------------------------------------------- - * "THE BEER-WARE LICENSE" (Revision 42): - * wrote this file. As long as you retain this notice you - * can do whatever you want with this stuff. If we meet some day, and you think - * this stuff is worth it, you can buy me a beer in return. Maxim Sobolev - * ---------------------------------------------------------------------------- - * - * $FreeBSD$ - * - */ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#define CLSTSIZE 16384 -#define DEFAULT_SUFX ".ulzma" - -#define USED_BLOCKSIZE DEV_BSIZE - -#define CLOOP_MAGIC_LEN 128 -/* Format L3.0, since we move to XZ API */ -static char CLOOP_MAGIC_START[] = - "#!/bin/sh\n" - "#L3.0\n" - "n=uncompress\n" - "m=geom_$n\n" - "(kldstat -m $m 2>&-||kldload $m)>&-&&" - "mount_cd9660 /dev/`mdconfig -af $0`.$n $1\n" - "exit $?\n"; - -static char *readblock(int, char *, u_int32_t); -static void usage(void); -static void *safe_malloc(size_t); -static void cleanup(void); - -static char *cleanfile = NULL; - -int main(int argc, char **argv) -{ - char *iname, *oname, *obuf, *ibuf; - int fdr, fdw, i, opt, verbose, tmp; - struct iovec iov[2]; - struct stat sb; - uint32_t destlen; - uint64_t offset; - uint64_t *toc; - lzma_filter filters[2]; - lzma_options_lzma opt_lzma; - lzma_ret ret; - lzma_stream strm = LZMA_STREAM_INIT; - struct cloop_header { - char magic[CLOOP_MAGIC_LEN]; /* cloop magic */ - uint32_t blksz; /* block size */ - uint32_t nblocks; /* number of blocks */ - } hdr; - - memset(&hdr, 0, sizeof(hdr)); - hdr.blksz = CLSTSIZE; - strcpy(hdr.magic, CLOOP_MAGIC_START); - oname = NULL; - verbose = 0; - - while((opt = getopt(argc, argv, "o:s:v")) != -1) { - switch(opt) { - case 'o': - oname = optarg; - break; - - case 's': - tmp = atoi(optarg); - if (tmp <= 0) { - errx(1, - "invalid cluster size specified: %s", - optarg); - /* Not reached */ - } - if (tmp % USED_BLOCKSIZE != 0) { - errx(1, - "cluster size should be multiple of %d", - USED_BLOCKSIZE); - /* Not reached */ - } - if ( tmp > MAXPHYS) { - errx(1, "cluster size is too large"); - /* Not reached */ - } - hdr.blksz = tmp; - break; - - case 'v': - verbose = 1; - break; - - default: - usage(); - /* Not reached */ - } - } - argc -= optind; - argv += optind; - - if (argc != 1) { - usage(); - /* Not reached */ - } - - iname = argv[0]; - if (oname == NULL) { - asprintf(&oname, "%s%s", iname, DEFAULT_SUFX); - if (oname == NULL) { - err(1, "can't allocate memory"); - /* Not reached */ - } - } - - obuf = safe_malloc(hdr.blksz*2); - ibuf = safe_malloc(hdr.blksz); - - signal(SIGHUP, exit); - signal(SIGINT, exit); - signal(SIGTERM, exit); - signal(SIGXCPU, exit); - signal(SIGXFSZ, exit); - atexit(cleanup); - - fdr = open(iname, O_RDONLY); - if (fdr < 0) { - err(1, "open(%s)", iname); - /* Not reached */ - } - if (fstat(fdr, &sb) != 0) { - err(1, "fstat(%s)", iname); - /* Not reached */ - } - if (S_ISCHR(sb.st_mode)) { - off_t ms; - - if (ioctl(fdr, DIOCGMEDIASIZE, &ms) < 0) { - err(1, "ioctl(DIOCGMEDIASIZE)"); - /* Not reached */ - } - sb.st_size = ms; - } else if (!S_ISREG(sb.st_mode)) { - fprintf(stderr, - "%s: not a character device or regular file\n", - iname); - exit(1); - } - hdr.nblocks = sb.st_size / hdr.blksz; - if ((sb.st_size % hdr.blksz) != 0) { - if (verbose != 0) - fprintf(stderr, "file size is not multiple " - "of %d, padding data\n", hdr.blksz); - hdr.nblocks++; - } - toc = safe_malloc((hdr.nblocks + 1) * sizeof(*toc)); - - fdw = open(oname, O_WRONLY | O_TRUNC | O_CREAT, - S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH); - if (fdw < 0) { - err(1, "open(%s)", oname); - /* Not reached */ - } - cleanfile = oname; - - /* - * Prepare header that we will write later when we have index ready. - */ - iov[0].iov_base = (char *)&hdr; - iov[0].iov_len = sizeof(hdr); - iov[1].iov_base = (char *)toc; - iov[1].iov_len = (hdr.nblocks + 1) * sizeof(*toc); - offset = iov[0].iov_len + iov[1].iov_len; - - /* Reserve space for header */ - lseek(fdw, offset, SEEK_SET); - - if (verbose != 0) - fprintf(stderr, "data size %ju bytes, number of clusters " - "%u, index length %zu bytes\n", sb.st_size, - hdr.nblocks, iov[1].iov_len); - - /* Init lzma encoder */ - if (lzma_lzma_preset(&opt_lzma, LZMA_PRESET_DEFAULT)) - errx(1, "Error loading LZMA preset"); - - filters[0].id = LZMA_FILTER_LZMA2; - filters[0].options = &opt_lzma; - filters[1].id = LZMA_VLI_UNKNOWN; - - for(i = 0; i == 0 || ibuf != NULL; i++) { - ibuf = readblock(fdr, ibuf, hdr.blksz); - if (ibuf != NULL) { - destlen = hdr.blksz*2; - - ret = lzma_stream_encoder(&strm, filters, - LZMA_CHECK_CRC32); - if (ret != LZMA_OK) { - if (ret == LZMA_MEMLIMIT_ERROR) - errx(1, "can't compress data: " - "LZMA_MEMLIMIT_ERROR"); - - errx(1, "can't compress data: " - "LZMA compressor ERROR"); - } - - strm.next_in = ibuf; - strm.avail_in = hdr.blksz; - strm.next_out = obuf; - strm.avail_out = hdr.blksz*2; - - ret = lzma_code(&strm, LZMA_FINISH); - - if (ret != LZMA_STREAM_END) { - /* Error */ - errx(1, "lzma_code FINISH failed, code=%d, " - "pos(in=%zd, out=%zd)", - ret, - (hdr.blksz - strm.avail_in), - (hdr.blksz*2 - strm.avail_out)); - } - - destlen -= strm.avail_out; - - lzma_end(&strm); - - if (verbose != 0) - fprintf(stderr, "cluster #%d, in %u bytes, " - "out %u bytes\n", i, hdr.blksz, destlen); - } else { - destlen = USED_BLOCKSIZE - (offset % USED_BLOCKSIZE); - memset(obuf, 0, destlen); - if (verbose != 0) - fprintf(stderr, "padding data with %u bytes" - " so that file size is multiple of %d\n", - destlen, - USED_BLOCKSIZE); - } - if (write(fdw, obuf, destlen) < 0) { - err(1, "write(%s)", oname); - /* Not reached */ - } - toc[i] = htobe64(offset); - offset += destlen; - } - close(fdr); - - if (verbose != 0) - fprintf(stderr, "compressed data to %ju bytes, saved %lld " - "bytes, %.2f%% decrease.\n", offset, - (long long)(sb.st_size - offset), - 100.0 * (long long)(sb.st_size - offset) / - (float)sb.st_size); - - /* Convert to big endian */ - hdr.blksz = htonl(hdr.blksz); - hdr.nblocks = htonl(hdr.nblocks); - /* Write headers into pre-allocated space */ - lseek(fdw, 0, SEEK_SET); - if (writev(fdw, iov, 2) < 0) { - err(1, "writev(%s)", oname); - /* Not reached */ - } - cleanfile = NULL; - close(fdw); - - exit(0); -} - -static char * -readblock(int fd, char *ibuf, u_int32_t clstsize) -{ - int numread; - - bzero(ibuf, clstsize); - numread = read(fd, ibuf, clstsize); - if (numread < 0) { - err(1, "read() failed"); - /* Not reached */ - } - if (numread == 0) { - return NULL; - } - return ibuf; -} - -static void -usage(void) -{ - - fprintf(stderr, "usage: mkulzma [-v] [-o outfile] [-s cluster_size] " - "infile\n"); - exit(1); -} - -static void * -safe_malloc(size_t size) -{ - void *retval; - - retval = malloc(size); - if (retval == NULL) { - err(1, "can't allocate memory"); - /* Not reached */ - } - return retval; -} - -static void -cleanup(void) -{ - - if (cleanfile != NULL) - unlink(cleanfile); -} From e84c2dea937a98fecd37b6c4bef00ad2ab368023 Mon Sep 17 00:00:00 2001 From: Eric Joyner Date: Wed, 24 Feb 2016 00:42:43 +0000 Subject: [PATCH 20/25] ixl(4): Fix potential driver interrupt setup issues and startup crash. - Limit queue autoconfiguration to 8 queues to prevent the driver from requesting a large number of MSI-X vectors at boot. - Fix potential kernel panic that occurs when the driver loads and cannot get all requested MSIX vectors. Instead, attach() will fail with an error. - Move taskqueue setup to later in attach() to prevent having to free taskqueues if some other error in attach() occurs. Differential Revision: https://reviews.freebsd.org/D5205 MFC after: 1 month Tested by: jeffrey.e.pieper@intel.com Sponsored by: Intel Corporation --- sys/dev/ixl/if_ixl.c | 102 ++++++++++++++++++++++++++++--------------- 1 file changed, 67 insertions(+), 35 deletions(-) diff --git a/sys/dev/ixl/if_ixl.c b/sys/dev/ixl/if_ixl.c index f972c5fe2a6..22d8f0118b0 100644 --- a/sys/dev/ixl/if_ixl.c +++ b/sys/dev/ixl/if_ixl.c @@ -115,6 +115,8 @@ static int ixl_init_msix(struct ixl_pf *); static void ixl_configure_msix(struct ixl_pf *); static void ixl_configure_itr(struct ixl_pf *); static void ixl_configure_legacy(struct ixl_pf *); +static void ixl_init_taskqueues(struct ixl_pf *); +static void ixl_free_taskqueues(struct ixl_pf *); static void ixl_free_pci_resources(struct ixl_pf *); static void ixl_local_timer(void *); static int ixl_setup_interface(device_t, struct ixl_vsi *); @@ -642,7 +644,7 @@ ixl_attach(device_t dev) else error = ixl_assign_vsi_legacy(pf); if (error) - goto err_late; + goto err_mac_hmc; if (((hw->aq.fw_maj_ver == 4) && (hw->aq.fw_min_ver < 33)) || (hw->aq.fw_maj_ver < 4)) { @@ -667,7 +669,7 @@ ixl_attach(device_t dev) error = ixl_switch_config(pf); if (error) { device_printf(dev, "Initial switch config failed: %d\n", error); - goto err_mac_hmc; + goto err_late; } /* Limit phy interrupts to link and modules failure */ @@ -680,6 +682,9 @@ ixl_attach(device_t dev) bus = ixl_get_bus_info(hw, dev); i40e_set_pci_config_data(hw, bus); + /* Initialize taskqueues */ + ixl_init_taskqueues(pf); + /* Initialize statistics */ ixl_pf_reset_stats(pf); ixl_update_stats_counters(pf); @@ -748,7 +753,6 @@ ixl_detach(device_t dev) struct ixl_pf *pf = device_get_softc(dev); struct i40e_hw *hw = &pf->hw; struct ixl_vsi *vsi = &pf->vsi; - struct ixl_queue *que = vsi->queues; i40e_status status; #ifdef PCI_IOV int error; @@ -777,13 +781,7 @@ ixl_detach(device_t dev) IXL_PF_UNLOCK(pf); } - for (int i = 0; i < vsi->num_queues; i++, que++) { - if (que->tq) { - taskqueue_drain(que->tq, &que->task); - taskqueue_drain(que->tq, &que->tx_task); - taskqueue_free(que->tq); - } - } + ixl_free_taskqueues(pf); /* Shutdown LAN HMC */ status = i40e_shutdown_lan_hmc(hw); @@ -1990,6 +1988,58 @@ ixl_assign_vsi_legacy(struct ixl_pf *pf) return (0); } +static void +ixl_init_taskqueues(struct ixl_pf *pf) +{ + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + device_t dev = pf->dev; + + /* Tasklet for Admin Queue */ + TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); +#ifdef PCI_IOV + /* VFLR Tasklet */ + TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf); +#endif + + /* Create and start PF taskqueue */ + pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, + taskqueue_thread_enqueue, &pf->tq); + taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq", + device_get_nameunit(dev)); + + /* Create queue tasks and start queue taskqueues */ + for (int i = 0; i < vsi->num_queues; i++, que++) { + TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); + TASK_INIT(&que->task, 0, ixl_handle_que, que); + que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, + taskqueue_thread_enqueue, &que->tq); +#ifdef RSS + CPU_SETOF(cpu_id, &cpu_mask); + taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, + &cpu_mask, "%s (bucket %d)", + device_get_nameunit(dev), cpu_id); +#else + taskqueue_start_threads(&que->tq, 1, PI_NET, + "%s (que %d)", device_get_nameunit(dev), que->me); +#endif + } + +} + +static void +ixl_free_taskqueues(struct ixl_pf *pf) +{ + struct ixl_vsi *vsi = &pf->vsi; + struct ixl_queue *que = vsi->queues; + + if (pf->tq) + taskqueue_free(pf->tq); + for (int i = 0; i < vsi->num_queues; i++, que++) { + if (que->tq) + taskqueue_free(que->tq); + } +} /********************************************************************* * @@ -2028,17 +2078,6 @@ ixl_assign_vsi_msix(struct ixl_pf *pf) } bus_describe_intr(dev, pf->res, pf->tag, "aq"); pf->admvec = vector; - /* Tasklet for Admin Queue */ - TASK_INIT(&pf->adminq, 0, ixl_do_adminq, pf); - -#ifdef PCI_IOV - TASK_INIT(&pf->vflr_task, 0, ixl_handle_vflr, pf); -#endif - - pf->tq = taskqueue_create_fast("ixl_adm", M_NOWAIT, - taskqueue_thread_enqueue, &pf->tq); - taskqueue_start_threads(&pf->tq, 1, PI_NET, "%s adminq", - device_get_nameunit(pf->dev)); ++vector; /* Now set up the stations */ @@ -2069,19 +2108,6 @@ ixl_assign_vsi_msix(struct ixl_pf *pf) #endif bus_bind_intr(dev, que->res, cpu_id); que->msix = vector; - TASK_INIT(&que->tx_task, 0, ixl_deferred_mq_start, que); - TASK_INIT(&que->task, 0, ixl_handle_que, que); - que->tq = taskqueue_create_fast("ixl_que", M_NOWAIT, - taskqueue_thread_enqueue, &que->tq); -#ifdef RSS - CPU_SETOF(cpu_id, &cpu_mask); - taskqueue_start_threads_cpuset(&que->tq, 1, PI_NET, - &cpu_mask, "%s (bucket %d)", - device_get_nameunit(dev), cpu_id); -#else - taskqueue_start_threads(&que->tq, 1, PI_NET, - "%s que", device_get_nameunit(dev)); -#endif } return (0); @@ -2144,9 +2170,15 @@ ixl_init_msix(struct ixl_pf *pf) /* Figure out a reasonable auto config value */ queues = (mp_ncpus > (available - 1)) ? (available - 1) : mp_ncpus; - /* Override with hardcoded value if sane */ + /* Override with hardcoded value if it's less than autoconfig count */ if ((ixl_max_queues != 0) && (ixl_max_queues <= queues)) queues = ixl_max_queues; + else if ((ixl_max_queues != 0) && (ixl_max_queues > queues)) + device_printf(dev, "ixl_max_queues > # of cpus, using " + "autoconfig amount...\n"); + /* Or limit maximum auto-configured queues to 8 */ + else if ((ixl_max_queues == 0) && (queues > 8)) + queues = 8; #ifdef RSS /* If we're doing RSS, clamp at the number of RSS buckets */ From e4c893dd013cd5b6330d2a053aa2fa8d942b3c5c Mon Sep 17 00:00:00 2001 From: Eric Joyner Date: Wed, 24 Feb 2016 01:11:51 +0000 Subject: [PATCH 21/25] ixl(4)/ixlv(4): Remove a couple unnecessary register writes/flushes. The one in ixl_xmit() has brought up to us as being unnecessary at least a couple times. Differential Revision: https://reviews.freebsd.org/D5208 Tested by: jeffrey.e.pieper@intel.com Sponsored by: Intel Corporation --- sys/dev/ixl/if_ixl.c | 1 - sys/dev/ixl/ixl_txrx.c | 1 - 2 files changed, 2 deletions(-) diff --git a/sys/dev/ixl/if_ixl.c b/sys/dev/ixl/if_ixl.c index 22d8f0118b0..3717cbf1a8e 100644 --- a/sys/dev/ixl/if_ixl.c +++ b/sys/dev/ixl/if_ixl.c @@ -2912,7 +2912,6 @@ ixl_initialize_vsi(struct ixl_vsi *vsi) device_printf(dev, "Fail in init_rx_ring %d\n", i); break; } - wr32(vsi->hw, I40E_QRX_TAIL(que->me), 0); #ifdef DEV_NETMAP /* preserve queue */ if (vsi->ifp->if_capenable & IFCAP_NETMAP) { diff --git a/sys/dev/ixl/ixl_txrx.c b/sys/dev/ixl/ixl_txrx.c index e78af119bfc..0ae7433e8e8 100644 --- a/sys/dev/ixl/ixl_txrx.c +++ b/sys/dev/ixl/ixl_txrx.c @@ -390,7 +390,6 @@ ixl_xmit(struct ixl_queue *que, struct mbuf **m_headp) ++txr->total_packets; wr32(hw, txr->tail, i); - ixl_flush(hw); /* Mark outstanding work */ if (que->busy == 0) que->busy = 1; From 3bead71e959dddbea4aa09bacea6615dcb5829d9 Mon Sep 17 00:00:00 2001 From: Marcelo Araujo Date: Wed, 24 Feb 2016 01:32:12 +0000 Subject: [PATCH 22/25] - Add a global option where we can protect processes when swap space is exhausted. How to use: Basically we need to add on rc.conf an another option like: If we want to protect only the main processes. syslogd_oomprotect="YES" If we want to protect all future children of the specified processes. syslogd_oomprotect="ALL" PR: 204741 (based on) Submitted by: eugen@grosbein.net Reviewed by: jhb, allanjude, rpokala and bapt MFC after: 4 weeks Relnotes: Yes Sponsored by: gandi.net Differential Revision: https://reviews.freebsd.org/D5176 --- etc/defaults/rc.conf | 1 + etc/rc.subr | 26 +++++++++++++++++++++++++- share/man/man8/rc.subr.8 | 12 +++++++++++- 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/etc/defaults/rc.conf b/etc/defaults/rc.conf index b9c6b5cc4d6..12bf3f21837 100644 --- a/etc/defaults/rc.conf +++ b/etc/defaults/rc.conf @@ -253,6 +253,7 @@ hostapd_enable="NO" # Run hostap daemon. syslogd_enable="YES" # Run syslog daemon (or NO). syslogd_program="/usr/sbin/syslogd" # path to syslogd, if you want a different one. syslogd_flags="-s" # Flags to syslogd (if enabled). +syslogd_oomprotect="YES" # Don't kill syslogd when swap space is exhausted. altlog_proglist="" # List of chrooted applicatioins in /var inetd_enable="NO" # Run the network daemon dispatcher (YES/NO). inetd_program="/usr/sbin/inetd" # path to inetd, if you want a different one. diff --git a/etc/rc.subr b/etc/rc.subr index a6de452ca5b..cdec9a24548 100644 --- a/etc/rc.subr +++ b/etc/rc.subr @@ -45,6 +45,7 @@ _rc_subr_loaded="YES" SYSCTL="/sbin/sysctl" SYSCTL_N="${SYSCTL} -n" SYSCTL_W="${SYSCTL}" +PROTECT="/usr/bin/protect" ID="/usr/bin/id" IDCMD="if [ -x $ID ]; then $ID -un; fi" PS="/bin/ps -ww" @@ -755,6 +756,8 @@ check_startmsgs() # # ${name}_nice n Nice level to run ${command} at. # +# ${name}_oomprotect n Don't kill ${command} when swap space is exhausted. +# # ${name}_user n User to run ${command} as, using su(1) if not # using ${name}_chroot. # Requires /usr to be mounted. @@ -944,7 +947,8 @@ run_rc_command() _nice=\$${name}_nice _user=\$${name}_user \ _group=\$${name}_group _groups=\$${name}_groups \ _fib=\$${name}_fib _env=\$${name}_env \ - _prepend=\$${name}_prepend _login_class=\${${name}_login_class:-daemon} + _prepend=\$${name}_prepend _login_class=\${${name}_login_class:-daemon} \ + _oomprotect=\$${name}_oomprotect if [ -n "$_user" ]; then # unset $_user if running as that user if [ "$_user" = "$(eval $IDCMD)" ]; then @@ -1182,6 +1186,26 @@ $command $rc_flags $command_args" ;; esac + + # Apply protect(1) to the PID if ${name}_oomprotect is set. + case "$rc_arg" in + start) + if [ -n "$_oomprotect" ]; then + if [ -f "${PROTECT}" ]; then + pid=$(check_process $command) + case $_oomprotect in + [Aa][Ll][Ll]) + ${PROTECT} -i -p ${pid} + ;; + [Yy][Ee][Ss]) + ${PROTECT} -p ${pid} + ;; + esac + fi + fi + ;; + esac + return $_return done diff --git a/share/man/man8/rc.subr.8 b/share/man/man8/rc.subr.8 index 47457ead904..47959f3ea0a 100644 --- a/share/man/man8/rc.subr.8 +++ b/share/man/man8/rc.subr.8 @@ -29,7 +29,7 @@ .\" .\" $FreeBSD$ .\" -.Dd October 12, 2014 +.Dd February 4, 2016 .Dt RC.SUBR 8 .Os .Sh NAME @@ -604,6 +604,16 @@ as. Only supported after .Pa /usr is mounted. +.It Va ${name}_oomprotect +.Xr protect 1 +.Va command +from being killed when swap space is exhausted. +If +.Em YES +is used, no child processes are protected. +If +.Em ALL , +protect all child processes. .It Va ${name}_program Full path to the command. Overrides From 24b6748c426d05d214be3a90c90f22c9fb82d80a Mon Sep 17 00:00:00 2001 From: Kirk McKusick Date: Wed, 24 Feb 2016 01:58:40 +0000 Subject: [PATCH 23/25] The UFS filesystem requires that the last block of a file always be allocated. When shortening the length of a file in which the new end of the file contains a hole, the hole must have a block allocated. Reported by: Maxim Sobolev Reviewed by: kib Tested by: Peter Holm --- sys/ufs/ffs/ffs_inode.c | 77 ++++++++++++++++++++++++++++++++--------- 1 file changed, 60 insertions(+), 17 deletions(-) diff --git a/sys/ufs/ffs/ffs_inode.c b/sys/ufs/ffs/ffs_inode.c index b41e04b05ad..c8dac1b4151 100644 --- a/sys/ufs/ffs/ffs_inode.c +++ b/sys/ufs/ffs/ffs_inode.c @@ -184,7 +184,7 @@ ffs_truncate(vp, length, flags, cred) struct inode *ip; ufs2_daddr_t bn, lbn, lastblock, lastiblock[NIADDR], indir_lbn[NIADDR]; ufs2_daddr_t oldblks[NDADDR + NIADDR], newblks[NDADDR + NIADDR]; - ufs2_daddr_t count, blocksreleased = 0, datablocks; + ufs2_daddr_t count, blocksreleased = 0, datablocks, blkno; struct bufobj *bo; struct fs *fs; struct buf *bp; @@ -192,7 +192,7 @@ ffs_truncate(vp, length, flags, cred) int softdeptrunc, journaltrunc; int needextclean, extblocks; int offset, size, level, nblocks; - int i, error, allerror; + int i, error, allerror, indiroff; off_t osize; ip = VTOI(vp); @@ -329,16 +329,57 @@ ffs_truncate(vp, length, flags, cred) ip->i_flag |= IN_CHANGE | IN_UPDATE; return (ffs_update(vp, !DOINGASYNC(vp))); } - if (DOINGSOFTDEP(vp)) { + /* + * Lookup block number for a given offset. Zero length files + * have no blocks, so return a blkno of -1. + */ + lbn = lblkno(fs, length - 1); + if (length == 0) { + blkno = -1; + } else if (lbn < NDADDR) { + blkno = DIP(ip, i_db[lbn]); + } else { + error = UFS_BALLOC(vp, lblktosize(fs, (off_t)lbn), fs->fs_bsize, + cred, BA_METAONLY, &bp); + if (error) + return (error); + indiroff = (lbn - NDADDR) % NINDIR(fs); + if (ip->i_ump->um_fstype == UFS1) + blkno = ((ufs1_daddr_t *)(bp->b_data))[indiroff]; + else + blkno = ((ufs2_daddr_t *)(bp->b_data))[indiroff]; + /* + * If the block number is non-zero, then the indirect block + * must have been previously allocated and need not be written. + * If the block number is zero, then we may have allocated + * the indirect block and hence need to write it out. + */ + if (blkno != 0) + brelse(bp); + else if (DOINGSOFTDEP(vp) || DOINGASYNC(vp)) + bdwrite(bp); + else + bwrite(bp); + } + /* + * If the block number at the new end of the file is zero, + * then we must allocate it to ensure that the last block of + * the file is allocated. Soft updates does not handle this + * case, so here we have to clean up the soft updates data + * structures describing the allocation past the truncation + * point. Finding and deallocating those structures is a lot of + * work. Since partial truncation with a hole at the end occurs + * rarely, we solve the problem by syncing the file so that it + * will have no soft updates data structures left. + */ + if (blkno == 0 && (error = ffs_syncvnode(vp, MNT_WAIT, 0)) != 0) + return (error); + if (blkno != 0 && DOINGSOFTDEP(vp)) { if (softdeptrunc == 0 && journaltrunc == 0) { /* - * If a file is only partially truncated, then - * we have to clean up the data structures - * describing the allocation past the truncation - * point. Finding and deallocating those structures - * is a lot of work. Since partial truncation occurs - * rarely, we solve the problem by syncing the file - * so that it will have no data structures left. + * If soft updates cannot handle this truncation, + * clean up soft dependency data structures and + * fall through to the synchronous truncation. */ if ((error = ffs_syncvnode(vp, MNT_WAIT, 0)) != 0) return (error); @@ -358,15 +399,17 @@ ffs_truncate(vp, length, flags, cred) } } /* - * Shorten the size of the file. If the file is not being - * truncated to a block boundary, the contents of the - * partial block following the end of the file must be - * zero'ed in case it ever becomes accessible again because - * of subsequent file growth. Directories however are not + * Shorten the size of the file. If the last block of the + * shortened file is unallocated, we must allocate it. + * Additionally, if the file is not being truncated to a + * block boundary, the contents of the partial block + * following the end of the file must be zero'ed in + * case it ever becomes accessible again because of + * subsequent file growth. Directories however are not * zero'ed as they should grow back initialized to empty. */ offset = blkoff(fs, length); - if (offset == 0) { + if (blkno != 0 && offset == 0) { ip->i_size = length; DIP_SET(ip, i_size, length); } else { @@ -390,7 +433,7 @@ ffs_truncate(vp, length, flags, cred) ip->i_size = length; DIP_SET(ip, i_size, length); size = blksize(fs, ip, lbn); - if (vp->v_type != VDIR) + if (vp->v_type != VDIR && offset != 0) bzero((char *)bp->b_data + offset, (u_int)(size - offset)); /* Kirk's code has reallocbuf(bp, size, 1) here */ From 443a0f85dda6042e82cc07f6626fcf1f3eb8156e Mon Sep 17 00:00:00 2001 From: Adrian Chadd Date: Wed, 24 Feb 2016 02:52:49 +0000 Subject: [PATCH 24/25] Fixes to make it compile under gcc-4.2. --- sys/geom/uzip/g_uzip.c | 4 ++-- sys/geom/uzip/g_uzip.h | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/sys/geom/uzip/g_uzip.c b/sys/geom/uzip/g_uzip.c index 17d179bc115..3660a92d61e 100644 --- a/sys/geom/uzip/g_uzip.c +++ b/sys/geom/uzip/g_uzip.c @@ -43,8 +43,6 @@ __FBSDID("$FreeBSD$"); #include -MALLOC_DEFINE(M_GEOM_UZIP, "geom_uzip", "GEOM UZIP data structures"); - #include #include #include @@ -53,6 +51,8 @@ MALLOC_DEFINE(M_GEOM_UZIP, "geom_uzip", "GEOM UZIP data structures"); #include #include +MALLOC_DEFINE(M_GEOM_UZIP, "geom_uzip", "GEOM UZIP data structures"); + FEATURE(geom_uzip, "GEOM read-only compressed disks support"); struct g_uzip_blk { diff --git a/sys/geom/uzip/g_uzip.h b/sys/geom/uzip/g_uzip.h index 7a16827e7eb..6decc41bab0 100644 --- a/sys/geom/uzip/g_uzip.h +++ b/sys/geom/uzip/g_uzip.h @@ -27,9 +27,11 @@ * * $FreeBSD$ */ +#ifndef __GEOM_G_UZIP_H__ +#define __GEOM_G_UZIP_H__ -#if !defined(M_GEOM_UZIP) MALLOC_DECLARE(M_GEOM_UZIP); -#endif #define DEFINE_RAW_METHOD(func, rval, args...) typedef rval (*func##_t)(args) + +#endif /* __GEOM_G_UZIP_H__ */ From dc2e095538de260bf8c62570423a6e144f5090c9 Mon Sep 17 00:00:00 2001 From: Adrian Chadd Date: Wed, 24 Feb 2016 03:06:26 +0000 Subject: [PATCH 25/25] Migrate UNCOMPRESS -> UZIP --- sys/mips/conf/DIR-825C1 | 2 +- sys/mips/conf/MT7620 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sys/mips/conf/DIR-825C1 b/sys/mips/conf/DIR-825C1 index 4c37e972926..c95b2aa5886 100644 --- a/sys/mips/conf/DIR-825C1 +++ b/sys/mips/conf/DIR-825C1 @@ -48,7 +48,7 @@ options AR71XX_ENV_UBOOT device geom_map # uzip - to boot natively from flash -options GEOM_UNCOMPRESS +options GEOM_UZIP options GEOM_PART_GPT # yes, this board has a PCI connected atheros device diff --git a/sys/mips/conf/MT7620 b/sys/mips/conf/MT7620 index 3ac1469e4d4..d915517961e 100644 --- a/sys/mips/conf/MT7620 +++ b/sys/mips/conf/MT7620 @@ -93,7 +93,7 @@ options FFS #Berkeley Fast Filesystem options MROUTING # Multicast routing options IPFIREWALL_DEFAULT_TO_ACCEPT -options GEOM_UNCOMPRESS +options GEOM_UZIP options MD_ROOT options ROOTDEVNAME=\"ufs:da0s1\"