From c57ff818f61483c3131dc4fb6dd87ede7ab29bba Mon Sep 17 00:00:00 2001 From: Sam James Date: Mon, 9 Oct 2023 20:50:06 +0100 Subject: [PATCH 01/15] config/zfs-build.m4: add Gentoo's bash-completion path Followup e69ade32e116e72d03068c03799924c3f1a15c95 by adding Gentoo's bash completion path. We should probably consider using/honouring the standard --with-bashcompletiondir autoconf option as well, but that's something to do later. Reviewed-by: Brian Behlendorf Reviewed-by: Umer Saleem Signed-off-by: Sam James Closes #15372 --- config/zfs-build.m4 | 1 + 1 file changed, 1 insertion(+) diff --git a/config/zfs-build.m4 b/config/zfs-build.m4 index e4197dc1424..5f36569fe25 100644 --- a/config/zfs-build.m4 +++ b/config/zfs-build.m4 @@ -626,6 +626,7 @@ AC_DEFUN([ZFS_AC_DEFAULT_PACKAGE], [ ubuntu) bashcompletiondir=/usr/share/bash-completion/completions ;; debian) bashcompletiondir=/usr/share/bash-completion/completions ;; freebsd) bashcompletiondir=$sysconfdir/bash_completion.d;; + gentoo) bashcompletiondir=/usr/share/bash-completion/completions ;; *) bashcompletiondir=/etc/bash_completion.d ;; esac AC_MSG_RESULT([$bashcompletiondir]) From 008baa091f39487fd053fece42be4fd23a09d4d4 Mon Sep 17 00:00:00 2001 From: Alexander Motin Date: Mon, 9 Oct 2023 16:27:18 -0400 Subject: [PATCH 02/15] FreeBSD: Reduce divergence from in-tree sources This includes random small tweaks, primarily a build fixes, required when ZFS is built as part of FreeBSD base. Reviewed-by: Brian Behlendorf Reviewed-by: Tino Reichardt Signed-off-by: Alexander Motin Sponsored by: iXsystems, Inc. Closes #15368 --- include/os/freebsd/spl/sys/atomic.h | 6 ++++-- include/os/freebsd/spl/sys/ccompat.h | 2 +- include/os/freebsd/spl/sys/ctype.h | 1 + include/os/freebsd/spl/sys/spl_condvar.h | 2 +- include/os/freebsd/zfs/sys/arc_os.h | 2 +- include/os/freebsd/zfs/sys/freebsd_event.h | 2 +- include/os/freebsd/zfs/sys/zfs_context_os.h | 2 +- include/sys/zfs_context.h | 2 ++ module/os/freebsd/zfs/event_os.c | 2 +- module/os/freebsd/zfs/kmod_core.c | 2 +- tests/zfs-tests/cmd/dosmode_readonly_write.c | 2 +- 11 files changed, 15 insertions(+), 10 deletions(-) diff --git a/include/os/freebsd/spl/sys/atomic.h b/include/os/freebsd/spl/sys/atomic.h index 8b9cec15c5e..40a67704fde 100644 --- a/include/os/freebsd/spl/sys/atomic.h +++ b/include/os/freebsd/spl/sys/atomic.h @@ -167,7 +167,7 @@ atomic_dec_64_nv(volatile uint64_t *target) return (atomic_add_64_nv(target, -1)); } -#if !defined(COMPAT_32BIT) && defined(__LP64__) +#ifdef __LP64__ static __inline void * atomic_cas_ptr(volatile void *target, void *cmp, void *newval) { @@ -181,7 +181,7 @@ atomic_cas_ptr(volatile void *target, void *cmp, void *newval) return ((void *)atomic_cas_32((volatile uint32_t *)target, (uint32_t)cmp, (uint32_t)newval)); } -#endif /* !defined(COMPAT_32BIT) && defined(__LP64__) */ +#endif /* __LP64__ */ #else /* _STANDALONE */ /* @@ -190,6 +190,8 @@ atomic_cas_ptr(volatile void *target, void *cmp, void *newval) */ #undef atomic_add_64 #define atomic_add_64(ptr, val) *(ptr) += val +#undef atomic_sub_64 +#define atomic_sub_64(ptr, val) *(ptr) -= val #endif /* !_STANDALONE */ #endif /* !_OPENSOLARIS_SYS_ATOMIC_H_ */ diff --git a/include/os/freebsd/spl/sys/ccompat.h b/include/os/freebsd/spl/sys/ccompat.h index eaee9159eab..e34bab7e896 100644 --- a/include/os/freebsd/spl/sys/ccompat.h +++ b/include/os/freebsd/spl/sys/ccompat.h @@ -1,5 +1,5 @@ /* - * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * SPDX-License-Identifier: BSD-2-Clause * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions diff --git a/include/os/freebsd/spl/sys/ctype.h b/include/os/freebsd/spl/sys/ctype.h index 53afd8b8bd4..b9ca0f8ef2c 100644 --- a/include/os/freebsd/spl/sys/ctype.h +++ b/include/os/freebsd/spl/sys/ctype.h @@ -39,5 +39,6 @@ ((C) >= 0x3A && (C) <= 0x40) || \ ((C) >= 0x5B && (C) <= 0x60) || \ ((C) >= 0x7B && (C) <= 0x7E)) +#define isspace(C) ((C) == 0x20 || ((C) >= 0x9 && (C) <= 0xD)) #endif diff --git a/include/os/freebsd/spl/sys/spl_condvar.h b/include/os/freebsd/spl/sys/spl_condvar.h index 7405f647d59..2835adafd41 100644 --- a/include/os/freebsd/spl/sys/spl_condvar.h +++ b/include/os/freebsd/spl/sys/spl_condvar.h @@ -1,5 +1,5 @@ /* - * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2000 Jake Burkholder . * All rights reserved. diff --git a/include/os/freebsd/zfs/sys/arc_os.h b/include/os/freebsd/zfs/sys/arc_os.h index a95618b91fe..ad2aba23b90 100644 --- a/include/os/freebsd/zfs/sys/arc_os.h +++ b/include/os/freebsd/zfs/sys/arc_os.h @@ -1,5 +1,5 @@ /* - * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Martin Matuska * diff --git a/include/os/freebsd/zfs/sys/freebsd_event.h b/include/os/freebsd/zfs/sys/freebsd_event.h index 544ff8b0f81..a32596d918a 100644 --- a/include/os/freebsd/zfs/sys/freebsd_event.h +++ b/include/os/freebsd/zfs/sys/freebsd_event.h @@ -1,5 +1,5 @@ /* - * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Rob Wing * diff --git a/include/os/freebsd/zfs/sys/zfs_context_os.h b/include/os/freebsd/zfs/sys/zfs_context_os.h index 1ce72330412..457fa3af814 100644 --- a/include/os/freebsd/zfs/sys/zfs_context_os.h +++ b/include/os/freebsd/zfs/sys/zfs_context_os.h @@ -78,7 +78,7 @@ extern int hz; extern int tick; typedef int fstrans_cookie_t; #define spl_fstrans_mark() (0) -#define spl_fstrans_unmark(x) (x = 0) +#define spl_fstrans_unmark(x) ((void)x) #define signal_pending(x) SIGPENDING(x) #define current curthread #define thread_join(x) diff --git a/include/sys/zfs_context.h b/include/sys/zfs_context.h index 0d31195447d..6a337b49edf 100644 --- a/include/sys/zfs_context.h +++ b/include/sys/zfs_context.h @@ -695,8 +695,10 @@ extern char *kmem_asprintf(const char *fmt, ...); #define kmem_strfree(str) kmem_free((str), strlen(str) + 1) #define kmem_strdup(s) strdup(s) +#ifndef __cplusplus extern int kmem_scnprintf(char *restrict str, size_t size, const char *restrict fmt, ...); +#endif /* * Hostname information diff --git a/module/os/freebsd/zfs/event_os.c b/module/os/freebsd/zfs/event_os.c index 97ac151e4fa..239d44d0cfe 100644 --- a/module/os/freebsd/zfs/event_os.c +++ b/module/os/freebsd/zfs/event_os.c @@ -1,5 +1,5 @@ /* - * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2022 Rob Wing * diff --git a/module/os/freebsd/zfs/kmod_core.c b/module/os/freebsd/zfs/kmod_core.c index f4c87013dbf..9a268573528 100644 --- a/module/os/freebsd/zfs/kmod_core.c +++ b/module/os/freebsd/zfs/kmod_core.c @@ -141,7 +141,7 @@ zfsdev_ioctl(struct cdev *dev, ulong_t zcmd, caddr_t arg, int flag, if (len != sizeof (zfs_iocparm_t)) return (EINVAL); - uaddr = (void *)zp->zfs_cmd; + uaddr = (void *)(uintptr_t)zp->zfs_cmd; zc = vmem_zalloc(sizeof (zfs_cmd_t), KM_SLEEP); #ifdef ZFS_LEGACY_SUPPORT /* diff --git a/tests/zfs-tests/cmd/dosmode_readonly_write.c b/tests/zfs-tests/cmd/dosmode_readonly_write.c index 0441d1c7b47..b45602d8065 100644 --- a/tests/zfs-tests/cmd/dosmode_readonly_write.c +++ b/tests/zfs-tests/cmd/dosmode_readonly_write.c @@ -1,5 +1,5 @@ /* - * SPDX-License-Identifier: BSD-2-Clause-FreeBSD + * SPDX-License-Identifier: BSD-2-Clause * * Copyright (c) 2021 iXsystems, Inc. * From 954a380e19c9905deb2152275b13a0fb787b6923 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Mon, 9 Oct 2023 17:22:44 -0700 Subject: [PATCH 03/15] ZTS: Move zpool_import_hostid_changed* tests to Linux runfile Relocate the zpool_import_hostid_changed* test cases to the Linux runfile until these tests are modified to run cleanly on FreeBSD. Reviewed-by: George Melikov Signed-off-by: Brian Behlendorf Closes #15377 --- tests/runfiles/common.run | 4 ---- tests/runfiles/linux.run | 7 +++++++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/runfiles/common.run b/tests/runfiles/common.run index 1435c55e8fc..ef787c65c0f 100644 --- a/tests/runfiles/common.run +++ b/tests/runfiles/common.run @@ -415,10 +415,6 @@ tests = ['zpool_import_001_pos', 'zpool_import_002_pos', 'zpool_import_rename_001_pos', 'zpool_import_all_001_pos', 'zpool_import_encrypted', 'zpool_import_encrypted_load', 'zpool_import_errata3', 'zpool_import_errata4', - 'zpool_import_hostid_changed', - 'zpool_import_hostid_changed_unclean_export', - 'zpool_import_hostid_changed_cachefile', - 'zpool_import_hostid_changed_cachefile_unclean_export', 'import_cachefile_device_added', 'import_cachefile_device_removed', 'import_cachefile_device_replaced', diff --git a/tests/runfiles/linux.run b/tests/runfiles/linux.run index 2c8d5cb0ecb..2252e46df3a 100644 --- a/tests/runfiles/linux.run +++ b/tests/runfiles/linux.run @@ -82,6 +82,13 @@ tests = ['zpool_expand_001_pos', 'zpool_expand_002_pos', 'zpool_expand_003_neg', 'zpool_expand_004_pos', 'zpool_expand_005_pos'] tags = ['functional', 'cli_root', 'zpool_expand'] +[tests/functional/cli_root/zpool_import:Linux] +tests = ['zpool_import_hostid_changed', + 'zpool_import_hostid_changed_unclean_export', + 'zpool_import_hostid_changed_cachefile', + 'zpool_import_hostid_changed_cachefile_unclean_export'] +tags = ['functional', 'cli_root', 'zpool_import'] + [tests/functional/cli_root/zpool_reopen:Linux] tests = ['zpool_reopen_001_pos', 'zpool_reopen_002_pos', 'zpool_reopen_003_pos', 'zpool_reopen_004_pos', 'zpool_reopen_005_pos', From 250349ffffbaefd356a28cc900ad73a3fe0e60c0 Mon Sep 17 00:00:00 2001 From: Umer Saleem Date: Tue, 10 Oct 2023 05:24:24 +0500 Subject: [PATCH 04/15] ZTS: Fix verify_fs_mount in delegate_common.kshlib verify_fs_mount expects the dataset to remain unmounted after updating the mountpoint property in delegate_common.kshlib. This commit updates verify_fs_mount and uses nomount parameter for zfs set to update the mountpoint property without mounting the dataset. This fixes the zfs_allow_010_pos test case, which was failing on FreeBSD after the behavior update in setting the mountpoint property. Reviewed-by: Brian Behlendorf Reviewed-by: Alexander Motin Signed-off-by: Umer Saleem Closes #15376 --- .../tests/functional/delegate/delegate_common.kshlib | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib b/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib index 3f2f089e817..5ddb6ca2ddc 100644 --- a/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib +++ b/tests/zfs-tests/tests/functional/delegate/delegate_common.kshlib @@ -861,7 +861,7 @@ function verify_fs_mount fi if ! ismounted $fs ; then - log_must zfs set mountpoint=$newmntpt $fs + log_must zfs set -u mountpoint=$newmntpt $fs log_must rm -rf $newmntpt log_must mkdir $newmntpt @@ -878,7 +878,7 @@ function verify_fs_mount fi log_must zfs umount $fs log_must rm -rf $newmntpt - log_must zfs set mountpoint=$mntpt $fs + log_must zfs set -u mountpoint=$mntpt $fs fi return 0 From dc1d3303d25b32fe1ef25be74bfa76f29c5b3027 Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sun, 6 Aug 2023 01:58:45 +1000 Subject: [PATCH 05/15] mailmap: initial, trying to tidy up a lot of the commit history This comes from the observation that a huge number of commit author fields look quite strange (to my eyes), but quite often the Signed-off-by: trailer has the correct name. For these I have updated the name where it was obvious how to do so, however, I have not created a mapping for the commit email to the Signed-off-by email, as whatever I choose for email will become the prime candidate for inclusion in the AUTHORS file, and care needs to be taken when acting without explicit consent. There's a small handful of commits that look like they were done on local machines, or CI hosts, or similar, where the git authorship config wasn't set up properly. Its obvious what this should look like, so I've just done them. The remainder is mapping Github noreply emails to either an obviously-correct Signed-off-by trailer, or to a an author from another commit. This was mostly done by hand, so there may be errors, but I think its close. I do not understand where these come from - I know that they're what commits made via Github web look like when there's no real address set on the account, but I find it hard to believe that so many of these came through the web, especially given the complexity of most of the changes. I suspect there's some kind of merge helper tool in play here. Regardless, the history is set now, and this tries to get it back on track. Obviously, all of this helps the history look tidy, but this also feeds into the AUTHORS update script. See next commit. Reviewed-by: Brian Behlendorf Reviewed-by: Tino Reichardt Signed-off-by: Rob Norris Closes #15374 --- .gitignore | 1 + .mailmap | 189 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 .mailmap diff --git a/.gitignore b/.gitignore index 1ef47d921c2..47d17ae16d3 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ !.cirrus.yml !.gitignore !.gitmodules +!.mailmap !AUTHORS !autogen.sh !CODE_OF_CONDUCT.md diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000000..46ef016b93f --- /dev/null +++ b/.mailmap @@ -0,0 +1,189 @@ +# This file maps the name+email seen in a commit back to a canonical +# name+email. Git will replace the commit name/email with the canonical version +# wherever it sees it. +# +# If there is a commit in the history with a "wrong" name or email, list it +# here. If you regularly commit with an alternate name or email address and +# would like to ensure that you are always listed consistently in the repo, add +# mapping here. +# +# On the other hand, if you use multiple names or email addresses legitimately +# (eg you use a company email address for your paid OpenZFS work, and a +# personal address for your evening side projects), then don't map one to the +# other here. +# +# The most common formats are: +# +# Canonical Name +# Canonical Name +# Canonical Name Commit Name +# +# See https://git-scm.com/docs/gitmailmap for more info. + +# These maps are making names consistent where they have varied but the email +# address has never changed. In most cases, the full name is in the +# Signed-off-by of a commit with a matching author. +Ahelenia Ziemiańska +Ahelenia Ziemiańska +Alex John +Andreas Dilger +Andrew Walker +Benedikt Neuffer +Chengfei Zhu +Chris Lindee +Colm Buckley +Crag Wang +Damian Szuberski +Daniel Kolesa +Debabrata Banerjee +Finix Yan +Gaurav Kumar +Gionatan Danti +Glenn Washburn +Gordan Bobic +Gregory Bartholomew +hedong zhang +InsanePrawn +Jason Cohen +Jason Harmening +Jeremy Faulkner +Jinshan Xiong +John Poduska +Justin Scholz +Ka Ho Ng +Kash Pande +Kay Pedersen +KernelOfTruth +Liu Hua +Liu Qing +loli10K +Matthias Blankertz +Michael Gmelin +Olivier Mazouffre +Piotr Kubaj +Quentin Zdanis +Roberto Ricci +Rob Norris +Rob Norris +Sam Lunt +Sanjeev Bagewadi +Stoiko Ivanov +Tamas TEVESZ +WHR +Yanping Gao +Youzhong Yang + +# Commits from strange places, long ago +Brian Behlendorf +Brian Behlendorf +Brian Behlendorf +Brian Behlendorf +Brian Behlendorf +Herb Wartens +Ned Bass +Tulsi Jain + +# Mappings from Github no-reply addresses +ajs124 +Alek Pinchuk +Alexander Lobakin +Alexey Smirnoff +Allen Holl <65494904+allen-4@users.noreply.github.com> +Ameer Hamza <106930537+ixhamza@users.noreply.github.com> +Andrew J. Hesford <48421688+ahesford@users.noreply.github.com>> +Andrew Sun +Aron Xu +Arun KV <65647132+arun-kv@users.noreply.github.com> +Ben Wolsieffer +bernie1995 <42413912+bernie1995@users.noreply.github.com> +Boris Protopopov +Brad Forschinger +Brandon Thetford +buzzingwires <131118055+buzzingwires@users.noreply.github.com> +Cedric Maunoury <38213715+cedricmaunoury@users.noreply.github.com> +Charles Suh +Dacian Reece-Stremtan <35844628+dacianstremtan@users.noreply.github.com> +Damian Szuberski <30863496+szubersk@users.noreply.github.com> +Daniel Hiepler <32984777+heeplr@users.noreply.github.com> +Daniel Kobras +Daniel Reichelt +David Quigley +DHE +Dmitri John Ledkov <19779+xnox@users.noreply.github.com> +Dries Michiels <32487486+driesmp@users.noreply.github.com> +Edmund Nadolski <137826107+ednadolski-ix@users.noreply.github.com> +Érico Nogueira <34201958+ericonr@users.noreply.github.com> +Fedor Uporov <60701163+fuporovvStack@users.noreply.github.com> +Felix Dörre +Felix Neumärker <34678034+xdch47@users.noreply.github.com> +Finix Yan +Gaurav Kumar +George Gaydarov +Georgy Yakovlev <168902+gyakovlev@users.noreply.github.com> +Gerardwx +Gian-Carlo DeFazio +Giuseppe Di Natale +Hajo Möller +Harry Mallon <1816667+hjmallon@users.noreply.github.com> +Hiếu Lê +Jake Howard +James Cowgill +Jason King +Jeff Dike <52420226+jdike@users.noreply.github.com> +Jitendra Patidar <53164267+jsai20@users.noreply.github.com> +João Carlos Mendes Luís +John Eismeier <32205350+jeis2497052@users.noreply.github.com> +John L. Hammond <35266395+jhammond-intel@users.noreply.github.com> +John-Mark Gurney +John Ramsden +Jonathon Fernyhough <559369+jonathonf@users.noreply.github.com> +Justin Hibbits +Kevin Jin <33590050+jxdking@users.noreply.github.com> +Kevin P. Fleming +Krzysztof Piecuch <3964215+pikrzysztof@users.noreply.github.com> +Kyle Evans +Laurențiu Nicola +loli10K +Lorenz Hüdepohl +Luís Henriques <73643340+lumigch@users.noreply.github.com> +Marcin Skarbek +Matt Fiddaman <81489167+matt-fidd@users.noreply.github.com> +Max Zettlmeißl <6818198+maxz@users.noreply.github.com> +Michael Niewöhner +Michael Zhivich <33133421+mzhivich@users.noreply.github.com> +Mo Zhou <5723047+cdluminate@users.noreply.github.com> +Nick Mattis +omni <79493359+omnivagant@users.noreply.github.com> +Pablo Correa Gómez <32678034+pablofsf@users.noreply.github.com> +Paul Zuchowski <31706010+PaulZ-98@users.noreply.github.com> +Peter Ashford +Peter Dave Hello +Peter Wirdemo <4224155+pewo@users.noreply.github.com> +Petros Koutoupis +Ping Huang <101400146+hpingfs@users.noreply.github.com> +Piotr P. Stefaniak +Richard Allen <33836503+belperite@users.noreply.github.com> +Rich Ercolani <214141+rincebrain@users.noreply.github.com> +Rob Wing <98866084+rob-wing@users.noreply.github.com> +Roman Strashkin +Ryan Hirasaki <4690732+RyanHir@users.noreply.github.com> +Samuel Wycliffe J <115969550+samwyc@users.noreply.github.com> +Samuel Wycliffe <50765275+npc203@users.noreply.github.com> +Savyasachee Jha +Scott Colby +Sean Eric Fagan +Spencer Kinny <30333052+Spencer-Kinny@users.noreply.github.com> +Srikanth N S <75025422+nssrikanth@users.noreply.github.com> +Thomas Geppert +Tim Crawford +Tom Matthews +Tony Perkins <62951051+tony-zfs@users.noreply.github.com> +Torsten Wörtwein +Tulsi Jain +Václav Skála <33496485+vaclavskala@users.noreply.github.com> +Violet Purcell <66446404+vimproved@users.noreply.github.com> +Vipin Kumar Verma <75025470+vermavipinkumar@users.noreply.github.com> +Wolfgang Bumiller +xtouqh <72357159+xtouqh@users.noreply.github.com> +Yuri Pankov <113725409+yuripv@users.noreply.github.com> +Yuri Pankov <82001006+yuripv@users.noreply.github.com> From 000cfca0689a988a5a0973245b9a90529e4778fb Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sun, 6 Aug 2023 02:10:31 +1000 Subject: [PATCH 06/15] update_authors: add missing names from commits to AUTHORS Full description of what's happening in comments. Reviewed-by: Brian Behlendorf Reviewed-by: Tino Reichardt Signed-off-by: Rob Norris Closes #15374 --- scripts/Makefile.am | 1 + scripts/update_authors.pl | 322 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 323 insertions(+) create mode 100755 scripts/update_authors.pl diff --git a/scripts/Makefile.am b/scripts/Makefile.am index 07543456643..8ec9ea4b5c3 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -9,6 +9,7 @@ dist_noinst_SCRIPTS = \ %D%/man-dates.sh \ %D%/mancheck.sh \ %D%/paxcheck.sh \ + %D%/update_authors.pl \ %D%/zfs-tests-color.sh scripts_scripts = \ diff --git a/scripts/update_authors.pl b/scripts/update_authors.pl new file mode 100755 index 00000000000..8dd49b5fb38 --- /dev/null +++ b/scripts/update_authors.pl @@ -0,0 +1,322 @@ +#!/usr/bin/env perl + +# SPDX-License-Identifier: MIT +# +# Copyright (c) 2023, Rob Norris +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + + +# This program will update the AUTHORS file to include commit authors that are +# in the git history but are not yet credited. +# +# The CONTRIBUTORS section of the AUTHORS file attempts to be a list of +# individual contributors to OpenZFS, with one name, address and line per +# person. This is good for readability, but does not really leave room for the +# that names and emails on commits from the same individual can be different, +# for all kinds of reasons, not limited to: +# +# - a person might change organisations, and so their email address changes +# +# - a person might be paid to work on OpenZFS for their employer, and then hack +# on personal projects in the evening, so commits legitimately come from +# different addresses +# +# - names change for all kinds of reasons +# +# To try and account for this, this program will try to find all the possible +# names and emails for a single contributor, and then select the "best" one to +# add to the AUTHORS file. +# +# The CONTRIBUTORS section of the AUTHORS file is considered the source of +# truth. Once an individual committer is listed in there, that line will not be +# removed regardless of what is discovered in the commit history. However, it +# can't just be _anything_. The name or email still has to match something seen +# in the commit history, so that we're able to undertand that its the same +# contributor. +# +# The bulk of the work is in running `git log` to fetch commit author names and +# emails. For each value, we generate a "slug" to use as an internal id for +# that value, which is mostly just the lowercase of the value with whitespace +# and punctuation removed. Two values with subtle differences can produce the +# same slug, so at this point we also try to keep the "best" pre-slug value as +# the display version. We use this slug to update two maps, one of email->name, +# the other of name->email. +# +# Once collected, we then walk all the emails we've seen and get all the names +# associated with every instance. Then for each of those names, we get all the +# emails associated, and so on until we've seen all the connected names and +# emails. This collection is every possible name and email for an individual +# contributor. +# +# Finaly, we consider these groups, and select the "best" name and email for +# the contributor, and add them to the author tables if they aren't there +# already. Once we've done everyone, we write out a new AUTHORS file, and +# that's the whole job. +# +# This is imperfect! Its necessary for the user to examine the diff and make +# sure its sensible. If it hasn't hooked up right, it may necessary to adjust +# the input data (via .mailmap) or improve the heuristics in this program. It +# took a long time to get into good shape when first written (355 new names +# added to AUTHORS!) but hopefully in the future we'll be running this +# regularly so it doesn't fall so far behind. + + +use 5.010; +use warnings; +use strict; + +# Storage for the "best looking" version of name or email, keyed on slug. +my %display_name; +my %display_email; + +# First, we load the existing AUTHORS file. We save everything before +# CONTRIBUTORS: line as-is so we can write it back out to the new file. Then +# we extract name,email pairs from the remainder and store them in a pair of +# hashtables, keyed on slug. +my %authors_name; +my %authors_email; + +my @authors_header; + +for my $line (do { local (@ARGV) = ('AUTHORS'); <> }) { + chomp $line; + state $in_header = 1; + if ($in_header) { + push @authors_header, $line; + $in_header = 0 if $line =~ m/^CONTRIBUTORS:/; + } else { + my ($name, $email) = $line =~ m/^\s+(.+)(?= <) <([^>]+)/; + next unless $name; + + my $semail = email_slug($email); + my $sname = name_slug($name); + + $authors_name{$semail} = $sname; + $authors_email{$sname} = $semail; + + # The name/email in AUTHORS is already the "best looking" + # version, by definition. + $display_name{$sname} = $name; + $display_email{$semail} = $email; + } +} + +# Next, we load all the commit authors. and form name<->email mappings, keyed +# on slug. Note that this format is getting the .mailmap-converted form. This +# lets us control the input to some extent by making changes there. +my %git_names; +my %git_emails; + +for my $line (reverse qx(git log --pretty=tformat:'%aN:::%aE')) { + chomp $line; + my ($name, $email) = $line =~ m/^(.*):::(.*)/; + next unless $name && $email; + + my $semail = email_slug($email); + my $sname = name_slug($name); + + $git_names{$semail}{$sname} = 1; + $git_emails{$sname}{$semail} = 1; + + # Update the "best looking" display value, but only if we don't already + # have something from the AUTHORS file. If we do, we must not change it. + if (!$authors_name{email_slug($email)}) { + update_display_email($email); + } + + if (!$authors_email{name_slug($name)}) { + update_display_name($name); + } +} + +# Now collect unique committers by all names+emails we've ever seen for them. +# We start with emails and resolve all possible names, then we resolve the +# emails for those names, and round and round until there's nothing left. +my @committers; +for my $start_email (sort keys %git_names) { + # it might have been deleted already through a cross-reference + next unless $git_names{$start_email}; + + my %emails; + my %names; + + my @check_emails = ($start_email); + my @check_names; + while (@check_emails || @check_names) { + while (my $email = shift @check_emails) { + next if $emails{$email}++; + push @check_names, + sort keys %{delete $git_names{$email}}; + } + while (my $name = shift @check_names) { + next if $names{$name}++; + push @check_emails, + sort keys %{delete $git_emails{$name}}; + } + } + + # A "committer" is the collection of connected names and emails. + push @committers, [[sort keys %emails], [sort keys %names]]; +} + +# Now we have our committers, we can work out what to add to AUTHORS. +for my $committer (@committers) { + my ($emails, $names) = @$committer; + + # If this commiter is already in AUTHORS, we must not touch. + next if grep { $authors_name{$_} } @$emails; + next if grep { $authors_email{$_} } @$names; + + # Decide on the "best" name and email to use + my $email = best_email(@$emails); + my $name = best_name(@$names); + + $authors_email{$name} = $email; + $authors_name{$email} = $name; +} + +# Now output the new AUTHORS file +open my $fh, '>', 'AUTHORS' or die "E: couldn't open AUTHORS for write: $!\n"; +#my $fh = \*STDOUT; +say $fh join("\n", @authors_header, ""); +for my $name (sort keys %authors_email) { + my $cname = $display_name{$name}; + my $cemail = $display_email{email_slug($authors_email{$name})}; + say $fh " $cname <$cemail>"; +} + +exit 0; + +# "Slugs" are used at the hashtable key for names and emails. They are used to +# making two variants of a value be the "same" for matching. Mostly this is +# to make upper and lower-case versions of a name or email compare the same, +# but we do a little bit of munging to handle some common cases. +# +# Note that these are only used for matching internally; for display, the +# slug will be used to look up the display form. +sub name_slug { + my ($name) = @_; + + # Remove spaces and dots, to handle differences in initials. + $name =~ s/[\s\.]//g; + + return lc $name; +} +sub email_slug { + my ($email) = @_; + + # Remove everything up to and including the first space, and the last + # space and everything after it. + $email =~ s/^(.*\s+)|(\s+.*)$//g; + + # Remove the leading userid+ on Github noreply addresses. They're + # optional and we want to treat them as the same thing. + $email =~ s/^[^\+]*\+//g if $email =~ m/\.noreply\.github\.com$/; + + return lc $email; +} + +sub update_display_name { + my ($name) = @_; + my $sname = name_slug($name); + + # For names, "more specific" means "has more non-lower-case characters" + # (in ASCII), guessing that if a person has gone to some effort to + # specialise their name in a later commit, they presumably care more + # about it. If this is wrong, its probably better to add a .mailmap + # entry. + + my $cname = $display_name{$sname}; + if (!$cname || + ($name =~ tr/a-z //) < ($cname =~ tr/a-z //)) { + $display_name{$sname} = $name; + } +} +sub update_display_email { + my ($email) = @_; + my $semail = email_slug($email); + + # Like names, we prefer uppercase when possible. We also remove any + # leading "plus address" for Github noreply addresses. + $email =~ s/^[^\+]*\+//g if $email =~ m/\.noreply\.github\.com$/; + + my $cemail = $display_email{$semail}; + if (!$cemail || + ($email =~ tr/a-z //) < ($cemail =~ tr/a-z //)) { + $display_email{$semail} = $email; + } +} + +sub best_name { + my @names = sort { + my $cmp; + my ($aa) = $display_name{$a}; + my ($bb) = $display_name{$b}; + + # The "best" name is very subjective, and a simple sort + # produced good-enough results, so I didn't try harder. Use of + # accented characters, punctuation and caps are probably an + # indicator of "better", but possibly we should also take into + # account the most recent name we saw, in case the committer + # has changed their name or nickname or similar. + # + # Really, .mailmap is the place to control this. + + return ($aa cmp $bb); + } @_; + + return shift @names; +} +sub best_email { + state $internal_re = qr/\.(?:internal|local|\(none\))$/; + state $noreply_re = qr/\.noreply\.github\.com$/; + state $freemail_re = qr/\@(?:gmail|hotmail)\.com$/; + + my @emails = sort { + my $cmp; + + # prefer address with a single @ over those without + $cmp = (($b =~ tr/@//) == 1) <=> (($a =~ tr/@//) == 1); + return $cmp unless $cmp == 0; + + # prefer any address over internal/local addresses + $cmp = (($a =~ $internal_re) <=> ($b =~ $internal_re)); + return $cmp unless $cmp == 0; + + # prefer any address over github noreply aliases + $cmp = (($a =~ $noreply_re) <=> ($b =~ $noreply_re)); + return $cmp unless $cmp == 0; + + # prefer any address over freemail providers + $cmp = (($a =~ $freemail_re) <=> ($b =~ $freemail_re)); + return $cmp unless $cmp == 0; + + # alphabetical by domain + my ($alocal, $adom) = split /\@/, $a; + my ($blocal, $bdom) = split /\@/, $b; + $cmp = ($adom cmp $bdom); + return $cmp unless $cmp == 0; + + # alphabetical by local part + return ($alocal cmp $blocal); + } @_; + + return shift @emails; +} From 28096c7c13a1945c5830c1b7f88de4fe8e88f09d Mon Sep 17 00:00:00 2001 From: Rob Norris Date: Sun, 6 Aug 2023 02:11:19 +1000 Subject: [PATCH 07/15] AUTHORS: update with missing names This is generated by scripts/update_authors.pl. I've looked over the results fairly closely and while I don't think they're bad, they could be improved somewhat, but also, I don't know if its good form to just update this without explicit consent from those named. Reviewed-by: Brian Behlendorf Reviewed-by: Tino Reichardt Signed-off-by: Rob Norris Closes #15374 --- AUTHORS | 365 ++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 339 insertions(+), 26 deletions(-) diff --git a/AUTHORS b/AUTHORS index c2af58d7508..be1efb87b34 100644 --- a/AUTHORS +++ b/AUTHORS @@ -10,228 +10,450 @@ PAST MAINTAINERS: CONTRIBUTORS: Aaron Fineman + Adam D. Moss Adam Leventhal Adam Stevko + adisbladis + Adrian Chadd + Ahelenia Ziemiańska Ahmed G + Aidan Harris + AJ Jordan + ajs124 Akash Ayare + Akash B Alan Somers Alar Aun Albert Lee Alec Salazar + Alejandro Colomar Alejandro R. Sedeño Alek Pinchuk Aleksa Sarai + Alexander Eremin + Alexander Lobakin + Alexander Motin + Alexander Pyhalov + Alexander Richardson + Alexander Stetsenko Alex Braunegg + Alexey Shvetsov + Alexey Smirnoff + Alex John Alex McWhirter Alex Reece Alex Wilson Alex Zhuravlev - Alexander Eremin - Alexander Motin - Alexander Pyhalov - Alexander Stetsenko - Alexey Shvetsov - Alexey Smirnoff Allan Jude + Allen Holl + alteriks + Alyssa Ross + Ameer Hamza + Anatoly Borodin AndCycle + Andrea Gelmini + Andrea Righi Andreas Buschmann Andreas Dilger + Andreas Vögele Andrew Barnes Andrew Hamilton + Andrew Innes + Andrew J. Hesford Andrew Reid Andrew Stormont + Andrew Sun Andrew Tselischev + Andrew Turner + Andrew Walker + Andrey Prokopenko Andrey Vesnovaty Andriy Gapon Andy Bakun + Andy Fiddaman Aniruddha Shankar + Anton Gubarkov Antonio Russo Arkadiusz Bubała + Armin Wehrfritz Arne Jansen Aron Xu + Arshad Hussain + Arun KV + Arvind Sankar + Attila Fülöp + Avatat Bart Coddens Basil Crow - Huang Liu + Bassu Ben Allen - Ben Rubson + Ben Cordero + Benedikt Neuffer Benjamin Albrecht + Benjamin Gentil + Ben McGough + Ben Rubson + Ben Wolsieffer + bernie1995 Bill McGonigle Bill Pijewski Boris Protopopov + Brad Forschinger Brad Lewis + Brandon Thetford + Brian Atkinson Brian Behlendorf Brian J. Murrell + Brooks Davis + BtbN + bunder2015 + buzzingwires + bzzz77 + cable2999 Caleb James DeLisle Cao Xuewen Carlo Landmeter Carlos Alberto Lopez Perez + Cedric Maunoury Chaoyu Zhang + Charles Suh Chen Can + Chengfei Zhu Chen Haiquan Chip Parker Chris Burroughs Chris Dunlap Chris Dunlop + Chris Lindee + Chris McDonough Chris Siden - Chris Wedgwood - Chris Williamson - Chris Zubrzycki - Christ Schlacta + Chris Siebenmann Christer Ekholm Christian Kohlschütter Christian Neukirchen Christian Schwarz Christopher Voltz + Christ Schlacta + Chris Wedgwood + Chris Williamson + Chris Zubrzycki + Chuck Tuffli Chunwei Chen Clemens Fruhwirth + Clemens Lang + Clint Armstrong Coleman Kane Colin Ian King + Colm Buckley + Crag Wang Craig Loomis Craig Sanders Cyril Plisko - DHE + Cy Schubert + Cédric Berger + Dacian Reece-Stremtan + Dag-Erling Smørgrav + Damiano Albani + Damian Szuberski Damian Wojsław + Daniel Hiepler + Daniel Hoffman + Daniel Kobras + Daniel Kolesa + Daniel Reichelt + Daniel Stevenson + Daniel Verite + Daniil Lunev Dan Kimmel Dan McDonald Dan Swartzendruber Dan Vatca - Daniel Hoffman - Daniel Verite - Daniil Lunev Darik Horn Dave Eddy + David Hedberg David Lamparter David Qian David Quigley Debabrata Banerjee + D. Ebdrup Denys Rtveliashvili Derek Dai + DHE + Didier Roche Dimitri John Ledkov + Dimitry Andric + Dirkjan Bussink Dmitry Khasanov + Dominic Pearson Dominik Hassler Dominik Honnef Don Brady + Doug Rabson Dr. András Korn + Dries Michiels + Edmund Nadolski + Eitan Adler Eli Rosenthal + Eli Schwartz Eric Desrochers Eric Dillmann Eric Schrock + Ethan Coe-Renner Etienne Dechamps + Evan Allrich + Evan Harris Evan Susarret Fabian Grünbichler + Fabio Buso + Fabio Scaccabarozzi Fajar A. Nugraha Fan Yong + fbynite + Fedor Uporov + Felix Dörre + Felix Neumärker Feng Sun + Finix Yan + Francesco Mazzoli Frederik Wessels Frédéric Vanniere + Gabriel A. Devenyi Garrett D'Amore + Garrett Fields Garrison Jensen Gary Mills Gaurav Kumar GeLiXin George Amanakis + George Diamantopoulos + George Gaydarov George Melikov George Wilson Georgy Yakovlev + Gerardwx + Gian-Carlo DeFazio + Gionatan Danti Giuseppe Di Natale + Glenn Washburn Gordan Bobic + Gordon Bergling Gordon Ross + Graham Christensen + Graham Perrin Gregor Kopka + Gregory Bartholomew + grembo Grischa Zengel + grodik Gunnar Beutner Gvozden Neskovic Hajo Möller + Han Gao Hans Rosenfeld + Harald van Dijk + Harry Mallon + Harry Sintonen + HC + hedong zhang + Heitor Alves de Siqueira + Henrik Riomar + Herb Wartens + Hiếu Lê + Huang Liu Håkan Johansson + Igor K Igor Kozhukhov Igor Lvovsky + ilbsmart + illiliti + ilovezfs + InsanePrawn Isaac Huang - JK Dingwall Jacek Fefliński + Jacob Adams + Jake Howard James Cowgill + James H James Lee James Pan + James Wah Jan Engelhardt Jan Kryl Jan Sanislo + Jason Cohen + Jason Harmening Jason King Jason Zaman Javen Wu + Jean-Baptiste Lallement + Jeff Dike + Jeremy Faulkner Jeremy Gill Jeremy Jones + Jeremy Visser Jerry Jelinek + Jessica Clarke Jinshan Xiong + Jitendra Patidar + JK Dingwall Joe Stein + John-Mark Gurney John Albietz John Eismeier - John L. Hammond + John Gallagher John Layman - John Paul Adrian Glaubitz - John Wren Kennedy + John L. Hammond + John M. Layman Johnny Stenback + John Paul Adrian Glaubitz + John Poduska + John Ramsden + John Wren Kennedy + jokersus + Jonathon Fernyhough Jorgen Lundman Josef 'Jeff' Sipek + Josh Soref Joshua M. Clulow + José Luis Salvador Rufo + João Carlos Mendes Luís + Julian Brunner + Julian Heuking + jumbi77 Justin Bedő + Justin Gottula + Justin Hibbits + Justin Keogh Justin Lecher + Justin Scholz Justin T. Gibbs + jyxent Jörg Thalheim - KORN Andras + ka7 + Ka Ho Ng Kamil Domański Karsten Kretschmer Kash Pande + Kay Pedersen Keith M Wesolowski + KernelOfTruth + Kevin Bowling + Kevin Jin + Kevin P. Fleming Kevin Tanguy KireinaHoro Kjeld Schouten-Lebbing + Kleber Tarcísio + Kody A Kantor Kohsuke Kawaguchi + Konstantin Khorenko + KORN Andras + Kristof Provost + Krzysztof Piecuch Kyle Blatter + Kyle Evans Kyle Fuller - Loli + Laevos + Lalufu Lars Johannsen + Laura Hild + Laurențiu Nicola + Lauri Tirkkonen + liaoyuxiangqin Li Dongyang + Liu Hua + Liu Qing Li Wei + Loli + lorddoskias + Lorenz Brun + Lorenz Hüdepohl + louwrentius Lukas Wunner + luozhengzheng + Luís Henriques Madhav Suresh + manfromafar Manoj Joseph Manuel Amador (Rudd-O) Marcel Huber + Marcel Menzel + Marcel Schilling Marcel Telka Marcel Wysocki + Marcin Skarbek + Mariusz Zaborski + Mark Johnston + Mark Maybee + Mark Roper Mark Shellenbaum + marku89 Mark Wright Martin Matuska + Martin Rüegg Massimo Maggi - Matt Johnston - Matt Kemp + Mateusz Guzik + Mateusz Piotrowski <0mp@FreeBSD.org> + Mathieu Velten + Matt Fiddaman Matthew Ahrens Matthew Thode + Matthias Blankertz + Matt Johnston + Matt Kemp + Matt Macy Matus Kral + Mauricio Faria de Oliveira Max Grossman Maximilian Mehnert + Max Zettlmeißl + Md Islam + megari + Michael D Labriola + Michael Franzl Michael Gebetsroither Michael Kjorling Michael Martin Michael Niewöhner + Michael Zhivich + Michal Vasilek Mike Gerdts Mike Harsch Mike Leddy Mike Swanson Milan Jurik + Minsoo Choo + Mohamed Tawfik Morgan Jones Moritz Maxeiner + Mo Zhou + naivekun + nathancheek Nathaniel Clark Nathaniel Wesley Filardo + Nathan Lewis Nav Ravindranath Neal Gompa (ニール・ゴンパ) Ned Bass Neependra Khare Neil Stockbridge + Nick Black Nick Garvey + Nick Mattis + Nick Terrell + Niklas Haas Nikolay Borisov + nordaux + ofthesun9 Olaf Faaland Oleg Drokin Oleg Stepura + Olivier Mazouffre + omni + Orivej Desh + Pablo Correa Gómez + Palash Gandhi + Patrick Mooney Patrik Greco Paul B. Henson Paul Dagnelie @@ -243,69 +465,160 @@ CONTRIBUTORS: Pedro Giffuni Peng Peter Ashford + Peter Dave Hello + Peter Levine + Peter Wirdemo + Petros Koutoupis + Philip Pokorny + Philipp Riederer + Phil Kauffman + Ping Huang + Piotr Kubaj + Piotr P. Stefaniak Prakash Surya Prasad Joshi + privb0x23 + P.SCH + Quentin Zdanis + Rafael Kitover + RageLtMan Ralf Ertzinger Randall Mason Remy Blank + renelson + Reno Reckling Ricardo M. Correia - Rich Ercolani + Riccardo Schirone + Richard Allen Richard Elling Richard Laager Richard Lowe Richard Sharpe Richard Yao + Rich Ercolani + Robert Novak + Roberto Ricci + Rob Norris + Rob Wing Rohan Puri Romain Dolbeau Roman Strashkin + Ross Williams Ruben Kerkhof + Ryan Hirasaki + Ryan Lahfa + Ryan Libby + Ryan Moeller + Sam Hathaway + Sam Lunt + Samuel VERSCHELDE + Samuel Wycliffe + Samuel Wycliffe J + Sanjeev Bagewadi + Sara Hartse Saso Kiselkov + Satadru Pramanik + Savyasachee Jha + Scott Colby Scot W. Stevenson Sean Eric Fagan Sebastian Gottschall + Sebastien Roy Sen Haerens Serapheim Dimitropoulos Seth Forshee + Shaan Nobee Shampavman + Shaun Tancheff Shen Yan Simon Guest Simon Klinkert Sowrabha Gopal + Spencer Kinny + Srikanth N S Stanislav Seletskiy Steffen Müthing Stephen Blinick + sterlingjensen Steve Dougherty + Steve Mokris Steven Burgess Steven Hartland Steven Johnson + Steven Noonan + stf Stian Ellingsen + Stoiko Ivanov + Stéphane Lesimple Suman Chakravartula Sydney Vanda Sören Tempel + Tamas TEVESZ + Teodor Spæren + TerraTech Thijs Cramer + Thomas Geppert + Thomas Lamprecht + Till Maas Tim Chase Tim Connors Tim Crawford Tim Haley + timor + Timothy Day + Tim Schumacher Tino Reichardt Tobin Harding Tom Caputi Tom Matthews - Tom Prince Tomohiro Kusumi + Tom Prince Tony Hutter + Tony Nguyen + Tony Perkins Toomas Soome + Torsten Wörtwein + Toyam Cox + Trevor Bautista Trey Dockendorf + Troels Nørgaard + Tulsi Jain Turbo Fredriksson Tyler J. Stachecki + Umer Saleem + Valmiky Arquissandas + Val Packett + Vince van Oosten + Violet Purcell + Vipin Kumar Verma Vitaut Bajaryn + Volker Mauel + Václav Skála + Walter Huf + Warner Losh Weigang Li + WHR Will Andrews Will Rouesnel + Windel Bouwman + Wojciech Małota-Wójcik Wolfgang Bumiller Xin Li + Xinliang Liu + xtouqh + Yann Collet + Yanping Gao Ying Zhu + Youzhong Yang + yparitcher + yuina822 YunQiang Su Yuri Pankov Yuxuan Shui Zachary Bedell + Zach Dykstra + zgock + Zhu Chuang + Érico Nogueira + Đoàn Trần Công Danh + 韩朴宇 From aefb6a2bd6c24597cde655e9ce69edd0a4c34357 Mon Sep 17 00:00:00 2001 From: Tony Hutter Date: Tue, 10 Oct 2023 08:57:48 -0700 Subject: [PATCH 08/15] zvol: Temporally disable blk-mq There was a report of zvol data loss (#15351) after enabling blk-mq on a zvol backed with 16k physical block sized disks. Out of an abundance of caution, do not allow the user to enable blk-mq until we can look into the issue. Note that blk-mq was not enabled by default on zvols. It was always opt-in via the zvol_use_blk_mq module parameter. Reviewed-by: Brian Behlendorf Reviewed-by: Tony Nguyen Signed-off-by: Tony Hutter Addresses: #15351 Closes #15378 --- man/man4/zfs.4 | 57 ---------------------------- module/os/linux/zfs/zvol_os.c | 12 ------ tests/zfs-tests/include/tunables.cfg | 2 +- 3 files changed, 1 insertion(+), 70 deletions(-) diff --git a/man/man4/zfs.4 b/man/man4/zfs.4 index cfadd79d87f..71a3e67ee67 100644 --- a/man/man4/zfs.4 +++ b/man/man4/zfs.4 @@ -2317,63 +2317,6 @@ If .Sy zvol_threads to the number of CPUs present or 32 (whichever is greater). . -.It Sy zvol_blk_mq_threads Ns = Ns Sy 0 Pq uint -The number of threads per zvol to use for queuing IO requests. -This parameter will only appear if your kernel supports -.Li blk-mq -and is only read and assigned to a zvol at zvol load time. -If -.Sy 0 -(the default) then internally set -.Sy zvol_blk_mq_threads -to the number of CPUs present. -. -.It Sy zvol_use_blk_mq Ns = Ns Sy 0 Ns | Ns 1 Pq uint -Set to -.Sy 1 -to use the -.Li blk-mq -API for zvols. -Set to -.Sy 0 -(the default) to use the legacy zvol APIs. -This setting can give better or worse zvol performance depending on -the workload. -This parameter will only appear if your kernel supports -.Li blk-mq -and is only read and assigned to a zvol at zvol load time. -. -.It Sy zvol_blk_mq_blocks_per_thread Ns = Ns Sy 8 Pq uint -If -.Sy zvol_use_blk_mq -is enabled, then process this number of -.Sy volblocksize Ns -sized blocks per zvol thread. -This tunable can be use to favor better performance for zvol reads (lower -values) or writes (higher values). -If set to -.Sy 0 , -then the zvol layer will process the maximum number of blocks -per thread that it can. -This parameter will only appear if your kernel supports -.Li blk-mq -and is only applied at each zvol's load time. -. -.It Sy zvol_blk_mq_queue_depth Ns = Ns Sy 0 Pq uint -The queue_depth value for the zvol -.Li blk-mq -interface. -This parameter will only appear if your kernel supports -.Li blk-mq -and is only applied at each zvol's load time. -If -.Sy 0 -(the default) then use the kernel's default queue depth. -Values are clamped to the kernel's -.Dv BLKDEV_MIN_RQ -and -.Dv BLKDEV_MAX_RQ Ns / Ns Dv BLKDEV_DEFAULT_RQ -limits. -. .It Sy zvol_volmode Ns = Ns Sy 1 Pq uint Defines zvol block devices behaviour when .Sy volmode Ns = Ns Sy default : diff --git a/module/os/linux/zfs/zvol_os.c b/module/os/linux/zfs/zvol_os.c index f94ce69fb9e..e145a8c876b 100644 --- a/module/os/linux/zfs/zvol_os.c +++ b/module/os/linux/zfs/zvol_os.c @@ -1626,18 +1626,6 @@ MODULE_PARM_DESC(zvol_prefetch_bytes, "Prefetch N bytes at zvol start+end"); module_param(zvol_volmode, uint, 0644); MODULE_PARM_DESC(zvol_volmode, "Default volmode property value"); -#ifdef HAVE_BLK_MQ -module_param(zvol_blk_mq_queue_depth, uint, 0644); -MODULE_PARM_DESC(zvol_blk_mq_queue_depth, "Default blk-mq queue depth"); - -module_param(zvol_use_blk_mq, uint, 0644); -MODULE_PARM_DESC(zvol_use_blk_mq, "Use the blk-mq API for zvols"); - -module_param(zvol_blk_mq_blocks_per_thread, uint, 0644); -MODULE_PARM_DESC(zvol_blk_mq_blocks_per_thread, - "Process volblocksize blocks per thread"); -#endif - #ifndef HAVE_BLKDEV_GET_ERESTARTSYS module_param(zvol_open_timeout_ms, uint, 0644); MODULE_PARM_DESC(zvol_open_timeout_ms, "Timeout for ZVOL open retries"); diff --git a/tests/zfs-tests/include/tunables.cfg b/tests/zfs-tests/include/tunables.cfg index 80e7bcb3bd0..8010a945159 100644 --- a/tests/zfs-tests/include/tunables.cfg +++ b/tests/zfs-tests/include/tunables.cfg @@ -89,7 +89,7 @@ VDEV_VALIDATE_SKIP vdev.validate_skip vdev_validate_skip VOL_INHIBIT_DEV UNSUPPORTED zvol_inhibit_dev VOL_MODE vol.mode zvol_volmode VOL_RECURSIVE vol.recursive UNSUPPORTED -VOL_USE_BLK_MQ UNSUPPORTED zvol_use_blk_mq +VOL_USE_BLK_MQ UNSUPPORTED UNSUPPORTED XATTR_COMPAT xattr_compat zfs_xattr_compat ZEVENT_LEN_MAX zevent.len_max zfs_zevent_len_max ZEVENT_RETAIN_MAX zevent.retain_max zfs_zevent_retain_max From 483ccf0c630deeb3a3376c6072c0a8fd83848283 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Tue, 10 Oct 2023 08:59:10 -0700 Subject: [PATCH 09/15] ZTS: Remove zfs_allow_010_pos expection for FreeBSD This issue should now be address by PR #15376 and the exception for this test case be removed. Reviewed-by: Alexander Motin Reviewed-by: Umer Saleem Signed-off-by: Brian Behlendorf Closes #15382 --- tests/test-runner/bin/zts-report.py.in | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test-runner/bin/zts-report.py.in b/tests/test-runner/bin/zts-report.py.in index 558e4b57279..5d1360380de 100755 --- a/tests/test-runner/bin/zts-report.py.in +++ b/tests/test-runner/bin/zts-report.py.in @@ -262,7 +262,6 @@ if sys.platform.startswith('freebsd'): 'cli_root/zfs_inherit/zfs_inherit_001_neg': ['FAIL', known_reason], 'cli_root/zpool_import/zpool_import_012_pos': ['FAIL', known_reason], 'delegate/zfs_allow_003_pos': ['FAIL', known_reason], - 'delegate/zfs_allow_010_pos': ['FAIL', known_reason], 'inheritance/inherit_001_pos': ['FAIL', 11829], 'resilver/resilver_restart_001': ['FAIL', known_reason], 'pool_checkpoint/checkpoint_big_rewind': ['FAIL', 12622], From bc29124b1bf90eae7e9539bcafeb109e4df821ac Mon Sep 17 00:00:00 2001 From: Daniel Berlin Date: Tue, 10 Oct 2023 14:04:32 -0400 Subject: [PATCH 10/15] Ensure we call fput when cloning fails due to different devices. Right now, zpl_ioctl_ficlone and zpl_ioctl_ficlonerange do not call put on the src fd if the source and destination are on two different devices. This leaves the source file held open in this case. Reviewed-by: Kay Pedersen Reviewed-by: Brian Behlendorf Reviewed-by: Alexander Motin Signed-off-by: Daniel Berlin Closes #15386 --- module/os/linux/zfs/zpl_file_range.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/module/os/linux/zfs/zpl_file_range.c b/module/os/linux/zfs/zpl_file_range.c index 2abbf44df58..c47fe99dacf 100644 --- a/module/os/linux/zfs/zpl_file_range.c +++ b/module/os/linux/zfs/zpl_file_range.c @@ -202,8 +202,10 @@ zpl_ioctl_ficlone(struct file *dst_file, void *arg) if (src_file == NULL) return (-EBADF); - if (dst_file->f_op != src_file->f_op) + if (dst_file->f_op != src_file->f_op) { + fput(src_file); return (-EXDEV); + } size_t len = i_size_read(file_inode(src_file)); @@ -237,8 +239,10 @@ zpl_ioctl_ficlonerange(struct file *dst_file, void __user *arg) if (src_file == NULL) return (-EBADF); - if (dst_file->f_op != src_file->f_op) + if (dst_file->f_op != src_file->f_op) { + fput(src_file); return (-EXDEV); + } size_t len = fcr.fcr_src_length; if (len == 0) From 822b32ee51ccf2e24a358e4927b4a218e9b81925 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Tue, 10 Oct 2023 13:31:15 -0700 Subject: [PATCH 11/15] CI: Move perl script to dist_noinst_DATA Everything listed in dist_noinst_SCRIPTS is assumed to be a shell script, this generates a shellcheck SC1071 error since perl is not supported. Move update_authors.pl to dist_noinst_DATA with the other perl scripts. Reviewed-by: Tony Hutter Reviewed-by: Rob N Signed-off-by: Brian Behlendorf Closes #15392 --- scripts/Makefile.am | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Makefile.am b/scripts/Makefile.am index 8ec9ea4b5c3..b43bf97dbdf 100644 --- a/scripts/Makefile.am +++ b/scripts/Makefile.am @@ -9,7 +9,6 @@ dist_noinst_SCRIPTS = \ %D%/man-dates.sh \ %D%/mancheck.sh \ %D%/paxcheck.sh \ - %D%/update_authors.pl \ %D%/zfs-tests-color.sh scripts_scripts = \ @@ -30,6 +29,7 @@ endif dist_noinst_DATA += \ %D%/cstyle.pl \ %D%/enum-extract.pl \ + %D%/update_authors.pl \ %D%/zfs2zol-patch.sed \ %D%/zol2zfs-patch.sed From 9facf2d1ad0c4857de1d47a2a14e882a3fb78b89 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Tue, 10 Oct 2023 13:32:33 -0700 Subject: [PATCH 12/15] ZTS: Debug zfs_share_concurrent_shares failure Update zfs_share_concurrent_shares test case to wait a few seconds and recheck that the filesystem isn't shared. The intent here is determine the nature of the error and if it may be a race. Reviewed-by: Tony Hutter Reviewed-by: Umer Saleem Signed-off-by: Brian Behlendorf Closes #15379 --- .../zfs_share/zfs_share_concurrent_shares.ksh | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_concurrent_shares.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_concurrent_shares.ksh index c226f56e3dc..d779689f83b 100755 --- a/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_concurrent_shares.ksh +++ b/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_concurrent_shares.ksh @@ -98,11 +98,26 @@ function test_share # filesystem zfs set sharenfs=on $filesystem || \ sub_fail "zfs set sharenfs=on $filesystem failed." - is_shared $mntp || \ - sub_fail "File system $filesystem is not shared (set sharenfs)." # - # Verify 'zfs share' works as well. + # Verify 'zfs share' results in a shared mount. We check this + # multiple times because of Fedora 37+ it's been observed in + # the CI that the share may not be immediately reported. + # + for retry in $(seq 1 10); do + is_shared $mntp && break + + log_note "Wait $retry / 10 for is_shared $mntp (set sharenfs)" + + if [[ $retry -eq 10 ]]; then + sub_fail "File system $filesystem is not shared (set sharenfs)." + fi + + sleep 1 + done + + # + # Verify 'zfs unshare' works as well. # zfs unshare $filesystem || \ sub_fail "zfs unshare $filesystem failed." @@ -112,9 +127,23 @@ function test_share # filesystem zfs share $filesystem || \ sub_fail "zfs share $filesystem failed." - is_shared $mntp || \ - sub_fail "file system $filesystem is not shared (zfs share)." + # + # Verify 'zfs share' results in a shared mount. We check this + # multiple times because of Fedora 37+ it's been observed in + # the CI that the share may not be immediately reported. + # + for retry in $(seq 1 10); do + is_shared $mntp && break + + log_note "Wait $retry / 10 for is_shared $mntp (zfs share)" + + if [[ $retry -eq 10 ]]; then + sub_fail "File system $filesystem is not shared (zfs share)." + fi + + sleep 1 + done #log_note "Sharing a shared file system fails." zfs share $filesystem && \ From 1b310dfb1d5aa90db7d8c4ec8d6a92b4430b0496 Mon Sep 17 00:00:00 2001 From: Alexander Motin Date: Wed, 11 Oct 2023 19:37:21 -0400 Subject: [PATCH 13/15] DMU: Do not pre-read holes during write dmu_tx_check_ioerr() pre-reads blocks that are going to be dirtied as part of transaction to both prefetch them and check for errors. But it makes no sense to do it for holes, since there are no disk reads to prefetch and there can be no errors. On the other side those blocks are anonymous, and they are freed immediately by the dbuf_rele() without even being put into dbuf cache, so we just burn CPU time on decompression and overheads and get absolutely no result at the end. Use of dbuf_hold_impl() with fail_sparse parameter allows to skip the extra work, and on my tests with sequential 8KB writes to empty ZVOL with 32KB blocks shows throughput increase from 1.7 to 2GB/s. Reviewed-by: Brian Atkinson Reviewed-by: Brian Behlendorf Signed-off-by: Alexander Motin Sponsored by: iXsystems, Inc. Closes #15371 --- module/zfs/dmu_tx.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/module/zfs/dmu_tx.c b/module/zfs/dmu_tx.c index 0eb8c17e331..8451b5082e8 100644 --- a/module/zfs/dmu_tx.c +++ b/module/zfs/dmu_tx.c @@ -210,10 +210,12 @@ dmu_tx_check_ioerr(zio_t *zio, dnode_t *dn, int level, uint64_t blkid) dmu_buf_impl_t *db; rw_enter(&dn->dn_struct_rwlock, RW_READER); - db = dbuf_hold_level(dn, level, blkid, FTAG); + err = dbuf_hold_impl(dn, level, blkid, TRUE, FALSE, FTAG, &db); rw_exit(&dn->dn_struct_rwlock); - if (db == NULL) - return (SET_ERROR(EIO)); + if (err == ENOENT) + return (0); + if (err != 0) + return (err); /* * PARTIAL_FIRST allows caching for uncacheable blocks. It will * be cleared after dmu_buf_will_dirty() call dbuf_read() again. From fd5128622783735b3f0334c668bca8013cbcd487 Mon Sep 17 00:00:00 2001 From: Brian Behlendorf Date: Wed, 11 Oct 2023 16:56:32 -0700 Subject: [PATCH 14/15] spec: define _bashcompletiondir if undefined Always define _bashcompletiondir in the spec file to a reasonable value when it is undefined. Required for `rpmbuild --rebuild `. Signed-off-by: Brian Behlendorf Closes #15396 --- rpm/generic/zfs.spec.in | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/rpm/generic/zfs.spec.in b/rpm/generic/zfs.spec.in index 711e6c751dc..2e89abd0edf 100644 --- a/rpm/generic/zfs.spec.in +++ b/rpm/generic/zfs.spec.in @@ -19,6 +19,15 @@ %endif %endif +# Set the default _bashcompletiondir directory based on distribution. +%if %{undefined _bashcompletiondir} +%if 0%{?rhel}%{?fedora}%{?centos}%{?suse_version}%{?openEuler} +%global _bashcompletiondir /etc/bash_completion.d +%else +%global _bashcompletiondir /usr/share/bash-completion +%endif +%endif + # Set the default dracut directory based on distribution. %if %{undefined _dracutdir} %if 0%{?rhel}%{?fedora}%{?centos}%{?suse_version}%{?openEuler} From 8a740701283495e59944839bb2607f80907d950e Mon Sep 17 00:00:00 2001 From: Jason King Date: Thu, 12 Oct 2023 13:01:54 -0500 Subject: [PATCH 15/15] Zpool can start allocating from metaslab before TRIMs have completed When doing a manual TRIM on a zpool, the metaslab being TRIMmed is potentially re-enabled before all queued TRIM zios for that metaslab have completed. Since TRIM zios have the lowest priority, it is possible to get into a situation where allocations occur from the just re-enabled metaslab and cut ahead of queued TRIMs to the same metaslab. If the ranges overlap, this will cause corruption. We were able to trigger this pretty consistently with a small single top-level vdev zpool (i.e. small number of metaslabs) with heavy parallel write activity while performing a manual TRIM against a somewhat 'slow' device (so TRIMs took a bit of time to complete). With the patch, we've not been able to recreate it since. It was on illumos, but inspection of the OpenZFS trim code looks like the relevant pieces are largely unchanged and so it appears it would be vulnerable to the same issue. Reviewed-by: Igor Kozhukhov Reviewed-by: Alexander Motin Reviewed-by: Brian Behlendorf Signed-off-by: Jason King Illumos-issue: https://www.illumos.org/issues/15939 Closes #15395 --- module/zfs/vdev_trim.c | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/module/zfs/vdev_trim.c b/module/zfs/vdev_trim.c index 0d71b943434..03e17db024e 100644 --- a/module/zfs/vdev_trim.c +++ b/module/zfs/vdev_trim.c @@ -23,6 +23,7 @@ * Copyright (c) 2016 by Delphix. All rights reserved. * Copyright (c) 2019 by Lawrence Livermore National Security, LLC. * Copyright (c) 2021 Hewlett Packard Enterprise Development LP + * Copyright 2023 RackTop Systems, Inc. */ #include @@ -591,6 +592,7 @@ vdev_trim_ranges(trim_args_t *ta) uint64_t extent_bytes_max = ta->trim_extent_bytes_max; uint64_t extent_bytes_min = ta->trim_extent_bytes_min; spa_t *spa = vd->vdev_spa; + int error = 0; ta->trim_start_time = gethrtime(); ta->trim_bytes_done = 0; @@ -610,19 +612,32 @@ vdev_trim_ranges(trim_args_t *ta) uint64_t writes_required = ((size - 1) / extent_bytes_max) + 1; for (uint64_t w = 0; w < writes_required; w++) { - int error; - error = vdev_trim_range(ta, VDEV_LABEL_START_SIZE + rs_get_start(rs, ta->trim_tree) + (w *extent_bytes_max), MIN(size - (w * extent_bytes_max), extent_bytes_max)); if (error != 0) { - return (error); + goto done; } } } - return (0); +done: + /* + * Make sure all TRIMs for this metaslab have completed before + * returning. TRIM zios have lower priority over regular or syncing + * zios, so all TRIM zios for this metaslab must complete before the + * metaslab is re-enabled. Otherwise it's possible write zios to + * this metaslab could cut ahead of still queued TRIM zios for this + * metaslab causing corruption if the ranges overlap. + */ + mutex_enter(&vd->vdev_trim_io_lock); + while (vd->vdev_trim_inflight[0] > 0) { + cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock); + } + mutex_exit(&vd->vdev_trim_io_lock); + + return (error); } static void @@ -941,11 +956,6 @@ vdev_trim_thread(void *arg) } spa_config_exit(spa, SCL_CONFIG, FTAG); - mutex_enter(&vd->vdev_trim_io_lock); - while (vd->vdev_trim_inflight[0] > 0) { - cv_wait(&vd->vdev_trim_io_cv, &vd->vdev_trim_io_lock); - } - mutex_exit(&vd->vdev_trim_io_lock); range_tree_destroy(ta.trim_tree);