riscv: support for Cache-Block Operations (CBO).

CBO represents a subset of Cache-Management Operations (CMO) spec.
While the CMO spec encompasses all operations on caches, the CBO subset
operates on cache blocks only.

Detect Zicbom, Zicboz and Zicbop extensions and provide cache invalidation
handlers based on Zicbom instructions.

Sponsored by:	UKRI
Differential Revision:	https://reviews.freebsd.org/D49852

sys/conf/files.riscv

@@ -46,6 +46,7 @@ riscv/riscv/bus_space_asm.S	standard
 riscv/riscv/busdma_bounce.c	standard
 riscv/riscv/busdma_machdep.c	standard
 riscv/riscv/cache.c		standard
+riscv/riscv/cbo.c		standard
 riscv/riscv/clock.c		standard
 riscv/riscv/copyinout.S		standard
 riscv/riscv/cpufunc_asm.S	standard

sys/riscv/include/cbo.h

@@ -0,0 +1,33 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2025 Ruslan Bukin <br@bsdpad.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef	_RISCV_CBO_H_
+#define	_RISCV_CBO_H_
+
+void cbo_zicbom_setup_cache(int cbom_block_size);
+
+#endif	/* _RISCV_CBO_H_ */

sys/riscv/riscv/cbo.c

@@ -0,0 +1,104 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2025 Ruslan Bukin <br@bsdpad.com>
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* Cache Block Operations. */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+
+#include <machine/cbo.h>
+
+static void
+cbo_zicbom_cpu_dcache_wbinv_range(vm_offset_t va, vm_size_t len)
+{
+	vm_offset_t addr;
+
+	/*
+	 * A flush operation atomically performs a clean operation followed by
+	 * an invalidate operation.
+	 */
+
+	va &= ~(dcache_line_size - 1);
+	for (addr = va; addr < va + len; addr += dcache_line_size)
+		__asm __volatile(".option push; .option arch, +zicbom\n"
+				 "cbo.flush (%0); .option pop\n" :: "r"(addr));
+}
+
+static void
+cbo_zicbom_cpu_dcache_inv_range(vm_offset_t va, vm_size_t len)
+{
+	vm_offset_t addr;
+
+	/*
+	 * An invalidate operation makes data from store operations performed by
+	 * a set of non-coherent agents visible to the set of coherent agents at
+	 * a point common to both sets by deallocating all copies of a cache
+	 * block from the set of coherent caches up to that point.
+	 */
+
+	va &= ~(dcache_line_size - 1);
+	for (addr = va; addr < va + len; addr += dcache_line_size)
+		__asm __volatile(".option push; .option arch, +zicbom\n"
+				 "cbo.inval (%0); .option pop\n" :: "r"(addr));
+}
+
+static void
+cbo_zicbom_cpu_dcache_wb_range(vm_offset_t va, vm_size_t len)
+{
+	vm_offset_t addr;
+
+	/*
+	 * A clean operation makes data from store operations performed by the
+	 * set of coherent agents visible to a set of non-coherent agents at a
+	 * point common to both sets by performing a write transfer of a copy of
+	 * a cache block to that point provided a coherent agent performed a
+	 * store operation that modified the data in the cache block since the
+	 * previous invalidate, clean, or flush operation on the cache block.
+	 */
+
+	va &= ~(dcache_line_size - 1);
+	for (addr = va; addr < va + len; addr += dcache_line_size)
+		__asm __volatile(".option push; .option arch, +zicbom\n"
+				 "cbo.clean (%0); .option pop\n" :: "r"(addr));
+}
+
+void
+cbo_zicbom_setup_cache(int cbom_block_size)
+{
+	struct riscv_cache_ops zicbom_ops;
+
+	if (cbom_block_size <= 0 || !powerof2(cbom_block_size)) {
+		printf("Zicbom: could not initialise (invalid cache line %d)\n",
+		    cbom_block_size);
+		return;
+	}
+
+	zicbom_ops.dcache_wbinv_range = cbo_zicbom_cpu_dcache_wbinv_range;
+	zicbom_ops.dcache_inv_range = cbo_zicbom_cpu_dcache_inv_range;
+	zicbom_ops.dcache_wb_range = cbo_zicbom_cpu_dcache_wb_range;
+	riscv_cache_install_hooks(&zicbom_ops, cbom_block_size);
+}

sys/riscv/riscv/identcpu.c

@@ -53,6 +53,7 @@
 #include <machine/elf.h>
 #include <machine/md_var.h>
 #include <machine/thead.h>
+#include <machine/cbo.h>
 
 #ifdef FDT
 #include <dev/fdt/fdt_common.h>
@@ -78,6 +79,11 @@ bool __read_frequently has_sstc;
 bool __read_frequently has_sscofpmf;
 bool has_svpbmt;
 
+/* Z-extensions support. */
+bool has_zicbom;
+bool has_zicboz;
+bool has_zicbop;
+
 struct cpu_desc {
 	const char	*cpu_mvendor_name;
 	const char	*cpu_march_name;
@@ -89,6 +95,12 @@ struct cpu_desc {
 #define	 SV_SVPBMT	(1 << 2)
 #define	 SV_SVINVAL	(1 << 3)
 #define	 SV_SSCOFPMF	(1 << 4)
+	u_int		z_extensions;		/* Multi-letter extensions. */
+#define	 Z_ZICBOM	(1 << 0)
+#define	 Z_ZICBOZ	(1 << 1)
+#define	 Z_ZICBOP	(1 << 2)
+	int		cbom_block_size;
+	int		cboz_block_size;
 };
 
 struct cpu_desc cpu_desc[MAXCPU];
@@ -196,11 +208,24 @@ parse_ext_x(struct cpu_desc *desc __unused, char *isa, int idx, int len)
 static __inline int
 parse_ext_z(struct cpu_desc *desc __unused, char *isa, int idx, int len)
 {
+#define	CHECK_Z_EXT(str, flag)						\
+	do {								\
+		if (strncmp(&isa[idx], (str),				\
+		    MIN(strlen(str), len - idx)) == 0) {		\
+			desc->z_extensions |= flag;			\
+			return (idx + strlen(str));			\
+		}							\
+	} while (0)
+
+	/* Check for known/supported extensions. */
+	CHECK_Z_EXT("zicbom",	Z_ZICBOM);
+	CHECK_Z_EXT("zicboz",	Z_ZICBOZ);
+	CHECK_Z_EXT("zicbop",	Z_ZICBOP);
+
+#undef CHECK_Z_EXT
 	/*
 	 * Proceed to the next multi-letter extension or the end of the
 	 * string.
-	 *
-	 * TODO: parse some of these.
 	 */
 	while (isa[idx] != '_' && idx < len) {
 		idx++;
@@ -321,6 +346,22 @@ parse_mmu_fdt(struct cpu_desc *desc, phandle_t node)
 	}
 }
 
+static void
+parse_cbo_fdt(struct cpu_desc *desc, phandle_t node)
+{
+	int error;
+
+	error = OF_getencprop(node, "riscv,cbom-block-size",
+	    &desc->cbom_block_size, sizeof(desc->cbom_block_size));
+	if (error == -1)
+		desc->cbom_block_size = 0;
+
+	error = OF_getencprop(node, "riscv,cboz-block-size",
+	    &desc->cboz_block_size, sizeof(desc->cboz_block_size));
+	if (error == -1)
+		desc->cboz_block_size = 0;
+}
+
 static void
 identify_cpu_features_fdt(u_int cpu, struct cpu_desc *desc)
 {
@@ -372,6 +413,9 @@ identify_cpu_features_fdt(u_int cpu, struct cpu_desc *desc)
 		/* Check MMU features. */
 		parse_mmu_fdt(desc, node);
 
+		/* Cache-block operations (CBO). */
+		parse_cbo_fdt(desc, node);
+
 		/* We are done. */
 		break;
 	}
@@ -422,6 +466,11 @@ update_global_capabilities(u_int cpu, struct cpu_desc *desc)
 	UPDATE_CAP(has_sscofpmf, (desc->smode_extensions & SV_SSCOFPMF) != 0);
 	UPDATE_CAP(has_svpbmt, (desc->smode_extensions & SV_SVPBMT) != 0);
 
+	/* Z extension support. */
+	UPDATE_CAP(has_zicbom, (desc->z_extensions & Z_ZICBOM) != 0);
+	UPDATE_CAP(has_zicboz, (desc->z_extensions & Z_ZICBOZ) != 0);
+	UPDATE_CAP(has_zicbop, (desc->z_extensions & Z_ZICBOP) != 0);
+
 #undef UPDATE_CAP
 }
 
@@ -506,6 +555,9 @@ identify_cpu(u_int cpu)
 
 	update_global_capabilities(cpu, desc);
 	handle_cpu_quirks(cpu, desc);
+
+	if (has_zicbom && cpu == 0)
+		cbo_zicbom_setup_cache(desc->cbom_block_size);
 }
 
 void