Skip to content

Commit 83704ca

Browse files
gburdhackorum
authored andcommitted
Add RISC-V CRC32C using the Zbc extension
This adds hardware-accelerated CRC-32C computation for RISC-V platforms with the Zbc (carry-less multiply) or Zbkc (crypto carry-less) extension. The implementation uses the clmul and clmulh instructions for polynomial folding with Barrett reduction to compute CRC-32C checksums. This provides approximately 20x speedup over the software slicing-by-8 implementation. The algorithm is based on the Google Abseil project's RISC-V CRC32C implementation (abseil/abseil-cpp#1986 in absl/crc/internal/crc_riscv.cc) that is Copyright 2025 The Abseil Authors licensed under the Apache License, Version 2.0. Runtime detection uses the Linux riscv_hwprobe syscall (kernel 6.4+) to check for Zbc/Zbkc support, falling back gracefully to software on older kernels or non-Linux platforms. Similar to ARMv8 CRC Extension and x86 SSE 4.2 support, this is compiled with '-march=rv64gc_zbc' and selected at runtime based on CPU capabilities.
1 parent 0349289 commit 83704ca

8 files changed

Lines changed: 482 additions & 7 deletions

File tree

config/c-compiler.m4

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -835,6 +835,47 @@ fi
835835
undefine([Ac_cachevar])dnl
836836
])# PGAC_LOONGARCH_CRC32C_INTRINSICS
837837

838+
# PGAC_RISCV_ZBC_CRC32C_INTRINSICS
839+
# ---------------------------------
840+
# Check if the compiler supports RISC-V Zbc (carry-less multiply) instructions
841+
# for CRC-32C computation, using inline assembly for clmul instruction.
842+
#
843+
# An optional compiler flag can be passed as argument (e.g. -march=rv64gc_zbc).
844+
# If the intrinsics are supported, sets pgac_riscv_zbc_crc32c_intrinsics and
845+
# CFLAGS_CRC.
846+
#
847+
# The Zbc extension provides clmul and clmulh instructions which are used with
848+
# polynomial folding to compute CRC-32C. This implementation is based on the
849+
# algorithm from Google Abseil (https://github.com/abseil/abseil-cpp/pull/1986).
850+
AC_DEFUN([PGAC_RISCV_ZBC_CRC32C_INTRINSICS],
851+
[define([Ac_cachevar], [AS_TR_SH([pgac_cv_riscv_zbc_crc32c_intrinsics_$1])])dnl
852+
AC_CACHE_CHECK([for RISC-V Zbc clmul with CFLAGS=$1], [Ac_cachevar],
853+
[pgac_save_CFLAGS=$CFLAGS
854+
CFLAGS="$pgac_save_CFLAGS $1"
855+
AC_LINK_IFELSE([AC_LANG_PROGRAM([
856+
#if !defined(__riscv) || !defined(__riscv_xlen) || __riscv_xlen != 64
857+
#error not RISC-V 64-bit
858+
#endif
859+
860+
static inline unsigned long clmul_test(unsigned long a, unsigned long b)
861+
{
862+
unsigned long result;
863+
__asm__("clmul %0, %1, %2" : "=r"(result) : "r"(a), "r"(b));
864+
return result;
865+
}],
866+
[unsigned long result = clmul_test(0x123, 0x456);
867+
/* return computed value, to prevent the above being optimized away */
868+
return result == 0;])],
869+
[Ac_cachevar=yes],
870+
[Ac_cachevar=no])
871+
CFLAGS="$pgac_save_CFLAGS"])
872+
if test x"$Ac_cachevar" = x"yes"; then
873+
CFLAGS_CRC="$1"
874+
pgac_riscv_zbc_crc32c_intrinsics=yes
875+
fi
876+
undefine([Ac_cachevar])dnl
877+
])# PGAC_RISCV_ZBC_CRC32C_INTRINSICS
878+
838879
# PGAC_XSAVE_INTRINSICS
839880
# ---------------------
840881
# Check if the compiler supports the XSAVE instructions using the _xgetbv

configure.ac

Lines changed: 29 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2247,6 +2247,17 @@ fi
22472247
# with the default compiler flags.
22482248
PGAC_LOONGARCH_CRC32C_INTRINSICS()
22492249

2250+
# Check for RISC-V Zbc (carry-less multiply) for CRC calculations.
2251+
#
2252+
# The Zbc extension provides clmul and clmulh instructions for hardware-
2253+
# accelerated CRC-32C computation using polynomial folding. Check if we
2254+
# can compile with -march=rv64gc_zbc flag. CFLAGS_CRC is set if the flag
2255+
# is required.
2256+
#
2257+
# This implementation is based on Google Abseil's algorithm:
2258+
# https://github.com/abseil/abseil-cpp/pull/1986
2259+
PGAC_RISCV_ZBC_CRC32C_INTRINSICS([-march=rv64gc_zbc])
2260+
22502261
AC_SUBST(CFLAGS_CRC)
22512262

22522263
# Select CRC-32C implementation.
@@ -2277,7 +2288,7 @@ AC_SUBST(CFLAGS_CRC)
22772288
#
22782289
# If we are targeting a LoongArch processor, CRC instructions are
22792290
# always available (at least on 64 bit), so no runtime check is needed.
2280-
if test x"$USE_SLICING_BY_8_CRC32C" = x"" && test x"$USE_SSE42_CRC32C" = x"" && test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_ARMV8_CRC32C" = x"" && test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_LOONGARCH_CRC32C" = x""; then
2291+
if test x"$USE_SLICING_BY_8_CRC32C" = x"" && test x"$USE_SSE42_CRC32C" = x"" && test x"$USE_SSE42_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_ARMV8_CRC32C" = x"" && test x"$USE_ARMV8_CRC32C_WITH_RUNTIME_CHECK" = x"" && test x"$USE_LOONGARCH_CRC32C" = x"" && test x"$USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK" = x""; then
22812292
# Use Intel SSE 4.2 if available.
22822293
if test x"$pgac_sse42_crc32_intrinsics" = x"yes" && test x"$SSE4_2_TARGETED" = x"1" ; then
22832294
USE_SSE42_CRC32C=1
@@ -2299,9 +2310,14 @@ if test x"$USE_SLICING_BY_8_CRC32C" = x"" && test x"$USE_SSE42_CRC32C" = x"" &&
22992310
if test x"$pgac_loongarch_crc32c_intrinsics" = x"yes"; then
23002311
USE_LOONGARCH_CRC32C=1
23012312
else
2302-
# fall back to slicing-by-8 algorithm, which doesn't require any
2303-
# special CPU support.
2304-
USE_SLICING_BY_8_CRC32C=1
2313+
# RISC-V Zbc CRC, with runtime check.
2314+
if test x"$pgac_riscv_zbc_crc32c_intrinsics" = x"yes"; then
2315+
USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK=1
2316+
else
2317+
# fall back to slicing-by-8 algorithm, which doesn't require any
2318+
# special CPU support.
2319+
USE_SLICING_BY_8_CRC32C=1
2320+
fi
23052321
fi
23062322
fi
23072323
fi
@@ -2336,9 +2352,15 @@ else
23362352
PG_CRC32C_OBJS="pg_crc32c_loongarch.o"
23372353
AC_MSG_RESULT(LoongArch CRCC instructions)
23382354
else
2339-
AC_DEFINE(USE_SLICING_BY_8_CRC32C, 1, [Define to 1 to use software CRC-32C implementation (slicing-by-8).])
2340-
PG_CRC32C_OBJS="pg_crc32c_sb8.o"
2341-
AC_MSG_RESULT(slicing-by-8)
2355+
if test x"$USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK" = x"1"; then
2356+
AC_DEFINE(USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK, 1, [Define to 1 to use RISC-V Zbc CRC instructions with a runtime check.])
2357+
PG_CRC32C_OBJS="pg_crc32c_riscv_zbc.o pg_crc32c_sb8.o pg_crc32c_riscv_choose.o"
2358+
AC_MSG_RESULT(RISC-V Zbc instructions with runtime check)
2359+
else
2360+
AC_DEFINE(USE_SLICING_BY_8_CRC32C, 1, [Define to 1 to use software CRC-32C implementation (slicing-by-8).])
2361+
PG_CRC32C_OBJS="pg_crc32c_sb8.o"
2362+
AC_MSG_RESULT(slicing-by-8)
2363+
fi
23422364
fi
23432365
fi
23442366
fi

meson.build

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2826,6 +2826,42 @@ int main(void)
28262826
have_optimized_crc = true
28272827
endif
28282828

2829+
elif host_cpu == 'riscv64'
2830+
2831+
# Check for RISC-V Zbc (carry-less multiply) extension for CRC-32C.
2832+
# The Zbc extension provides clmul and clmulh instructions used for
2833+
# hardware-accelerated CRC computation via polynomial folding.
2834+
#
2835+
# This implementation is based on Google Abseil's algorithm:
2836+
# https://github.com/abseil/abseil-cpp/pull/1986
2837+
2838+
prog = '''
2839+
#if !defined(__riscv) || !defined(__riscv_xlen) || __riscv_xlen != 64
2840+
#error not RISC-V 64-bit
2841+
#endif
2842+
2843+
static inline unsigned long clmul(unsigned long a, unsigned long b)
2844+
{
2845+
unsigned long result;
2846+
__asm__("clmul %0, %1, %2" : "=r"(result) : "r"(a), "r"(b));
2847+
return result;
2848+
}
2849+
2850+
int main(void)
2851+
{
2852+
unsigned long result = clmul(0x123, 0x456);
2853+
return result == 0;
2854+
}
2855+
'''
2856+
2857+
if cc.links(prog, name: 'RISC-V Zbc clmul with -march=rv64gc_zbc',
2858+
args: test_c_args + ['-march=rv64gc_zbc'])
2859+
# Use RISC-V Zbc CRC, with runtime check
2860+
cflags_crc += '-march=rv64gc_zbc'
2861+
cdata.set('USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK', 1)
2862+
have_optimized_crc = true
2863+
endif
2864+
28292865
endif
28302866

28312867
if not have_optimized_crc

src/include/port/pg_crc32c.h

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,20 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le
166166
extern pg_crc32c pg_comp_crc32c_pmull(pg_crc32c crc, const void *data, size_t len);
167167
#endif
168168

169+
#elif defined(USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK)
170+
171+
/*
172+
* Use RISC-V Zbc instructions, but perform a runtime check first
173+
* to check that they are available.
174+
*/
175+
#define COMP_CRC32C(crc, data, len) \
176+
((crc) = pg_comp_crc32c((crc), (data), (len)))
177+
#define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF)
178+
179+
extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len);
180+
extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len);
181+
extern pg_crc32c pg_comp_crc32c_riscv_zbc(pg_crc32c crc, const void *data, size_t len);
182+
169183
#else
170184
/*
171185
* Use slicing-by-8 algorithm.

src/port/meson.build

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,9 @@ replace_funcs_pos = [
101101
['pg_crc32c_loongarch', 'USE_LOONGARCH_CRC32C'],
102102

103103
# riscv
104+
['pg_crc32c_riscv_zbc', 'USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK', 'crc'],
105+
['pg_crc32c_riscv_choose', 'USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK'],
106+
['pg_crc32c_sb8', 'USE_RISCV_ZBC_CRC32C_WITH_RUNTIME_CHECK'],
104107
['pg_popcount_riscv', 'USE_RISCV_ZBB_WITH_RUNTIME_CHECK', 'zbb'],
105108

106109
# generic fallback

src/port/pg_crc32c_riscv_choose.c

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*-------------------------------------------------------------------------
2+
*
3+
* pg_crc32c_riscv_choose.c
4+
* Choose between RISC-V Zbc and software CRC-32C implementation.
5+
*
6+
* On first call, checks if the CPU supports the RISC-V Zbc (or Zbkc) extension.
7+
* If it does, use carry-less multiply instructions for CRC-32C computation.
8+
* Otherwise, fall back to the pure software implementation (slicing-by-8).
9+
*
10+
* Portions Copyright (c) 1996-2026, PostgreSQL Global Development Group
11+
* Portions Copyright (c) 1994, Regents of the University of California
12+
*
13+
*
14+
* IDENTIFICATION
15+
* src/port/pg_crc32c_riscv_choose.c
16+
*
17+
*-------------------------------------------------------------------------
18+
*/
19+
20+
#ifndef FRONTEND
21+
#include "postgres.h"
22+
#else
23+
#include "postgres_fe.h"
24+
#endif
25+
26+
#include <sys/syscall.h>
27+
#include <unistd.h>
28+
29+
#include "port/pg_crc32c.h"
30+
31+
/*
32+
* RISC-V hardware probing definitions
33+
*/
34+
#ifndef __NR_riscv_hwprobe
35+
#define __NR_riscv_hwprobe 258
36+
#endif
37+
38+
#ifndef RISCV_HWPROBE_KEY_IMA_EXT_0
39+
#define RISCV_HWPROBE_KEY_IMA_EXT_0 4
40+
#endif
41+
42+
#ifndef RISCV_HWPROBE_EXT_ZBC
43+
#define RISCV_HWPROBE_EXT_ZBC (1ULL << 7)
44+
#endif
45+
46+
#ifndef RISCV_HWPROBE_EXT_ZBKC
47+
#define RISCV_HWPROBE_EXT_ZBKC (1ULL << 27)
48+
#endif
49+
50+
struct riscv_hwprobe
51+
{
52+
int64 key;
53+
uint64 value;
54+
};
55+
56+
/*
57+
* Check if RISC-V Zbc or Zbkc extension is available
58+
*
59+
* Uses the riscv_hwprobe syscall which is available on Linux kernel 6.4+
60+
* Falls back to software if the syscall fails or extensions are not available.
61+
*/
62+
static bool
63+
pg_crc32c_riscv_zbc_available(void)
64+
{
65+
#if defined(__linux__) && defined(__riscv) && (__riscv_xlen == 64)
66+
struct riscv_hwprobe pair = {.key = RISCV_HWPROBE_KEY_IMA_EXT_0};
67+
68+
/*
69+
* Make the syscall. If it fails (e.g., old kernel, non-Linux), fall back
70+
* to software.
71+
*/
72+
if (syscall(__NR_riscv_hwprobe, &pair, 1, 0, NULL, 0) != 0)
73+
return false;
74+
75+
/*
76+
* Check if either Zbc (general bitmanip carry-less) or Zbkc (crypto
77+
* carry-less) is available. Both provide clmul/clmulh instructions.
78+
*/
79+
return (pair.value & (RISCV_HWPROBE_EXT_ZBC | RISCV_HWPROBE_EXT_ZBKC)) != 0;
80+
#else
81+
/* Not on RISC-V Linux, or not 64-bit - use software fallback */
82+
return false;
83+
#endif
84+
}
85+
86+
/*
87+
* This gets called on the first call. It replaces the function pointer
88+
* so that subsequent calls are routed directly to the chosen implementation.
89+
*/
90+
static pg_crc32c
91+
pg_comp_crc32c_choose(pg_crc32c crc, const void *data, size_t len)
92+
{
93+
if (pg_crc32c_riscv_zbc_available())
94+
pg_comp_crc32c = pg_comp_crc32c_riscv_zbc;
95+
else
96+
pg_comp_crc32c = pg_comp_crc32c_sb8;
97+
98+
return pg_comp_crc32c(crc, data, len);
99+
}
100+
101+
pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len) = pg_comp_crc32c_choose;

0 commit comments

Comments
 (0)