From 476a56de0783eb62536d4a587441b2778113736a Mon Sep 17 00:00:00 2001 From: Sean Parkinson Date: Mon, 17 Aug 2026 07:28:27 +1000 Subject: [PATCH] Regression testing fixes Fix guard around inclunde of chacha20_poly1305.h in internal.h for when session ticket is not ChaCha20-Poly1305. Fix port allocation and readiness testing in scripts. Fix other issues in the scripts as well. Update test.h code around ready file. For LTO compiles, all functions must be used. Make sure the assembly functions are marked as 'used' when not called internally. Fix a race in the example client that intermittently failed scripts/unit.test on TLS 1.3 non-blocking session resumption. RISC-V 64-bit: s0 needed but functions need to omit frame pointer. Have source matching generated code again. --- cmake/functions.cmake | 10 +- configure.ac | 16 +- examples/client/client.c | 85 +- scripts/benchmark.test | 29 +- scripts/dtls.test | 192 +++- scripts/dtlscid.test | 127 ++- scripts/multi-msg-record.py | 19 +- scripts/ocsp-responder-openssl-interop.test | 14 +- .../ocsp-stapling-with-ca-as-responder.test | 14 +- .../ocsp-stapling-with-wolfssl-responder.test | 14 +- scripts/ocsp-stapling.test | 23 +- scripts/ocsp-stapling2.test | 23 +- scripts/ocsp-stapling_tls13multi.test | 14 +- scripts/openssl.test | 9 +- scripts/openssl_srtp.test | 9 +- scripts/rsapss.test | 9 +- src/include.am | 15 + wolfcrypt/src/aes.c | 39 +- wolfcrypt/src/aes_xts_x86_asm.S | 841 ++++++++++++++++++ wolfcrypt/src/aes_xts_x86_asm.asm | 835 +++++++++++++++++ wolfcrypt/src/fe_448.c | 1 + wolfcrypt/src/fe_x25519_asm.asm | 520 +++++++++++ wolfcrypt/src/ge_448.c | 2 +- wolfcrypt/src/include.am | 1 + wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c | 16 +- .../src/port/arm/armv8-32-curve25519_c.c | 60 +- wolfcrypt/src/port/arm/armv8-32-sha3-asm.S | 6 + wolfcrypt/src/port/arm/armv8-32-sha3-asm_c.c | 6 + wolfcrypt/src/port/arm/armv8-curve25519_c.c | 4 +- wolfcrypt/src/port/arm/thumb2-aes-asm_c.c | 16 +- wolfcrypt/src/port/arm/thumb2-curve25519_c.c | 60 +- wolfcrypt/src/port/arm/thumb2-sha3-asm.S | 6 + wolfcrypt/src/port/arm/thumb2-sha3-asm_c.c | 6 + .../src/port/riscv64/riscv-64-aes-asm_c.c | 20 +- .../src/port/riscv64/riscv-64-sha256-asm_c.c | 7 +- .../src/port/riscv64/riscv-64-sha3-asm.S | 7 + .../src/port/riscv64/riscv-64-sha3-asm_c.c | 9 +- wolfcrypt/src/sha256_asm.asm | 4 + wolfcrypt/src/sha512_asm.asm | 6 +- wolfcrypt/src/sp_x86_64_asm.S | 4 +- wolfcrypt/src/sp_x86_64_asm.asm | 4 +- wolfssl-VS2022.vcxproj | 22 + wolfssl.vcxproj | 22 + wolfssl/internal.h | 2 +- wolfssl/test.h | 56 +- wolfssl/wolfcrypt/wc_port.h | 13 + 46 files changed, 3007 insertions(+), 210 deletions(-) create mode 100644 wolfcrypt/src/aes_xts_x86_asm.S create mode 100644 wolfcrypt/src/aes_xts_x86_asm.asm diff --git a/cmake/functions.cmake b/cmake/functions.cmake index 1856dcb22c3..b0cb0628b96 100644 --- a/cmake/functions.cmake +++ b/cmake/functions.cmake @@ -468,6 +468,10 @@ function(generate_lib_src_list LIB_SOURCES) if(BUILD_AESNI) list(APPEND LIB_SOURCES wolfcrypt/src/aes_asm.S) + # 32-bit x86 AES-XTS. The file guards its own contents on + # WOLFSSL_AES_XTS and WOLFSSL_X86_BUILD, so it compiles to + # nothing on any other target or without XTS. + list(APPEND LIB_SOURCES wolfcrypt/src/aes_xts_x86_asm.S) if(BUILD_INTELASM) list(APPEND LIB_SOURCES wolfcrypt/src/aes_gcm_asm.S) @@ -914,9 +918,13 @@ function(generate_lib_src_list LIB_SOURCES) endif() if(NOT BUILD_FIPS_V2 AND BUILD_AESNI) + # aes_xts_x86_asm.S guards its own contents on WOLFSSL_AES_XTS and + # WOLFSSL_X86_BUILD, so it compiles to nothing on any other target + # or without XTS. list(APPEND LIB_SOURCES wolfcrypt/src/aes_asm.S - wolfcrypt/src/aes_gcm_asm.S) + wolfcrypt/src/aes_gcm_asm.S + wolfcrypt/src/aes_xts_x86_asm.S) endif() if(BUILD_CAMELLIA) diff --git a/configure.ac b/configure.ac index 3b997f801ba..1c87d8e42d2 100644 --- a/configure.ac +++ b/configure.ac @@ -1215,10 +1215,13 @@ if (test "$host_cpu" = "x86_64" || test "$host_cpu" = "amd64") && then AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_X86_64_BUILD" fi -if test "$host_cpu" = "x86" -then +# config.sub normalises 32-bit x86 to i386/i486/i586/i686, so matching only +# "x86" never fires for a real host triple. +case "$host_cpu" in + x86 | i?86) AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_X86_BUILD" -fi + ;; +esac AC_ARG_ENABLE([leanpsk], @@ -4801,11 +4804,12 @@ then then AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_X86_64_BUILD" fi - if test "$host_cpu" = "x86" - then + case "$host_cpu" in + x86 | i?86) AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_X86_BUILD" ENABLED_X86_ASM=yes - fi + ;; + esac fi AC_SUBST([ENABLED_AESNI]) AC_SUBST([ENABLED_AESNI_WITH_AVX]) diff --git a/examples/client/client.c b/examples/client/client.c index 822da1afd7d..3f8102aeb8d 100644 --- a/examples/client/client.c +++ b/examples/client/client.c @@ -89,6 +89,9 @@ static const char *wolfsentry_config_path = NULL; #ifndef MAX_NON_BLOCK_SEC #define MAX_NON_BLOCK_SEC 10 #endif +/* How long a single wait for the socket blocks before the loop re-checks its + * overall budget. Short enough that the budget is still honoured closely. */ +#define NON_BLOCK_POLL_SEC 1 #define OCSP_STAPLING 1 #define OCSP_STAPLINGV2 2 @@ -1124,12 +1127,28 @@ static int ClientWrite(WOLFSSL* ssl, const char* msg, int msgSz, const char* str return err; } -static int ClientRead(WOLFSSL* ssl, char* reply, int replyLen, int mustRead, - const char* str, int exitWithRet) +/* Read a reply. On a non-blocking socket a WANT_READ only means the reply has + * not arrived yet, so wait for it rather than returning on the first poll - + * returning early lets the caller shut the connection down while the peer is + * still writing, which the peer then reports as a transport error. + * + * replyRequired says whether a missing reply is a failure, not whether one is + * worth waiting for: it selects the wait budget and controls whether giving up + * is reported as an error. */ +static int ClientRead(WOLFSSL* ssl, char* reply, int replyLen, + int replyRequired, const char* str, int exitWithRet) { int ret, err; char buffer[WOLFSSL_MAX_ERROR_SZ]; double start = current_time(1), elapsed; + /* A required reply gets the full non-blocking budget, an optional one the + * shorter of the two - MAX_NON_BLOCK_SEC is overridable and may be set + * below DEFAULT_TIMEOUT_SEC, which would otherwise invert the two. */ + double maxWait = MAX_NON_BLOCK_SEC; + + if (!replyRequired && DEFAULT_TIMEOUT_SEC < MAX_NON_BLOCK_SEC) { + maxWait = DEFAULT_TIMEOUT_SEC; + } do { err = 0; /* reset error */ @@ -1158,17 +1177,49 @@ static int ClientRead(WOLFSSL* ssl, char* reply, int replyLen, int mustRead, } } - if (mustRead && - (err == WOLFSSL_ERROR_WANT_READ - || err == WOLFSSL_ERROR_WANT_WRITE)) { + if (err == WOLFSSL_ERROR_WANT_READ + || err == WOLFSSL_ERROR_WANT_WRITE) { + int selectRet; + elapsed = current_time(0) - start; - if (elapsed > MAX_NON_BLOCK_SEC) { - LOG_ERROR("Nonblocking read timeout\n"); + if (elapsed > maxWait) { + if (replyRequired) { + LOG_ERROR("Nonblocking read timeout\n"); + } + ret = WOLFSSL_FATAL_ERROR; + break; + } + + /* Wait for the socket instead of spinning on it. */ + if (err == WOLFSSL_ERROR_WANT_WRITE) { + selectRet = tcp_select_tx(wolfSSL_get_fd(ssl), + NON_BLOCK_POLL_SEC); + } + else { + selectRet = tcp_select(wolfSSL_get_fd(ssl), + NON_BLOCK_POLL_SEC); + } + + #ifdef WOLFSSL_DTLS + /* A DTLS timeout means the peer's datagram was lost - let the + * library retransmit rather than waiting for something that is + * never coming (see NonBlockingSSL_Connect). */ + if (selectRet == TEST_TIMEOUT && wolfSSL_dtls(ssl)) { + if (wolfSSL_dtls_got_timeout(ssl) != WOLFSSL_SUCCESS) { + err = wolfSSL_get_error(ssl, WOLFSSL_FATAL_ERROR); + break; + } + } + else + #endif + /* select() itself failed - retrying would spin, not wait. */ + if (selectRet == TEST_SELECT_FAIL) { + LOG_ERROR("%s tcp_select error\n", str); ret = WOLFSSL_FATAL_ERROR; break; } } - } while ((mustRead && err == WOLFSSL_ERROR_WANT_READ) + } while (err == WOLFSSL_ERROR_WANT_READ || err == WOLFSSL_ERROR_WANT_WRITE #ifdef WOLFSSL_ASYNC_CRYPT || err == WC_NO_ERR_TRACE(WC_PENDING_E) @@ -1183,11 +1234,15 @@ static int ClientRead(WOLFSSL* ssl, char* reply, int replyLen, int mustRead, return err; } +/* replyRequired: whether a missing reply fails the exchange. See ClientRead. */ static int ClientWriteRead(WOLFSSL* ssl, const char* msg, int msgSz, - char* reply, int replyLen, int mustRead, + char* reply, int replyLen, int replyRequired, const char* str, int exitWithRet) { int ret = 0; + /* Which half of the exchange the error below came from - the message used + * to say SSL_write for a failure returned by ClientRead. */ + const char* stage = "SSL_write"; do { ret = ClientWrite(ssl, msg, msgSz, str, exitWithRet); @@ -1207,6 +1262,7 @@ static int ClientWriteRead(WOLFSSL* ssl, const char* msg, int msgSz, } else { LOG_ERROR("%s tcp_select error\n", str); + stage = "tcp_select"; if (!exitWithRet) err_sys("tcp_select failed"); else @@ -1214,8 +1270,9 @@ static int ClientWriteRead(WOLFSSL* ssl, const char* msg, int msgSz, break; } } - ret = ClientRead(ssl, reply, replyLen, mustRead, str, exitWithRet); - if (mustRead && ret != 0) { + stage = "SSL_read"; + ret = ClientRead(ssl, reply, replyLen, replyRequired, str, exitWithRet); + if (replyRequired && ret != 0) { if (!exitWithRet) err_sys("ClientRead failed"); else @@ -1224,9 +1281,11 @@ static int ClientWriteRead(WOLFSSL* ssl, const char* msg, int msgSz, break; } while (1); - if (ret != 0) { + /* A failed optional read is not an error - the caller asked for the reply + * only if one turned up - so do not log one. */ + if (ret != 0 && (replyRequired || XSTRCMP(stage, "SSL_read") != 0)) { char buffer[WOLFSSL_MAX_ERROR_SZ]; - LOG_ERROR("SSL_write%s msg error %d, %s\n", str, ret, + LOG_ERROR("%s%s msg error %d, %s\n", stage, str, ret, wolfSSL_ERR_error_string((unsigned long)ret, buffer)); } diff --git a/scripts/benchmark.test b/scripts/benchmark.test index f120a15dc15..801d57cde4f 100755 --- a/scripts/benchmark.test +++ b/scripts/benchmark.test @@ -53,17 +53,20 @@ if [ "$#" -lt 2 ]; then exit 1 fi -# Use unique benchmark port so it won't conflict with any other tests -bench_port=11113 +# Bind an ephemeral port and read it back from the ready file below, rather +# than hoping a fixed number is free. The ready file is per-run for the same +# reason - a fixed path is shared by any concurrent run. +bench_port=0 +ready_file=/tmp/wolfssl_server_ready$$ no_pid=-1 server_pid=$no_pid counter=0 client_result=-1 remove_ready_file() { - if test -e /tmp/wolfssl_server_ready; then + if test -e "$ready_file"; then echo "removing existing server_ready file" - rm /tmp/wolfssl_server_ready + rm "$ready_file" fi } @@ -96,7 +99,7 @@ remove_ready_file if [ $1 -eq 1 ] then # start server in loop mode with port - ./examples/server/server -i -p $bench_port $4 & + ./examples/server/server -i -p $bench_port -R "$ready_file" $4 & server_pid=$! fi @@ -104,7 +107,7 @@ fi if [ $1 -eq 2 ] then # start server in loop mode, non-blocking, benchmark throughput with port - ./examples/server/server -i -N -B $2 -p $bench_port $4 & + ./examples/server/server -i -N -B $2 -p $bench_port -R "$ready_file" $4 & server_pid=$! fi @@ -112,11 +115,23 @@ fi # with "time", bear in mind that those 2 seconds will be reflected in # the "real" time. echo "Waiting for server_ready file..." -while [ ! -s /tmp/wolfssl_server_ready -a "$counter" -lt 20 ]; do +while [ ! -s "$ready_file" -a "$counter" -lt 20 ]; do sleep 0.1 counter=$((counter+ 1)) done +if [ ! -s "$ready_file" ]; then + echo "Server never reported a port!" + do_cleanup + exit 1 +fi + +# sleep for an additional 0.1 to mitigate race on write/read of $ready_file: +sleep 0.1 + +bench_port=$(cat "$ready_file") +echo "Server listening on port $bench_port" + # benchmark connections if [ $1 -eq 1 ] then diff --git a/scripts/dtls.test b/scripts/dtls.test index 9a1f621ebfb..eb1d835bf81 100755 --- a/scripts/dtls.test +++ b/scripts/dtls.test @@ -22,6 +22,9 @@ DO_EXTENDED_SERVER_PERMUTATION_TEST=${DO_EXTENDED_SERVER_PERMUTATION_TEST:-0} DO_DELAY_TEST=${DO_DELAY_TEST:-0} # An example use would be: DTLS_VERSION=-v3 scripts/dtls.test +# Prerequisites, each of which makes the script skip (exit 77) when missing: +# the udp_proxy helper, python3, and a server build that can report an +# ephemeral port through -p 0 with -R. # Note the output also consists of a single PCAP file which has a set of # three packets (1-byte, strlen()-byte, 1-byte payload) deliniating each test. @@ -40,6 +43,23 @@ if ./examples/server/server -? 2>&1 | grep "Server not compiled in!" ; then exit 77 fi +# Every test runs through the udp_proxy helper. Without it each one fails at the +# client, so skip the script rather than report a wall of failures. +# command -v matches how the proxy is actually launched below: it searches +# $PATH for a bare name and tests executability for a path with a slash. An +# extra -x test would accept a ./udp_proxy that the launch could never find. +if ! command -v "$UDP_PROXY_BIN" >/dev/null 2>&1; then + echo "skipping dtls.test because $UDP_PROXY_BIN was not found." 1>&2 + exit 77 +fi + +# python3 picks the proxy port for every test, on top of generating the +# permutation and delay lists, so it is a hard requirement of the whole script. +if ! command -v python3 >/dev/null 2>&1; then + echo 'skipping dtls.test because python3 was not found.' 1>&2 + exit 77 +fi + # bwrap execution environment to avoid port conflicts if [ "${AM_BWRAPPED-}" != "yes" ]; then bwrap_path="$(command -v bwrap)" @@ -73,20 +93,117 @@ cleanup () { sleep 1 kill $TCPDUMP_PID fi + + rm -f "$READY_FILE" } trap cleanup err exit -PROXY_PORT=1234 -SERVER_PORT=4321 +# Nothing ever binds the marker port - the three datagrams sent to it only tag +# the capture - so a fixed number is safe here. The server and proxy ports are +# chosen per test below. +MARKER_PORT=4321 KEY_UPDATE_SIZE=35 NUM_TESTS_FAILED=0 NUM_TESTS_RUN=0 +# The server writes the port it bound to this file. The pid keeps concurrent +# runs from sharing one. +READY_FILE="$WOLFSSL_ROOT/wolfssl_dtls_ready$$" + +HAVE_SS=no +command -v ss >/dev/null 2>&1 && HAVE_SS=yes + +# A free UDP port for the proxy. udp_proxy has no equivalent of the server's -R +# ready file, so the port is picked here and the proxy polled below until it is +# actually bound. +free_udp_port() { + python3 -c 'import socket +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s.bind(("127.0.0.1", 0)) +print(s.getsockname()[1]) +s.close()' +} + +# A usable port is a non-zero decimal below 65536. Worth checking rather than +# assuming: a SINGLE_THREADED build never resolves the ephemeral port in +# udp_accept(), so the ready file legitimately contains 0 there, and pointing +# the proxy at port 0 would report every test as a DTLS failure instead. +valid_port() { + case "$1" in + ''|*[!0-9]*) return 1 ;; + esac + [ "$1" -gt 0 ] && [ "$1" -lt 65536 ] +} + +# Wait for $1 to be bound for UDP. Where ss is not available to be asked this +# cannot be checked, so say so rather than let a blind sleep pass for a check. +wait_udp_bound() { + local counter=0 + + if [ "$HAVE_SS" != "yes" ]; then + echo "[warning] ss not available - cannot confirm the proxy bound $1" + sleep 0.5 + return 0 + fi + + while [ "$counter" -lt 50 ]; do + # Match on the local port specifically - a bare grep for the number can + # be satisfied by the peer column or an unrelated socket. + if [ -n "$(ss -lunH "sport = :$1" 2>/dev/null)" ]; then + return 0 + fi + sleep 0.1 + counter=$((counter + 1)) + done + return 1 +} if [ "$DTLS_VERSION" = "-v4" ]; then UDP_PROXY_EXTRA_ARGS="-u" fi +# Everything below depends on the server being able to report the port it bound. +# A SINGLE_THREADED or WOLFSSL_TIRTOS build never resolves an ephemeral port in +# udp_accept(), and a NO_FILESYSTEM or NETOS build compiles write_ready_file() +# to a no-op, so probe the capability once here rather than rediscovering it on +# every one of the tests below. +check_ready_file_support() { + local probe_pid counter port + + rm -f "$READY_FILE" + $WOLFSSL_ROOT/examples/server/server -u -p 0 -R "$READY_FILE" $DTLS_VERSION \ + >/dev/null 2>&1 & + probe_pid=$! + + counter=0 + while [ ! -s "$READY_FILE" -a "$counter" -lt 50 ]; do + if ! kill -0 "$probe_pid" 2>&- && [ ! -s "$READY_FILE" ]; then + break + fi + sleep 0.1 + counter=$((counter + 1)) + done + + port="" + if [ -s "$READY_FILE" ]; then + sleep 0.1 + port=$(cat "$READY_FILE") + fi + + kill -9 "$probe_pid" 2>&- + wait "$probe_pid" 2>&- + rm -f "$READY_FILE" + + valid_port "$port" +} + +if ! check_ready_file_support; then + echo 'skipping dtls.test because the server cannot report an ephemeral port' 1>&2 + echo 'with -p 0 -R (needs threads and a filesystem in the build).' 1>&2 + exit 77 +fi + + # $WOLFSSL_ROOT/tests/unit.test tests/test-dtls13.conf set -o pipefail @@ -94,21 +211,72 @@ prepend() { # Usage: cmd 2>&1 | prepend "sometext " while read line; do echo "${1}${line}"; done } +# Record a failed test. Plain assignment rather than ((n++)) so the count is +# not also the command's exit status. +fail_test() { + echo "***Test failed***${1:+ $1}" + NUM_TESTS_FAILED=$((NUM_TESTS_FAILED + 1)) +} + run_test() { # usage: run_test "" "" "" "" ((NUM_TESTS_RUN++)) - echo "" | nc -u -w 1 127.0.0.1 $SERVER_PORT # This is a marker for the PCAP file - echo "$1" | nc -u -w 1 127.0.0.1 $SERVER_PORT # This is a marker for the PCAP file - echo "" | nc -u -w 1 127.0.0.1 $SERVER_PORT # This is a marker for the PCAP file + echo "" | nc -u -w 1 127.0.0.1 $MARKER_PORT # This is a marker for the PCAP file + echo "$1" | nc -u -w 1 127.0.0.1 $MARKER_PORT # This is a marker for the PCAP file + echo "" | nc -u -w 1 127.0.0.1 $MARKER_PORT # This is a marker for the PCAP file echo -e "\n${1}\n" - stdbuf -oL -eL $WOLFSSL_ROOT/examples/server/server -u -p$SERVER_PORT $DTLS_VERSION $3 2>&1 | prepend "[server] " & - sleep 0.2 + + # Bind an ephemeral port and wait for the server to report it, rather than + # taking a fixed port and assuming a fixed sleep covers the bind. + rm -f "$READY_FILE" + stdbuf -oL -eL $WOLFSSL_ROOT/examples/server/server -u -p 0 -R "$READY_FILE" $DTLS_VERSION $3 2>&1 | prepend "[server] " & + # The pid is prepend's, but it exits on EOF when the server dies, so it + # still tells us whether the server is alive - and saves the full poll when + # it is not, which matters at 5040 tests with the extended permutations. + SERVER_JOB_PID=$! + + counter=0 + while [ ! -s "$READY_FILE" -a "$counter" -lt 50 ]; do + if ! kill -0 "$SERVER_JOB_PID" 2>&- && [ ! -s "$READY_FILE" ]; then + fail_test "server exited before creating the ready file" + kill_server + return + fi + sleep 0.1 + counter=$((counter + 1)) + done + if [ ! -s "$READY_FILE" ]; then + fail_test "server did not become ready" + kill_server + return + fi + + # sleep for an additional 0.1 to mitigate race on write/read of $READY_FILE: + sleep 0.1 + + SERVER_PORT=$(cat "$READY_FILE") + if ! valid_port "$SERVER_PORT"; then + fail_test "server reported unusable port '$SERVER_PORT'" + kill_server + return + fi + + PROXY_PORT=$(free_udp_port) + if ! valid_port "$PROXY_PORT"; then + fail_test "could not obtain a free UDP port for the proxy" + kill_server + return + fi stdbuf -oL -eL $UDP_PROXY_BIN -p $PROXY_PORT -s 127.0.0.1:$SERVER_PORT $UDP_PROXY_EXTRA_ARGS $2 2>&1 | prepend "[udp-proxy] " & - sleep 0.2 + if ! wait_udp_bound $PROXY_PORT; then + fail_test "udp-proxy did not bind port $PROXY_PORT" + kill_server + return + fi + # Wrap this command in a timeout so that a deadlock won't bring down the entire test $TIMEOUT_KILL_1M stdbuf -oL -eL $WOLFSSL_ROOT/examples/client/client -u -p$PROXY_PORT $DTLS_VERSION $4 2>&1 | prepend "[client] " if [ $? != 0 ]; then - echo "***Test failed***" - ((NUM_TESTS_FAILED++)) + fail_test fi kill_server } @@ -167,7 +335,9 @@ EOF } echo "Starting capture" -tcpdump -i lo -n port ${SERVER_PORT} -w ${PCAP_FILENAME} -U & +# The server port is ephemeral and changes per test, so capture by protocol and +# address rather than one fixed port. +tcpdump -i lo -n udp and host 127.0.0.1 -w ${PCAP_FILENAME} -U & TCPDUMP_PID=$! sleep 0.5 diff --git a/scripts/dtlscid.test b/scripts/dtlscid.test index a44ee60340a..93a4d98d792 100755 --- a/scripts/dtlscid.test +++ b/scripts/dtlscid.test @@ -7,8 +7,10 @@ # wrapper and orphan the server it was meant to kill. if command -v timeout >/dev/null 2>&1; then TIMEOUT_KILL_2M="timeout -s KILL 2m" + TIMEOUT_KILL_1M="timeout -s KILL 1m" else TIMEOUT_KILL_2M="" + TIMEOUT_KILL_1M="" fi @@ -52,6 +54,7 @@ cleanup () { echo "Killing server $SERVER_PID" kill $SERVER_PID fi + rm -f "$READY_FILE" } trap cleanup err exit @@ -61,19 +64,133 @@ SCID="BB" HEXCID=$(printf $CCID | od -An -tx1 | tr -d ' \n') HEXSCID=$(printf $SCID | od -An -tx1 | tr -d ' \n') WOLFSSL_ROOT=$(pwd) +# The server reports the port it bound in this file. The pid keeps concurrent +# runs - a second "make check" in another source tree, say - from sharing one. +READY_FILE="$WOLFSSL_ROOT/wolfssl_dtlscid_ready$$" + +# A usable port is a non-zero decimal below 65536. Worth checking rather than +# assuming: a SINGLE_THREADED or WOLFSSL_TIRTOS build never resolves the +# ephemeral port in udp_accept(), so the ready file legitimately contains 0. +valid_port() { + case "$1" in + ''|*[!0-9]*) return 1 ;; + esac + [ "$1" -gt 0 ] && [ "$1" -lt 65536 ] +} + +# The test below depends on the server being able to report the port it bound. +# Besides the SINGLE_THREADED/WOLFSSL_TIRTOS case above, a NO_FILESYSTEM or +# NETOS build compiles write_ready_file() to a no-op and never creates the file +# at all. Probe the capability once and skip, rather than report a build-wide +# limitation as a CID test failure. +check_ready_file_support() { + local probe_pid counter port + + rm -f "$READY_FILE" + $WOLFSSL_ROOT/examples/server/server -v4 -u -p 0 -R "$READY_FILE" \ + >/dev/null 2>&1 & + probe_pid=$! + + counter=0 + while [ ! -s "$READY_FILE" -a "$counter" -lt 50 ]; do + if ! kill -0 "$probe_pid" 2>&- && [ ! -s "$READY_FILE" ]; then + break + fi + sleep 0.1 + counter=$((counter + 1)) + done + + port="" + if [ -s "$READY_FILE" ]; then + sleep 0.1 + port=$(cat "$READY_FILE") + fi + + kill -9 "$probe_pid" 2>&- + wait "$probe_pid" 2>&- + rm -f "$READY_FILE" + + valid_port "$port" +} + +if ! check_ready_file_support; then + echo 'skipping dtlscid.test because the server cannot report an ephemeral' 1>&2 + echo 'port with -p 0 -R (needs threads and a filesystem in the build).' 1>&2 + exit 77 +fi + +# Report a failed check, with the output of both sides, and stop the test. +fail () { + echo "$1" + echo "--- server output ---" + cat "$SERVER_FILE" + echo "--- client output ---" + cat "$CLIENT_FILE" + rm -f "$SERVER_FILE" "$CLIENT_FILE" + exit 1 +} test_cid () { echo "Running test_cid" SERVER_FILE=$(mktemp) CLIENT_FILE=$(mktemp) - $TIMEOUT_KILL_2M $WOLFSSL_ROOT/examples/server/server -v4 -u --cid $SCID 1> $SERVER_FILE & + rm -f "$READY_FILE" + + # Bind an ephemeral port (-p 0) and wait for the server to report it, rather + # than taking the default port and assuming a fixed sleep is long enough for + # the bind. The default port also collides with any other test using it when + # the network namespace could not be unshared above. + $TIMEOUT_KILL_2M $WOLFSSL_ROOT/examples/server/server -v4 -u \ + --cid $SCID -p 0 -R "$READY_FILE" 1> $SERVER_FILE & SERVER_PID=$! - sleep 0.2 - $TIMEOUT_KILL_2M $WOLFSSL_ROOT/examples/client/client -v4 -u --cid $CCID 1> $CLIENT_FILE + + counter=0 + while [ ! -s "$READY_FILE" -a "$counter" -lt 50 ]; do + sleep 0.1 + counter=$((counter + 1)) + done + if [ ! -s "$READY_FILE" ]; then + fail "server did not become ready" + fi + + # sleep for an additional 0.1 to mitigate race on write/read of $READY_FILE: + sleep 0.1 + + PORT=$(cat "$READY_FILE") + if ! valid_port "$PORT"; then + fail "server reported unusable port '$PORT'" + fi + echo "Server listening on port $PORT" + + # Bound below the server's timeout: a DTLS client with nothing to talk to + # retransmits for over two minutes, so without this the server would be + # killed first and the client left running against nothing. + $TIMEOUT_KILL_1M $WOLFSSL_ROOT/examples/client/client -v4 -u \ + --cid $CCID -p $PORT 1> $CLIENT_FILE + CLIENT_RESULT=$? + if [ $CLIENT_RESULT != 0 ]; then + fail "client failed with $CLIENT_RESULT" + fi + wait $SERVER_PID + SERVER_RESULT=$? SERVER_PID= - grep "Sending CID is ${HEXSCID}" $CLIENT_FILE > /dev/null - grep "Sending CID is ${HEXCID}" $SERVER_FILE > /dev/null + if [ $SERVER_RESULT != 0 ]; then + fail "server failed with $SERVER_RESULT" + fi + + # Each side reports the CID it sends, which is the one its peer chose. These + # checks are the point of the test, so a mismatch has to fail the script - + # as the last commands in the function their status was previously + # discarded by the echo below. + if ! grep -q "Sending CID is ${HEXSCID}" $CLIENT_FILE; then + fail "client did not report sending CID ${HEXSCID}" + fi + if ! grep -q "Sending CID is ${HEXCID}" $SERVER_FILE; then + fail "server did not report sending CID ${HEXCID}" + fi + + rm -f "$SERVER_FILE" "$CLIENT_FILE" echo "test_cid has passed" } diff --git a/scripts/multi-msg-record.py b/scripts/multi-msg-record.py index 8145f62f030..6832bc3e841 100755 --- a/scripts/multi-msg-record.py +++ b/scripts/multi-msg-record.py @@ -193,22 +193,19 @@ def _parse_hs_types(data): return msgs -def _get_free_port(): - """Get an available TCP port.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("127.0.0.1", 0)) - return s.getsockname()[1] - - def _listen_socket(): - """Bind a listening TCP socket on localhost with the standard test timeout.""" - port = _get_free_port() + """Bind a listening TCP socket on localhost with the standard test timeout. + + The port is read back from the socket that keeps it, rather than from a + throwaway one that is closed first: closing it leaves a window in which + anything else on the machine can take the port before the real bind. + """ srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - srv.bind(("127.0.0.1", port)) + srv.bind(("127.0.0.1", 0)) srv.listen(1) srv.settimeout(15) - return srv, port + return srv, srv.getsockname()[1] def _run_wolf_client(port, version, cipher, extra=()): diff --git a/scripts/ocsp-responder-openssl-interop.test b/scripts/ocsp-responder-openssl-interop.test index 83e96897baa..751532e8f47 100755 --- a/scripts/ocsp-responder-openssl-interop.test +++ b/scripts/ocsp-responder-openssl-interop.test @@ -150,6 +150,18 @@ print_responder_logs() { # already bound, to avoid the birthday problem in case other # instances are sharing this host. +# True when anything holds the port - listening or established, TCP or UDP. +# nc -z below only finds TCP listeners, so on its own it misses a UDP responder +# socket and a port held by an unrelated outbound connection, both of which +# still make bind() fail. +port_bound() { + if command -v ss >/dev/null 2>&1; then + [ -n "$(ss -Hantu "sport = :$1" 2>/dev/null)" ] + else + return 1 + fi +} + get_first_free_port() { local ret="$1" local scanned=0 @@ -157,7 +169,7 @@ get_first_free_port() { if [[ "$ret" -ge 65536 ]]; then ret=1024 fi - if ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then + if ! port_bound "$ret" && ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then break fi ret=$((ret+1)) diff --git a/scripts/ocsp-stapling-with-ca-as-responder.test b/scripts/ocsp-stapling-with-ca-as-responder.test index 7845e2f523c..61fece79605 100755 --- a/scripts/ocsp-stapling-with-ca-as-responder.test +++ b/scripts/ocsp-stapling-with-ca-as-responder.test @@ -206,6 +206,18 @@ ca=certs/external/DigiCertGlobalRootCA.pem # already bound, to avoid the birthday problem in case other # instances are sharing this host. +# True when anything holds the port - listening or established, TCP or UDP. +# nc -z below only finds TCP listeners, so on its own it misses a UDP responder +# socket and a port held by an unrelated outbound connection, both of which +# still make bind() fail. +port_bound() { + if command -v ss >/dev/null 2>&1; then + [ -n "$(ss -Hantu "sport = :$1" 2>/dev/null)" ] + else + return 1 + fi +} + get_first_free_port() { local ret="$1" local scanned=0 @@ -213,7 +225,7 @@ get_first_free_port() { if [[ "$ret" -ge 65536 ]]; then ret=1024 fi - if ! nc -w 1 -z 127.0.0.1 "$ret"; then + if ! port_bound "$ret" && ! nc -w 1 -z 127.0.0.1 "$ret"; then break fi ret=$((ret+1)) diff --git a/scripts/ocsp-stapling-with-wolfssl-responder.test b/scripts/ocsp-stapling-with-wolfssl-responder.test index 1b9fcfe0127..eefcbf064a3 100755 --- a/scripts/ocsp-stapling-with-wolfssl-responder.test +++ b/scripts/ocsp-stapling-with-wolfssl-responder.test @@ -352,6 +352,18 @@ fi # already bound, to avoid the birthday problem in case other # instances are sharing this host. +# True when anything holds the port - listening or established, TCP or UDP. +# nc -z below only finds TCP listeners, so on its own it misses a UDP responder +# socket and a port held by an unrelated outbound connection, both of which +# still make bind() fail. +port_bound() { + if command -v ss >/dev/null 2>&1; then + [ -n "$(ss -Hantu "sport = :$1" 2>/dev/null)" ] + else + return 1 + fi +} + get_first_free_port() { local ret="$1" local scanned=0 @@ -359,7 +371,7 @@ get_first_free_port() { if [[ "$ret" -ge 65536 ]]; then ret=1024 fi - if ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then + if ! port_bound "$ret" && ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then break fi ret=$((ret+1)) diff --git a/scripts/ocsp-stapling.test b/scripts/ocsp-stapling.test index 83aa25df5ea..79f92c550df 100755 --- a/scripts/ocsp-stapling.test +++ b/scripts/ocsp-stapling.test @@ -278,6 +278,18 @@ fi # already bound, to avoid the birthday problem in case other # instances are sharing this host. +# True when anything holds the port - listening or established, TCP or UDP. +# nc -z below only finds TCP listeners, so on its own it misses a UDP responder +# socket and a port held by an unrelated outbound connection, both of which +# still make bind() fail. +port_bound() { + if command -v ss >/dev/null 2>&1; then + [ -n "$(ss -Hantu "sport = :$1" 2>/dev/null)" ] + else + return 1 + fi +} + get_first_free_port() { local ret="$1" local scanned=0 @@ -285,7 +297,7 @@ get_first_free_port() { if [[ "$ret" -ge 65536 ]]; then ret=1024 fi - if ! nc -w 1 -z $V4V6_FLAG $LOCALHOST_FOR_NC "$ret"; then + if ! port_bound "$ret" && ! nc -w 1 -z $V4V6_FLAG $LOCALHOST_FOR_NC "$ret"; then break fi ret=$((ret+1)) @@ -534,10 +546,15 @@ generate_port() { # Also check if the port is already bound on this system if [ $collision -eq 0 ]; then + # TCP and UDP, established as well as listening, matched on the + # local port. -lnt saw only TCP listeners, so it was blind to the + # UDP sockets the DTLS cases here bind and to a port held by an + # unrelated outbound connection - both of which still block bind. + # A false positive only costs another iteration of this loop. if command -v ss &>/dev/null; then - ss -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + [ -n "$(ss -Hantu "sport = :${p}" 2>/dev/null)" ] && collision=1 elif command -v netstat &>/dev/null; then - netstat -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + netstat -an 2>/dev/null | grep -q "[:.]${p}[[:space:]]" && collision=1 fi fi diff --git a/scripts/ocsp-stapling2.test b/scripts/ocsp-stapling2.test index 3fdf66cd5d6..567770fce85 100755 --- a/scripts/ocsp-stapling2.test +++ b/scripts/ocsp-stapling2.test @@ -261,6 +261,18 @@ fi # already bound, to avoid the birthday problem in case other # instances are sharing this host. +# True when anything holds the port - listening or established, TCP or UDP. +# nc -z below only finds TCP listeners, so on its own it misses a UDP responder +# socket and a port held by an unrelated outbound connection, both of which +# still make bind() fail. +port_bound() { + if command -v ss >/dev/null 2>&1; then + [ -n "$(ss -Hantu "sport = :$1" 2>/dev/null)" ] + else + return 1 + fi +} + get_first_free_port() { local ret="$1" local scanned=0 @@ -268,7 +280,7 @@ get_first_free_port() { if [[ "$ret" -ge 65536 ]]; then ret=1024 fi - if ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then + if ! port_bound "$ret" && ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then break fi ret=$((ret+1)) @@ -532,10 +544,15 @@ generate_port() { # Also check if the port is already bound on this system if [ $collision -eq 0 ]; then + # TCP and UDP, established as well as listening, matched on the + # local port. -lnt saw only TCP listeners, so it was blind to the + # UDP sockets the DTLS cases here bind and to a port held by an + # unrelated outbound connection - both of which still block bind. + # A false positive only costs another iteration of this loop. if command -v ss &>/dev/null; then - ss -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + [ -n "$(ss -Hantu "sport = :${p}" 2>/dev/null)" ] && collision=1 elif command -v netstat &>/dev/null; then - netstat -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + netstat -an 2>/dev/null | grep -q "[:.]${p}[[:space:]]" && collision=1 fi fi diff --git a/scripts/ocsp-stapling_tls13multi.test b/scripts/ocsp-stapling_tls13multi.test index 59c93eebba2..bb1b0a32c9e 100755 --- a/scripts/ocsp-stapling_tls13multi.test +++ b/scripts/ocsp-stapling_tls13multi.test @@ -291,6 +291,18 @@ fi # already bound, to avoid the birthday problem in case other # instances are sharing this host. +# True when anything holds the port - listening or established, TCP or UDP. +# nc -z below only finds TCP listeners, so on its own it misses a UDP responder +# socket and a port held by an unrelated outbound connection, both of which +# still make bind() fail. +port_bound() { + if command -v ss >/dev/null 2>&1; then + [ -n "$(ss -Hantu "sport = :$1" 2>/dev/null)" ] + else + return 1 + fi +} + get_first_free_port() { local ret="$1" local scanned=0 @@ -298,7 +310,7 @@ get_first_free_port() { if [[ "$ret" -ge 65536 ]]; then ret=1024 fi - if ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then + if ! port_bound "$ret" && ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then break fi ret=$((ret+1)) diff --git a/scripts/openssl.test b/scripts/openssl.test index f64e22fe836..84be7eca5bd 100755 --- a/scripts/openssl.test +++ b/scripts/openssl.test @@ -93,10 +93,15 @@ generate_port() { # Also check if the port is already bound on this system if [ "$collision" -eq 0 ]; then + # TCP and UDP, established as well as listening, matched on the + # local port. -lnt saw only TCP listeners, so it was blind to the + # UDP sockets the DTLS cases here bind and to a port held by an + # unrelated outbound connection - both of which still block bind. + # A false positive only costs another iteration of this loop. if command -v ss &>/dev/null; then - ss -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + [ -n "$(ss -Hantu "sport = :${p}" 2>/dev/null)" ] && collision=1 elif command -v netstat &>/dev/null; then - netstat -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + netstat -an 2>/dev/null | grep -q "[:.]${p}[[:space:]]" && collision=1 fi fi diff --git a/scripts/openssl_srtp.test b/scripts/openssl_srtp.test index 5fb907fb3a8..a882c3f81df 100755 --- a/scripts/openssl_srtp.test +++ b/scripts/openssl_srtp.test @@ -71,10 +71,15 @@ generate_port() { # Also check if the port is already bound on this system if [ "$collision" -eq 0 ]; then + # TCP and UDP, established as well as listening, matched on the + # local port. -lnt saw only TCP listeners, so it was blind to the + # UDP sockets the DTLS cases here bind and to a port held by an + # unrelated outbound connection - both of which still block bind. + # A false positive only costs another iteration of this loop. if command -v ss &>/dev/null; then - ss -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + [ -n "$(ss -Hantu "sport = :${p}" 2>/dev/null)" ] && collision=1 elif command -v netstat &>/dev/null; then - netstat -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + netstat -an 2>/dev/null | grep -q "[:.]${p}[[:space:]]" && collision=1 fi fi diff --git a/scripts/rsapss.test b/scripts/rsapss.test index 53e6da2371c..b80325f0697 100755 --- a/scripts/rsapss.test +++ b/scripts/rsapss.test @@ -95,10 +95,15 @@ generate_port() { # Also check if the port is already bound on this system if [ "$collision" -eq 0 ]; then + # TCP and UDP, established as well as listening, matched on the + # local port. -lnt saw only TCP listeners, so it was blind to the + # UDP sockets the DTLS cases here bind and to a port held by an + # unrelated outbound connection - both of which still block bind. + # A false positive only costs another iteration of this loop. if command -v ss &>/dev/null; then - ss -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + [ -n "$(ss -Hantu "sport = :${p}" 2>/dev/null)" ] && collision=1 elif command -v netstat &>/dev/null; then - netstat -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + netstat -an 2>/dev/null | grep -q "[:.]${p}[[:space:]]" && collision=1 fi fi diff --git a/src/include.am b/src/include.am index 12133f62e33..6361d15ff28 100644 --- a/src/include.am +++ b/src/include.am @@ -115,6 +115,9 @@ if BUILD_AESNI src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S +if BUILD_AESXTS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_x86_asm.S +endif else if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S @@ -277,6 +280,9 @@ if BUILD_AESNI src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S +if BUILD_AESXTS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_x86_asm.S +endif else if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S @@ -613,6 +619,9 @@ if BUILD_AESNI src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S +if BUILD_AESXTS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_x86_asm.S +endif else if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S @@ -1005,6 +1014,9 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_x86_64_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S +if BUILD_AESXTS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_x86_asm.S +endif else if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S @@ -2011,6 +2023,9 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_x86_64_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S +if BUILD_AESXTS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_x86_asm.S +endif else if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index e96e4b1842f..50c80ed0418 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -18176,16 +18176,19 @@ int wc_AesXtsDecryptSector(XtsAes* aes, byte* out, const byte* in, word32 sz, } #endif -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) #if defined(USE_INTEL_SPEEDUP_FOR_AES) && !defined(USE_INTEL_SPEEDUP) #define USE_INTEL_SPEEDUP #endif -#if defined(USE_INTEL_SPEEDUP) +/* aes_xts_x86_asm.S provides the AES-NI routines for 32-bit x86 but has no + * AVX1/VAES/AVX512 variants, so do not let the wider paths be declared there - + * they would be undefined at link time. */ +#if defined(USE_INTEL_SPEEDUP) && !defined(WOLFSSL_X86_BUILD) #define HAVE_INTEL_AVX1 #define HAVE_INTEL_AVX2 -#endif /* USE_INTEL_SPEEDUP */ +#endif /* USE_INTEL_SPEEDUP && !WOLFSSL_X86_BUILD */ void AES_XTS_encrypt_aesni(const unsigned char *in, unsigned char *out, word32 sz, const unsigned char* i, const unsigned char* key, @@ -18294,7 +18297,7 @@ void AES_XTS_decrypt_update_avx512(const unsigned char *in, unsigned char *out, #endif /* HAVE_INTEL_AVX512 */ #endif /* HAVE_AES_DECRYPT */ -#endif /* WOLFSSL_AESNI && !WOLFSSL_X86_BUILD */ +#endif /* WOLFSSL_AESNI */ #ifdef HAVE_AES_ECB #if (!defined(WOLFSSL_ARMASM) || (!defined(__aarch64__) && \ @@ -18570,7 +18573,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds); ret = 0; #endif -#elif defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#elif defined(WOLFSSL_AESNI) if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) @@ -18692,7 +18695,7 @@ int wc_AesXtsEncryptInit(XtsAes* xaes, const byte* i, word32 iSz, stream->bytes_crypted_with_this_tweak = 0; { -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) @@ -18731,7 +18734,7 @@ int wc_AesXtsEncryptInit(XtsAes* xaes, const byte* i, word32 iSz, RESTORE_VECTOR_REGISTERS(); } else -#endif /* WOLFSSL_AESNI && !WOLFSSL_X86_BUILD */ +#endif /* WOLFSSL_AESNI */ { ret = AesXtsInitTweak_sw(xaes, stream->tweak_block); } @@ -18761,7 +18764,7 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s { int ret; -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) Aes *aes; #endif @@ -18769,7 +18772,7 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s return BAD_FUNC_ARG; } -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) aes = &xaes->aes; #endif @@ -18805,7 +18808,7 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s } #endif { -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) @@ -18848,7 +18851,7 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s RESTORE_VECTOR_REGISTERS(); } else -#endif /* WOLFSSL_AESNI && !WOLFSSL_X86_BUILD */ +#endif /* WOLFSSL_AESNI */ { ret = AesXtsEncryptUpdate_sw(xaes, out, in, sz, stream->tweak_block); } @@ -19134,7 +19137,7 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds); ret = 0; #endif -#elif defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#elif defined(WOLFSSL_AESNI) if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) @@ -19259,7 +19262,7 @@ int wc_AesXtsDecryptInit(XtsAes* xaes, const byte* i, word32 iSz, stream->bytes_crypted_with_this_tweak = 0; { -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) @@ -19298,7 +19301,7 @@ int wc_AesXtsDecryptInit(XtsAes* xaes, const byte* i, word32 iSz, RESTORE_VECTOR_REGISTERS(); } else -#endif /* WOLFSSL_AESNI && !WOLFSSL_X86_BUILD */ +#endif /* WOLFSSL_AESNI */ { ret = AesXtsInitTweak_sw(xaes, stream->tweak_block); } @@ -19326,7 +19329,7 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s struct XtsAesStreamData *stream) { int ret; -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) Aes *aes; #endif @@ -19334,7 +19337,7 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s return BAD_FUNC_ARG; } -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS aes = &xaes->aes_decrypt; #else @@ -19364,7 +19367,7 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s #endif { -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) @@ -19407,7 +19410,7 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s RESTORE_VECTOR_REGISTERS(); } else -#endif /* WOLFSSL_AESNI && !WOLFSSL_X86_BUILD */ +#endif /* WOLFSSL_AESNI */ { ret = AesXtsDecryptUpdate_sw(xaes, out, in, sz, stream->tweak_block); diff --git a/wolfcrypt/src/aes_xts_x86_asm.S b/wolfcrypt/src/aes_xts_x86_asm.S new file mode 100644 index 00000000000..3a35f9bb169 --- /dev/null +++ b/wolfcrypt/src/aes_xts_x86_asm.S @@ -0,0 +1,841 @@ +/* aes_xts_x86_asm + * + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_AES_XTS_X86_ASM_S + +#include + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ + +#ifdef WOLFSSL_AES_XTS +#ifdef WOLFSSL_X86_BUILD + # 32-bit (i386) AES-NI AES-XTS: single-block ports of the x86_64 + # AES_XTS_*_aesni routines (xmm0-7, cdecl ABI); GF const on stack for PIC. + # void AES_XTS_init_aesni(unsigned char* i, const unsigned char* tweak_key, + # int tweak_nr); +.text +.globl AES_XTS_init_aesni +WC_ASM_ATT_HIDDEN(AES_XTS_init_aesni) +.type AES_XTS_init_aesni,@function +.align 16 +AES_XTS_init_aesni: + movl 4(%esp), %eax + movdqu (%eax), %xmm2 + movl 8(%esp), %ecx + pxor (%ecx), %xmm2 + movdqu 16(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 32(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 48(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 64(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 80(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 96(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 112(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 128(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 144(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + cmpl $11, 12(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_init_aesni_enclast_1 + aesenc %xmm0, %xmm2 + movdqu 176(%ecx), %xmm1 + aesenc %xmm1, %xmm2 + cmpl $13, 12(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_init_aesni_enclast_1 + aesenc %xmm0, %xmm2 + movdqu 208(%ecx), %xmm1 + aesenc %xmm1, %xmm2 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_init_aesni_enclast_1: + aesenclast %xmm0, %xmm2 + movdqu %xmm2, (%eax) + ret +.size AES_XTS_init_aesni,.-AES_XTS_init_aesni + # void AES_XTS_encrypt_aesni(const unsigned char* in, unsigned char* out, + # word32 sz, const unsigned char* i, const unsigned char* key, + # const unsigned char* key2, int nr); +.text +.globl AES_XTS_encrypt_aesni +WC_ASM_ATT_HIDDEN(AES_XTS_encrypt_aesni) +.type AES_XTS_encrypt_aesni,@function +.align 16 +AES_XTS_encrypt_aesni: + pushl %edi + pushl %ebx + subl $16, %esp + movl $0x87, (%esp) + movl $0x01, 4(%esp) + movl $0x01, 8(%esp) + movl $0x01, 12(%esp) + movdqu (%esp), %xmm6 + movl 40(%esp), %eax + movdqu (%eax), %xmm2 + movl 48(%esp), %ecx + pxor (%ecx), %xmm2 + movdqu 16(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 32(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 48(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 64(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 80(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 96(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 112(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 128(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 144(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + cmpl $11, 52(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_encrypt_aesni_enclast_2 + aesenc %xmm0, %xmm2 + movdqu 176(%ecx), %xmm1 + aesenc %xmm1, %xmm2 + cmpl $13, 52(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_encrypt_aesni_enclast_2 + aesenc %xmm0, %xmm2 + movdqu 208(%ecx), %xmm1 + aesenc %xmm1, %xmm2 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_encrypt_aesni_enclast_2: + aesenclast %xmm0, %xmm2 + xorl %edi, %edi + movl 36(%esp), %edx + andl $0xfffffff0, %edx +L_AES_XTS_encrypt_aesni_loop: + cmpl %edx, %edi + jae L_AES_XTS_encrypt_aesni_loop_done + movl 28(%esp), %eax + movdqu (%eax,%edi,1), %xmm3 + pxor %xmm2, %xmm3 + movl 44(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + cmpl $11, 52(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_encrypt_aesni_enclast_3 + aesenc %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + cmpl $13, 52(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_encrypt_aesni_enclast_3 + aesenc %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_encrypt_aesni_enclast_3: + aesenclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) + movdqa %xmm2, %xmm4 + psrad $31, %xmm4 + pslld $0x01, %xmm2 + pshufd $0x93, %xmm4, %xmm4 + pand %xmm6, %xmm4 + pxor %xmm4, %xmm2 + addl $16, %edi + jmp L_AES_XTS_encrypt_aesni_loop +L_AES_XTS_encrypt_aesni_loop_done: + movl 36(%esp), %eax + cmpl %eax, %edi + je L_AES_XTS_encrypt_aesni_done + subl $16, %edi + movl 32(%esp), %eax + movdqu (%eax,%edi,1), %xmm5 + addl $16, %edi + movdqu %xmm5, (%esp) + xorl %edx, %edx +L_AES_XTS_encrypt_aesni_cts: + movzbl (%esp,%edx,1), %ecx + movl 28(%esp), %eax + movzbl (%eax,%edi,1), %ebx + movl 32(%esp), %eax + movb %cl, (%eax,%edi,1) + movb %bl, (%esp,%edx,1) + incl %edi + incl %edx + movl 36(%esp), %eax + cmpl %eax, %edi + jb L_AES_XTS_encrypt_aesni_cts + subl %edx, %edi + movdqu (%esp), %xmm3 + subl $16, %edi + pxor %xmm2, %xmm3 + movl 44(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + cmpl $11, 52(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_encrypt_aesni_enclast_4 + aesenc %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + cmpl $13, 52(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_encrypt_aesni_enclast_4 + aesenc %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_encrypt_aesni_enclast_4: + aesenclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) +L_AES_XTS_encrypt_aesni_done: + addl $16, %esp + popl %ebx + popl %edi + ret +.size AES_XTS_encrypt_aesni,.-AES_XTS_encrypt_aesni + # void AES_XTS_encrypt_update_aesni(const unsigned char* in, + # unsigned char* out, word32 sz, const unsigned char* key, + # unsigned char* i, int nr); Tweak is read (already encrypted) from *i + # and the advanced tweak written back to *i. +.text +.globl AES_XTS_encrypt_update_aesni +WC_ASM_ATT_HIDDEN(AES_XTS_encrypt_update_aesni) +.type AES_XTS_encrypt_update_aesni,@function +.align 16 +AES_XTS_encrypt_update_aesni: + pushl %edi + pushl %ebx + subl $16, %esp + movl $0x87, (%esp) + movl $0x01, 4(%esp) + movl $0x01, 8(%esp) + movl $0x01, 12(%esp) + movdqu (%esp), %xmm6 + movl 44(%esp), %eax + movdqu (%eax), %xmm2 + xorl %edi, %edi + movl 36(%esp), %edx + andl $0xfffffff0, %edx +L_AES_XTS_encrypt_update_aesni_loop: + cmpl %edx, %edi + jae L_AES_XTS_encrypt_update_aesni_loop_done + movl 28(%esp), %eax + movdqu (%eax,%edi,1), %xmm3 + pxor %xmm2, %xmm3 + movl 40(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + cmpl $11, 48(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_encrypt_update_aesni_enclast_5 + aesenc %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + cmpl $13, 48(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_encrypt_update_aesni_enclast_5 + aesenc %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_encrypt_update_aesni_enclast_5: + aesenclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) + movdqa %xmm2, %xmm4 + psrad $31, %xmm4 + pslld $0x01, %xmm2 + pshufd $0x93, %xmm4, %xmm4 + pand %xmm6, %xmm4 + pxor %xmm4, %xmm2 + addl $16, %edi + jmp L_AES_XTS_encrypt_update_aesni_loop +L_AES_XTS_encrypt_update_aesni_loop_done: + movl 36(%esp), %eax + cmpl %eax, %edi + je L_AES_XTS_encrypt_update_aesni_done + subl $16, %edi + movl 32(%esp), %eax + movdqu (%eax,%edi,1), %xmm5 + addl $16, %edi + movdqu %xmm5, (%esp) + xorl %edx, %edx +L_AES_XTS_encrypt_update_aesni_cts: + movzbl (%esp,%edx,1), %ecx + movl 28(%esp), %eax + movzbl (%eax,%edi,1), %ebx + movl 32(%esp), %eax + movb %cl, (%eax,%edi,1) + movb %bl, (%esp,%edx,1) + incl %edi + incl %edx + movl 36(%esp), %eax + cmpl %eax, %edi + jb L_AES_XTS_encrypt_update_aesni_cts + subl %edx, %edi + movdqu (%esp), %xmm3 + subl $16, %edi + pxor %xmm2, %xmm3 + movl 40(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + cmpl $11, 48(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_encrypt_update_aesni_enclast_6 + aesenc %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + cmpl $13, 48(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_encrypt_update_aesni_enclast_6 + aesenc %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_encrypt_update_aesni_enclast_6: + aesenclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) +L_AES_XTS_encrypt_update_aesni_done: + movl 44(%esp), %eax + movdqu %xmm2, (%eax) + addl $16, %esp + popl %ebx + popl %edi + ret +.size AES_XTS_encrypt_update_aesni,.-AES_XTS_encrypt_update_aesni + # void AES_XTS_decrypt_aesni(const unsigned char* in, unsigned char* out, + # word32 sz, const unsigned char* i, const unsigned char* key, + # const unsigned char* key2, int nr); +.text +.globl AES_XTS_decrypt_aesni +WC_ASM_ATT_HIDDEN(AES_XTS_decrypt_aesni) +.type AES_XTS_decrypt_aesni,@function +.align 16 +AES_XTS_decrypt_aesni: + pushl %edi + pushl %ebx + subl $16, %esp + movl $0x87, (%esp) + movl $0x01, 4(%esp) + movl $0x01, 8(%esp) + movl $0x01, 12(%esp) + movdqu (%esp), %xmm6 + movl 40(%esp), %eax + movdqu (%eax), %xmm2 + movl 48(%esp), %ecx + pxor (%ecx), %xmm2 + movdqu 16(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 32(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 48(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 64(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 80(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 96(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 112(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 128(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 144(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + cmpl $11, 52(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_enclast_7 + aesenc %xmm0, %xmm2 + movdqu 176(%ecx), %xmm1 + aesenc %xmm1, %xmm2 + cmpl $13, 52(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_enclast_7 + aesenc %xmm0, %xmm2 + movdqu 208(%ecx), %xmm1 + aesenc %xmm1, %xmm2 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_decrypt_aesni_enclast_7: + aesenclast %xmm0, %xmm2 + xorl %edi, %edi + movl 36(%esp), %eax + movl %eax, %edx + andl $0xfffffff0, %edx + cmpl %eax, %edx + je L_AES_XTS_decrypt_aesni_bound + subl $16, %edx +L_AES_XTS_decrypt_aesni_bound: +L_AES_XTS_decrypt_aesni_loop: + cmpl %edx, %edi + jae L_AES_XTS_decrypt_aesni_loop_done + movl 28(%esp), %eax + movdqu (%eax,%edi,1), %xmm3 + pxor %xmm2, %xmm3 + movl 44(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + cmpl $11, 52(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_declast_8 + aesdec %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + cmpl $13, 52(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_declast_8 + aesdec %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_decrypt_aesni_declast_8: + aesdeclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) + movdqa %xmm2, %xmm4 + psrad $31, %xmm4 + pslld $0x01, %xmm2 + pshufd $0x93, %xmm4, %xmm4 + pand %xmm6, %xmm4 + pxor %xmm4, %xmm2 + addl $16, %edi + jmp L_AES_XTS_decrypt_aesni_loop +L_AES_XTS_decrypt_aesni_loop_done: + movl 36(%esp), %eax + cmpl %eax, %edi + je L_AES_XTS_decrypt_aesni_done + movdqa %xmm2, %xmm4 + movdqa %xmm2, %xmm5 + psrad $31, %xmm4 + pslld $0x01, %xmm5 + pshufd $0x93, %xmm4, %xmm4 + pand %xmm6, %xmm4 + pxor %xmm4, %xmm5 + movl 28(%esp), %eax + movdqu (%eax,%edi,1), %xmm3 + pxor %xmm5, %xmm3 + movl 44(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + cmpl $11, 52(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_declast_9 + aesdec %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + cmpl $13, 52(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_declast_9 + aesdec %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_decrypt_aesni_declast_9: + aesdeclast %xmm0, %xmm3 + pxor %xmm5, %xmm3 + movdqu %xmm3, (%esp) + addl $16, %edi + xorl %edx, %edx +L_AES_XTS_decrypt_aesni_cts: + movzbl (%esp,%edx,1), %ecx + movl 28(%esp), %eax + movzbl (%eax,%edi,1), %ebx + movl 32(%esp), %eax + movb %cl, (%eax,%edi,1) + movb %bl, (%esp,%edx,1) + incl %edi + incl %edx + movl 36(%esp), %eax + cmpl %eax, %edi + jb L_AES_XTS_decrypt_aesni_cts + subl %edx, %edi + movdqu (%esp), %xmm3 + pxor %xmm2, %xmm3 + movl 44(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + cmpl $11, 52(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_declast_10 + aesdec %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + cmpl $13, 52(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_declast_10 + aesdec %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_decrypt_aesni_declast_10: + aesdeclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + subl $16, %edi + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) +L_AES_XTS_decrypt_aesni_done: + addl $16, %esp + popl %ebx + popl %edi + ret +.size AES_XTS_decrypt_aesni,.-AES_XTS_decrypt_aesni + # void AES_XTS_decrypt_update_aesni(const unsigned char* in, + # unsigned char* out, word32 sz, const unsigned char* key, + # unsigned char* i, int nr); Tweak is read from *i and the advanced + # tweak written back to *i. +.text +.globl AES_XTS_decrypt_update_aesni +WC_ASM_ATT_HIDDEN(AES_XTS_decrypt_update_aesni) +.type AES_XTS_decrypt_update_aesni,@function +.align 16 +AES_XTS_decrypt_update_aesni: + pushl %edi + pushl %ebx + subl $16, %esp + movl $0x87, (%esp) + movl $0x01, 4(%esp) + movl $0x01, 8(%esp) + movl $0x01, 12(%esp) + movdqu (%esp), %xmm6 + movl 44(%esp), %eax + movdqu (%eax), %xmm2 + xorl %edi, %edi + movl 36(%esp), %eax + movl %eax, %edx + andl $0xfffffff0, %edx + cmpl %eax, %edx + je L_AES_XTS_decrypt_update_aesni_bound + subl $16, %edx +L_AES_XTS_decrypt_update_aesni_bound: +L_AES_XTS_decrypt_update_aesni_loop: + cmpl %edx, %edi + jae L_AES_XTS_decrypt_update_aesni_loop_done + movl 28(%esp), %eax + movdqu (%eax,%edi,1), %xmm3 + pxor %xmm2, %xmm3 + movl 40(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + cmpl $11, 48(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_decrypt_update_aesni_declast_11 + aesdec %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + cmpl $13, 48(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_decrypt_update_aesni_declast_11 + aesdec %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_decrypt_update_aesni_declast_11: + aesdeclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) + movdqa %xmm2, %xmm4 + psrad $31, %xmm4 + pslld $0x01, %xmm2 + pshufd $0x93, %xmm4, %xmm4 + pand %xmm6, %xmm4 + pxor %xmm4, %xmm2 + addl $16, %edi + jmp L_AES_XTS_decrypt_update_aesni_loop +L_AES_XTS_decrypt_update_aesni_loop_done: + movl 36(%esp), %eax + cmpl %eax, %edi + je L_AES_XTS_decrypt_update_aesni_done + movdqa %xmm2, %xmm4 + movdqa %xmm2, %xmm5 + psrad $31, %xmm4 + pslld $0x01, %xmm5 + pshufd $0x93, %xmm4, %xmm4 + pand %xmm6, %xmm4 + pxor %xmm4, %xmm5 + movl 28(%esp), %eax + movdqu (%eax,%edi,1), %xmm3 + pxor %xmm5, %xmm3 + movl 40(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + cmpl $11, 48(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_decrypt_update_aesni_declast_12 + aesdec %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + cmpl $13, 48(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_decrypt_update_aesni_declast_12 + aesdec %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_decrypt_update_aesni_declast_12: + aesdeclast %xmm0, %xmm3 + pxor %xmm5, %xmm3 + movdqu %xmm3, (%esp) + addl $16, %edi + xorl %edx, %edx +L_AES_XTS_decrypt_update_aesni_cts: + movzbl (%esp,%edx,1), %ecx + movl 28(%esp), %eax + movzbl (%eax,%edi,1), %ebx + movl 32(%esp), %eax + movb %cl, (%eax,%edi,1) + movb %bl, (%esp,%edx,1) + incl %edi + incl %edx + movl 36(%esp), %eax + cmpl %eax, %edi + jb L_AES_XTS_decrypt_update_aesni_cts + subl %edx, %edi + movdqu (%esp), %xmm3 + pxor %xmm2, %xmm3 + movl 40(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + cmpl $11, 48(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_decrypt_update_aesni_declast_13 + aesdec %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + cmpl $13, 48(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_decrypt_update_aesni_declast_13 + aesdec %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_decrypt_update_aesni_declast_13: + aesdeclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + subl $16, %edi + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) +L_AES_XTS_decrypt_update_aesni_done: + movl 44(%esp), %eax + movdqu %xmm2, (%eax) + addl $16, %esp + popl %ebx + popl %edi + ret +.size AES_XTS_decrypt_update_aesni,.-AES_XTS_decrypt_update_aesni +#endif /* WOLFSSL_X86_BUILD */ +#endif /* WOLFSSL_AES_XTS */ + +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/wolfcrypt/src/aes_xts_x86_asm.asm b/wolfcrypt/src/aes_xts_x86_asm.asm new file mode 100644 index 00000000000..bc7f43f7aa2 --- /dev/null +++ b/wolfcrypt/src/aes_xts_x86_asm.asm @@ -0,0 +1,835 @@ +; /* aes_xts_x86_asm +; * +; * Copyright (C) 2006-2026 wolfSSL Inc. +; * +; * This file is part of wolfSSL. +; * +; * wolfSSL is free software; you can redistribute it and/or modify +; * it under the terms of the GNU General Public License as published by +; * the Free Software Foundation; either version 3 of the License, or +; * (at your option) any later version. +; * +; * wolfSSL is distributed in the hope that it will be useful, +; * but WITHOUT ANY WARRANTY; without even the implied warranty of +; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +; * GNU General Public License for more details. +; * +; * You should have received a copy of the GNU General Public License +; * along with this program; if not, write to the Free Software +; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA +; */ + +IF @Version LT 1200 +; AVX2 instructions not recognized by old versions of MASM +IFNDEF NO_AVX2_SUPPORT +NO_AVX2_SUPPORT = 1 +ENDIF +; MOVBE instruction not recognized by old versions of MASM +IFNDEF NO_MOVBE_SUPPORT +NO_MOVBE_SUPPORT = 1 +ENDIF +ENDIF + +IFNDEF HAVE_INTEL_AVX1 +HAVE_INTEL_AVX1 = 1 +ENDIF +IFNDEF NO_AVX2_SUPPORT +HAVE_INTEL_AVX2 = 1 +ENDIF + +IFNDEF _WIN32 +_WIN32 = 1 +ENDIF + +.686P +.XMM +.MODEL FLAT, C + +IFDEF WOLFSSL_AES_XTS +IFDEF WOLFSSL_X86_BUILD + ; 32-bit (i386) AES-NI AES-XTS: single-block ports of the x86_64 + ; AES_XTS_*_aesni routines (xmm0-7, cdecl ABI); GF const on stack for PIC. + ; void AES_XTS_init_aesni(unsigned char* i, const unsigned char* tweak_key, + ; int tweak_nr); +_TEXT SEGMENT READONLY PARA +AES_XTS_init_aesni PROC + mov eax, DWORD PTR [esp+4] + movdqu xmm2, OWORD PTR [eax] + mov ecx, DWORD PTR [esp+8] + pxor xmm2, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesenc xmm2, xmm0 + cmp DWORD PTR [esp+12], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_init_aesni_enclast_1 + aesenc xmm2, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesenc xmm2, xmm1 + cmp DWORD PTR [esp+12], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_init_aesni_enclast_1 + aesenc xmm2, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesenc xmm2, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_init_aesni_enclast_1: + aesenclast xmm2, xmm0 + movdqu OWORD PTR [eax], xmm2 + ret +AES_XTS_init_aesni ENDP +_TEXT ENDS + ; void AES_XTS_encrypt_aesni(const unsigned char* in, unsigned char* out, + ; word32 sz, const unsigned char* i, const unsigned char* key, + ; const unsigned char* key2, int nr); +_TEXT SEGMENT READONLY PARA +AES_XTS_encrypt_aesni PROC + push edi + push ebx + sub esp, 16 + mov DWORD PTR [esp], 135 + mov DWORD PTR [esp+4], 1 + mov DWORD PTR [esp+8], 1 + mov DWORD PTR [esp+12], 1 + movdqu xmm6, OWORD PTR [esp] + mov eax, DWORD PTR [esp+40] + movdqu xmm2, OWORD PTR [eax] + mov ecx, DWORD PTR [esp+48] + pxor xmm2, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesenc xmm2, xmm0 + cmp DWORD PTR [esp+52], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_encrypt_aesni_enclast_2 + aesenc xmm2, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesenc xmm2, xmm1 + cmp DWORD PTR [esp+52], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_encrypt_aesni_enclast_2 + aesenc xmm2, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesenc xmm2, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_encrypt_aesni_enclast_2: + aesenclast xmm2, xmm0 + xor edi, edi + mov edx, DWORD PTR [esp+36] + and edx, 4294967280 +L_AES_XTS_encrypt_aesni_loop: + cmp edi, edx + jae L_AES_XTS_encrypt_aesni_loop_done + mov eax, DWORD PTR [esp+28] + movdqu xmm3, OWORD PTR [eax+edi] + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+44] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesenc xmm3, xmm0 + cmp DWORD PTR [esp+52], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_encrypt_aesni_enclast_3 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesenc xmm3, xmm1 + cmp DWORD PTR [esp+52], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_encrypt_aesni_enclast_3 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesenc xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_encrypt_aesni_enclast_3: + aesenclast xmm3, xmm0 + pxor xmm3, xmm2 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 + movdqa xmm4, xmm2 + psrad xmm4, 31 + pslld xmm2, 1 + pshufd xmm4, xmm4, 147 + pand xmm4, xmm6 + pxor xmm2, xmm4 + add edi, 16 + jmp L_AES_XTS_encrypt_aesni_loop +L_AES_XTS_encrypt_aesni_loop_done: + mov eax, DWORD PTR [esp+36] + cmp edi, eax + je L_AES_XTS_encrypt_aesni_done + sub edi, 16 + mov eax, DWORD PTR [esp+32] + movdqu xmm5, OWORD PTR [eax+edi] + add edi, 16 + movdqu OWORD PTR [esp], xmm5 + xor edx, edx +L_AES_XTS_encrypt_aesni_cts: + movzx ecx, BYTE PTR [esp+edx] + mov eax, DWORD PTR [esp+28] + movzx ebx, BYTE PTR [eax+edi] + mov eax, DWORD PTR [esp+32] + mov BYTE PTR [eax+edi], cl + mov BYTE PTR [esp+edx], bl + inc edi + inc edx + mov eax, DWORD PTR [esp+36] + cmp edi, eax + jb L_AES_XTS_encrypt_aesni_cts + sub edi, edx + movdqu xmm3, OWORD PTR [esp] + sub edi, 16 + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+44] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesenc xmm3, xmm0 + cmp DWORD PTR [esp+52], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_encrypt_aesni_enclast_4 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesenc xmm3, xmm1 + cmp DWORD PTR [esp+52], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_encrypt_aesni_enclast_4 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesenc xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_encrypt_aesni_enclast_4: + aesenclast xmm3, xmm0 + pxor xmm3, xmm2 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 +L_AES_XTS_encrypt_aesni_done: + add esp, 16 + pop ebx + pop edi + ret +AES_XTS_encrypt_aesni ENDP +_TEXT ENDS + ; void AES_XTS_encrypt_update_aesni(const unsigned char* in, + ; unsigned char* out, word32 sz, const unsigned char* key, + ; unsigned char* i, int nr); Tweak is read (already encrypted) from *i + ; and the advanced tweak written back to *i. +_TEXT SEGMENT READONLY PARA +AES_XTS_encrypt_update_aesni PROC + push edi + push ebx + sub esp, 16 + mov DWORD PTR [esp], 135 + mov DWORD PTR [esp+4], 1 + mov DWORD PTR [esp+8], 1 + mov DWORD PTR [esp+12], 1 + movdqu xmm6, OWORD PTR [esp] + mov eax, DWORD PTR [esp+44] + movdqu xmm2, OWORD PTR [eax] + xor edi, edi + mov edx, DWORD PTR [esp+36] + and edx, 4294967280 +L_AES_XTS_encrypt_update_aesni_loop: + cmp edi, edx + jae L_AES_XTS_encrypt_update_aesni_loop_done + mov eax, DWORD PTR [esp+28] + movdqu xmm3, OWORD PTR [eax+edi] + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+40] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesenc xmm3, xmm0 + cmp DWORD PTR [esp+48], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_encrypt_update_aesni_enclast_5 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesenc xmm3, xmm1 + cmp DWORD PTR [esp+48], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_encrypt_update_aesni_enclast_5 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesenc xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_encrypt_update_aesni_enclast_5: + aesenclast xmm3, xmm0 + pxor xmm3, xmm2 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 + movdqa xmm4, xmm2 + psrad xmm4, 31 + pslld xmm2, 1 + pshufd xmm4, xmm4, 147 + pand xmm4, xmm6 + pxor xmm2, xmm4 + add edi, 16 + jmp L_AES_XTS_encrypt_update_aesni_loop +L_AES_XTS_encrypt_update_aesni_loop_done: + mov eax, DWORD PTR [esp+36] + cmp edi, eax + je L_AES_XTS_encrypt_update_aesni_done + sub edi, 16 + mov eax, DWORD PTR [esp+32] + movdqu xmm5, OWORD PTR [eax+edi] + add edi, 16 + movdqu OWORD PTR [esp], xmm5 + xor edx, edx +L_AES_XTS_encrypt_update_aesni_cts: + movzx ecx, BYTE PTR [esp+edx] + mov eax, DWORD PTR [esp+28] + movzx ebx, BYTE PTR [eax+edi] + mov eax, DWORD PTR [esp+32] + mov BYTE PTR [eax+edi], cl + mov BYTE PTR [esp+edx], bl + inc edi + inc edx + mov eax, DWORD PTR [esp+36] + cmp edi, eax + jb L_AES_XTS_encrypt_update_aesni_cts + sub edi, edx + movdqu xmm3, OWORD PTR [esp] + sub edi, 16 + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+40] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesenc xmm3, xmm0 + cmp DWORD PTR [esp+48], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_encrypt_update_aesni_enclast_6 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesenc xmm3, xmm1 + cmp DWORD PTR [esp+48], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_encrypt_update_aesni_enclast_6 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesenc xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_encrypt_update_aesni_enclast_6: + aesenclast xmm3, xmm0 + pxor xmm3, xmm2 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 +L_AES_XTS_encrypt_update_aesni_done: + mov eax, DWORD PTR [esp+44] + movdqu OWORD PTR [eax], xmm2 + add esp, 16 + pop ebx + pop edi + ret +AES_XTS_encrypt_update_aesni ENDP +_TEXT ENDS + ; void AES_XTS_decrypt_aesni(const unsigned char* in, unsigned char* out, + ; word32 sz, const unsigned char* i, const unsigned char* key, + ; const unsigned char* key2, int nr); +_TEXT SEGMENT READONLY PARA +AES_XTS_decrypt_aesni PROC + push edi + push ebx + sub esp, 16 + mov DWORD PTR [esp], 135 + mov DWORD PTR [esp+4], 1 + mov DWORD PTR [esp+8], 1 + mov DWORD PTR [esp+12], 1 + movdqu xmm6, OWORD PTR [esp] + mov eax, DWORD PTR [esp+40] + movdqu xmm2, OWORD PTR [eax] + mov ecx, DWORD PTR [esp+48] + pxor xmm2, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesenc xmm2, xmm0 + cmp DWORD PTR [esp+52], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_decrypt_aesni_enclast_7 + aesenc xmm2, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesenc xmm2, xmm1 + cmp DWORD PTR [esp+52], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_decrypt_aesni_enclast_7 + aesenc xmm2, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesenc xmm2, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_decrypt_aesni_enclast_7: + aesenclast xmm2, xmm0 + xor edi, edi + mov eax, DWORD PTR [esp+36] + mov edx, eax + and edx, 4294967280 + cmp edx, eax + je L_AES_XTS_decrypt_aesni_bound + sub edx, 16 +L_AES_XTS_decrypt_aesni_bound: +L_AES_XTS_decrypt_aesni_loop: + cmp edi, edx + jae L_AES_XTS_decrypt_aesni_loop_done + mov eax, DWORD PTR [esp+28] + movdqu xmm3, OWORD PTR [eax+edi] + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+44] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesdec xmm3, xmm0 + cmp DWORD PTR [esp+52], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_decrypt_aesni_declast_8 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesdec xmm3, xmm1 + cmp DWORD PTR [esp+52], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_decrypt_aesni_declast_8 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesdec xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_decrypt_aesni_declast_8: + aesdeclast xmm3, xmm0 + pxor xmm3, xmm2 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 + movdqa xmm4, xmm2 + psrad xmm4, 31 + pslld xmm2, 1 + pshufd xmm4, xmm4, 147 + pand xmm4, xmm6 + pxor xmm2, xmm4 + add edi, 16 + jmp L_AES_XTS_decrypt_aesni_loop +L_AES_XTS_decrypt_aesni_loop_done: + mov eax, DWORD PTR [esp+36] + cmp edi, eax + je L_AES_XTS_decrypt_aesni_done + movdqa xmm4, xmm2 + movdqa xmm5, xmm2 + psrad xmm4, 31 + pslld xmm5, 1 + pshufd xmm4, xmm4, 147 + pand xmm4, xmm6 + pxor xmm5, xmm4 + mov eax, DWORD PTR [esp+28] + movdqu xmm3, OWORD PTR [eax+edi] + pxor xmm3, xmm5 + mov ecx, DWORD PTR [esp+44] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesdec xmm3, xmm0 + cmp DWORD PTR [esp+52], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_decrypt_aesni_declast_9 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesdec xmm3, xmm1 + cmp DWORD PTR [esp+52], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_decrypt_aesni_declast_9 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesdec xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_decrypt_aesni_declast_9: + aesdeclast xmm3, xmm0 + pxor xmm3, xmm5 + movdqu OWORD PTR [esp], xmm3 + add edi, 16 + xor edx, edx +L_AES_XTS_decrypt_aesni_cts: + movzx ecx, BYTE PTR [esp+edx] + mov eax, DWORD PTR [esp+28] + movzx ebx, BYTE PTR [eax+edi] + mov eax, DWORD PTR [esp+32] + mov BYTE PTR [eax+edi], cl + mov BYTE PTR [esp+edx], bl + inc edi + inc edx + mov eax, DWORD PTR [esp+36] + cmp edi, eax + jb L_AES_XTS_decrypt_aesni_cts + sub edi, edx + movdqu xmm3, OWORD PTR [esp] + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+44] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesdec xmm3, xmm0 + cmp DWORD PTR [esp+52], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_decrypt_aesni_declast_10 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesdec xmm3, xmm1 + cmp DWORD PTR [esp+52], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_decrypt_aesni_declast_10 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesdec xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_decrypt_aesni_declast_10: + aesdeclast xmm3, xmm0 + pxor xmm3, xmm2 + sub edi, 16 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 +L_AES_XTS_decrypt_aesni_done: + add esp, 16 + pop ebx + pop edi + ret +AES_XTS_decrypt_aesni ENDP +_TEXT ENDS + ; void AES_XTS_decrypt_update_aesni(const unsigned char* in, + ; unsigned char* out, word32 sz, const unsigned char* key, + ; unsigned char* i, int nr); Tweak is read from *i and the advanced + ; tweak written back to *i. +_TEXT SEGMENT READONLY PARA +AES_XTS_decrypt_update_aesni PROC + push edi + push ebx + sub esp, 16 + mov DWORD PTR [esp], 135 + mov DWORD PTR [esp+4], 1 + mov DWORD PTR [esp+8], 1 + mov DWORD PTR [esp+12], 1 + movdqu xmm6, OWORD PTR [esp] + mov eax, DWORD PTR [esp+44] + movdqu xmm2, OWORD PTR [eax] + xor edi, edi + mov eax, DWORD PTR [esp+36] + mov edx, eax + and edx, 4294967280 + cmp edx, eax + je L_AES_XTS_decrypt_update_aesni_bound + sub edx, 16 +L_AES_XTS_decrypt_update_aesni_bound: +L_AES_XTS_decrypt_update_aesni_loop: + cmp edi, edx + jae L_AES_XTS_decrypt_update_aesni_loop_done + mov eax, DWORD PTR [esp+28] + movdqu xmm3, OWORD PTR [eax+edi] + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+40] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesdec xmm3, xmm0 + cmp DWORD PTR [esp+48], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_decrypt_update_aesni_declast_11 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesdec xmm3, xmm1 + cmp DWORD PTR [esp+48], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_decrypt_update_aesni_declast_11 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesdec xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_decrypt_update_aesni_declast_11: + aesdeclast xmm3, xmm0 + pxor xmm3, xmm2 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 + movdqa xmm4, xmm2 + psrad xmm4, 31 + pslld xmm2, 1 + pshufd xmm4, xmm4, 147 + pand xmm4, xmm6 + pxor xmm2, xmm4 + add edi, 16 + jmp L_AES_XTS_decrypt_update_aesni_loop +L_AES_XTS_decrypt_update_aesni_loop_done: + mov eax, DWORD PTR [esp+36] + cmp edi, eax + je L_AES_XTS_decrypt_update_aesni_done + movdqa xmm4, xmm2 + movdqa xmm5, xmm2 + psrad xmm4, 31 + pslld xmm5, 1 + pshufd xmm4, xmm4, 147 + pand xmm4, xmm6 + pxor xmm5, xmm4 + mov eax, DWORD PTR [esp+28] + movdqu xmm3, OWORD PTR [eax+edi] + pxor xmm3, xmm5 + mov ecx, DWORD PTR [esp+40] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesdec xmm3, xmm0 + cmp DWORD PTR [esp+48], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_decrypt_update_aesni_declast_12 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesdec xmm3, xmm1 + cmp DWORD PTR [esp+48], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_decrypt_update_aesni_declast_12 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesdec xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_decrypt_update_aesni_declast_12: + aesdeclast xmm3, xmm0 + pxor xmm3, xmm5 + movdqu OWORD PTR [esp], xmm3 + add edi, 16 + xor edx, edx +L_AES_XTS_decrypt_update_aesni_cts: + movzx ecx, BYTE PTR [esp+edx] + mov eax, DWORD PTR [esp+28] + movzx ebx, BYTE PTR [eax+edi] + mov eax, DWORD PTR [esp+32] + mov BYTE PTR [eax+edi], cl + mov BYTE PTR [esp+edx], bl + inc edi + inc edx + mov eax, DWORD PTR [esp+36] + cmp edi, eax + jb L_AES_XTS_decrypt_update_aesni_cts + sub edi, edx + movdqu xmm3, OWORD PTR [esp] + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+40] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesdec xmm3, xmm0 + cmp DWORD PTR [esp+48], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_decrypt_update_aesni_declast_13 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesdec xmm3, xmm1 + cmp DWORD PTR [esp+48], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_decrypt_update_aesni_declast_13 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesdec xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_decrypt_update_aesni_declast_13: + aesdeclast xmm3, xmm0 + pxor xmm3, xmm2 + sub edi, 16 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 +L_AES_XTS_decrypt_update_aesni_done: + mov eax, DWORD PTR [esp+44] + movdqu OWORD PTR [eax], xmm2 + add esp, 16 + pop ebx + pop edi + ret +AES_XTS_decrypt_update_aesni ENDP +_TEXT ENDS +ENDIF +ENDIF +END diff --git a/wolfcrypt/src/fe_448.c b/wolfcrypt/src/fe_448.c index c12d8faf440..42b8eaaf463 100644 --- a/wolfcrypt/src/fe_448.c +++ b/wolfcrypt/src/fe_448.c @@ -1126,6 +1126,7 @@ int curve448(byte* r, const byte* n, const byte* a) fe448_reduce(x3); fe448_sqr(x3, x3); fe448_sub(z3, z3, t1); + fe448_reduce(z3); fe448_sqr(z3, z3); fe448_mul(z3, z3, x1); fe448_sub(t1, t0, x2); diff --git a/wolfcrypt/src/fe_x25519_asm.asm b/wolfcrypt/src/fe_x25519_asm.asm index cbbb99a44e5..3bd50cd61c3 100644 --- a/wolfcrypt/src/fe_x25519_asm.asm +++ b/wolfcrypt/src/fe_x25519_asm.asm @@ -1290,116 +1290,178 @@ fe_invert_x64 PROC mov QWORD PTR [rsp+136], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+136] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+136] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rdx, QWORD PTR [rsp+136] mov rcx, QWORD PTR [rsp+128] add rsp, 152 @@ -3314,116 +3376,178 @@ L_curve25519_base_x64_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov r8, QWORD PTR [rsp+160] ; Multiply ; A[0] * B[0] @@ -5609,116 +5733,178 @@ L_curve25519_x64_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov r9, QWORD PTR [rsp+168] ; Multiply ; A[0] * B[0] @@ -5899,115 +6085,177 @@ fe_pow22523_x64 PROC mov QWORD PTR [rsp+104], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+104] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+104] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 mov rcx, QWORD PTR [rsp+96] mov rdx, rsp mov r8, QWORD PTR [rsp+104] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rdx, QWORD PTR [rsp+104] mov rcx, QWORD PTR [rsp+96] add rsp, 120 @@ -11543,116 +11791,178 @@ fe_invert_avx2 PROC mov QWORD PTR [rsp+136], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+136] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+136] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rdx, QWORD PTR [rsp+136] mov rcx, QWORD PTR [rsp+128] add rsp, 152 @@ -13206,116 +13516,178 @@ L_curve25519_base_avx2_last_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov r8, QWORD PTR [rsp+160] mov rax, QWORD PTR [r8] ; Multiply @@ -15095,116 +15467,178 @@ L_curve25519_avx2_last_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov r9, QWORD PTR [rsp+168] mov rax, QWORD PTR [r9] ; Multiply @@ -15356,115 +15790,177 @@ fe_pow22523_avx2 PROC mov QWORD PTR [rsp+104], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+104] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+104] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 mov rcx, QWORD PTR [rsp+96] mov rdx, rsp mov r8, QWORD PTR [rsp+104] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rdx, QWORD PTR [rsp+104] mov rcx, QWORD PTR [rsp+96] add rsp, 120 @@ -20577,16 +21073,22 @@ L_curve25519_base_avx512_ifma_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+736] vmovdqu xmm7, OWORD PTR [rsp+752] @@ -21395,16 +21897,22 @@ L_curve25519_avx512_ifma_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+744] vmovdqu xmm7, OWORD PTR [rsp+760] @@ -22147,16 +22655,22 @@ L_curve25519_base_avx512_ifma_dq_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+736] vmovdqu xmm7, OWORD PTR [rsp+752] @@ -22920,16 +23434,22 @@ L_curve25519_avx512_ifma_dq_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+744] vmovdqu xmm7, OWORD PTR [rsp+760] diff --git a/wolfcrypt/src/ge_448.c b/wolfcrypt/src/ge_448.c index f3a895f23ee..d2c77638dc0 100644 --- a/wolfcrypt/src/ge_448.c +++ b/wolfcrypt/src/ge_448.c @@ -10837,7 +10837,7 @@ static WC_INLINE void ge448_sub(ge448_p2 *r, const ge448_p2 *p, fe448_mul(t3, t1, t2); /* t3 = E1 = C*D */ fe448_mul39081(t3, t3); /* t3 = E = d*C*D */ fe448_mul(r->Z, p->Z, q->Z); /* r->Z = A = Z1*Z2 */ - fe448_sqr(t0, p->Z); /* t0 = B = A^2 */ + fe448_sqr(t0, r->Z); /* t0 = B = A^2 */ fe448_sub(t4, t0, t3); /* t4 = F = B-(--E) */ fe448_add(t0, t0, t3); /* t0 = G = B+(--E) */ fe448_reduce(t0); diff --git a/wolfcrypt/src/include.am b/wolfcrypt/src/include.am index f0e7ca3c717..13b6d70c025 100644 --- a/wolfcrypt/src/include.am +++ b/wolfcrypt/src/include.am @@ -21,6 +21,7 @@ EXTRA_DIST += wolfcrypt/src/aes_x86_64_asm.asm EXTRA_DIST += wolfcrypt/src/aes_gcm_asm.asm EXTRA_DIST += wolfcrypt/src/aes_gcm_x86_asm.asm EXTRA_DIST += wolfcrypt/src/aes_xts_asm.asm +EXTRA_DIST += wolfcrypt/src/aes_xts_x86_asm.asm EXTRA_DIST += wolfcrypt/src/chacha_asm.asm EXTRA_DIST += wolfcrypt/src/poly1305_asm.asm EXTRA_DIST += wolfcrypt/src/fe_x25519_asm.asm diff --git a/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c b/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c index 4d0c5a01bef..da717f8837d 100644 --- a/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c @@ -946,11 +946,11 @@ WC_OMIT_FRAME_POINTER void AES_set_encrypt_key(const unsigned char* key, void AES_encrypt_block(const word32* te_p, int nr_p, int len_p, const word32* ks_p); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te_p, int nr_p, - int len_p, const word32* ks_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te_p, + int nr_p, int len_p, const word32* ks_p) #else -WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te, int nr, int len, - const word32* ks) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te, + int nr, int len, const word32* ks) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -8237,11 +8237,11 @@ WC_OMIT_FRAME_POINTER void AES_CTR_encrypt(const unsigned char* in, #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE void AES_decrypt_block(const word32* td_p, int nr_p, const byte* td4_p); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td_p, int nr_p, - const byte* td4_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td_p, + int nr_p, const byte* td4_p) #else -WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td, int nr, - const byte* td4) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td, + int nr, const byte* td4) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG diff --git a/wolfcrypt/src/port/arm/armv8-32-curve25519_c.c b/wolfcrypt/src/port/arm/armv8-32-curve25519_c.c index 133265ae98f..7b7d52a557b 100644 --- a/wolfcrypt/src/port/arm/armv8-32-curve25519_c.c +++ b/wolfcrypt/src/port/arm/armv8-32-curve25519_c.c @@ -84,9 +84,9 @@ WC_OMIT_FRAME_POINTER void fe_init(void) void fe_add_sub_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) #else -WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -299,9 +299,9 @@ WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) void fe_sub_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sub_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sub_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -375,9 +375,9 @@ WC_OMIT_FRAME_POINTER void fe_sub(fe r, const fe a, const fe b) void fe_add_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_add_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_op(void) #else -WC_OMIT_FRAME_POINTER void fe_add_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2501,9 +2501,9 @@ WC_OMIT_FRAME_POINTER void fe_cmov_table(fe* r, const fe* base, signed char b) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) void fe_mul_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2895,9 +2895,9 @@ WC_OMIT_FRAME_POINTER void fe_mul_op(void) void fe_mul_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3313,9 +3313,9 @@ WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #else void fe_mul_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3466,9 +3466,9 @@ WC_OMIT_FRAME_POINTER void fe_mul_op(void) void fe_mul_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3670,9 +3670,9 @@ WC_OMIT_FRAME_POINTER void fe_mul(fe r, const fe a, const fe b) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) void fe_sq_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3957,9 +3957,9 @@ WC_OMIT_FRAME_POINTER void fe_sq_op(void) void fe_sq_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -4268,9 +4268,9 @@ WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #else void fe_sq_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -4407,9 +4407,9 @@ WC_OMIT_FRAME_POINTER void fe_sq_op(void) void fe_sq_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -4596,9 +4596,9 @@ WC_OMIT_FRAME_POINTER void fe_sq(fe r, const fe a) #ifdef HAVE_CURVE25519 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -4680,9 +4680,9 @@ WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) #else #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -5701,9 +5701,9 @@ WC_OMIT_FRAME_POINTER void fe_invert(fe r, const fe a) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -6055,9 +6055,9 @@ WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) #else #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG diff --git a/wolfcrypt/src/port/arm/armv8-32-sha3-asm.S b/wolfcrypt/src/port/arm/armv8-32-sha3-asm.S index aa7192c67ac..c5be47e5692 100644 --- a/wolfcrypt/src/port/arm/armv8-32-sha3-asm.S +++ b/wolfcrypt/src/port/arm/armv8-32-sha3-asm.S @@ -30,6 +30,12 @@ #include +/* Honor WC_SHA3_NO_ASM as sha3.c does: suppress this NEON BlockSha3 so it + * doesn't multiply-define against sha3.c's C BlockSha3 on arm32. */ +#ifdef WC_SHA3_NO_ASM + #undef WOLFSSL_ARMASM +#endif + #ifdef WOLFSSL_ARMASM #if !defined(__aarch64__) && !defined(WOLFSSL_ARMASM_THUMB2) #ifndef WOLFSSL_ARMASM_INLINE diff --git a/wolfcrypt/src/port/arm/armv8-32-sha3-asm_c.c b/wolfcrypt/src/port/arm/armv8-32-sha3-asm_c.c index abc1a9fe09e..da28f24fb4d 100644 --- a/wolfcrypt/src/port/arm/armv8-32-sha3-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-32-sha3-asm_c.c @@ -31,6 +31,12 @@ #include #include +/* Honor WC_SHA3_NO_ASM as sha3.c does: suppress this NEON BlockSha3 so it + * doesn't multiply-define against sha3.c's C BlockSha3 on arm32. */ +#ifdef WC_SHA3_NO_ASM + #undef WOLFSSL_ARMASM +#endif + #ifdef WOLFSSL_ARMASM #if !defined(__aarch64__) && !defined(WOLFSSL_ARMASM_THUMB2) #ifdef WOLFSSL_ARMASM_INLINE diff --git a/wolfcrypt/src/port/arm/armv8-curve25519_c.c b/wolfcrypt/src/port/arm/armv8-curve25519_c.c index 53a573a12fe..168260f4167 100644 --- a/wolfcrypt/src/port/arm/armv8-curve25519_c.c +++ b/wolfcrypt/src/port/arm/armv8-curve25519_c.c @@ -732,7 +732,7 @@ void fe_invert_nct(fe r, const fe a) ); } -void fe_mul(fe r, const fe a, const fe b) +WC_KEEP_FOR_ASM void fe_mul(fe r, const fe a, const fe b) { __asm__ __volatile__ ( /* Multiply */ @@ -869,7 +869,7 @@ void fe_mul(fe r, const fe a, const fe b) ); } -void fe_sq(fe r, const fe a) +WC_KEEP_FOR_ASM void fe_sq(fe r, const fe a) { __asm__ __volatile__ ( /* Square */ diff --git a/wolfcrypt/src/port/arm/thumb2-aes-asm_c.c b/wolfcrypt/src/port/arm/thumb2-aes-asm_c.c index 9405f6e8215..8feaac55ed9 100644 --- a/wolfcrypt/src/port/arm/thumb2-aes-asm_c.c +++ b/wolfcrypt/src/port/arm/thumb2-aes-asm_c.c @@ -656,11 +656,11 @@ WC_OMIT_FRAME_POINTER void AES_set_encrypt_key(const unsigned char* key, void AES_encrypt_block(const word32* te_p, int nr_p, int len_p, const word32* ks_p); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te_p, int nr_p, - int len_p, const word32* ks_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te_p, + int nr_p, int len_p, const word32* ks_p) #else -WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te, int nr, int len, - const word32* ks) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te, + int nr, int len, const word32* ks) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3605,11 +3605,11 @@ WC_OMIT_FRAME_POINTER void AES_CTR_encrypt(const unsigned char* in, #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE void AES_decrypt_block(const word32* td_p, int nr_p, const byte* td4_p); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td_p, int nr_p, - const byte* td4_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td_p, + int nr_p, const byte* td4_p) #else -WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td, int nr, - const byte* td4) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td, + int nr, const byte* td4) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG diff --git a/wolfcrypt/src/port/arm/thumb2-curve25519_c.c b/wolfcrypt/src/port/arm/thumb2-curve25519_c.c index c2607724d57..02e83cb18ab 100644 --- a/wolfcrypt/src/port/arm/thumb2-curve25519_c.c +++ b/wolfcrypt/src/port/arm/thumb2-curve25519_c.c @@ -84,9 +84,9 @@ WC_OMIT_FRAME_POINTER void fe_init(void) void fe_add_sub_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) #else -WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -192,9 +192,9 @@ WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) void fe_sub_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sub_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sub_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -264,9 +264,9 @@ WC_OMIT_FRAME_POINTER void fe_sub(fe r, const fe a, const fe b) void fe_add_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_add_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_op(void) #else -WC_OMIT_FRAME_POINTER void fe_add_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -1766,9 +1766,9 @@ WC_OMIT_FRAME_POINTER void fe_cmov_table(fe* r, const fe* base, signed char b) #ifdef WOLFSSL_ARM_ARCH_7M void fe_mul_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2156,9 +2156,9 @@ WC_OMIT_FRAME_POINTER void fe_mul_op(void) void fe_mul_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2566,9 +2566,9 @@ WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #else void fe_mul_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2710,9 +2710,9 @@ WC_OMIT_FRAME_POINTER void fe_mul_op(void) void fe_mul_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2901,9 +2901,9 @@ WC_OMIT_FRAME_POINTER void fe_mul(fe r, const fe a, const fe b) #ifdef WOLFSSL_ARM_ARCH_7M void fe_sq_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3184,9 +3184,9 @@ WC_OMIT_FRAME_POINTER void fe_sq_op(void) void fe_sq_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3487,9 +3487,9 @@ WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #else void fe_sq_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3617,9 +3617,9 @@ WC_OMIT_FRAME_POINTER void fe_sq_op(void) void fe_sq_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3793,9 +3793,9 @@ WC_OMIT_FRAME_POINTER void fe_sq(fe r, const fe a) #ifdef HAVE_CURVE25519 #ifdef WOLFSSL_ARM_ARCH_7M #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3858,9 +3858,9 @@ WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) #else #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -5123,9 +5123,9 @@ WC_OMIT_FRAME_POINTER void fe_invert(fe r, const fe a) #ifdef WOLFSSL_ARM_ARCH_7M #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -5461,9 +5461,9 @@ WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) #else #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG diff --git a/wolfcrypt/src/port/arm/thumb2-sha3-asm.S b/wolfcrypt/src/port/arm/thumb2-sha3-asm.S index 8df840e51d4..b885c8f8736 100644 --- a/wolfcrypt/src/port/arm/thumb2-sha3-asm.S +++ b/wolfcrypt/src/port/arm/thumb2-sha3-asm.S @@ -30,6 +30,12 @@ #include +/* Honor WC_SHA3_NO_ASM as sha3.c does: suppress this BlockSha3 so it + * doesn't multiply-define against sha3.c's C BlockSha3 on thumb2. */ +#ifdef WC_SHA3_NO_ASM + #undef WOLFSSL_ARMASM +#endif + #ifdef WOLFSSL_ARMASM #ifdef WOLFSSL_ARMASM_THUMB2 #ifndef WOLFSSL_ARMASM_INLINE diff --git a/wolfcrypt/src/port/arm/thumb2-sha3-asm_c.c b/wolfcrypt/src/port/arm/thumb2-sha3-asm_c.c index b01ed75c021..531431d37ca 100644 --- a/wolfcrypt/src/port/arm/thumb2-sha3-asm_c.c +++ b/wolfcrypt/src/port/arm/thumb2-sha3-asm_c.c @@ -31,6 +31,12 @@ #include #include +/* Honor WC_SHA3_NO_ASM as sha3.c does: suppress this BlockSha3 so it + * doesn't multiply-define against sha3.c's C BlockSha3 on thumb2. */ +#ifdef WC_SHA3_NO_ASM + #undef WOLFSSL_ARMASM +#endif + #ifdef WOLFSSL_ARMASM #ifdef WOLFSSL_ARMASM_THUMB2 #ifdef WOLFSSL_ARMASM_INLINE diff --git a/wolfcrypt/src/port/riscv64/riscv-64-aes-asm_c.c b/wolfcrypt/src/port/riscv64/riscv-64-aes-asm_c.c index b64c17d99e0..7230894c083 100644 --- a/wolfcrypt/src/port/riscv64/riscv-64-aes-asm_c.c +++ b/wolfcrypt/src/port/riscv64/riscv-64-aes-asm_c.c @@ -30949,8 +30949,8 @@ void AES_CBC_encrypt_RISCV64(const byte* in, byte* out, word32 sz, byte* reg, } #ifdef HAVE_AES_DECRYPT -void AES_CBC_decrypt_RISCV64(const byte* in, byte* out, word32 sz, byte* reg, - byte* key, int nr) +WC_OMIT_FRAME_POINTER void AES_CBC_decrypt_RISCV64(const byte* in, byte* out, + word32 sz, byte* reg, byte* key, int nr) { const word32* td = L_AES_base_td; const word8* td4 = L_AES_base_td4; @@ -48005,10 +48005,10 @@ void AES_XTS_decrypt_RISCV64(const byte* in, byte* out, word32 sz, #endif /* HAVE_AES_DECRYPT */ #endif /* WOLFSSL_AES_XTS */ #ifdef HAVE_AESGCM -void AES_GCM_encrypt_RISCV64(const byte* in_p, byte* out_p, word32 sz_p, - const byte* nonce_p, word32 nonceSz_p, byte* tag_p, word32 tagSz_p, - const byte* aad_p, word32 aadSz_p, byte* key_p, byte* m0_p, byte* tmp_p, - byte* reg_p, int nr_p) +WC_OMIT_FRAME_POINTER void AES_GCM_encrypt_RISCV64(const byte* in_p, + byte* out_p, word32 sz_p, const byte* nonce_p, word32 nonceSz_p, + byte* tag_p, word32 tagSz_p, const byte* aad_p, word32 aadSz_p, byte* key_p, + byte* m0_p, byte* tmp_p, byte* reg_p, int nr_p) { register const byte* in __asm__ ("a0") = (const byte*)in_p; register byte* out __asm__ ("a1") = (byte*)out_p; @@ -58260,10 +58260,10 @@ void AES_GCM_encrypt_RISCV64(const byte* in_p, byte* out_p, word32 sz_p, } #ifdef HAVE_AES_DECRYPT -int AES_GCM_decrypt_RISCV64(byte* in_p, byte* out_p, word32 sz_p, - const byte* nonce_p, word32 nonceSz_p, const byte* tag_p, word32 tagSz_p, - const byte* aad_p, word32 aadSz_p, byte* key_p, byte* m0_p, byte* tmp_p, - byte* reg_p, int nr_p) +WC_OMIT_FRAME_POINTER int AES_GCM_decrypt_RISCV64(byte* in_p, byte* out_p, + word32 sz_p, const byte* nonce_p, word32 nonceSz_p, const byte* tag_p, + word32 tagSz_p, const byte* aad_p, word32 aadSz_p, byte* key_p, byte* m0_p, + byte* tmp_p, byte* reg_p, int nr_p) { register byte* in __asm__ ("a0") = (byte*)in_p; register byte* out __asm__ ("a1") = (byte*)out_p; diff --git a/wolfcrypt/src/port/riscv64/riscv-64-sha256-asm_c.c b/wolfcrypt/src/port/riscv64/riscv-64-sha256-asm_c.c index a1fa7616f73..f042dc77a3f 100644 --- a/wolfcrypt/src/port/riscv64/riscv-64-sha256-asm_c.c +++ b/wolfcrypt/src/port/riscv64/riscv-64-sha256-asm_c.c @@ -254,8 +254,8 @@ XALIGNED(8) static const word32 L_SHA256_transform_riscv_crypto_len_k[] = { 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, }; -void Transform_Sha256_Len_riscv_crypto(wc_Sha256* sha256, const byte* data, - word32 len) +WC_OMIT_FRAME_POINTER void Transform_Sha256_Len_riscv_crypto(wc_Sha256* sha256, + const byte* data, word32 len) { const word32* k = L_SHA256_transform_riscv_crypto_len_k; @@ -1320,7 +1320,8 @@ XALIGNED(8) static const word32 L_SHA256_transform_riscv_len_k[] = { 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, }; -void Transform_Sha256_Len_riscv(wc_Sha256* sha256, const byte* data, word32 len) +WC_OMIT_FRAME_POINTER void Transform_Sha256_Len_riscv(wc_Sha256* sha256, + const byte* data, word32 len) { const word32* k = L_SHA256_transform_riscv_len_k; diff --git a/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm.S b/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm.S index c4b50ca5f9a..dfd89b0fd57 100644 --- a/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm.S +++ b/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm.S @@ -29,6 +29,13 @@ #define _WC_BUILDING_RISCV_64_SHA3_ASM_S #include + +/* Honor WC_SHA3_NO_ASM as sha3.c does: suppress this BlockSha3 so it + * doesn't multiply-define against sha3.c's C BlockSha3 on riscv64. */ +#ifdef WC_SHA3_NO_ASM + #undef WOLFSSL_RISCV_ASM +#endif + #ifdef WOLFSSL_RISCV_ASM #ifndef WOLFSSL_RISCV_ASM_INLINE #ifdef WOLFSSL_SHA3 diff --git a/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm_c.c b/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm_c.c index a2b8616dfb0..21f3cf4df3b 100644 --- a/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm_c.c +++ b/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm_c.c @@ -30,6 +30,13 @@ #include #include + +/* Honor WC_SHA3_NO_ASM as sha3.c does: suppress this BlockSha3 so it + * doesn't multiply-define against sha3.c's C BlockSha3 on riscv64. */ +#ifdef WC_SHA3_NO_ASM + #undef WOLFSSL_RISCV_ASM +#endif + #ifdef WOLFSSL_RISCV_ASM #ifdef WOLFSSL_RISCV_ASM_INLINE #include @@ -585,7 +592,7 @@ XALIGNED(16) static const word64 L_SHA3_transform_riscv_r[] = { 0x0000000080000001UL, 0x8000000080008008UL, }; -void BlockSha3(word64* s) +WC_OMIT_FRAME_POINTER void BlockSha3(word64* s) { const word64* r = L_SHA3_transform_riscv_r; diff --git a/wolfcrypt/src/sha256_asm.asm b/wolfcrypt/src/sha256_asm.asm index 4fa488d79e8..be9750ab763 100644 --- a/wolfcrypt/src/sha256_asm.asm +++ b/wolfcrypt/src/sha256_asm.asm @@ -12952,7 +12952,9 @@ Transform_Sha256_AVX2_Len PROC vmovdqu ymm1, YMMWORD PTR [rsi+32] vmovups YMMWORD PTR [rdi+32], ymm0 vmovups YMMWORD PTR [rdi+64], ymm1 + sub rsp, 32 call Transform_Sha256_AVX2 + add rsp, 32 add rsi, 64 sub DWORD PTR [rsp+512], 64 jz L_sha256_len_avx2_done @@ -19521,7 +19523,9 @@ Transform_Sha256_AVX2_RORX_Len PROC vmovdqu ymm1, YMMWORD PTR [rsi+32] vmovups YMMWORD PTR [rdi+32], ymm0 vmovups YMMWORD PTR [rdi+64], ymm1 + sub rsp, 32 call Transform_Sha256_AVX2_RORX + add rsp, 32 add rsi, 64 sub DWORD PTR [rsp+512], 64 jz L_sha256_len_avx2_rorx_done diff --git a/wolfcrypt/src/sha512_asm.asm b/wolfcrypt/src/sha512_asm.asm index 0b95a23dae0..b7cdf467fee 100644 --- a/wolfcrypt/src/sha512_asm.asm +++ b/wolfcrypt/src/sha512_asm.asm @@ -6443,7 +6443,9 @@ Transform_Sha512_AVX2_Len PROC vmovups YMMWORD PTR [rdi+96], ymm1 vmovups YMMWORD PTR [rdi+128], ymm2 vmovups YMMWORD PTR [rdi+160], ymm3 + sub rsp, 32 call Transform_Sha512_AVX2 + add rsp, 32 add QWORD PTR [rdi+224], 128 sub ebp, 128 jz L_sha512_len_avx2_done @@ -9241,9 +9243,9 @@ Transform_Sha512_AVX2_RORX_Len PROC vmovups YMMWORD PTR [rdi+96], ymm1 vmovups YMMWORD PTR [rdi+128], ymm2 vmovups YMMWORD PTR [rdi+160], ymm3 - sub rsp, 8 + sub rsp, 40 call Transform_Sha512_AVX2_RORX - add rsp, 8 + add rsp, 40 pop rsi add QWORD PTR [rdi+224], 128 sub esi, 128 diff --git a/wolfcrypt/src/sp_x86_64_asm.S b/wolfcrypt/src/sp_x86_64_asm.S index 3994867b4e0..61ca591a626 100644 --- a/wolfcrypt/src/sp_x86_64_asm.S +++ b/wolfcrypt/src/sp_x86_64_asm.S @@ -45182,7 +45182,7 @@ _sp_256_mont_tpl_4: movq $0xffffffff00000001, %r10 adcq 16(%rsi), %rcx adcq 24(%rsi), %r8 - sbbq $0x00, %r11 + sbbq %r11, %r11 movl %r11d, %r9d andq %r11, %r10 subq %r11, %rdx @@ -45361,7 +45361,7 @@ _sp_256_mont_rsb_sub_dbl_4: movq $0xffffffff00000001, %r15 sbbq %r12, %r8 sbbq %r13, %r9 - sbbq $0x00, %rsi + sbbq %rsi, %rsi movl %esi, %r14d andq %rsi, %r15 addq %rsi, %rax diff --git a/wolfcrypt/src/sp_x86_64_asm.asm b/wolfcrypt/src/sp_x86_64_asm.asm index 690beda8957..f58813f6841 100644 --- a/wolfcrypt/src/sp_x86_64_asm.asm +++ b/wolfcrypt/src/sp_x86_64_asm.asm @@ -43853,7 +43853,7 @@ sp_256_mont_tpl_4 PROC mov r12, 18446744069414584321 adc r9, QWORD PTR [rdx+16] adc r10, QWORD PTR [rdx+24] - sbb r13, 0 + sbb r13, r13 mov r11d, r13d and r12, r13 sub rax, r13 @@ -44005,7 +44005,7 @@ sp_256_mont_rsb_sub_dbl_4 PROC mov rsi, 18446744069414584321 sbb r10, r14 sbb r11, r15 - sbb rdx, 0 + sbb rdx, rdx mov edi, edx and rsi, rdx add rax, rdx diff --git a/wolfssl-VS2022.vcxproj b/wolfssl-VS2022.vcxproj index 9e7f78151f7..928cd6c1b66 100644 --- a/wolfssl-VS2022.vcxproj +++ b/wolfssl-VS2022.vcxproj @@ -562,6 +562,28 @@ true true + + false + false + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(OutDir)%(Filename).obj" %(Identity) + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(IntDir)%(Filename).obj" %(Identity) + $(OutDir)%(Filename).obj + $(IntDir)%(Filename).obj + false + false + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(OutDir)%(Filename).obj" %(Identity) + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(IntDir)%(Filename).obj" %(Identity) + $(OutDir)%(Filename).obj + $(IntDir)%(Filename).obj + true + true + true + true + true + true + true + true + false false diff --git a/wolfssl.vcxproj b/wolfssl.vcxproj index 30c290811a5..0ec821dbe32 100644 --- a/wolfssl.vcxproj +++ b/wolfssl.vcxproj @@ -562,6 +562,28 @@ true true + + false + false + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(OutDir)%(Filename).obj" %(Identity) + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(IntDir)%(Filename).obj" %(Identity) + $(OutDir)%(Filename).obj + $(IntDir)%(Filename).obj + false + false + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(OutDir)%(Filename).obj" %(Identity) + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(IntDir)%(Filename).obj" %(Identity) + $(OutDir)%(Filename).obj + $(IntDir)%(Filename).obj + true + true + true + true + true + true + true + true + false false diff --git a/wolfssl/internal.h b/wolfssl/internal.h index 2f96892e06e..3f6dd0ddddb 100644 --- a/wolfssl/internal.h +++ b/wolfssl/internal.h @@ -53,7 +53,7 @@ #ifdef HAVE_POLY1305 #include #endif -#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) && defined(OPENSSL_EXTRA) +#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) #include #endif #ifdef HAVE_ARIA diff --git a/wolfssl/test.h b/wolfssl/test.h index 23c4ce194fa..055b5b43916 100644 --- a/wolfssl/test.h +++ b/wolfssl/test.h @@ -1896,8 +1896,35 @@ static WC_INLINE int udp_read_connect(SOCKET_T sockfd) } #endif +/* Write the port the server bound to the ready file named by -R, so a script + * driving the server can wait for the file and learn which port an ephemeral + * (-p 0) bind landed on. Shared by the TCP and UDP accept paths - a DTLS server + * needs it just as much as a TLS one. */ +static WC_INLINE void write_ready_file(func_args* args, word16 port) +{ +#if (!defined(NO_FILESYSTEM) || defined(FORCE_BUFFER_TEST)) && !defined(NETOS) + tcp_ready* ready = NULL; + + if (args) + ready = args->signal; + + if (ready && ready->srfName) { + XFILE srf = XFOPEN(ready->srfName, "w"); + + if (srf) { + LIBCALL_CHECK_RET(fprintf(srf, "%d\n", (int)port)); + fclose(srf); + } + } +#else + (void)args; + (void)port; +#endif +} + static WC_INLINE void udp_accept(SOCKET_T* sockfd, SOCKET_T* clientfd, - int useAnyAddr, word16 port, func_args* args) + int useAnyAddr, word16 port, func_args* args, + int ready_file) { SOCKADDR_IN_T addr; @@ -1964,6 +1991,10 @@ static WC_INLINE void udp_accept(SOCKET_T* sockfd, SOCKET_T* clientfd, fprintf(stderr, "args or args->signal was NULL. Not setting ready info."); } + if (ready_file) { + write_ready_file(args, port); + } + *clientfd = *sockfd; } @@ -1977,7 +2008,7 @@ static WC_INLINE void tcp_accept(SOCKET_T* sockfd, SOCKET_T* clientfd, (void) ready; /* Account for case when "ready" is not used */ if (udp) { - udp_accept(sockfd, clientfd, useAnyAddr, port, args); + udp_accept(sockfd, clientfd, useAnyAddr, port, args, ready_file); return; } @@ -2002,26 +2033,7 @@ static WC_INLINE void tcp_accept(SOCKET_T* sockfd, SOCKET_T* clientfd, #endif /* !SINGLE_THREADED */ if (ready_file) { - #if (!defined(NO_FILESYSTEM) || defined(FORCE_BUFFER_TEST)) && \ - !defined(NETOS) - XFILE srf = (XFILE)NULL; - if (args) - ready = args->signal; - - if (ready) { - srf = XFOPEN(ready->srfName, "w"); - - if (srf) { - /* let's write port sever is listening on to ready file - external monitor can then do ephemeral ports by passing - -p 0 to server on supported platforms with -R ready_file - client can then wait for existence of ready_file and see - which port the server is listening on. */ - LIBCALL_CHECK_RET(fprintf(srf, "%d\n", (int)port)); - fclose(srf); - } - } - #endif + write_ready_file(args, port); } } diff --git a/wolfssl/wolfcrypt/wc_port.h b/wolfssl/wolfcrypt/wc_port.h index 3d3faf24ac6..b3a29567cb0 100644 --- a/wolfssl/wolfcrypt/wc_port.h +++ b/wolfssl/wolfcrypt/wc_port.h @@ -176,6 +176,19 @@ #endif #endif +/* Marks a function that is only ever reached from a branch inside an inline + * assembly string. The compiler does not parse those strings, so it sees no + * reference to the function: without this, -flto decides the definition is + * unreachable, drops the body, and the branch is left with nothing to bind to + * at link time. Costs nothing in a non-LTO build. */ +#ifndef WC_KEEP_FOR_ASM + #if defined(__GNUC__) || defined(__clang__) + #define WC_KEEP_FOR_ASM __attribute__((used)) + #else + #define WC_KEEP_FOR_ASM + #endif +#endif + /* THREADING/MUTEX SECTION */ #if defined(SINGLE_THREADED) && defined(NO_FILESYSTEM) /* No system headers required for build. */