diff --git a/cmake/functions.cmake b/cmake/functions.cmake index 1856dcb22c..b0cb0628b9 100644 --- a/cmake/functions.cmake +++ b/cmake/functions.cmake @@ -468,6 +468,10 @@ function(generate_lib_src_list LIB_SOURCES) if(BUILD_AESNI) list(APPEND LIB_SOURCES wolfcrypt/src/aes_asm.S) + # 32-bit x86 AES-XTS. The file guards its own contents on + # WOLFSSL_AES_XTS and WOLFSSL_X86_BUILD, so it compiles to + # nothing on any other target or without XTS. + list(APPEND LIB_SOURCES wolfcrypt/src/aes_xts_x86_asm.S) if(BUILD_INTELASM) list(APPEND LIB_SOURCES wolfcrypt/src/aes_gcm_asm.S) @@ -914,9 +918,13 @@ function(generate_lib_src_list LIB_SOURCES) endif() if(NOT BUILD_FIPS_V2 AND BUILD_AESNI) + # aes_xts_x86_asm.S guards its own contents on WOLFSSL_AES_XTS and + # WOLFSSL_X86_BUILD, so it compiles to nothing on any other target + # or without XTS. list(APPEND LIB_SOURCES wolfcrypt/src/aes_asm.S - wolfcrypt/src/aes_gcm_asm.S) + wolfcrypt/src/aes_gcm_asm.S + wolfcrypt/src/aes_xts_x86_asm.S) endif() if(BUILD_CAMELLIA) diff --git a/configure.ac b/configure.ac index 3b997f801b..1c87d8e42d 100644 --- a/configure.ac +++ b/configure.ac @@ -1215,10 +1215,13 @@ if (test "$host_cpu" = "x86_64" || test "$host_cpu" = "amd64") && then AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_X86_64_BUILD" fi -if test "$host_cpu" = "x86" -then +# config.sub normalises 32-bit x86 to i386/i486/i586/i686, so matching only +# "x86" never fires for a real host triple. +case "$host_cpu" in + x86 | i?86) AM_CFLAGS="$AM_CFLAGS -DWOLFSSL_X86_BUILD" -fi + ;; +esac AC_ARG_ENABLE([leanpsk], @@ -4801,11 +4804,12 @@ then then AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_X86_64_BUILD" fi - if test "$host_cpu" = "x86" - then + case "$host_cpu" in + x86 | i?86) AM_CCASFLAGS="$AM_CCASFLAGS -DWOLFSSL_X86_BUILD" ENABLED_X86_ASM=yes - fi + ;; + esac fi AC_SUBST([ENABLED_AESNI]) AC_SUBST([ENABLED_AESNI_WITH_AVX]) diff --git a/examples/client/client.c b/examples/client/client.c index 822da1afd7..3f8102aeb8 100644 --- a/examples/client/client.c +++ b/examples/client/client.c @@ -89,6 +89,9 @@ static const char *wolfsentry_config_path = NULL; #ifndef MAX_NON_BLOCK_SEC #define MAX_NON_BLOCK_SEC 10 #endif +/* How long a single wait for the socket blocks before the loop re-checks its + * overall budget. Short enough that the budget is still honoured closely. */ +#define NON_BLOCK_POLL_SEC 1 #define OCSP_STAPLING 1 #define OCSP_STAPLINGV2 2 @@ -1124,12 +1127,28 @@ static int ClientWrite(WOLFSSL* ssl, const char* msg, int msgSz, const char* str return err; } -static int ClientRead(WOLFSSL* ssl, char* reply, int replyLen, int mustRead, - const char* str, int exitWithRet) +/* Read a reply. On a non-blocking socket a WANT_READ only means the reply has + * not arrived yet, so wait for it rather than returning on the first poll - + * returning early lets the caller shut the connection down while the peer is + * still writing, which the peer then reports as a transport error. + * + * replyRequired says whether a missing reply is a failure, not whether one is + * worth waiting for: it selects the wait budget and controls whether giving up + * is reported as an error. */ +static int ClientRead(WOLFSSL* ssl, char* reply, int replyLen, + int replyRequired, const char* str, int exitWithRet) { int ret, err; char buffer[WOLFSSL_MAX_ERROR_SZ]; double start = current_time(1), elapsed; + /* A required reply gets the full non-blocking budget, an optional one the + * shorter of the two - MAX_NON_BLOCK_SEC is overridable and may be set + * below DEFAULT_TIMEOUT_SEC, which would otherwise invert the two. */ + double maxWait = MAX_NON_BLOCK_SEC; + + if (!replyRequired && DEFAULT_TIMEOUT_SEC < MAX_NON_BLOCK_SEC) { + maxWait = DEFAULT_TIMEOUT_SEC; + } do { err = 0; /* reset error */ @@ -1158,17 +1177,49 @@ static int ClientRead(WOLFSSL* ssl, char* reply, int replyLen, int mustRead, } } - if (mustRead && - (err == WOLFSSL_ERROR_WANT_READ - || err == WOLFSSL_ERROR_WANT_WRITE)) { + if (err == WOLFSSL_ERROR_WANT_READ + || err == WOLFSSL_ERROR_WANT_WRITE) { + int selectRet; + elapsed = current_time(0) - start; - if (elapsed > MAX_NON_BLOCK_SEC) { - LOG_ERROR("Nonblocking read timeout\n"); + if (elapsed > maxWait) { + if (replyRequired) { + LOG_ERROR("Nonblocking read timeout\n"); + } + ret = WOLFSSL_FATAL_ERROR; + break; + } + + /* Wait for the socket instead of spinning on it. */ + if (err == WOLFSSL_ERROR_WANT_WRITE) { + selectRet = tcp_select_tx(wolfSSL_get_fd(ssl), + NON_BLOCK_POLL_SEC); + } + else { + selectRet = tcp_select(wolfSSL_get_fd(ssl), + NON_BLOCK_POLL_SEC); + } + + #ifdef WOLFSSL_DTLS + /* A DTLS timeout means the peer's datagram was lost - let the + * library retransmit rather than waiting for something that is + * never coming (see NonBlockingSSL_Connect). */ + if (selectRet == TEST_TIMEOUT && wolfSSL_dtls(ssl)) { + if (wolfSSL_dtls_got_timeout(ssl) != WOLFSSL_SUCCESS) { + err = wolfSSL_get_error(ssl, WOLFSSL_FATAL_ERROR); + break; + } + } + else + #endif + /* select() itself failed - retrying would spin, not wait. */ + if (selectRet == TEST_SELECT_FAIL) { + LOG_ERROR("%s tcp_select error\n", str); ret = WOLFSSL_FATAL_ERROR; break; } } - } while ((mustRead && err == WOLFSSL_ERROR_WANT_READ) + } while (err == WOLFSSL_ERROR_WANT_READ || err == WOLFSSL_ERROR_WANT_WRITE #ifdef WOLFSSL_ASYNC_CRYPT || err == WC_NO_ERR_TRACE(WC_PENDING_E) @@ -1183,11 +1234,15 @@ static int ClientRead(WOLFSSL* ssl, char* reply, int replyLen, int mustRead, return err; } +/* replyRequired: whether a missing reply fails the exchange. See ClientRead. */ static int ClientWriteRead(WOLFSSL* ssl, const char* msg, int msgSz, - char* reply, int replyLen, int mustRead, + char* reply, int replyLen, int replyRequired, const char* str, int exitWithRet) { int ret = 0; + /* Which half of the exchange the error below came from - the message used + * to say SSL_write for a failure returned by ClientRead. */ + const char* stage = "SSL_write"; do { ret = ClientWrite(ssl, msg, msgSz, str, exitWithRet); @@ -1207,6 +1262,7 @@ static int ClientWriteRead(WOLFSSL* ssl, const char* msg, int msgSz, } else { LOG_ERROR("%s tcp_select error\n", str); + stage = "tcp_select"; if (!exitWithRet) err_sys("tcp_select failed"); else @@ -1214,8 +1270,9 @@ static int ClientWriteRead(WOLFSSL* ssl, const char* msg, int msgSz, break; } } - ret = ClientRead(ssl, reply, replyLen, mustRead, str, exitWithRet); - if (mustRead && ret != 0) { + stage = "SSL_read"; + ret = ClientRead(ssl, reply, replyLen, replyRequired, str, exitWithRet); + if (replyRequired && ret != 0) { if (!exitWithRet) err_sys("ClientRead failed"); else @@ -1224,9 +1281,11 @@ static int ClientWriteRead(WOLFSSL* ssl, const char* msg, int msgSz, break; } while (1); - if (ret != 0) { + /* A failed optional read is not an error - the caller asked for the reply + * only if one turned up - so do not log one. */ + if (ret != 0 && (replyRequired || XSTRCMP(stage, "SSL_read") != 0)) { char buffer[WOLFSSL_MAX_ERROR_SZ]; - LOG_ERROR("SSL_write%s msg error %d, %s\n", str, ret, + LOG_ERROR("%s%s msg error %d, %s\n", stage, str, ret, wolfSSL_ERR_error_string((unsigned long)ret, buffer)); } diff --git a/scripts/benchmark.test b/scripts/benchmark.test index f120a15dc1..801d57cde4 100755 --- a/scripts/benchmark.test +++ b/scripts/benchmark.test @@ -53,17 +53,20 @@ if [ "$#" -lt 2 ]; then exit 1 fi -# Use unique benchmark port so it won't conflict with any other tests -bench_port=11113 +# Bind an ephemeral port and read it back from the ready file below, rather +# than hoping a fixed number is free. The ready file is per-run for the same +# reason - a fixed path is shared by any concurrent run. +bench_port=0 +ready_file=/tmp/wolfssl_server_ready$$ no_pid=-1 server_pid=$no_pid counter=0 client_result=-1 remove_ready_file() { - if test -e /tmp/wolfssl_server_ready; then + if test -e "$ready_file"; then echo "removing existing server_ready file" - rm /tmp/wolfssl_server_ready + rm "$ready_file" fi } @@ -96,7 +99,7 @@ remove_ready_file if [ $1 -eq 1 ] then # start server in loop mode with port - ./examples/server/server -i -p $bench_port $4 & + ./examples/server/server -i -p $bench_port -R "$ready_file" $4 & server_pid=$! fi @@ -104,7 +107,7 @@ fi if [ $1 -eq 2 ] then # start server in loop mode, non-blocking, benchmark throughput with port - ./examples/server/server -i -N -B $2 -p $bench_port $4 & + ./examples/server/server -i -N -B $2 -p $bench_port -R "$ready_file" $4 & server_pid=$! fi @@ -112,11 +115,23 @@ fi # with "time", bear in mind that those 2 seconds will be reflected in # the "real" time. echo "Waiting for server_ready file..." -while [ ! -s /tmp/wolfssl_server_ready -a "$counter" -lt 20 ]; do +while [ ! -s "$ready_file" -a "$counter" -lt 20 ]; do sleep 0.1 counter=$((counter+ 1)) done +if [ ! -s "$ready_file" ]; then + echo "Server never reported a port!" + do_cleanup + exit 1 +fi + +# sleep for an additional 0.1 to mitigate race on write/read of $ready_file: +sleep 0.1 + +bench_port=$(cat "$ready_file") +echo "Server listening on port $bench_port" + # benchmark connections if [ $1 -eq 1 ] then diff --git a/scripts/dtls.test b/scripts/dtls.test index 9a1f621ebf..eb1d835bf8 100755 --- a/scripts/dtls.test +++ b/scripts/dtls.test @@ -22,6 +22,9 @@ DO_EXTENDED_SERVER_PERMUTATION_TEST=${DO_EXTENDED_SERVER_PERMUTATION_TEST:-0} DO_DELAY_TEST=${DO_DELAY_TEST:-0} # An example use would be: DTLS_VERSION=-v3 scripts/dtls.test +# Prerequisites, each of which makes the script skip (exit 77) when missing: +# the udp_proxy helper, python3, and a server build that can report an +# ephemeral port through -p 0 with -R. # Note the output also consists of a single PCAP file which has a set of # three packets (1-byte, strlen()-byte, 1-byte payload) deliniating each test. @@ -40,6 +43,23 @@ if ./examples/server/server -? 2>&1 | grep "Server not compiled in!" ; then exit 77 fi +# Every test runs through the udp_proxy helper. Without it each one fails at the +# client, so skip the script rather than report a wall of failures. +# command -v matches how the proxy is actually launched below: it searches +# $PATH for a bare name and tests executability for a path with a slash. An +# extra -x test would accept a ./udp_proxy that the launch could never find. +if ! command -v "$UDP_PROXY_BIN" >/dev/null 2>&1; then + echo "skipping dtls.test because $UDP_PROXY_BIN was not found." 1>&2 + exit 77 +fi + +# python3 picks the proxy port for every test, on top of generating the +# permutation and delay lists, so it is a hard requirement of the whole script. +if ! command -v python3 >/dev/null 2>&1; then + echo 'skipping dtls.test because python3 was not found.' 1>&2 + exit 77 +fi + # bwrap execution environment to avoid port conflicts if [ "${AM_BWRAPPED-}" != "yes" ]; then bwrap_path="$(command -v bwrap)" @@ -73,20 +93,117 @@ cleanup () { sleep 1 kill $TCPDUMP_PID fi + + rm -f "$READY_FILE" } trap cleanup err exit -PROXY_PORT=1234 -SERVER_PORT=4321 +# Nothing ever binds the marker port - the three datagrams sent to it only tag +# the capture - so a fixed number is safe here. The server and proxy ports are +# chosen per test below. +MARKER_PORT=4321 KEY_UPDATE_SIZE=35 NUM_TESTS_FAILED=0 NUM_TESTS_RUN=0 +# The server writes the port it bound to this file. The pid keeps concurrent +# runs from sharing one. +READY_FILE="$WOLFSSL_ROOT/wolfssl_dtls_ready$$" + +HAVE_SS=no +command -v ss >/dev/null 2>&1 && HAVE_SS=yes + +# A free UDP port for the proxy. udp_proxy has no equivalent of the server's -R +# ready file, so the port is picked here and the proxy polled below until it is +# actually bound. +free_udp_port() { + python3 -c 'import socket +s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +s.bind(("127.0.0.1", 0)) +print(s.getsockname()[1]) +s.close()' +} + +# A usable port is a non-zero decimal below 65536. Worth checking rather than +# assuming: a SINGLE_THREADED build never resolves the ephemeral port in +# udp_accept(), so the ready file legitimately contains 0 there, and pointing +# the proxy at port 0 would report every test as a DTLS failure instead. +valid_port() { + case "$1" in + ''|*[!0-9]*) return 1 ;; + esac + [ "$1" -gt 0 ] && [ "$1" -lt 65536 ] +} + +# Wait for $1 to be bound for UDP. Where ss is not available to be asked this +# cannot be checked, so say so rather than let a blind sleep pass for a check. +wait_udp_bound() { + local counter=0 + + if [ "$HAVE_SS" != "yes" ]; then + echo "[warning] ss not available - cannot confirm the proxy bound $1" + sleep 0.5 + return 0 + fi + + while [ "$counter" -lt 50 ]; do + # Match on the local port specifically - a bare grep for the number can + # be satisfied by the peer column or an unrelated socket. + if [ -n "$(ss -lunH "sport = :$1" 2>/dev/null)" ]; then + return 0 + fi + sleep 0.1 + counter=$((counter + 1)) + done + return 1 +} if [ "$DTLS_VERSION" = "-v4" ]; then UDP_PROXY_EXTRA_ARGS="-u" fi +# Everything below depends on the server being able to report the port it bound. +# A SINGLE_THREADED or WOLFSSL_TIRTOS build never resolves an ephemeral port in +# udp_accept(), and a NO_FILESYSTEM or NETOS build compiles write_ready_file() +# to a no-op, so probe the capability once here rather than rediscovering it on +# every one of the tests below. +check_ready_file_support() { + local probe_pid counter port + + rm -f "$READY_FILE" + $WOLFSSL_ROOT/examples/server/server -u -p 0 -R "$READY_FILE" $DTLS_VERSION \ + >/dev/null 2>&1 & + probe_pid=$! + + counter=0 + while [ ! -s "$READY_FILE" -a "$counter" -lt 50 ]; do + if ! kill -0 "$probe_pid" 2>&- && [ ! -s "$READY_FILE" ]; then + break + fi + sleep 0.1 + counter=$((counter + 1)) + done + + port="" + if [ -s "$READY_FILE" ]; then + sleep 0.1 + port=$(cat "$READY_FILE") + fi + + kill -9 "$probe_pid" 2>&- + wait "$probe_pid" 2>&- + rm -f "$READY_FILE" + + valid_port "$port" +} + +if ! check_ready_file_support; then + echo 'skipping dtls.test because the server cannot report an ephemeral port' 1>&2 + echo 'with -p 0 -R (needs threads and a filesystem in the build).' 1>&2 + exit 77 +fi + + # $WOLFSSL_ROOT/tests/unit.test tests/test-dtls13.conf set -o pipefail @@ -94,21 +211,72 @@ prepend() { # Usage: cmd 2>&1 | prepend "sometext " while read line; do echo "${1}${line}"; done } +# Record a failed test. Plain assignment rather than ((n++)) so the count is +# not also the command's exit status. +fail_test() { + echo "***Test failed***${1:+ $1}" + NUM_TESTS_FAILED=$((NUM_TESTS_FAILED + 1)) +} + run_test() { # usage: run_test "" "" "" "" ((NUM_TESTS_RUN++)) - echo "" | nc -u -w 1 127.0.0.1 $SERVER_PORT # This is a marker for the PCAP file - echo "$1" | nc -u -w 1 127.0.0.1 $SERVER_PORT # This is a marker for the PCAP file - echo "" | nc -u -w 1 127.0.0.1 $SERVER_PORT # This is a marker for the PCAP file + echo "" | nc -u -w 1 127.0.0.1 $MARKER_PORT # This is a marker for the PCAP file + echo "$1" | nc -u -w 1 127.0.0.1 $MARKER_PORT # This is a marker for the PCAP file + echo "" | nc -u -w 1 127.0.0.1 $MARKER_PORT # This is a marker for the PCAP file echo -e "\n${1}\n" - stdbuf -oL -eL $WOLFSSL_ROOT/examples/server/server -u -p$SERVER_PORT $DTLS_VERSION $3 2>&1 | prepend "[server] " & - sleep 0.2 + + # Bind an ephemeral port and wait for the server to report it, rather than + # taking a fixed port and assuming a fixed sleep covers the bind. + rm -f "$READY_FILE" + stdbuf -oL -eL $WOLFSSL_ROOT/examples/server/server -u -p 0 -R "$READY_FILE" $DTLS_VERSION $3 2>&1 | prepend "[server] " & + # The pid is prepend's, but it exits on EOF when the server dies, so it + # still tells us whether the server is alive - and saves the full poll when + # it is not, which matters at 5040 tests with the extended permutations. + SERVER_JOB_PID=$! + + counter=0 + while [ ! -s "$READY_FILE" -a "$counter" -lt 50 ]; do + if ! kill -0 "$SERVER_JOB_PID" 2>&- && [ ! -s "$READY_FILE" ]; then + fail_test "server exited before creating the ready file" + kill_server + return + fi + sleep 0.1 + counter=$((counter + 1)) + done + if [ ! -s "$READY_FILE" ]; then + fail_test "server did not become ready" + kill_server + return + fi + + # sleep for an additional 0.1 to mitigate race on write/read of $READY_FILE: + sleep 0.1 + + SERVER_PORT=$(cat "$READY_FILE") + if ! valid_port "$SERVER_PORT"; then + fail_test "server reported unusable port '$SERVER_PORT'" + kill_server + return + fi + + PROXY_PORT=$(free_udp_port) + if ! valid_port "$PROXY_PORT"; then + fail_test "could not obtain a free UDP port for the proxy" + kill_server + return + fi stdbuf -oL -eL $UDP_PROXY_BIN -p $PROXY_PORT -s 127.0.0.1:$SERVER_PORT $UDP_PROXY_EXTRA_ARGS $2 2>&1 | prepend "[udp-proxy] " & - sleep 0.2 + if ! wait_udp_bound $PROXY_PORT; then + fail_test "udp-proxy did not bind port $PROXY_PORT" + kill_server + return + fi + # Wrap this command in a timeout so that a deadlock won't bring down the entire test $TIMEOUT_KILL_1M stdbuf -oL -eL $WOLFSSL_ROOT/examples/client/client -u -p$PROXY_PORT $DTLS_VERSION $4 2>&1 | prepend "[client] " if [ $? != 0 ]; then - echo "***Test failed***" - ((NUM_TESTS_FAILED++)) + fail_test fi kill_server } @@ -167,7 +335,9 @@ EOF } echo "Starting capture" -tcpdump -i lo -n port ${SERVER_PORT} -w ${PCAP_FILENAME} -U & +# The server port is ephemeral and changes per test, so capture by protocol and +# address rather than one fixed port. +tcpdump -i lo -n udp and host 127.0.0.1 -w ${PCAP_FILENAME} -U & TCPDUMP_PID=$! sleep 0.5 diff --git a/scripts/dtlscid.test b/scripts/dtlscid.test index a44ee60340..93a4d98d79 100755 --- a/scripts/dtlscid.test +++ b/scripts/dtlscid.test @@ -7,8 +7,10 @@ # wrapper and orphan the server it was meant to kill. if command -v timeout >/dev/null 2>&1; then TIMEOUT_KILL_2M="timeout -s KILL 2m" + TIMEOUT_KILL_1M="timeout -s KILL 1m" else TIMEOUT_KILL_2M="" + TIMEOUT_KILL_1M="" fi @@ -52,6 +54,7 @@ cleanup () { echo "Killing server $SERVER_PID" kill $SERVER_PID fi + rm -f "$READY_FILE" } trap cleanup err exit @@ -61,19 +64,133 @@ SCID="BB" HEXCID=$(printf $CCID | od -An -tx1 | tr -d ' \n') HEXSCID=$(printf $SCID | od -An -tx1 | tr -d ' \n') WOLFSSL_ROOT=$(pwd) +# The server reports the port it bound in this file. The pid keeps concurrent +# runs - a second "make check" in another source tree, say - from sharing one. +READY_FILE="$WOLFSSL_ROOT/wolfssl_dtlscid_ready$$" + +# A usable port is a non-zero decimal below 65536. Worth checking rather than +# assuming: a SINGLE_THREADED or WOLFSSL_TIRTOS build never resolves the +# ephemeral port in udp_accept(), so the ready file legitimately contains 0. +valid_port() { + case "$1" in + ''|*[!0-9]*) return 1 ;; + esac + [ "$1" -gt 0 ] && [ "$1" -lt 65536 ] +} + +# The test below depends on the server being able to report the port it bound. +# Besides the SINGLE_THREADED/WOLFSSL_TIRTOS case above, a NO_FILESYSTEM or +# NETOS build compiles write_ready_file() to a no-op and never creates the file +# at all. Probe the capability once and skip, rather than report a build-wide +# limitation as a CID test failure. +check_ready_file_support() { + local probe_pid counter port + + rm -f "$READY_FILE" + $WOLFSSL_ROOT/examples/server/server -v4 -u -p 0 -R "$READY_FILE" \ + >/dev/null 2>&1 & + probe_pid=$! + + counter=0 + while [ ! -s "$READY_FILE" -a "$counter" -lt 50 ]; do + if ! kill -0 "$probe_pid" 2>&- && [ ! -s "$READY_FILE" ]; then + break + fi + sleep 0.1 + counter=$((counter + 1)) + done + + port="" + if [ -s "$READY_FILE" ]; then + sleep 0.1 + port=$(cat "$READY_FILE") + fi + + kill -9 "$probe_pid" 2>&- + wait "$probe_pid" 2>&- + rm -f "$READY_FILE" + + valid_port "$port" +} + +if ! check_ready_file_support; then + echo 'skipping dtlscid.test because the server cannot report an ephemeral' 1>&2 + echo 'port with -p 0 -R (needs threads and a filesystem in the build).' 1>&2 + exit 77 +fi + +# Report a failed check, with the output of both sides, and stop the test. +fail () { + echo "$1" + echo "--- server output ---" + cat "$SERVER_FILE" + echo "--- client output ---" + cat "$CLIENT_FILE" + rm -f "$SERVER_FILE" "$CLIENT_FILE" + exit 1 +} test_cid () { echo "Running test_cid" SERVER_FILE=$(mktemp) CLIENT_FILE=$(mktemp) - $TIMEOUT_KILL_2M $WOLFSSL_ROOT/examples/server/server -v4 -u --cid $SCID 1> $SERVER_FILE & + rm -f "$READY_FILE" + + # Bind an ephemeral port (-p 0) and wait for the server to report it, rather + # than taking the default port and assuming a fixed sleep is long enough for + # the bind. The default port also collides with any other test using it when + # the network namespace could not be unshared above. + $TIMEOUT_KILL_2M $WOLFSSL_ROOT/examples/server/server -v4 -u \ + --cid $SCID -p 0 -R "$READY_FILE" 1> $SERVER_FILE & SERVER_PID=$! - sleep 0.2 - $TIMEOUT_KILL_2M $WOLFSSL_ROOT/examples/client/client -v4 -u --cid $CCID 1> $CLIENT_FILE + + counter=0 + while [ ! -s "$READY_FILE" -a "$counter" -lt 50 ]; do + sleep 0.1 + counter=$((counter + 1)) + done + if [ ! -s "$READY_FILE" ]; then + fail "server did not become ready" + fi + + # sleep for an additional 0.1 to mitigate race on write/read of $READY_FILE: + sleep 0.1 + + PORT=$(cat "$READY_FILE") + if ! valid_port "$PORT"; then + fail "server reported unusable port '$PORT'" + fi + echo "Server listening on port $PORT" + + # Bound below the server's timeout: a DTLS client with nothing to talk to + # retransmits for over two minutes, so without this the server would be + # killed first and the client left running against nothing. + $TIMEOUT_KILL_1M $WOLFSSL_ROOT/examples/client/client -v4 -u \ + --cid $CCID -p $PORT 1> $CLIENT_FILE + CLIENT_RESULT=$? + if [ $CLIENT_RESULT != 0 ]; then + fail "client failed with $CLIENT_RESULT" + fi + wait $SERVER_PID + SERVER_RESULT=$? SERVER_PID= - grep "Sending CID is ${HEXSCID}" $CLIENT_FILE > /dev/null - grep "Sending CID is ${HEXCID}" $SERVER_FILE > /dev/null + if [ $SERVER_RESULT != 0 ]; then + fail "server failed with $SERVER_RESULT" + fi + + # Each side reports the CID it sends, which is the one its peer chose. These + # checks are the point of the test, so a mismatch has to fail the script - + # as the last commands in the function their status was previously + # discarded by the echo below. + if ! grep -q "Sending CID is ${HEXSCID}" $CLIENT_FILE; then + fail "client did not report sending CID ${HEXSCID}" + fi + if ! grep -q "Sending CID is ${HEXCID}" $SERVER_FILE; then + fail "server did not report sending CID ${HEXCID}" + fi + + rm -f "$SERVER_FILE" "$CLIENT_FILE" echo "test_cid has passed" } diff --git a/scripts/multi-msg-record.py b/scripts/multi-msg-record.py index 8145f62f03..6832bc3e84 100755 --- a/scripts/multi-msg-record.py +++ b/scripts/multi-msg-record.py @@ -193,22 +193,19 @@ def _parse_hs_types(data): return msgs -def _get_free_port(): - """Get an available TCP port.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("127.0.0.1", 0)) - return s.getsockname()[1] - - def _listen_socket(): - """Bind a listening TCP socket on localhost with the standard test timeout.""" - port = _get_free_port() + """Bind a listening TCP socket on localhost with the standard test timeout. + + The port is read back from the socket that keeps it, rather than from a + throwaway one that is closed first: closing it leaves a window in which + anything else on the machine can take the port before the real bind. + """ srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - srv.bind(("127.0.0.1", port)) + srv.bind(("127.0.0.1", 0)) srv.listen(1) srv.settimeout(15) - return srv, port + return srv, srv.getsockname()[1] def _run_wolf_client(port, version, cipher, extra=()): diff --git a/scripts/ocsp-responder-openssl-interop.test b/scripts/ocsp-responder-openssl-interop.test index 83e96897ba..751532e8f4 100755 --- a/scripts/ocsp-responder-openssl-interop.test +++ b/scripts/ocsp-responder-openssl-interop.test @@ -150,6 +150,18 @@ print_responder_logs() { # already bound, to avoid the birthday problem in case other # instances are sharing this host. +# True when anything holds the port - listening or established, TCP or UDP. +# nc -z below only finds TCP listeners, so on its own it misses a UDP responder +# socket and a port held by an unrelated outbound connection, both of which +# still make bind() fail. +port_bound() { + if command -v ss >/dev/null 2>&1; then + [ -n "$(ss -Hantu "sport = :$1" 2>/dev/null)" ] + else + return 1 + fi +} + get_first_free_port() { local ret="$1" local scanned=0 @@ -157,7 +169,7 @@ get_first_free_port() { if [[ "$ret" -ge 65536 ]]; then ret=1024 fi - if ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then + if ! port_bound "$ret" && ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then break fi ret=$((ret+1)) diff --git a/scripts/ocsp-stapling-with-ca-as-responder.test b/scripts/ocsp-stapling-with-ca-as-responder.test index 7845e2f523..61fece7960 100755 --- a/scripts/ocsp-stapling-with-ca-as-responder.test +++ b/scripts/ocsp-stapling-with-ca-as-responder.test @@ -206,6 +206,18 @@ ca=certs/external/DigiCertGlobalRootCA.pem # already bound, to avoid the birthday problem in case other # instances are sharing this host. +# True when anything holds the port - listening or established, TCP or UDP. +# nc -z below only finds TCP listeners, so on its own it misses a UDP responder +# socket and a port held by an unrelated outbound connection, both of which +# still make bind() fail. +port_bound() { + if command -v ss >/dev/null 2>&1; then + [ -n "$(ss -Hantu "sport = :$1" 2>/dev/null)" ] + else + return 1 + fi +} + get_first_free_port() { local ret="$1" local scanned=0 @@ -213,7 +225,7 @@ get_first_free_port() { if [[ "$ret" -ge 65536 ]]; then ret=1024 fi - if ! nc -w 1 -z 127.0.0.1 "$ret"; then + if ! port_bound "$ret" && ! nc -w 1 -z 127.0.0.1 "$ret"; then break fi ret=$((ret+1)) diff --git a/scripts/ocsp-stapling-with-wolfssl-responder.test b/scripts/ocsp-stapling-with-wolfssl-responder.test index 1b9fcfe012..eefcbf064a 100755 --- a/scripts/ocsp-stapling-with-wolfssl-responder.test +++ b/scripts/ocsp-stapling-with-wolfssl-responder.test @@ -352,6 +352,18 @@ fi # already bound, to avoid the birthday problem in case other # instances are sharing this host. +# True when anything holds the port - listening or established, TCP or UDP. +# nc -z below only finds TCP listeners, so on its own it misses a UDP responder +# socket and a port held by an unrelated outbound connection, both of which +# still make bind() fail. +port_bound() { + if command -v ss >/dev/null 2>&1; then + [ -n "$(ss -Hantu "sport = :$1" 2>/dev/null)" ] + else + return 1 + fi +} + get_first_free_port() { local ret="$1" local scanned=0 @@ -359,7 +371,7 @@ get_first_free_port() { if [[ "$ret" -ge 65536 ]]; then ret=1024 fi - if ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then + if ! port_bound "$ret" && ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then break fi ret=$((ret+1)) diff --git a/scripts/ocsp-stapling.test b/scripts/ocsp-stapling.test index 83aa25df5e..79f92c550d 100755 --- a/scripts/ocsp-stapling.test +++ b/scripts/ocsp-stapling.test @@ -278,6 +278,18 @@ fi # already bound, to avoid the birthday problem in case other # instances are sharing this host. +# True when anything holds the port - listening or established, TCP or UDP. +# nc -z below only finds TCP listeners, so on its own it misses a UDP responder +# socket and a port held by an unrelated outbound connection, both of which +# still make bind() fail. +port_bound() { + if command -v ss >/dev/null 2>&1; then + [ -n "$(ss -Hantu "sport = :$1" 2>/dev/null)" ] + else + return 1 + fi +} + get_first_free_port() { local ret="$1" local scanned=0 @@ -285,7 +297,7 @@ get_first_free_port() { if [[ "$ret" -ge 65536 ]]; then ret=1024 fi - if ! nc -w 1 -z $V4V6_FLAG $LOCALHOST_FOR_NC "$ret"; then + if ! port_bound "$ret" && ! nc -w 1 -z $V4V6_FLAG $LOCALHOST_FOR_NC "$ret"; then break fi ret=$((ret+1)) @@ -534,10 +546,15 @@ generate_port() { # Also check if the port is already bound on this system if [ $collision -eq 0 ]; then + # TCP and UDP, established as well as listening, matched on the + # local port. -lnt saw only TCP listeners, so it was blind to the + # UDP sockets the DTLS cases here bind and to a port held by an + # unrelated outbound connection - both of which still block bind. + # A false positive only costs another iteration of this loop. if command -v ss &>/dev/null; then - ss -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + [ -n "$(ss -Hantu "sport = :${p}" 2>/dev/null)" ] && collision=1 elif command -v netstat &>/dev/null; then - netstat -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + netstat -an 2>/dev/null | grep -q "[:.]${p}[[:space:]]" && collision=1 fi fi diff --git a/scripts/ocsp-stapling2.test b/scripts/ocsp-stapling2.test index 3fdf66cd5d..567770fce8 100755 --- a/scripts/ocsp-stapling2.test +++ b/scripts/ocsp-stapling2.test @@ -261,6 +261,18 @@ fi # already bound, to avoid the birthday problem in case other # instances are sharing this host. +# True when anything holds the port - listening or established, TCP or UDP. +# nc -z below only finds TCP listeners, so on its own it misses a UDP responder +# socket and a port held by an unrelated outbound connection, both of which +# still make bind() fail. +port_bound() { + if command -v ss >/dev/null 2>&1; then + [ -n "$(ss -Hantu "sport = :$1" 2>/dev/null)" ] + else + return 1 + fi +} + get_first_free_port() { local ret="$1" local scanned=0 @@ -268,7 +280,7 @@ get_first_free_port() { if [[ "$ret" -ge 65536 ]]; then ret=1024 fi - if ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then + if ! port_bound "$ret" && ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then break fi ret=$((ret+1)) @@ -532,10 +544,15 @@ generate_port() { # Also check if the port is already bound on this system if [ $collision -eq 0 ]; then + # TCP and UDP, established as well as listening, matched on the + # local port. -lnt saw only TCP listeners, so it was blind to the + # UDP sockets the DTLS cases here bind and to a port held by an + # unrelated outbound connection - both of which still block bind. + # A false positive only costs another iteration of this loop. if command -v ss &>/dev/null; then - ss -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + [ -n "$(ss -Hantu "sport = :${p}" 2>/dev/null)" ] && collision=1 elif command -v netstat &>/dev/null; then - netstat -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + netstat -an 2>/dev/null | grep -q "[:.]${p}[[:space:]]" && collision=1 fi fi diff --git a/scripts/ocsp-stapling_tls13multi.test b/scripts/ocsp-stapling_tls13multi.test index 59c93eebba..bb1b0a32c9 100755 --- a/scripts/ocsp-stapling_tls13multi.test +++ b/scripts/ocsp-stapling_tls13multi.test @@ -291,6 +291,18 @@ fi # already bound, to avoid the birthday problem in case other # instances are sharing this host. +# True when anything holds the port - listening or established, TCP or UDP. +# nc -z below only finds TCP listeners, so on its own it misses a UDP responder +# socket and a port held by an unrelated outbound connection, both of which +# still make bind() fail. +port_bound() { + if command -v ss >/dev/null 2>&1; then + [ -n "$(ss -Hantu "sport = :$1" 2>/dev/null)" ] + else + return 1 + fi +} + get_first_free_port() { local ret="$1" local scanned=0 @@ -298,7 +310,7 @@ get_first_free_port() { if [[ "$ret" -ge 65536 ]]; then ret=1024 fi - if ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then + if ! port_bound "$ret" && ! nc -w 1 -z ${LOCALHOST_FOR_NC} "$ret"; then break fi ret=$((ret+1)) diff --git a/scripts/openssl.test b/scripts/openssl.test index f64e22fe83..84be7eca5b 100755 --- a/scripts/openssl.test +++ b/scripts/openssl.test @@ -93,10 +93,15 @@ generate_port() { # Also check if the port is already bound on this system if [ "$collision" -eq 0 ]; then + # TCP and UDP, established as well as listening, matched on the + # local port. -lnt saw only TCP listeners, so it was blind to the + # UDP sockets the DTLS cases here bind and to a port held by an + # unrelated outbound connection - both of which still block bind. + # A false positive only costs another iteration of this loop. if command -v ss &>/dev/null; then - ss -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + [ -n "$(ss -Hantu "sport = :${p}" 2>/dev/null)" ] && collision=1 elif command -v netstat &>/dev/null; then - netstat -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + netstat -an 2>/dev/null | grep -q "[:.]${p}[[:space:]]" && collision=1 fi fi diff --git a/scripts/openssl_srtp.test b/scripts/openssl_srtp.test index 5fb907fb3a..a882c3f81d 100755 --- a/scripts/openssl_srtp.test +++ b/scripts/openssl_srtp.test @@ -71,10 +71,15 @@ generate_port() { # Also check if the port is already bound on this system if [ "$collision" -eq 0 ]; then + # TCP and UDP, established as well as listening, matched on the + # local port. -lnt saw only TCP listeners, so it was blind to the + # UDP sockets the DTLS cases here bind and to a port held by an + # unrelated outbound connection - both of which still block bind. + # A false positive only costs another iteration of this loop. if command -v ss &>/dev/null; then - ss -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + [ -n "$(ss -Hantu "sport = :${p}" 2>/dev/null)" ] && collision=1 elif command -v netstat &>/dev/null; then - netstat -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + netstat -an 2>/dev/null | grep -q "[:.]${p}[[:space:]]" && collision=1 fi fi diff --git a/scripts/rsapss.test b/scripts/rsapss.test index 53e6da2371..b80325f069 100755 --- a/scripts/rsapss.test +++ b/scripts/rsapss.test @@ -95,10 +95,15 @@ generate_port() { # Also check if the port is already bound on this system if [ "$collision" -eq 0 ]; then + # TCP and UDP, established as well as listening, matched on the + # local port. -lnt saw only TCP listeners, so it was blind to the + # UDP sockets the DTLS cases here bind and to a port held by an + # unrelated outbound connection - both of which still block bind. + # A false positive only costs another iteration of this loop. if command -v ss &>/dev/null; then - ss -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + [ -n "$(ss -Hantu "sport = :${p}" 2>/dev/null)" ] && collision=1 elif command -v netstat &>/dev/null; then - netstat -lnt 2>/dev/null | grep -q ":${p}[[:space:]]" && collision=1 + netstat -an 2>/dev/null | grep -q "[:.]${p}[[:space:]]" && collision=1 fi fi diff --git a/src/include.am b/src/include.am index 12133f62e3..6361d15ff2 100644 --- a/src/include.am +++ b/src/include.am @@ -115,6 +115,9 @@ if BUILD_AESNI src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S +if BUILD_AESXTS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_x86_asm.S +endif else if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S @@ -277,6 +280,9 @@ if BUILD_AESNI src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S +if BUILD_AESXTS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_x86_asm.S +endif else if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S @@ -613,6 +619,9 @@ if BUILD_AESNI src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S +if BUILD_AESXTS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_x86_asm.S +endif else if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S @@ -1005,6 +1014,9 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_x86_64_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S +if BUILD_AESXTS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_x86_asm.S +endif else if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S @@ -2011,6 +2023,9 @@ src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_asm.S src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_x86_64_asm.S if BUILD_X86_ASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_x86_asm.S +if BUILD_AESXTS +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_xts_x86_asm.S +endif else if BUILD_AESGCM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/aes_gcm_asm.S diff --git a/wolfcrypt/src/aes.c b/wolfcrypt/src/aes.c index e96e4b1842..50c80ed041 100644 --- a/wolfcrypt/src/aes.c +++ b/wolfcrypt/src/aes.c @@ -18176,16 +18176,19 @@ int wc_AesXtsDecryptSector(XtsAes* aes, byte* out, const byte* in, word32 sz, } #endif -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) #if defined(USE_INTEL_SPEEDUP_FOR_AES) && !defined(USE_INTEL_SPEEDUP) #define USE_INTEL_SPEEDUP #endif -#if defined(USE_INTEL_SPEEDUP) +/* aes_xts_x86_asm.S provides the AES-NI routines for 32-bit x86 but has no + * AVX1/VAES/AVX512 variants, so do not let the wider paths be declared there - + * they would be undefined at link time. */ +#if defined(USE_INTEL_SPEEDUP) && !defined(WOLFSSL_X86_BUILD) #define HAVE_INTEL_AVX1 #define HAVE_INTEL_AVX2 -#endif /* USE_INTEL_SPEEDUP */ +#endif /* USE_INTEL_SPEEDUP && !WOLFSSL_X86_BUILD */ void AES_XTS_encrypt_aesni(const unsigned char *in, unsigned char *out, word32 sz, const unsigned char* i, const unsigned char* key, @@ -18294,7 +18297,7 @@ void AES_XTS_decrypt_update_avx512(const unsigned char *in, unsigned char *out, #endif /* HAVE_INTEL_AVX512 */ #endif /* HAVE_AES_DECRYPT */ -#endif /* WOLFSSL_AESNI && !WOLFSSL_X86_BUILD */ +#endif /* WOLFSSL_AESNI */ #ifdef HAVE_AES_ECB #if (!defined(WOLFSSL_ARMASM) || (!defined(__aarch64__) && \ @@ -18570,7 +18573,7 @@ int wc_AesXtsEncrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds); ret = 0; #endif -#elif defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#elif defined(WOLFSSL_AESNI) if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) @@ -18692,7 +18695,7 @@ int wc_AesXtsEncryptInit(XtsAes* xaes, const byte* i, word32 iSz, stream->bytes_crypted_with_this_tweak = 0; { -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) @@ -18731,7 +18734,7 @@ int wc_AesXtsEncryptInit(XtsAes* xaes, const byte* i, word32 iSz, RESTORE_VECTOR_REGISTERS(); } else -#endif /* WOLFSSL_AESNI && !WOLFSSL_X86_BUILD */ +#endif /* WOLFSSL_AESNI */ { ret = AesXtsInitTweak_sw(xaes, stream->tweak_block); } @@ -18761,7 +18764,7 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s { int ret; -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) Aes *aes; #endif @@ -18769,7 +18772,7 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s return BAD_FUNC_ARG; } -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) aes = &xaes->aes; #endif @@ -18805,7 +18808,7 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s } #endif { -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) @@ -18848,7 +18851,7 @@ static int AesXtsEncryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s RESTORE_VECTOR_REGISTERS(); } else -#endif /* WOLFSSL_AESNI && !WOLFSSL_X86_BUILD */ +#endif /* WOLFSSL_AESNI */ { ret = AesXtsEncryptUpdate_sw(xaes, out, in, sz, stream->tweak_block); } @@ -19134,7 +19137,7 @@ int wc_AesXtsDecrypt(XtsAes* xaes, byte* out, const byte* in, word32 sz, (byte*)xaes->tweak.key, (byte*)xaes->aes.tmp, xaes->aes.rounds); ret = 0; #endif -#elif defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#elif defined(WOLFSSL_AESNI) if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) @@ -19259,7 +19262,7 @@ int wc_AesXtsDecryptInit(XtsAes* xaes, const byte* i, word32 iSz, stream->bytes_crypted_with_this_tweak = 0; { -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) @@ -19298,7 +19301,7 @@ int wc_AesXtsDecryptInit(XtsAes* xaes, const byte* i, word32 iSz, RESTORE_VECTOR_REGISTERS(); } else -#endif /* WOLFSSL_AESNI && !WOLFSSL_X86_BUILD */ +#endif /* WOLFSSL_AESNI */ { ret = AesXtsInitTweak_sw(xaes, stream->tweak_block); } @@ -19326,7 +19329,7 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s struct XtsAesStreamData *stream) { int ret; -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) Aes *aes; #endif @@ -19334,7 +19337,7 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s return BAD_FUNC_ARG; } -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) #ifdef WC_AES_XTS_SUPPORT_SIMULTANEOUS_ENC_AND_DEC_KEYS aes = &xaes->aes_decrypt; #else @@ -19364,7 +19367,7 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s #endif { -#if defined(WOLFSSL_AESNI) && !defined(WOLFSSL_X86_BUILD) +#if defined(WOLFSSL_AESNI) if (aes->use_aesni) { SAVE_VECTOR_REGISTERS(return _svr_ret;); #if defined(HAVE_INTEL_AVX512) @@ -19407,7 +19410,7 @@ static int AesXtsDecryptUpdate(XtsAes* xaes, byte* out, const byte* in, word32 s RESTORE_VECTOR_REGISTERS(); } else -#endif /* WOLFSSL_AESNI && !WOLFSSL_X86_BUILD */ +#endif /* WOLFSSL_AESNI */ { ret = AesXtsDecryptUpdate_sw(xaes, out, in, sz, stream->tweak_block); diff --git a/wolfcrypt/src/aes_xts_x86_asm.S b/wolfcrypt/src/aes_xts_x86_asm.S new file mode 100644 index 0000000000..3a35f9bb16 --- /dev/null +++ b/wolfcrypt/src/aes_xts_x86_asm.S @@ -0,0 +1,841 @@ +/* aes_xts_x86_asm + * + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_AES_XTS_X86_ASM_S + +#include + +#ifndef HAVE_INTEL_AVX1 +#define HAVE_INTEL_AVX1 +#endif /* HAVE_INTEL_AVX1 */ +#ifndef NO_AVX2_SUPPORT +#ifndef HAVE_INTEL_AVX2 +#define HAVE_INTEL_AVX2 +#endif /* HAVE_INTEL_AVX2 */ +#endif /* NO_AVX2_SUPPORT */ + +#ifdef WOLFSSL_AES_XTS +#ifdef WOLFSSL_X86_BUILD + # 32-bit (i386) AES-NI AES-XTS: single-block ports of the x86_64 + # AES_XTS_*_aesni routines (xmm0-7, cdecl ABI); GF const on stack for PIC. + # void AES_XTS_init_aesni(unsigned char* i, const unsigned char* tweak_key, + # int tweak_nr); +.text +.globl AES_XTS_init_aesni +WC_ASM_ATT_HIDDEN(AES_XTS_init_aesni) +.type AES_XTS_init_aesni,@function +.align 16 +AES_XTS_init_aesni: + movl 4(%esp), %eax + movdqu (%eax), %xmm2 + movl 8(%esp), %ecx + pxor (%ecx), %xmm2 + movdqu 16(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 32(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 48(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 64(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 80(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 96(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 112(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 128(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 144(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + cmpl $11, 12(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_init_aesni_enclast_1 + aesenc %xmm0, %xmm2 + movdqu 176(%ecx), %xmm1 + aesenc %xmm1, %xmm2 + cmpl $13, 12(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_init_aesni_enclast_1 + aesenc %xmm0, %xmm2 + movdqu 208(%ecx), %xmm1 + aesenc %xmm1, %xmm2 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_init_aesni_enclast_1: + aesenclast %xmm0, %xmm2 + movdqu %xmm2, (%eax) + ret +.size AES_XTS_init_aesni,.-AES_XTS_init_aesni + # void AES_XTS_encrypt_aesni(const unsigned char* in, unsigned char* out, + # word32 sz, const unsigned char* i, const unsigned char* key, + # const unsigned char* key2, int nr); +.text +.globl AES_XTS_encrypt_aesni +WC_ASM_ATT_HIDDEN(AES_XTS_encrypt_aesni) +.type AES_XTS_encrypt_aesni,@function +.align 16 +AES_XTS_encrypt_aesni: + pushl %edi + pushl %ebx + subl $16, %esp + movl $0x87, (%esp) + movl $0x01, 4(%esp) + movl $0x01, 8(%esp) + movl $0x01, 12(%esp) + movdqu (%esp), %xmm6 + movl 40(%esp), %eax + movdqu (%eax), %xmm2 + movl 48(%esp), %ecx + pxor (%ecx), %xmm2 + movdqu 16(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 32(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 48(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 64(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 80(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 96(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 112(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 128(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 144(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + cmpl $11, 52(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_encrypt_aesni_enclast_2 + aesenc %xmm0, %xmm2 + movdqu 176(%ecx), %xmm1 + aesenc %xmm1, %xmm2 + cmpl $13, 52(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_encrypt_aesni_enclast_2 + aesenc %xmm0, %xmm2 + movdqu 208(%ecx), %xmm1 + aesenc %xmm1, %xmm2 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_encrypt_aesni_enclast_2: + aesenclast %xmm0, %xmm2 + xorl %edi, %edi + movl 36(%esp), %edx + andl $0xfffffff0, %edx +L_AES_XTS_encrypt_aesni_loop: + cmpl %edx, %edi + jae L_AES_XTS_encrypt_aesni_loop_done + movl 28(%esp), %eax + movdqu (%eax,%edi,1), %xmm3 + pxor %xmm2, %xmm3 + movl 44(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + cmpl $11, 52(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_encrypt_aesni_enclast_3 + aesenc %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + cmpl $13, 52(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_encrypt_aesni_enclast_3 + aesenc %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_encrypt_aesni_enclast_3: + aesenclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) + movdqa %xmm2, %xmm4 + psrad $31, %xmm4 + pslld $0x01, %xmm2 + pshufd $0x93, %xmm4, %xmm4 + pand %xmm6, %xmm4 + pxor %xmm4, %xmm2 + addl $16, %edi + jmp L_AES_XTS_encrypt_aesni_loop +L_AES_XTS_encrypt_aesni_loop_done: + movl 36(%esp), %eax + cmpl %eax, %edi + je L_AES_XTS_encrypt_aesni_done + subl $16, %edi + movl 32(%esp), %eax + movdqu (%eax,%edi,1), %xmm5 + addl $16, %edi + movdqu %xmm5, (%esp) + xorl %edx, %edx +L_AES_XTS_encrypt_aesni_cts: + movzbl (%esp,%edx,1), %ecx + movl 28(%esp), %eax + movzbl (%eax,%edi,1), %ebx + movl 32(%esp), %eax + movb %cl, (%eax,%edi,1) + movb %bl, (%esp,%edx,1) + incl %edi + incl %edx + movl 36(%esp), %eax + cmpl %eax, %edi + jb L_AES_XTS_encrypt_aesni_cts + subl %edx, %edi + movdqu (%esp), %xmm3 + subl $16, %edi + pxor %xmm2, %xmm3 + movl 44(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + cmpl $11, 52(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_encrypt_aesni_enclast_4 + aesenc %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + cmpl $13, 52(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_encrypt_aesni_enclast_4 + aesenc %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_encrypt_aesni_enclast_4: + aesenclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) +L_AES_XTS_encrypt_aesni_done: + addl $16, %esp + popl %ebx + popl %edi + ret +.size AES_XTS_encrypt_aesni,.-AES_XTS_encrypt_aesni + # void AES_XTS_encrypt_update_aesni(const unsigned char* in, + # unsigned char* out, word32 sz, const unsigned char* key, + # unsigned char* i, int nr); Tweak is read (already encrypted) from *i + # and the advanced tweak written back to *i. +.text +.globl AES_XTS_encrypt_update_aesni +WC_ASM_ATT_HIDDEN(AES_XTS_encrypt_update_aesni) +.type AES_XTS_encrypt_update_aesni,@function +.align 16 +AES_XTS_encrypt_update_aesni: + pushl %edi + pushl %ebx + subl $16, %esp + movl $0x87, (%esp) + movl $0x01, 4(%esp) + movl $0x01, 8(%esp) + movl $0x01, 12(%esp) + movdqu (%esp), %xmm6 + movl 44(%esp), %eax + movdqu (%eax), %xmm2 + xorl %edi, %edi + movl 36(%esp), %edx + andl $0xfffffff0, %edx +L_AES_XTS_encrypt_update_aesni_loop: + cmpl %edx, %edi + jae L_AES_XTS_encrypt_update_aesni_loop_done + movl 28(%esp), %eax + movdqu (%eax,%edi,1), %xmm3 + pxor %xmm2, %xmm3 + movl 40(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + cmpl $11, 48(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_encrypt_update_aesni_enclast_5 + aesenc %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + cmpl $13, 48(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_encrypt_update_aesni_enclast_5 + aesenc %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_encrypt_update_aesni_enclast_5: + aesenclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) + movdqa %xmm2, %xmm4 + psrad $31, %xmm4 + pslld $0x01, %xmm2 + pshufd $0x93, %xmm4, %xmm4 + pand %xmm6, %xmm4 + pxor %xmm4, %xmm2 + addl $16, %edi + jmp L_AES_XTS_encrypt_update_aesni_loop +L_AES_XTS_encrypt_update_aesni_loop_done: + movl 36(%esp), %eax + cmpl %eax, %edi + je L_AES_XTS_encrypt_update_aesni_done + subl $16, %edi + movl 32(%esp), %eax + movdqu (%eax,%edi,1), %xmm5 + addl $16, %edi + movdqu %xmm5, (%esp) + xorl %edx, %edx +L_AES_XTS_encrypt_update_aesni_cts: + movzbl (%esp,%edx,1), %ecx + movl 28(%esp), %eax + movzbl (%eax,%edi,1), %ebx + movl 32(%esp), %eax + movb %cl, (%eax,%edi,1) + movb %bl, (%esp,%edx,1) + incl %edi + incl %edx + movl 36(%esp), %eax + cmpl %eax, %edi + jb L_AES_XTS_encrypt_update_aesni_cts + subl %edx, %edi + movdqu (%esp), %xmm3 + subl $16, %edi + pxor %xmm2, %xmm3 + movl 40(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesenc %xmm0, %xmm3 + cmpl $11, 48(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_encrypt_update_aesni_enclast_6 + aesenc %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + cmpl $13, 48(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_encrypt_update_aesni_enclast_6 + aesenc %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesenc %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_encrypt_update_aesni_enclast_6: + aesenclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) +L_AES_XTS_encrypt_update_aesni_done: + movl 44(%esp), %eax + movdqu %xmm2, (%eax) + addl $16, %esp + popl %ebx + popl %edi + ret +.size AES_XTS_encrypt_update_aesni,.-AES_XTS_encrypt_update_aesni + # void AES_XTS_decrypt_aesni(const unsigned char* in, unsigned char* out, + # word32 sz, const unsigned char* i, const unsigned char* key, + # const unsigned char* key2, int nr); +.text +.globl AES_XTS_decrypt_aesni +WC_ASM_ATT_HIDDEN(AES_XTS_decrypt_aesni) +.type AES_XTS_decrypt_aesni,@function +.align 16 +AES_XTS_decrypt_aesni: + pushl %edi + pushl %ebx + subl $16, %esp + movl $0x87, (%esp) + movl $0x01, 4(%esp) + movl $0x01, 8(%esp) + movl $0x01, 12(%esp) + movdqu (%esp), %xmm6 + movl 40(%esp), %eax + movdqu (%eax), %xmm2 + movl 48(%esp), %ecx + pxor (%ecx), %xmm2 + movdqu 16(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 32(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 48(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 64(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 80(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 96(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 112(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 128(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + movdqu 144(%ecx), %xmm0 + aesenc %xmm0, %xmm2 + cmpl $11, 52(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_enclast_7 + aesenc %xmm0, %xmm2 + movdqu 176(%ecx), %xmm1 + aesenc %xmm1, %xmm2 + cmpl $13, 52(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_enclast_7 + aesenc %xmm0, %xmm2 + movdqu 208(%ecx), %xmm1 + aesenc %xmm1, %xmm2 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_decrypt_aesni_enclast_7: + aesenclast %xmm0, %xmm2 + xorl %edi, %edi + movl 36(%esp), %eax + movl %eax, %edx + andl $0xfffffff0, %edx + cmpl %eax, %edx + je L_AES_XTS_decrypt_aesni_bound + subl $16, %edx +L_AES_XTS_decrypt_aesni_bound: +L_AES_XTS_decrypt_aesni_loop: + cmpl %edx, %edi + jae L_AES_XTS_decrypt_aesni_loop_done + movl 28(%esp), %eax + movdqu (%eax,%edi,1), %xmm3 + pxor %xmm2, %xmm3 + movl 44(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + cmpl $11, 52(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_declast_8 + aesdec %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + cmpl $13, 52(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_declast_8 + aesdec %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_decrypt_aesni_declast_8: + aesdeclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) + movdqa %xmm2, %xmm4 + psrad $31, %xmm4 + pslld $0x01, %xmm2 + pshufd $0x93, %xmm4, %xmm4 + pand %xmm6, %xmm4 + pxor %xmm4, %xmm2 + addl $16, %edi + jmp L_AES_XTS_decrypt_aesni_loop +L_AES_XTS_decrypt_aesni_loop_done: + movl 36(%esp), %eax + cmpl %eax, %edi + je L_AES_XTS_decrypt_aesni_done + movdqa %xmm2, %xmm4 + movdqa %xmm2, %xmm5 + psrad $31, %xmm4 + pslld $0x01, %xmm5 + pshufd $0x93, %xmm4, %xmm4 + pand %xmm6, %xmm4 + pxor %xmm4, %xmm5 + movl 28(%esp), %eax + movdqu (%eax,%edi,1), %xmm3 + pxor %xmm5, %xmm3 + movl 44(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + cmpl $11, 52(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_declast_9 + aesdec %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + cmpl $13, 52(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_declast_9 + aesdec %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_decrypt_aesni_declast_9: + aesdeclast %xmm0, %xmm3 + pxor %xmm5, %xmm3 + movdqu %xmm3, (%esp) + addl $16, %edi + xorl %edx, %edx +L_AES_XTS_decrypt_aesni_cts: + movzbl (%esp,%edx,1), %ecx + movl 28(%esp), %eax + movzbl (%eax,%edi,1), %ebx + movl 32(%esp), %eax + movb %cl, (%eax,%edi,1) + movb %bl, (%esp,%edx,1) + incl %edi + incl %edx + movl 36(%esp), %eax + cmpl %eax, %edi + jb L_AES_XTS_decrypt_aesni_cts + subl %edx, %edi + movdqu (%esp), %xmm3 + pxor %xmm2, %xmm3 + movl 44(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + cmpl $11, 52(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_declast_10 + aesdec %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + cmpl $13, 52(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_decrypt_aesni_declast_10 + aesdec %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_decrypt_aesni_declast_10: + aesdeclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + subl $16, %edi + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) +L_AES_XTS_decrypt_aesni_done: + addl $16, %esp + popl %ebx + popl %edi + ret +.size AES_XTS_decrypt_aesni,.-AES_XTS_decrypt_aesni + # void AES_XTS_decrypt_update_aesni(const unsigned char* in, + # unsigned char* out, word32 sz, const unsigned char* key, + # unsigned char* i, int nr); Tweak is read from *i and the advanced + # tweak written back to *i. +.text +.globl AES_XTS_decrypt_update_aesni +WC_ASM_ATT_HIDDEN(AES_XTS_decrypt_update_aesni) +.type AES_XTS_decrypt_update_aesni,@function +.align 16 +AES_XTS_decrypt_update_aesni: + pushl %edi + pushl %ebx + subl $16, %esp + movl $0x87, (%esp) + movl $0x01, 4(%esp) + movl $0x01, 8(%esp) + movl $0x01, 12(%esp) + movdqu (%esp), %xmm6 + movl 44(%esp), %eax + movdqu (%eax), %xmm2 + xorl %edi, %edi + movl 36(%esp), %eax + movl %eax, %edx + andl $0xfffffff0, %edx + cmpl %eax, %edx + je L_AES_XTS_decrypt_update_aesni_bound + subl $16, %edx +L_AES_XTS_decrypt_update_aesni_bound: +L_AES_XTS_decrypt_update_aesni_loop: + cmpl %edx, %edi + jae L_AES_XTS_decrypt_update_aesni_loop_done + movl 28(%esp), %eax + movdqu (%eax,%edi,1), %xmm3 + pxor %xmm2, %xmm3 + movl 40(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + cmpl $11, 48(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_decrypt_update_aesni_declast_11 + aesdec %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + cmpl $13, 48(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_decrypt_update_aesni_declast_11 + aesdec %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_decrypt_update_aesni_declast_11: + aesdeclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) + movdqa %xmm2, %xmm4 + psrad $31, %xmm4 + pslld $0x01, %xmm2 + pshufd $0x93, %xmm4, %xmm4 + pand %xmm6, %xmm4 + pxor %xmm4, %xmm2 + addl $16, %edi + jmp L_AES_XTS_decrypt_update_aesni_loop +L_AES_XTS_decrypt_update_aesni_loop_done: + movl 36(%esp), %eax + cmpl %eax, %edi + je L_AES_XTS_decrypt_update_aesni_done + movdqa %xmm2, %xmm4 + movdqa %xmm2, %xmm5 + psrad $31, %xmm4 + pslld $0x01, %xmm5 + pshufd $0x93, %xmm4, %xmm4 + pand %xmm6, %xmm4 + pxor %xmm4, %xmm5 + movl 28(%esp), %eax + movdqu (%eax,%edi,1), %xmm3 + pxor %xmm5, %xmm3 + movl 40(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + cmpl $11, 48(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_decrypt_update_aesni_declast_12 + aesdec %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + cmpl $13, 48(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_decrypt_update_aesni_declast_12 + aesdec %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_decrypt_update_aesni_declast_12: + aesdeclast %xmm0, %xmm3 + pxor %xmm5, %xmm3 + movdqu %xmm3, (%esp) + addl $16, %edi + xorl %edx, %edx +L_AES_XTS_decrypt_update_aesni_cts: + movzbl (%esp,%edx,1), %ecx + movl 28(%esp), %eax + movzbl (%eax,%edi,1), %ebx + movl 32(%esp), %eax + movb %cl, (%eax,%edi,1) + movb %bl, (%esp,%edx,1) + incl %edi + incl %edx + movl 36(%esp), %eax + cmpl %eax, %edi + jb L_AES_XTS_decrypt_update_aesni_cts + subl %edx, %edi + movdqu (%esp), %xmm3 + pxor %xmm2, %xmm3 + movl 40(%esp), %ecx + pxor (%ecx), %xmm3 + movdqu 16(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 32(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 48(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 64(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 80(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 96(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 112(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 128(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + movdqu 144(%ecx), %xmm0 + aesdec %xmm0, %xmm3 + cmpl $11, 48(%esp) + movdqu 160(%ecx), %xmm0 + jl L_AES_XTS_decrypt_update_aesni_declast_13 + aesdec %xmm0, %xmm3 + movdqu 176(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + cmpl $13, 48(%esp) + movdqu 192(%ecx), %xmm0 + jl L_AES_XTS_decrypt_update_aesni_declast_13 + aesdec %xmm0, %xmm3 + movdqu 208(%ecx), %xmm1 + aesdec %xmm1, %xmm3 + movdqu 224(%ecx), %xmm0 +L_AES_XTS_decrypt_update_aesni_declast_13: + aesdeclast %xmm0, %xmm3 + pxor %xmm2, %xmm3 + subl $16, %edi + movl 32(%esp), %eax + movdqu %xmm3, (%eax,%edi,1) +L_AES_XTS_decrypt_update_aesni_done: + movl 44(%esp), %eax + movdqu %xmm2, (%eax) + addl $16, %esp + popl %ebx + popl %edi + ret +.size AES_XTS_decrypt_update_aesni,.-AES_XTS_decrypt_update_aesni +#endif /* WOLFSSL_X86_BUILD */ +#endif /* WOLFSSL_AES_XTS */ + +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif diff --git a/wolfcrypt/src/aes_xts_x86_asm.asm b/wolfcrypt/src/aes_xts_x86_asm.asm new file mode 100644 index 0000000000..bc7f43f7aa --- /dev/null +++ b/wolfcrypt/src/aes_xts_x86_asm.asm @@ -0,0 +1,835 @@ +; /* aes_xts_x86_asm +; * +; * Copyright (C) 2006-2026 wolfSSL Inc. +; * +; * This file is part of wolfSSL. +; * +; * wolfSSL is free software; you can redistribute it and/or modify +; * it under the terms of the GNU General Public License as published by +; * the Free Software Foundation; either version 3 of the License, or +; * (at your option) any later version. +; * +; * wolfSSL is distributed in the hope that it will be useful, +; * but WITHOUT ANY WARRANTY; without even the implied warranty of +; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +; * GNU General Public License for more details. +; * +; * You should have received a copy of the GNU General Public License +; * along with this program; if not, write to the Free Software +; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA +; */ + +IF @Version LT 1200 +; AVX2 instructions not recognized by old versions of MASM +IFNDEF NO_AVX2_SUPPORT +NO_AVX2_SUPPORT = 1 +ENDIF +; MOVBE instruction not recognized by old versions of MASM +IFNDEF NO_MOVBE_SUPPORT +NO_MOVBE_SUPPORT = 1 +ENDIF +ENDIF + +IFNDEF HAVE_INTEL_AVX1 +HAVE_INTEL_AVX1 = 1 +ENDIF +IFNDEF NO_AVX2_SUPPORT +HAVE_INTEL_AVX2 = 1 +ENDIF + +IFNDEF _WIN32 +_WIN32 = 1 +ENDIF + +.686P +.XMM +.MODEL FLAT, C + +IFDEF WOLFSSL_AES_XTS +IFDEF WOLFSSL_X86_BUILD + ; 32-bit (i386) AES-NI AES-XTS: single-block ports of the x86_64 + ; AES_XTS_*_aesni routines (xmm0-7, cdecl ABI); GF const on stack for PIC. + ; void AES_XTS_init_aesni(unsigned char* i, const unsigned char* tweak_key, + ; int tweak_nr); +_TEXT SEGMENT READONLY PARA +AES_XTS_init_aesni PROC + mov eax, DWORD PTR [esp+4] + movdqu xmm2, OWORD PTR [eax] + mov ecx, DWORD PTR [esp+8] + pxor xmm2, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesenc xmm2, xmm0 + cmp DWORD PTR [esp+12], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_init_aesni_enclast_1 + aesenc xmm2, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesenc xmm2, xmm1 + cmp DWORD PTR [esp+12], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_init_aesni_enclast_1 + aesenc xmm2, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesenc xmm2, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_init_aesni_enclast_1: + aesenclast xmm2, xmm0 + movdqu OWORD PTR [eax], xmm2 + ret +AES_XTS_init_aesni ENDP +_TEXT ENDS + ; void AES_XTS_encrypt_aesni(const unsigned char* in, unsigned char* out, + ; word32 sz, const unsigned char* i, const unsigned char* key, + ; const unsigned char* key2, int nr); +_TEXT SEGMENT READONLY PARA +AES_XTS_encrypt_aesni PROC + push edi + push ebx + sub esp, 16 + mov DWORD PTR [esp], 135 + mov DWORD PTR [esp+4], 1 + mov DWORD PTR [esp+8], 1 + mov DWORD PTR [esp+12], 1 + movdqu xmm6, OWORD PTR [esp] + mov eax, DWORD PTR [esp+40] + movdqu xmm2, OWORD PTR [eax] + mov ecx, DWORD PTR [esp+48] + pxor xmm2, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesenc xmm2, xmm0 + cmp DWORD PTR [esp+52], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_encrypt_aesni_enclast_2 + aesenc xmm2, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesenc xmm2, xmm1 + cmp DWORD PTR [esp+52], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_encrypt_aesni_enclast_2 + aesenc xmm2, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesenc xmm2, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_encrypt_aesni_enclast_2: + aesenclast xmm2, xmm0 + xor edi, edi + mov edx, DWORD PTR [esp+36] + and edx, 4294967280 +L_AES_XTS_encrypt_aesni_loop: + cmp edi, edx + jae L_AES_XTS_encrypt_aesni_loop_done + mov eax, DWORD PTR [esp+28] + movdqu xmm3, OWORD PTR [eax+edi] + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+44] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesenc xmm3, xmm0 + cmp DWORD PTR [esp+52], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_encrypt_aesni_enclast_3 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesenc xmm3, xmm1 + cmp DWORD PTR [esp+52], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_encrypt_aesni_enclast_3 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesenc xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_encrypt_aesni_enclast_3: + aesenclast xmm3, xmm0 + pxor xmm3, xmm2 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 + movdqa xmm4, xmm2 + psrad xmm4, 31 + pslld xmm2, 1 + pshufd xmm4, xmm4, 147 + pand xmm4, xmm6 + pxor xmm2, xmm4 + add edi, 16 + jmp L_AES_XTS_encrypt_aesni_loop +L_AES_XTS_encrypt_aesni_loop_done: + mov eax, DWORD PTR [esp+36] + cmp edi, eax + je L_AES_XTS_encrypt_aesni_done + sub edi, 16 + mov eax, DWORD PTR [esp+32] + movdqu xmm5, OWORD PTR [eax+edi] + add edi, 16 + movdqu OWORD PTR [esp], xmm5 + xor edx, edx +L_AES_XTS_encrypt_aesni_cts: + movzx ecx, BYTE PTR [esp+edx] + mov eax, DWORD PTR [esp+28] + movzx ebx, BYTE PTR [eax+edi] + mov eax, DWORD PTR [esp+32] + mov BYTE PTR [eax+edi], cl + mov BYTE PTR [esp+edx], bl + inc edi + inc edx + mov eax, DWORD PTR [esp+36] + cmp edi, eax + jb L_AES_XTS_encrypt_aesni_cts + sub edi, edx + movdqu xmm3, OWORD PTR [esp] + sub edi, 16 + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+44] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesenc xmm3, xmm0 + cmp DWORD PTR [esp+52], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_encrypt_aesni_enclast_4 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesenc xmm3, xmm1 + cmp DWORD PTR [esp+52], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_encrypt_aesni_enclast_4 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesenc xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_encrypt_aesni_enclast_4: + aesenclast xmm3, xmm0 + pxor xmm3, xmm2 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 +L_AES_XTS_encrypt_aesni_done: + add esp, 16 + pop ebx + pop edi + ret +AES_XTS_encrypt_aesni ENDP +_TEXT ENDS + ; void AES_XTS_encrypt_update_aesni(const unsigned char* in, + ; unsigned char* out, word32 sz, const unsigned char* key, + ; unsigned char* i, int nr); Tweak is read (already encrypted) from *i + ; and the advanced tweak written back to *i. +_TEXT SEGMENT READONLY PARA +AES_XTS_encrypt_update_aesni PROC + push edi + push ebx + sub esp, 16 + mov DWORD PTR [esp], 135 + mov DWORD PTR [esp+4], 1 + mov DWORD PTR [esp+8], 1 + mov DWORD PTR [esp+12], 1 + movdqu xmm6, OWORD PTR [esp] + mov eax, DWORD PTR [esp+44] + movdqu xmm2, OWORD PTR [eax] + xor edi, edi + mov edx, DWORD PTR [esp+36] + and edx, 4294967280 +L_AES_XTS_encrypt_update_aesni_loop: + cmp edi, edx + jae L_AES_XTS_encrypt_update_aesni_loop_done + mov eax, DWORD PTR [esp+28] + movdqu xmm3, OWORD PTR [eax+edi] + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+40] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesenc xmm3, xmm0 + cmp DWORD PTR [esp+48], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_encrypt_update_aesni_enclast_5 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesenc xmm3, xmm1 + cmp DWORD PTR [esp+48], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_encrypt_update_aesni_enclast_5 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesenc xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_encrypt_update_aesni_enclast_5: + aesenclast xmm3, xmm0 + pxor xmm3, xmm2 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 + movdqa xmm4, xmm2 + psrad xmm4, 31 + pslld xmm2, 1 + pshufd xmm4, xmm4, 147 + pand xmm4, xmm6 + pxor xmm2, xmm4 + add edi, 16 + jmp L_AES_XTS_encrypt_update_aesni_loop +L_AES_XTS_encrypt_update_aesni_loop_done: + mov eax, DWORD PTR [esp+36] + cmp edi, eax + je L_AES_XTS_encrypt_update_aesni_done + sub edi, 16 + mov eax, DWORD PTR [esp+32] + movdqu xmm5, OWORD PTR [eax+edi] + add edi, 16 + movdqu OWORD PTR [esp], xmm5 + xor edx, edx +L_AES_XTS_encrypt_update_aesni_cts: + movzx ecx, BYTE PTR [esp+edx] + mov eax, DWORD PTR [esp+28] + movzx ebx, BYTE PTR [eax+edi] + mov eax, DWORD PTR [esp+32] + mov BYTE PTR [eax+edi], cl + mov BYTE PTR [esp+edx], bl + inc edi + inc edx + mov eax, DWORD PTR [esp+36] + cmp edi, eax + jb L_AES_XTS_encrypt_update_aesni_cts + sub edi, edx + movdqu xmm3, OWORD PTR [esp] + sub edi, 16 + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+40] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesenc xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesenc xmm3, xmm0 + cmp DWORD PTR [esp+48], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_encrypt_update_aesni_enclast_6 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesenc xmm3, xmm1 + cmp DWORD PTR [esp+48], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_encrypt_update_aesni_enclast_6 + aesenc xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesenc xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_encrypt_update_aesni_enclast_6: + aesenclast xmm3, xmm0 + pxor xmm3, xmm2 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 +L_AES_XTS_encrypt_update_aesni_done: + mov eax, DWORD PTR [esp+44] + movdqu OWORD PTR [eax], xmm2 + add esp, 16 + pop ebx + pop edi + ret +AES_XTS_encrypt_update_aesni ENDP +_TEXT ENDS + ; void AES_XTS_decrypt_aesni(const unsigned char* in, unsigned char* out, + ; word32 sz, const unsigned char* i, const unsigned char* key, + ; const unsigned char* key2, int nr); +_TEXT SEGMENT READONLY PARA +AES_XTS_decrypt_aesni PROC + push edi + push ebx + sub esp, 16 + mov DWORD PTR [esp], 135 + mov DWORD PTR [esp+4], 1 + mov DWORD PTR [esp+8], 1 + mov DWORD PTR [esp+12], 1 + movdqu xmm6, OWORD PTR [esp] + mov eax, DWORD PTR [esp+40] + movdqu xmm2, OWORD PTR [eax] + mov ecx, DWORD PTR [esp+48] + pxor xmm2, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesenc xmm2, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesenc xmm2, xmm0 + cmp DWORD PTR [esp+52], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_decrypt_aesni_enclast_7 + aesenc xmm2, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesenc xmm2, xmm1 + cmp DWORD PTR [esp+52], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_decrypt_aesni_enclast_7 + aesenc xmm2, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesenc xmm2, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_decrypt_aesni_enclast_7: + aesenclast xmm2, xmm0 + xor edi, edi + mov eax, DWORD PTR [esp+36] + mov edx, eax + and edx, 4294967280 + cmp edx, eax + je L_AES_XTS_decrypt_aesni_bound + sub edx, 16 +L_AES_XTS_decrypt_aesni_bound: +L_AES_XTS_decrypt_aesni_loop: + cmp edi, edx + jae L_AES_XTS_decrypt_aesni_loop_done + mov eax, DWORD PTR [esp+28] + movdqu xmm3, OWORD PTR [eax+edi] + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+44] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesdec xmm3, xmm0 + cmp DWORD PTR [esp+52], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_decrypt_aesni_declast_8 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesdec xmm3, xmm1 + cmp DWORD PTR [esp+52], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_decrypt_aesni_declast_8 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesdec xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_decrypt_aesni_declast_8: + aesdeclast xmm3, xmm0 + pxor xmm3, xmm2 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 + movdqa xmm4, xmm2 + psrad xmm4, 31 + pslld xmm2, 1 + pshufd xmm4, xmm4, 147 + pand xmm4, xmm6 + pxor xmm2, xmm4 + add edi, 16 + jmp L_AES_XTS_decrypt_aesni_loop +L_AES_XTS_decrypt_aesni_loop_done: + mov eax, DWORD PTR [esp+36] + cmp edi, eax + je L_AES_XTS_decrypt_aesni_done + movdqa xmm4, xmm2 + movdqa xmm5, xmm2 + psrad xmm4, 31 + pslld xmm5, 1 + pshufd xmm4, xmm4, 147 + pand xmm4, xmm6 + pxor xmm5, xmm4 + mov eax, DWORD PTR [esp+28] + movdqu xmm3, OWORD PTR [eax+edi] + pxor xmm3, xmm5 + mov ecx, DWORD PTR [esp+44] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesdec xmm3, xmm0 + cmp DWORD PTR [esp+52], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_decrypt_aesni_declast_9 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesdec xmm3, xmm1 + cmp DWORD PTR [esp+52], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_decrypt_aesni_declast_9 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesdec xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_decrypt_aesni_declast_9: + aesdeclast xmm3, xmm0 + pxor xmm3, xmm5 + movdqu OWORD PTR [esp], xmm3 + add edi, 16 + xor edx, edx +L_AES_XTS_decrypt_aesni_cts: + movzx ecx, BYTE PTR [esp+edx] + mov eax, DWORD PTR [esp+28] + movzx ebx, BYTE PTR [eax+edi] + mov eax, DWORD PTR [esp+32] + mov BYTE PTR [eax+edi], cl + mov BYTE PTR [esp+edx], bl + inc edi + inc edx + mov eax, DWORD PTR [esp+36] + cmp edi, eax + jb L_AES_XTS_decrypt_aesni_cts + sub edi, edx + movdqu xmm3, OWORD PTR [esp] + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+44] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesdec xmm3, xmm0 + cmp DWORD PTR [esp+52], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_decrypt_aesni_declast_10 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesdec xmm3, xmm1 + cmp DWORD PTR [esp+52], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_decrypt_aesni_declast_10 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesdec xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_decrypt_aesni_declast_10: + aesdeclast xmm3, xmm0 + pxor xmm3, xmm2 + sub edi, 16 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 +L_AES_XTS_decrypt_aesni_done: + add esp, 16 + pop ebx + pop edi + ret +AES_XTS_decrypt_aesni ENDP +_TEXT ENDS + ; void AES_XTS_decrypt_update_aesni(const unsigned char* in, + ; unsigned char* out, word32 sz, const unsigned char* key, + ; unsigned char* i, int nr); Tweak is read from *i and the advanced + ; tweak written back to *i. +_TEXT SEGMENT READONLY PARA +AES_XTS_decrypt_update_aesni PROC + push edi + push ebx + sub esp, 16 + mov DWORD PTR [esp], 135 + mov DWORD PTR [esp+4], 1 + mov DWORD PTR [esp+8], 1 + mov DWORD PTR [esp+12], 1 + movdqu xmm6, OWORD PTR [esp] + mov eax, DWORD PTR [esp+44] + movdqu xmm2, OWORD PTR [eax] + xor edi, edi + mov eax, DWORD PTR [esp+36] + mov edx, eax + and edx, 4294967280 + cmp edx, eax + je L_AES_XTS_decrypt_update_aesni_bound + sub edx, 16 +L_AES_XTS_decrypt_update_aesni_bound: +L_AES_XTS_decrypt_update_aesni_loop: + cmp edi, edx + jae L_AES_XTS_decrypt_update_aesni_loop_done + mov eax, DWORD PTR [esp+28] + movdqu xmm3, OWORD PTR [eax+edi] + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+40] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesdec xmm3, xmm0 + cmp DWORD PTR [esp+48], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_decrypt_update_aesni_declast_11 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesdec xmm3, xmm1 + cmp DWORD PTR [esp+48], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_decrypt_update_aesni_declast_11 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesdec xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_decrypt_update_aesni_declast_11: + aesdeclast xmm3, xmm0 + pxor xmm3, xmm2 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 + movdqa xmm4, xmm2 + psrad xmm4, 31 + pslld xmm2, 1 + pshufd xmm4, xmm4, 147 + pand xmm4, xmm6 + pxor xmm2, xmm4 + add edi, 16 + jmp L_AES_XTS_decrypt_update_aesni_loop +L_AES_XTS_decrypt_update_aesni_loop_done: + mov eax, DWORD PTR [esp+36] + cmp edi, eax + je L_AES_XTS_decrypt_update_aesni_done + movdqa xmm4, xmm2 + movdqa xmm5, xmm2 + psrad xmm4, 31 + pslld xmm5, 1 + pshufd xmm4, xmm4, 147 + pand xmm4, xmm6 + pxor xmm5, xmm4 + mov eax, DWORD PTR [esp+28] + movdqu xmm3, OWORD PTR [eax+edi] + pxor xmm3, xmm5 + mov ecx, DWORD PTR [esp+40] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesdec xmm3, xmm0 + cmp DWORD PTR [esp+48], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_decrypt_update_aesni_declast_12 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesdec xmm3, xmm1 + cmp DWORD PTR [esp+48], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_decrypt_update_aesni_declast_12 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesdec xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_decrypt_update_aesni_declast_12: + aesdeclast xmm3, xmm0 + pxor xmm3, xmm5 + movdqu OWORD PTR [esp], xmm3 + add edi, 16 + xor edx, edx +L_AES_XTS_decrypt_update_aesni_cts: + movzx ecx, BYTE PTR [esp+edx] + mov eax, DWORD PTR [esp+28] + movzx ebx, BYTE PTR [eax+edi] + mov eax, DWORD PTR [esp+32] + mov BYTE PTR [eax+edi], cl + mov BYTE PTR [esp+edx], bl + inc edi + inc edx + mov eax, DWORD PTR [esp+36] + cmp edi, eax + jb L_AES_XTS_decrypt_update_aesni_cts + sub edi, edx + movdqu xmm3, OWORD PTR [esp] + pxor xmm3, xmm2 + mov ecx, DWORD PTR [esp+40] + pxor xmm3, [ecx] + movdqu xmm0, OWORD PTR [ecx+16] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+32] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+48] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+64] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+80] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+96] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+112] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+128] + aesdec xmm3, xmm0 + movdqu xmm0, OWORD PTR [ecx+144] + aesdec xmm3, xmm0 + cmp DWORD PTR [esp+48], 11 + movdqu xmm0, OWORD PTR [ecx+160] + jl L_AES_XTS_decrypt_update_aesni_declast_13 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+176] + aesdec xmm3, xmm1 + cmp DWORD PTR [esp+48], 13 + movdqu xmm0, OWORD PTR [ecx+192] + jl L_AES_XTS_decrypt_update_aesni_declast_13 + aesdec xmm3, xmm0 + movdqu xmm1, OWORD PTR [ecx+208] + aesdec xmm3, xmm1 + movdqu xmm0, OWORD PTR [ecx+224] +L_AES_XTS_decrypt_update_aesni_declast_13: + aesdeclast xmm3, xmm0 + pxor xmm3, xmm2 + sub edi, 16 + mov eax, DWORD PTR [esp+32] + movdqu OWORD PTR [eax+edi], xmm3 +L_AES_XTS_decrypt_update_aesni_done: + mov eax, DWORD PTR [esp+44] + movdqu OWORD PTR [eax], xmm2 + add esp, 16 + pop ebx + pop edi + ret +AES_XTS_decrypt_update_aesni ENDP +_TEXT ENDS +ENDIF +ENDIF +END diff --git a/wolfcrypt/src/fe_448.c b/wolfcrypt/src/fe_448.c index c12d8faf44..42b8eaaf46 100644 --- a/wolfcrypt/src/fe_448.c +++ b/wolfcrypt/src/fe_448.c @@ -1126,6 +1126,7 @@ int curve448(byte* r, const byte* n, const byte* a) fe448_reduce(x3); fe448_sqr(x3, x3); fe448_sub(z3, z3, t1); + fe448_reduce(z3); fe448_sqr(z3, z3); fe448_mul(z3, z3, x1); fe448_sub(t1, t0, x2); diff --git a/wolfcrypt/src/fe_x25519_asm.asm b/wolfcrypt/src/fe_x25519_asm.asm index cbbb99a44e..3bd50cd61c 100644 --- a/wolfcrypt/src/fe_x25519_asm.asm +++ b/wolfcrypt/src/fe_x25519_asm.asm @@ -1290,116 +1290,178 @@ fe_invert_x64 PROC mov QWORD PTR [rsp+136], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+136] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+136] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rdx, QWORD PTR [rsp+136] mov rcx, QWORD PTR [rsp+128] add rsp, 152 @@ -3314,116 +3376,178 @@ L_curve25519_base_x64_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov r8, QWORD PTR [rsp+160] ; Multiply ; A[0] * B[0] @@ -5609,116 +5733,178 @@ L_curve25519_x64_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov r9, QWORD PTR [rsp+168] ; Multiply ; A[0] * B[0] @@ -5899,115 +6085,177 @@ fe_pow22523_x64 PROC mov QWORD PTR [rsp+104], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+104] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+104] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 19 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 99 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_x64 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_x64 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_x64 + add rsp, 32 mov rcx, QWORD PTR [rsp+96] mov rdx, rsp mov r8, QWORD PTR [rsp+104] + sub rsp, 32 call fe_mul_x64 + add rsp, 32 mov rdx, QWORD PTR [rsp+104] mov rcx, QWORD PTR [rsp+96] add rsp, 120 @@ -11543,116 +11791,178 @@ fe_invert_avx2 PROC mov QWORD PTR [rsp+136], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+136] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+136] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rdx, QWORD PTR [rsp+136] mov rcx, QWORD PTR [rsp+128] add rsp, 152 @@ -13206,116 +13516,178 @@ L_curve25519_base_avx2_last_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov r8, QWORD PTR [rsp+160] mov rax, QWORD PTR [r8] ; Multiply @@ -15095,116 +15467,178 @@ L_curve25519_avx2_last_3: ; Invert lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] mov rdx, rsp lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+128] lea rdx, QWORD PTR [rsp+128] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+128] lea r8, QWORD PTR [rsp+96] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+96] lea rdx, QWORD PTR [rsp+96] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+96] lea r8, QWORD PTR [rsp+64] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov r9, QWORD PTR [rsp+168] mov rax, QWORD PTR [r9] ; Multiply @@ -15356,115 +15790,177 @@ fe_pow22523_avx2 PROC mov QWORD PTR [rsp+104], rdx mov rcx, rsp mov rdx, QWORD PTR [rsp+104] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, QWORD PTR [rsp+104] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 4 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 19 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 9 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+64] lea rdx, QWORD PTR [rsp+64] mov r8, 99 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+64] lea r8, QWORD PTR [rsp+32] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 lea rcx, QWORD PTR [rsp+32] lea rdx, QWORD PTR [rsp+32] mov r8, 49 + sub rsp, 32 call fe_sq_n_avx2 + add rsp, 32 mov rcx, rsp lea rdx, QWORD PTR [rsp+32] mov r8, rsp + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 mov rcx, rsp mov rdx, rsp + sub rsp, 32 call fe_sq_avx2 + add rsp, 32 mov rcx, QWORD PTR [rsp+96] mov rdx, rsp mov r8, QWORD PTR [rsp+104] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 mov rdx, QWORD PTR [rsp+104] mov rcx, QWORD PTR [rsp+96] add rsp, 120 @@ -20577,16 +21073,22 @@ L_curve25519_base_avx512_ifma_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+736] vmovdqu xmm7, OWORD PTR [rsp+752] @@ -21395,16 +21897,22 @@ L_curve25519_avx512_ifma_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+744] vmovdqu xmm7, OWORD PTR [rsp+760] @@ -22147,16 +22655,22 @@ L_curve25519_base_avx512_ifma_dq_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+736] vmovdqu xmm7, OWORD PTR [rsp+752] @@ -22920,16 +23434,22 @@ L_curve25519_avx512_ifma_dq_bits: ; z2 = 1 / z2 lea rcx, QWORD PTR [rsp+672] lea rdx, QWORD PTR [rsp+672] + sub rsp, 32 call fe_invert_avx2 + add rsp, 32 ; x2 = x2 * z2 lea rcx, QWORD PTR [rsp+640] lea rdx, QWORD PTR [rsp+640] lea r8, QWORD PTR [rsp+672] + sub rsp, 32 call fe_mul_avx2 + add rsp, 32 ; Store fully reduced result mov rcx, QWORD PTR [rsp+712] lea rdx, QWORD PTR [rsp+640] + sub rsp, 32 call fe_tobytes + add rsp, 32 xor rax, rax vmovdqu xmm6, OWORD PTR [rsp+744] vmovdqu xmm7, OWORD PTR [rsp+760] diff --git a/wolfcrypt/src/ge_448.c b/wolfcrypt/src/ge_448.c index f3a895f23e..d2c77638dc 100644 --- a/wolfcrypt/src/ge_448.c +++ b/wolfcrypt/src/ge_448.c @@ -10837,7 +10837,7 @@ static WC_INLINE void ge448_sub(ge448_p2 *r, const ge448_p2 *p, fe448_mul(t3, t1, t2); /* t3 = E1 = C*D */ fe448_mul39081(t3, t3); /* t3 = E = d*C*D */ fe448_mul(r->Z, p->Z, q->Z); /* r->Z = A = Z1*Z2 */ - fe448_sqr(t0, p->Z); /* t0 = B = A^2 */ + fe448_sqr(t0, r->Z); /* t0 = B = A^2 */ fe448_sub(t4, t0, t3); /* t4 = F = B-(--E) */ fe448_add(t0, t0, t3); /* t0 = G = B+(--E) */ fe448_reduce(t0); diff --git a/wolfcrypt/src/include.am b/wolfcrypt/src/include.am index f0e7ca3c71..13b6d70c02 100644 --- a/wolfcrypt/src/include.am +++ b/wolfcrypt/src/include.am @@ -21,6 +21,7 @@ EXTRA_DIST += wolfcrypt/src/aes_x86_64_asm.asm EXTRA_DIST += wolfcrypt/src/aes_gcm_asm.asm EXTRA_DIST += wolfcrypt/src/aes_gcm_x86_asm.asm EXTRA_DIST += wolfcrypt/src/aes_xts_asm.asm +EXTRA_DIST += wolfcrypt/src/aes_xts_x86_asm.asm EXTRA_DIST += wolfcrypt/src/chacha_asm.asm EXTRA_DIST += wolfcrypt/src/poly1305_asm.asm EXTRA_DIST += wolfcrypt/src/fe_x25519_asm.asm diff --git a/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c b/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c index 4d0c5a01be..da717f8837 100644 --- a/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-32-aes-asm_c.c @@ -946,11 +946,11 @@ WC_OMIT_FRAME_POINTER void AES_set_encrypt_key(const unsigned char* key, void AES_encrypt_block(const word32* te_p, int nr_p, int len_p, const word32* ks_p); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te_p, int nr_p, - int len_p, const word32* ks_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te_p, + int nr_p, int len_p, const word32* ks_p) #else -WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te, int nr, int len, - const word32* ks) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te, + int nr, int len, const word32* ks) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -8237,11 +8237,11 @@ WC_OMIT_FRAME_POINTER void AES_CTR_encrypt(const unsigned char* in, #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE void AES_decrypt_block(const word32* td_p, int nr_p, const byte* td4_p); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td_p, int nr_p, - const byte* td4_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td_p, + int nr_p, const byte* td4_p) #else -WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td, int nr, - const byte* td4) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td, + int nr, const byte* td4) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG diff --git a/wolfcrypt/src/port/arm/armv8-32-curve25519_c.c b/wolfcrypt/src/port/arm/armv8-32-curve25519_c.c index 133265ae98..7b7d52a557 100644 --- a/wolfcrypt/src/port/arm/armv8-32-curve25519_c.c +++ b/wolfcrypt/src/port/arm/armv8-32-curve25519_c.c @@ -84,9 +84,9 @@ WC_OMIT_FRAME_POINTER void fe_init(void) void fe_add_sub_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) #else -WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -299,9 +299,9 @@ WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) void fe_sub_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sub_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sub_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -375,9 +375,9 @@ WC_OMIT_FRAME_POINTER void fe_sub(fe r, const fe a, const fe b) void fe_add_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_add_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_op(void) #else -WC_OMIT_FRAME_POINTER void fe_add_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2501,9 +2501,9 @@ WC_OMIT_FRAME_POINTER void fe_cmov_table(fe* r, const fe* base, signed char b) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) void fe_mul_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2895,9 +2895,9 @@ WC_OMIT_FRAME_POINTER void fe_mul_op(void) void fe_mul_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3313,9 +3313,9 @@ WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #else void fe_mul_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3466,9 +3466,9 @@ WC_OMIT_FRAME_POINTER void fe_mul_op(void) void fe_mul_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3670,9 +3670,9 @@ WC_OMIT_FRAME_POINTER void fe_mul(fe r, const fe a, const fe b) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) void fe_sq_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3957,9 +3957,9 @@ WC_OMIT_FRAME_POINTER void fe_sq_op(void) void fe_sq_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -4268,9 +4268,9 @@ WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #else void fe_sq_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -4407,9 +4407,9 @@ WC_OMIT_FRAME_POINTER void fe_sq_op(void) void fe_sq_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -4596,9 +4596,9 @@ WC_OMIT_FRAME_POINTER void fe_sq(fe r, const fe a) #ifdef HAVE_CURVE25519 #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -4680,9 +4680,9 @@ WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) #else #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -5701,9 +5701,9 @@ WC_OMIT_FRAME_POINTER void fe_invert(fe r, const fe a) #if defined(WOLFSSL_ARM_ARCH) && (WOLFSSL_ARM_ARCH < 6) #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -6055,9 +6055,9 @@ WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) #else #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG diff --git a/wolfcrypt/src/port/arm/armv8-32-sha3-asm.S b/wolfcrypt/src/port/arm/armv8-32-sha3-asm.S index aa7192c67a..c5be47e569 100644 --- a/wolfcrypt/src/port/arm/armv8-32-sha3-asm.S +++ b/wolfcrypt/src/port/arm/armv8-32-sha3-asm.S @@ -30,6 +30,12 @@ #include +/* Honor WC_SHA3_NO_ASM as sha3.c does: suppress this NEON BlockSha3 so it + * doesn't multiply-define against sha3.c's C BlockSha3 on arm32. */ +#ifdef WC_SHA3_NO_ASM + #undef WOLFSSL_ARMASM +#endif + #ifdef WOLFSSL_ARMASM #if !defined(__aarch64__) && !defined(WOLFSSL_ARMASM_THUMB2) #ifndef WOLFSSL_ARMASM_INLINE diff --git a/wolfcrypt/src/port/arm/armv8-32-sha3-asm_c.c b/wolfcrypt/src/port/arm/armv8-32-sha3-asm_c.c index abc1a9fe09..da28f24fb4 100644 --- a/wolfcrypt/src/port/arm/armv8-32-sha3-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-32-sha3-asm_c.c @@ -31,6 +31,12 @@ #include #include +/* Honor WC_SHA3_NO_ASM as sha3.c does: suppress this NEON BlockSha3 so it + * doesn't multiply-define against sha3.c's C BlockSha3 on arm32. */ +#ifdef WC_SHA3_NO_ASM + #undef WOLFSSL_ARMASM +#endif + #ifdef WOLFSSL_ARMASM #if !defined(__aarch64__) && !defined(WOLFSSL_ARMASM_THUMB2) #ifdef WOLFSSL_ARMASM_INLINE diff --git a/wolfcrypt/src/port/arm/armv8-curve25519_c.c b/wolfcrypt/src/port/arm/armv8-curve25519_c.c index 53a573a12f..168260f416 100644 --- a/wolfcrypt/src/port/arm/armv8-curve25519_c.c +++ b/wolfcrypt/src/port/arm/armv8-curve25519_c.c @@ -732,7 +732,7 @@ void fe_invert_nct(fe r, const fe a) ); } -void fe_mul(fe r, const fe a, const fe b) +WC_KEEP_FOR_ASM void fe_mul(fe r, const fe a, const fe b) { __asm__ __volatile__ ( /* Multiply */ @@ -869,7 +869,7 @@ void fe_mul(fe r, const fe a, const fe b) ); } -void fe_sq(fe r, const fe a) +WC_KEEP_FOR_ASM void fe_sq(fe r, const fe a) { __asm__ __volatile__ ( /* Square */ diff --git a/wolfcrypt/src/port/arm/thumb2-aes-asm_c.c b/wolfcrypt/src/port/arm/thumb2-aes-asm_c.c index 9405f6e821..8feaac55ed 100644 --- a/wolfcrypt/src/port/arm/thumb2-aes-asm_c.c +++ b/wolfcrypt/src/port/arm/thumb2-aes-asm_c.c @@ -656,11 +656,11 @@ WC_OMIT_FRAME_POINTER void AES_set_encrypt_key(const unsigned char* key, void AES_encrypt_block(const word32* te_p, int nr_p, int len_p, const word32* ks_p); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te_p, int nr_p, - int len_p, const word32* ks_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te_p, + int nr_p, int len_p, const word32* ks_p) #else -WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te, int nr, int len, - const word32* ks) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_encrypt_block(const word32* te, + int nr, int len, const word32* ks) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3605,11 +3605,11 @@ WC_OMIT_FRAME_POINTER void AES_CTR_encrypt(const unsigned char* in, #ifndef WOLFSSL_ARMASM_AES_BLOCK_INLINE void AES_decrypt_block(const word32* td_p, int nr_p, const byte* td4_p); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td_p, int nr_p, - const byte* td4_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td_p, + int nr_p, const byte* td4_p) #else -WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td, int nr, - const byte* td4) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void AES_decrypt_block(const word32* td, + int nr, const byte* td4) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG diff --git a/wolfcrypt/src/port/arm/thumb2-curve25519_c.c b/wolfcrypt/src/port/arm/thumb2-curve25519_c.c index c2607724d5..02e83cb18a 100644 --- a/wolfcrypt/src/port/arm/thumb2-curve25519_c.c +++ b/wolfcrypt/src/port/arm/thumb2-curve25519_c.c @@ -84,9 +84,9 @@ WC_OMIT_FRAME_POINTER void fe_init(void) void fe_add_sub_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) #else -WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -192,9 +192,9 @@ WC_OMIT_FRAME_POINTER void fe_add_sub_op(void) void fe_sub_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sub_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sub_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sub_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -264,9 +264,9 @@ WC_OMIT_FRAME_POINTER void fe_sub(fe r, const fe a, const fe b) void fe_add_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_add_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_op(void) #else -WC_OMIT_FRAME_POINTER void fe_add_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_add_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -1766,9 +1766,9 @@ WC_OMIT_FRAME_POINTER void fe_cmov_table(fe* r, const fe* base, signed char b) #ifdef WOLFSSL_ARM_ARCH_7M void fe_mul_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2156,9 +2156,9 @@ WC_OMIT_FRAME_POINTER void fe_mul_op(void) void fe_mul_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2566,9 +2566,9 @@ WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #else void fe_mul_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2710,9 +2710,9 @@ WC_OMIT_FRAME_POINTER void fe_mul_op(void) void fe_mul_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -2901,9 +2901,9 @@ WC_OMIT_FRAME_POINTER void fe_mul(fe r, const fe a, const fe b) #ifdef WOLFSSL_ARM_ARCH_7M void fe_sq_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3184,9 +3184,9 @@ WC_OMIT_FRAME_POINTER void fe_sq_op(void) void fe_sq_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3487,9 +3487,9 @@ WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #else void fe_sq_op(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3617,9 +3617,9 @@ WC_OMIT_FRAME_POINTER void fe_sq_op(void) void fe_sq_op_full_red(void); #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #else -WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq_op_full_red(void) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3793,9 +3793,9 @@ WC_OMIT_FRAME_POINTER void fe_sq(fe r, const fe a) #ifdef HAVE_CURVE25519 #ifdef WOLFSSL_ARM_ARCH_7M #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -3858,9 +3858,9 @@ WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) #else #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r_p, fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_mul121666(fe r, fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -5123,9 +5123,9 @@ WC_OMIT_FRAME_POINTER void fe_invert(fe r, const fe a) #ifdef WOLFSSL_ARM_ARCH_7M #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG @@ -5461,9 +5461,9 @@ WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) #else #ifndef WOLFSSL_NO_VAR_ASSIGN_REG -WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r_p, const fe a_p) #else -WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) +WC_KEEP_FOR_ASM WC_OMIT_FRAME_POINTER void fe_sq2(fe r, const fe a) #endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ { #ifndef WOLFSSL_NO_VAR_ASSIGN_REG diff --git a/wolfcrypt/src/port/arm/thumb2-sha3-asm.S b/wolfcrypt/src/port/arm/thumb2-sha3-asm.S index 8df840e51d..b885c8f873 100644 --- a/wolfcrypt/src/port/arm/thumb2-sha3-asm.S +++ b/wolfcrypt/src/port/arm/thumb2-sha3-asm.S @@ -30,6 +30,12 @@ #include +/* Honor WC_SHA3_NO_ASM as sha3.c does: suppress this BlockSha3 so it + * doesn't multiply-define against sha3.c's C BlockSha3 on thumb2. */ +#ifdef WC_SHA3_NO_ASM + #undef WOLFSSL_ARMASM +#endif + #ifdef WOLFSSL_ARMASM #ifdef WOLFSSL_ARMASM_THUMB2 #ifndef WOLFSSL_ARMASM_INLINE diff --git a/wolfcrypt/src/port/arm/thumb2-sha3-asm_c.c b/wolfcrypt/src/port/arm/thumb2-sha3-asm_c.c index b01ed75c02..531431d37c 100644 --- a/wolfcrypt/src/port/arm/thumb2-sha3-asm_c.c +++ b/wolfcrypt/src/port/arm/thumb2-sha3-asm_c.c @@ -31,6 +31,12 @@ #include #include +/* Honor WC_SHA3_NO_ASM as sha3.c does: suppress this BlockSha3 so it + * doesn't multiply-define against sha3.c's C BlockSha3 on thumb2. */ +#ifdef WC_SHA3_NO_ASM + #undef WOLFSSL_ARMASM +#endif + #ifdef WOLFSSL_ARMASM #ifdef WOLFSSL_ARMASM_THUMB2 #ifdef WOLFSSL_ARMASM_INLINE diff --git a/wolfcrypt/src/port/riscv64/riscv-64-aes-asm_c.c b/wolfcrypt/src/port/riscv64/riscv-64-aes-asm_c.c index b64c17d99e..7230894c08 100644 --- a/wolfcrypt/src/port/riscv64/riscv-64-aes-asm_c.c +++ b/wolfcrypt/src/port/riscv64/riscv-64-aes-asm_c.c @@ -30949,8 +30949,8 @@ void AES_CBC_encrypt_RISCV64(const byte* in, byte* out, word32 sz, byte* reg, } #ifdef HAVE_AES_DECRYPT -void AES_CBC_decrypt_RISCV64(const byte* in, byte* out, word32 sz, byte* reg, - byte* key, int nr) +WC_OMIT_FRAME_POINTER void AES_CBC_decrypt_RISCV64(const byte* in, byte* out, + word32 sz, byte* reg, byte* key, int nr) { const word32* td = L_AES_base_td; const word8* td4 = L_AES_base_td4; @@ -48005,10 +48005,10 @@ void AES_XTS_decrypt_RISCV64(const byte* in, byte* out, word32 sz, #endif /* HAVE_AES_DECRYPT */ #endif /* WOLFSSL_AES_XTS */ #ifdef HAVE_AESGCM -void AES_GCM_encrypt_RISCV64(const byte* in_p, byte* out_p, word32 sz_p, - const byte* nonce_p, word32 nonceSz_p, byte* tag_p, word32 tagSz_p, - const byte* aad_p, word32 aadSz_p, byte* key_p, byte* m0_p, byte* tmp_p, - byte* reg_p, int nr_p) +WC_OMIT_FRAME_POINTER void AES_GCM_encrypt_RISCV64(const byte* in_p, + byte* out_p, word32 sz_p, const byte* nonce_p, word32 nonceSz_p, + byte* tag_p, word32 tagSz_p, const byte* aad_p, word32 aadSz_p, byte* key_p, + byte* m0_p, byte* tmp_p, byte* reg_p, int nr_p) { register const byte* in __asm__ ("a0") = (const byte*)in_p; register byte* out __asm__ ("a1") = (byte*)out_p; @@ -58260,10 +58260,10 @@ void AES_GCM_encrypt_RISCV64(const byte* in_p, byte* out_p, word32 sz_p, } #ifdef HAVE_AES_DECRYPT -int AES_GCM_decrypt_RISCV64(byte* in_p, byte* out_p, word32 sz_p, - const byte* nonce_p, word32 nonceSz_p, const byte* tag_p, word32 tagSz_p, - const byte* aad_p, word32 aadSz_p, byte* key_p, byte* m0_p, byte* tmp_p, - byte* reg_p, int nr_p) +WC_OMIT_FRAME_POINTER int AES_GCM_decrypt_RISCV64(byte* in_p, byte* out_p, + word32 sz_p, const byte* nonce_p, word32 nonceSz_p, const byte* tag_p, + word32 tagSz_p, const byte* aad_p, word32 aadSz_p, byte* key_p, byte* m0_p, + byte* tmp_p, byte* reg_p, int nr_p) { register byte* in __asm__ ("a0") = (byte*)in_p; register byte* out __asm__ ("a1") = (byte*)out_p; diff --git a/wolfcrypt/src/port/riscv64/riscv-64-sha256-asm_c.c b/wolfcrypt/src/port/riscv64/riscv-64-sha256-asm_c.c index a1fa7616f7..f042dc77a3 100644 --- a/wolfcrypt/src/port/riscv64/riscv-64-sha256-asm_c.c +++ b/wolfcrypt/src/port/riscv64/riscv-64-sha256-asm_c.c @@ -254,8 +254,8 @@ XALIGNED(8) static const word32 L_SHA256_transform_riscv_crypto_len_k[] = { 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, }; -void Transform_Sha256_Len_riscv_crypto(wc_Sha256* sha256, const byte* data, - word32 len) +WC_OMIT_FRAME_POINTER void Transform_Sha256_Len_riscv_crypto(wc_Sha256* sha256, + const byte* data, word32 len) { const word32* k = L_SHA256_transform_riscv_crypto_len_k; @@ -1320,7 +1320,8 @@ XALIGNED(8) static const word32 L_SHA256_transform_riscv_len_k[] = { 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2, }; -void Transform_Sha256_Len_riscv(wc_Sha256* sha256, const byte* data, word32 len) +WC_OMIT_FRAME_POINTER void Transform_Sha256_Len_riscv(wc_Sha256* sha256, + const byte* data, word32 len) { const word32* k = L_SHA256_transform_riscv_len_k; diff --git a/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm.S b/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm.S index c4b50ca5f9..dfd89b0fd5 100644 --- a/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm.S +++ b/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm.S @@ -29,6 +29,13 @@ #define _WC_BUILDING_RISCV_64_SHA3_ASM_S #include + +/* Honor WC_SHA3_NO_ASM as sha3.c does: suppress this BlockSha3 so it + * doesn't multiply-define against sha3.c's C BlockSha3 on riscv64. */ +#ifdef WC_SHA3_NO_ASM + #undef WOLFSSL_RISCV_ASM +#endif + #ifdef WOLFSSL_RISCV_ASM #ifndef WOLFSSL_RISCV_ASM_INLINE #ifdef WOLFSSL_SHA3 diff --git a/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm_c.c b/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm_c.c index a2b8616dfb..21f3cf4df3 100644 --- a/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm_c.c +++ b/wolfcrypt/src/port/riscv64/riscv-64-sha3-asm_c.c @@ -30,6 +30,13 @@ #include #include + +/* Honor WC_SHA3_NO_ASM as sha3.c does: suppress this BlockSha3 so it + * doesn't multiply-define against sha3.c's C BlockSha3 on riscv64. */ +#ifdef WC_SHA3_NO_ASM + #undef WOLFSSL_RISCV_ASM +#endif + #ifdef WOLFSSL_RISCV_ASM #ifdef WOLFSSL_RISCV_ASM_INLINE #include @@ -585,7 +592,7 @@ XALIGNED(16) static const word64 L_SHA3_transform_riscv_r[] = { 0x0000000080000001UL, 0x8000000080008008UL, }; -void BlockSha3(word64* s) +WC_OMIT_FRAME_POINTER void BlockSha3(word64* s) { const word64* r = L_SHA3_transform_riscv_r; diff --git a/wolfcrypt/src/sha256_asm.asm b/wolfcrypt/src/sha256_asm.asm index 4fa488d79e..be9750ab76 100644 --- a/wolfcrypt/src/sha256_asm.asm +++ b/wolfcrypt/src/sha256_asm.asm @@ -12952,7 +12952,9 @@ Transform_Sha256_AVX2_Len PROC vmovdqu ymm1, YMMWORD PTR [rsi+32] vmovups YMMWORD PTR [rdi+32], ymm0 vmovups YMMWORD PTR [rdi+64], ymm1 + sub rsp, 32 call Transform_Sha256_AVX2 + add rsp, 32 add rsi, 64 sub DWORD PTR [rsp+512], 64 jz L_sha256_len_avx2_done @@ -19521,7 +19523,9 @@ Transform_Sha256_AVX2_RORX_Len PROC vmovdqu ymm1, YMMWORD PTR [rsi+32] vmovups YMMWORD PTR [rdi+32], ymm0 vmovups YMMWORD PTR [rdi+64], ymm1 + sub rsp, 32 call Transform_Sha256_AVX2_RORX + add rsp, 32 add rsi, 64 sub DWORD PTR [rsp+512], 64 jz L_sha256_len_avx2_rorx_done diff --git a/wolfcrypt/src/sha512_asm.asm b/wolfcrypt/src/sha512_asm.asm index 0b95a23dae..b7cdf467fe 100644 --- a/wolfcrypt/src/sha512_asm.asm +++ b/wolfcrypt/src/sha512_asm.asm @@ -6443,7 +6443,9 @@ Transform_Sha512_AVX2_Len PROC vmovups YMMWORD PTR [rdi+96], ymm1 vmovups YMMWORD PTR [rdi+128], ymm2 vmovups YMMWORD PTR [rdi+160], ymm3 + sub rsp, 32 call Transform_Sha512_AVX2 + add rsp, 32 add QWORD PTR [rdi+224], 128 sub ebp, 128 jz L_sha512_len_avx2_done @@ -9241,9 +9243,9 @@ Transform_Sha512_AVX2_RORX_Len PROC vmovups YMMWORD PTR [rdi+96], ymm1 vmovups YMMWORD PTR [rdi+128], ymm2 vmovups YMMWORD PTR [rdi+160], ymm3 - sub rsp, 8 + sub rsp, 40 call Transform_Sha512_AVX2_RORX - add rsp, 8 + add rsp, 40 pop rsi add QWORD PTR [rdi+224], 128 sub esi, 128 diff --git a/wolfcrypt/src/sp_x86_64_asm.S b/wolfcrypt/src/sp_x86_64_asm.S index 3994867b4e..61ca591a62 100644 --- a/wolfcrypt/src/sp_x86_64_asm.S +++ b/wolfcrypt/src/sp_x86_64_asm.S @@ -45182,7 +45182,7 @@ _sp_256_mont_tpl_4: movq $0xffffffff00000001, %r10 adcq 16(%rsi), %rcx adcq 24(%rsi), %r8 - sbbq $0x00, %r11 + sbbq %r11, %r11 movl %r11d, %r9d andq %r11, %r10 subq %r11, %rdx @@ -45361,7 +45361,7 @@ _sp_256_mont_rsb_sub_dbl_4: movq $0xffffffff00000001, %r15 sbbq %r12, %r8 sbbq %r13, %r9 - sbbq $0x00, %rsi + sbbq %rsi, %rsi movl %esi, %r14d andq %rsi, %r15 addq %rsi, %rax diff --git a/wolfcrypt/src/sp_x86_64_asm.asm b/wolfcrypt/src/sp_x86_64_asm.asm index 690beda895..f58813f684 100644 --- a/wolfcrypt/src/sp_x86_64_asm.asm +++ b/wolfcrypt/src/sp_x86_64_asm.asm @@ -43853,7 +43853,7 @@ sp_256_mont_tpl_4 PROC mov r12, 18446744069414584321 adc r9, QWORD PTR [rdx+16] adc r10, QWORD PTR [rdx+24] - sbb r13, 0 + sbb r13, r13 mov r11d, r13d and r12, r13 sub rax, r13 @@ -44005,7 +44005,7 @@ sp_256_mont_rsb_sub_dbl_4 PROC mov rsi, 18446744069414584321 sbb r10, r14 sbb r11, r15 - sbb rdx, 0 + sbb rdx, rdx mov edi, edx and rsi, rdx add rax, rdx diff --git a/wolfssl-VS2022.vcxproj b/wolfssl-VS2022.vcxproj index 9e7f78151f..928cd6c1b6 100644 --- a/wolfssl-VS2022.vcxproj +++ b/wolfssl-VS2022.vcxproj @@ -562,6 +562,28 @@ true true + + false + false + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(OutDir)%(Filename).obj" %(Identity) + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(IntDir)%(Filename).obj" %(Identity) + $(OutDir)%(Filename).obj + $(IntDir)%(Filename).obj + false + false + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(OutDir)%(Filename).obj" %(Identity) + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(IntDir)%(Filename).obj" %(Identity) + $(OutDir)%(Filename).obj + $(IntDir)%(Filename).obj + true + true + true + true + true + true + true + true + false false diff --git a/wolfssl.vcxproj b/wolfssl.vcxproj index 30c290811a..0ec821dbe3 100644 --- a/wolfssl.vcxproj +++ b/wolfssl.vcxproj @@ -562,6 +562,28 @@ true true + + false + false + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(OutDir)%(Filename).obj" %(Identity) + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(IntDir)%(Filename).obj" %(Identity) + $(OutDir)%(Filename).obj + $(IntDir)%(Filename).obj + false + false + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(OutDir)%(Filename).obj" %(Identity) + ml.exe /c /safeseh /DWOLFSSL_AES_XTS /DWOLFSSL_X86_BUILD /Zi /Fo"$(IntDir)%(Filename).obj" %(Identity) + $(OutDir)%(Filename).obj + $(IntDir)%(Filename).obj + true + true + true + true + true + true + true + true + false false diff --git a/wolfssl/internal.h b/wolfssl/internal.h index 2f96892e06..3f6dd0dddd 100644 --- a/wolfssl/internal.h +++ b/wolfssl/internal.h @@ -53,7 +53,7 @@ #ifdef HAVE_POLY1305 #include #endif -#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) && defined(OPENSSL_EXTRA) +#if defined(HAVE_CHACHA) && defined(HAVE_POLY1305) #include #endif #ifdef HAVE_ARIA diff --git a/wolfssl/test.h b/wolfssl/test.h index 23c4ce194f..055b5b4391 100644 --- a/wolfssl/test.h +++ b/wolfssl/test.h @@ -1896,8 +1896,35 @@ static WC_INLINE int udp_read_connect(SOCKET_T sockfd) } #endif +/* Write the port the server bound to the ready file named by -R, so a script + * driving the server can wait for the file and learn which port an ephemeral + * (-p 0) bind landed on. Shared by the TCP and UDP accept paths - a DTLS server + * needs it just as much as a TLS one. */ +static WC_INLINE void write_ready_file(func_args* args, word16 port) +{ +#if (!defined(NO_FILESYSTEM) || defined(FORCE_BUFFER_TEST)) && !defined(NETOS) + tcp_ready* ready = NULL; + + if (args) + ready = args->signal; + + if (ready && ready->srfName) { + XFILE srf = XFOPEN(ready->srfName, "w"); + + if (srf) { + LIBCALL_CHECK_RET(fprintf(srf, "%d\n", (int)port)); + fclose(srf); + } + } +#else + (void)args; + (void)port; +#endif +} + static WC_INLINE void udp_accept(SOCKET_T* sockfd, SOCKET_T* clientfd, - int useAnyAddr, word16 port, func_args* args) + int useAnyAddr, word16 port, func_args* args, + int ready_file) { SOCKADDR_IN_T addr; @@ -1964,6 +1991,10 @@ static WC_INLINE void udp_accept(SOCKET_T* sockfd, SOCKET_T* clientfd, fprintf(stderr, "args or args->signal was NULL. Not setting ready info."); } + if (ready_file) { + write_ready_file(args, port); + } + *clientfd = *sockfd; } @@ -1977,7 +2008,7 @@ static WC_INLINE void tcp_accept(SOCKET_T* sockfd, SOCKET_T* clientfd, (void) ready; /* Account for case when "ready" is not used */ if (udp) { - udp_accept(sockfd, clientfd, useAnyAddr, port, args); + udp_accept(sockfd, clientfd, useAnyAddr, port, args, ready_file); return; } @@ -2002,26 +2033,7 @@ static WC_INLINE void tcp_accept(SOCKET_T* sockfd, SOCKET_T* clientfd, #endif /* !SINGLE_THREADED */ if (ready_file) { - #if (!defined(NO_FILESYSTEM) || defined(FORCE_BUFFER_TEST)) && \ - !defined(NETOS) - XFILE srf = (XFILE)NULL; - if (args) - ready = args->signal; - - if (ready) { - srf = XFOPEN(ready->srfName, "w"); - - if (srf) { - /* let's write port sever is listening on to ready file - external monitor can then do ephemeral ports by passing - -p 0 to server on supported platforms with -R ready_file - client can then wait for existence of ready_file and see - which port the server is listening on. */ - LIBCALL_CHECK_RET(fprintf(srf, "%d\n", (int)port)); - fclose(srf); - } - } - #endif + write_ready_file(args, port); } } diff --git a/wolfssl/wolfcrypt/wc_port.h b/wolfssl/wolfcrypt/wc_port.h index 3d3faf24ac..b3a29567cb 100644 --- a/wolfssl/wolfcrypt/wc_port.h +++ b/wolfssl/wolfcrypt/wc_port.h @@ -176,6 +176,19 @@ #endif #endif +/* Marks a function that is only ever reached from a branch inside an inline + * assembly string. The compiler does not parse those strings, so it sees no + * reference to the function: without this, -flto decides the definition is + * unreachable, drops the body, and the branch is left with nothing to bind to + * at link time. Costs nothing in a non-LTO build. */ +#ifndef WC_KEEP_FOR_ASM + #if defined(__GNUC__) || defined(__clang__) + #define WC_KEEP_FOR_ASM __attribute__((used)) + #else + #define WC_KEEP_FOR_ASM + #endif +#endif + /* THREADING/MUTEX SECTION */ #if defined(SINGLE_THREADED) && defined(NO_FILESYSTEM) /* No system headers required for build. */