From 357d17db0fad2db18990639b236e8956d1e32900 Mon Sep 17 00:00:00 2001 From: Sean Parkinson Date: Fri, 14 Aug 2026 09:15:25 +1000 Subject: [PATCH] ML-DSA AArch64/32: assembly implementation Add optimized AArch64 NEON assembly implementation Add optimized AArch32 NEON assembly implementation --- .github/configs/pq-all.json | 40 + .github/workflows/pq-all.yml | 2 +- src/include.am | 32 + wolfcrypt/src/include.am | 1 + wolfcrypt/src/port/arm/armv8-32-mldsa-asm.S | 10786 ++++++ wolfcrypt/src/port/arm/armv8-32-mldsa-asm_c.c | 11087 ++++++ wolfcrypt/src/port/arm/armv8-32-sha3-asm.S | 1253 + wolfcrypt/src/port/arm/armv8-32-sha3-asm_c.c | 1310 + wolfcrypt/src/port/arm/armv8-aes-asm.S | 30 + wolfcrypt/src/port/arm/armv8-aes-asm.asm | 10 + wolfcrypt/src/port/arm/armv8-aes-asm_c.c | 30 + wolfcrypt/src/port/arm/armv8-mldsa-asm.S | 29288 ++++++++++++++++ wolfcrypt/src/port/arm/armv8-mldsa-asm.asm | 27751 +++++++++++++++ wolfcrypt/src/port/arm/armv8-mldsa-asm_c.c | 27660 +++++++++++++++ wolfcrypt/src/port/arm/armv8-mlkem-asm.S | 2334 -- wolfcrypt/src/port/arm/armv8-mlkem-asm.asm | 2220 -- wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c | 2154 -- wolfcrypt/src/port/arm/armv8-sha3-asm.S | 3127 ++ wolfcrypt/src/port/arm/armv8-sha3-asm.asm | 2971 ++ wolfcrypt/src/port/arm/armv8-sha3-asm_c.c | 2891 ++ wolfcrypt/src/wc_mldsa.c | 1777 +- wolfcrypt/src/wc_mlkem_poly.c | 163 +- wolfssl-VS2022.vcxproj | 8 +- wolfssl.vcxproj | 8 +- wolfssl/wolfcrypt/sha3.h | 29 +- wolfssl/wolfcrypt/wc_mldsa.h | 140 + wolfssl/wolfcrypt/wc_mlkem.h | 10 - 27 files changed, 120220 insertions(+), 6892 deletions(-) create mode 100644 wolfcrypt/src/port/arm/armv8-32-mldsa-asm.S create mode 100644 wolfcrypt/src/port/arm/armv8-32-mldsa-asm_c.c create mode 100644 wolfcrypt/src/port/arm/armv8-mldsa-asm.S create mode 100644 wolfcrypt/src/port/arm/armv8-mldsa-asm.asm create mode 100644 wolfcrypt/src/port/arm/armv8-mldsa-asm_c.c diff --git a/.github/configs/pq-all.json b/.github/configs/pq-all.json index 0a147cdbd13..7fcb97e71f1 100644 --- a/.github/configs/pq-all.json +++ b/.github/configs/pq-all.json @@ -189,6 +189,46 @@ "configure": ["--host=aarch64-linux-gnu", "--enable-cryptonly", "--enable-armasm=inline", "--enable-experimental", "--enable-frodokem=aes,ephemeral,sve,sme", "--disable-examples"]}, +{"name": "mldsa-mlkem-arm32-armasm", "minutes": 0.5, "check": false, + "cc": "ccache arm-linux-gnueabihf-gcc", + "comment": "AArch32 NEON ML-DSA + ML-KEM and the batched Keccak they share. Runs the KATs under qemu: these are wrong-answer risks, so a build-only job would not catch them.", + "configure": ["--host=arm-linux-gnueabihf", "--enable-cryptonly", + "--enable-armasm", "--enable-mldsa", "--enable-mlkem", + "--disable-examples"], + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf", + "./wolfcrypt/test/testwolfcrypt"]]}, +{"name": "mldsa-mlkem-arm32-armasm-inline", "minutes": 0.5, "check": false, + "cc": "ccache arm-linux-gnueabihf-gcc", + "comment": "Same, through the inline-asm backend. It is a separate code path from the .S - operand register allocation differs - and has carried defects the .S build did not.", + "configure": ["--host=arm-linux-gnueabihf", "--enable-cryptonly", + "--enable-armasm=inline", "--enable-mldsa", "--enable-mlkem", + "--disable-examples"], + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf", + "./wolfcrypt/test/testwolfcrypt"]]}, +{"name": "mldsa-mlkem-arm32-armasm-no-neon", "minutes": 0.5, "check": false, + "cc": "ccache arm-linux-gnueabihf-gcc", + "comment": "WOLFSSL_ARMASM_NO_NEON: scalar SHA-3 assembly with the C ML-DSA, and the only AArch32 job covering the non-NEON dispatch.", + "configure": ["--host=arm-linux-gnueabihf", "--enable-cryptonly", + "--enable-armasm", "--enable-mldsa", "--enable-mlkem", + "--disable-examples", "CPPFLAGS=-DWOLFSSL_ARMASM_NO_NEON"], + "run": [["env", "QEMU_LD_PREFIX=/usr/arm-linux-gnueabihf", + "./wolfcrypt/test/testwolfcrypt"]]}, +{"name": "mldsa-mlkem-aarch64-armasm", "minutes": 0.5, "check": false, + "cc": "ccache aarch64-linux-gnu-gcc", + "comment": "AArch64 NEON ML-DSA + ML-KEM. QEMU_CPU=max so the FEAT_RDM and FEAT_SHA3 dispatchers take their instruction-extension paths.", + "configure": ["--host=aarch64-linux-gnu", "--enable-cryptonly", + "--enable-armasm", "--enable-mldsa", "--enable-mlkem", + "--disable-examples"], + "run": [["env", "QEMU_LD_PREFIX=/usr/aarch64-linux-gnu", "QEMU_CPU=max", + "./wolfcrypt/test/testwolfcrypt"]]}, +{"name": "mldsa-mlkem-aarch64-armasm-inline", "minutes": 0.5, "check": false, + "cc": "ccache aarch64-linux-gnu-gcc", + "comment": "Same, through the inline-asm backend.", + "configure": ["--host=aarch64-linux-gnu", "--enable-cryptonly", + "--enable-armasm=inline", "--enable-mldsa", "--enable-mlkem", + "--disable-examples"], + "run": [["env", "QEMU_LD_PREFIX=/usr/aarch64-linux-gnu", "QEMU_CPU=max", + "./wolfcrypt/test/testwolfcrypt"]]}, {"name": "mlkem-old-ids", "minutes": 0.4, "configure": ["--enable-intelasm", "--enable-sp-asm", "--enable-mlkem=yes,kyber,ml-kem", diff --git a/.github/workflows/pq-all.yml b/.github/workflows/pq-all.yml index 71cfa85b3f5..dcf0da5a3a8 100644 --- a/.github/workflows/pq-all.yml +++ b/.github/workflows/pq-all.yml @@ -49,7 +49,7 @@ jobs: - name: Install dependencies uses: ./.github/actions/install-apt-deps with: - packages: autoconf automake libtool build-essential bubblewrap crossbuild-essential-arm64 crossbuild-essential-armhf + packages: autoconf automake libtool build-essential bubblewrap crossbuild-essential-arm64 crossbuild-essential-armhf qemu-user ghcr-debs-tag: ubuntu-24.04-minimal # ccache via the cross-platform composite; the script passes the diff --git a/src/include.am b/src/include.am index 12133f62e33..e0997b29275 100644 --- a/src/include.am +++ b/src/include.am @@ -1351,6 +1351,22 @@ if BUILD_INTELASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_mldsa_asm.S endif BUILD_INTELASM endif !BUILD_X86_ASM +if BUILD_ARMASM +if !BUILD_ARM_THUMB +if BUILD_ARMASM_INLINE +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/arm/armv8-32-mldsa-asm_c.c +else +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/arm/armv8-32-mldsa-asm.S +endif !BUILD_ARMASM_INLINE +endif !BUILD_ARM_THUMB +endif BUILD_ARMASM +if BUILD_ARMASM_NEON +if BUILD_ARMASM_INLINE +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/arm/armv8-mldsa-asm_c.c +else +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/arm/armv8-mldsa-asm.S +endif !BUILD_ARMASM_INLINE +endif BUILD_ARMASM_NEON endif if BUILD_WC_LMS @@ -2152,6 +2168,22 @@ if BUILD_INTELASM src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/wc_mldsa_asm.S endif BUILD_INTELASM endif !BUILD_X86_ASM +if BUILD_ARMASM +if !BUILD_ARM_THUMB +if BUILD_ARMASM_INLINE +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/arm/armv8-32-mldsa-asm_c.c +else +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/arm/armv8-32-mldsa-asm.S +endif !BUILD_ARMASM_INLINE +endif !BUILD_ARM_THUMB +endif BUILD_ARMASM +if BUILD_ARMASM_NEON +if BUILD_ARMASM_INLINE +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/arm/armv8-mldsa-asm_c.c +else +src_libwolfssl@LIBSUFFIX@_la_SOURCES += wolfcrypt/src/port/arm/armv8-mldsa-asm.S +endif !BUILD_ARMASM_INLINE +endif BUILD_ARMASM_NEON endif if BUILD_WC_LMS diff --git a/wolfcrypt/src/include.am b/wolfcrypt/src/include.am index f0e7ca3c717..26af42baa1a 100644 --- a/wolfcrypt/src/include.am +++ b/wolfcrypt/src/include.am @@ -32,6 +32,7 @@ EXTRA_DIST += wolfcrypt/src/wc_mldsa_asm.asm EXTRA_DIST += wolfcrypt/src/port/arm/armv8-aes-asm.asm EXTRA_DIST += wolfcrypt/src/port/arm/armv8-chacha-asm.asm EXTRA_DIST += wolfcrypt/src/port/arm/armv8-curve25519.asm +EXTRA_DIST += wolfcrypt/src/port/arm/armv8-mldsa-asm.asm EXTRA_DIST += wolfcrypt/src/port/arm/armv8-mlkem-asm.asm EXTRA_DIST += wolfcrypt/src/port/arm/armv8-poly1305-asm.asm EXTRA_DIST += wolfcrypt/src/port/arm/armv8-sha256-asm.asm diff --git a/wolfcrypt/src/port/arm/armv8-32-mldsa-asm.S b/wolfcrypt/src/port/arm/armv8-32-mldsa-asm.S new file mode 100644 index 00000000000..f0830178e0c --- /dev/null +++ b/wolfcrypt/src/port/arm/armv8-32-mldsa-asm.S @@ -0,0 +1,10786 @@ +/* armv8-32-mldsa-asm + * + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +/* Generated using (from wolfssl): + * cd ../scripts + * ruby ./mldsa/mldsa.rb arm32 \ + * ../wolfssl/wolfcrypt/src/port/arm/armv8-32-mldsa-asm.S + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_ARMV8_32_MLDSA_ASM_S + +#include + +#ifdef WOLFSSL_ARMASM +#if !defined(__aarch64__) && !defined(WOLFSSL_ARMASM_THUMB2) +#ifndef WOLFSSL_ARMASM_INLINE +#ifdef WOLFSSL_HAVE_MLDSA +#ifndef WOLFSSL_ARMASM_NO_NEON +#ifndef __APPLE__ + .text + .type L_mldsa_arm32_consts, %object + .size L_mldsa_arm32_consts, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_arm32_consts: + .long 0x007fe001,0x03802001,0x00000000,0x00000000 + .text + .align 4 + .globl mldsa_mul_neon + WC_ASM_ATT_HIDDEN(mldsa_mul_neon) + .type mldsa_mul_neon, %function +mldsa_mul_neon: + vpush {d8-d15} + adr r3, L_mldsa_arm32_consts + vld1.8 {q0}, [r3] + mov r12, #32 +L_mldsa_arm32_mul_loop: + vld1.8 {q1-q2}, [r1]! + vld1.8 {q3-q4}, [r2]! + vmul.i32 q7, q3, d0[1] + vmul.i32 q8, q4, d0[1] + vmul.i32 q9, q1, q7 + vmul.i32 q10, q2, q8 + vqrdmulh.s32 q5, q1, q3 + vqrdmulh.s32 q6, q2, q4 + vqrdmulh.s32 q9, q9, d0[0] + vqrdmulh.s32 q10, q10, d0[0] + vsub.i32 q5, q5, q9 + vsub.i32 q6, q6, q10 + vshr.s32 q5, q5, #1 + vshr.s32 q6, q6, #1 + vst1.8 {q5-q6}, [r0]! + subs r12, r12, #1 + bne L_mldsa_arm32_mul_loop + vpop {d8-d15} + bx lr + .size mldsa_mul_neon,.-mldsa_mul_neon +#ifndef __APPLE__ + .text + .type L_mldsa_arm32_ntt_zetas, %object + .size L_mldsa_arm32_ntt_zetas, 1024 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_arm32_ntt_zetas: + .long 0xffff5c06,0x000064f7,0xffd83102,0xfff81503 + .long 0x00039e44,0xfff42118,0xfff2a128,0x00071e24 + .long 0x001bde2b,0x0023e92b,0xfffa84ad,0xffe0147f + .long 0x002f9a75,0xffd3fb09,0x002f7a49,0x0028e527 + .long 0x00299658,0x000fa070,0xffef85a4,0x0036b788 + .long 0xfff79d90,0xffeeeaa0,0x0027f968,0xffdfd37b + .long 0xffdfadd6,0xffc51ae7,0xffeaa4f7,0xffcdfc98 + .long 0x001ad035,0xffffb422,0x003d3201,0x000445c5 + .long 0x00294a67,0x00017620,0x002ef4cd,0x0035dec5 + .long 0xffe6a503,0xffc9302c,0xffd947d4,0x003bbeaf + .long 0xffc51585,0xffd18e7c,0x00368a96,0xffd43e41 + .long 0x00360400,0xfffb6a4d,0x0023d69c,0xfff7c55d + .long 0xffe6123d,0xffe6ead6,0x00357e1e,0xffc5af59 + .long 0x0035843f,0xffdf5617,0xffe7945c,0x0038738c + .long 0x000c63a8,0x00081b9a,0x000e8f76,0x003b3853 + .long 0x003b8534,0xffd8fc30,0x001f9d54,0xffd54f2d + .long 0xffc406e5,0xffe8ac81,0xffc7e1cf,0xffd19819 + .long 0xffe9d65d,0x003509ee,0x002135c7,0xffe7cfbb + .long 0xffeccf75,0x001d9772,0xffc1b072,0xfff0bcf6 + .long 0xffcf5280,0xffcfd2ae,0xffc890e0,0x0001efca + .long 0x003410f2,0xfff0fe85,0x0020c638,0x00296e9f + .long 0xffd2b7a3,0xffc7a44b,0xfff9ba6d,0xffda3409 + .long 0xfff5c282,0xffed4113,0xffffa63b,0xffec09f7 + .long 0xfffa2bdd,0x001495d4,0x001c4563,0xffea2c62 + .long 0xffccfbe9,0x00040af0,0x0007c417,0x002f4588 + .long 0x0000ad00,0xffef36be,0x000dcd44,0x003c675a + .long 0xffc72bca,0xffffde7e,0x00193948,0xffce69c0 + .long 0x0024756c,0xfffcc7df,0x000b98a1,0xffebe808 + .long 0x0002e46c,0xffc9c808,0x003036c2,0xffe3bff6 + .long 0xffdb3c93,0xfffd4ae0,0x00141305,0x00147792 + .long 0x00139e25,0xffe7d0e0,0xfff39944,0xffea0802 + .long 0xffd1eea2,0xffc4c79c,0xffc8a057,0x003a97d9 + .long 0x001fea93,0x0033ff5a,0x002358d4,0x003a41f8 + .long 0xffccff72,0x00223dfb,0xffdaab9f,0xffc9a422 + .long 0x000412f5,0x00252587,0xffed24f0,0x00359b5d + .long 0xffca48a0,0xffc6a2fc,0xffedbb56,0xffcf45de + .long 0x000dbe5e,0x001c5e1a,0x000de0e6,0x000c7f5a + .long 0x00078f83,0xffe7628a,0xffff5704,0xfff806fc + .long 0xfff60021,0xffd05af6,0x001f0084,0x0030ef86 + .long 0xffc9b97d,0xfff7fcd6,0xfff44592,0xffc921c2 + .long 0x00053919,0x0004610c,0xffdacd41,0x003eb01b + .long 0x003472e7,0xffcd003b,0x001a7cc7,0x00031924 + .long 0x002b5ee5,0x00291199,0xffd87a3a,0x00134d71 + .long 0x003de11c,0x00130984,0x0025f051,0x00185a46 + .long 0xffc68518,0x001314be,0x00283891,0xffc9db90 + .long 0xffd25089,0x001c853f,0x001d0b4b,0xffeff6a6 + .long 0xffeba8be,0x0012e11b,0xffcd5e3e,0xffea2d2f + .long 0xfff91de4,0x001406c7,0x00327283,0xffe20d6e + .long 0xffec7953,0x001d4099,0xffd92578,0xffeb05ad + .long 0x0016e405,0x000bdbe7,0x00221de8,0x0033f8cf + .long 0xfff7b934,0xffd4ca0c,0xffe67ff8,0xffe3d157 + .long 0xffd8911b,0xffc72c12,0x000910d8,0xffc65e1f + .long 0xffe14658,0x00251d8b,0x002573b7,0xfffd7c8f + .long 0x001ddd98,0x00336898,0x0002d4bb,0xffed93a7 + .long 0xffcf6cbe,0x00027c1c,0x0018aa08,0x002dfd71 + .long 0x000c5ca5,0x0019379a,0xffc7a167,0xffe48c3d + .long 0xffd1a13c,0x0035c539,0x003b0115,0x00041dc0 + .long 0x0021c4f7,0xfff11bf4,0x001a35e7,0x0007340e + .long 0xfff97d45,0x001a4cd0,0xffe47cae,0x001d2668 + .long 0xffe68e98,0xffef2633,0xfffc05da,0xffc57fdb + .long 0xffd32764,0xffdde1af,0xfff993dd,0xffdd1d09 + .long 0x0002cc93,0xfff11805,0x00189c2a,0xffc9e5a9 + .long 0xfff78a50,0x003bcf2c,0xffff434e,0xffeb36df + .long 0x003c15ca,0x00155e68,0xfff316b6,0x001e29ce +#ifndef __APPLE__ + .text + .type L_mldsa_arm32_ntt_consts, %object + .size L_mldsa_arm32_ntt_consts, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_arm32_ntt_consts: + .long 0x007fe001,0x03802001,0x00000000,0x00000000 +#ifndef __APPLE__ + .text + .type L_mldsa_arm32_ntt_zetas_p, %object + .size L_mldsa_arm32_ntt_zetas_p, 12 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_arm32_ntt_zetas_p: + .long L_mldsa_arm32_ntt_zetas + .text + .align 4 + .globl mldsa_ntt_neon + WC_ASM_ATT_HIDDEN(mldsa_ntt_neon) + .type mldsa_ntt_neon, %function +mldsa_ntt_neon: + vpush {d8-d15} + adr r1, L_mldsa_arm32_ntt_consts + vld1.8 {q0}, [r1] + vdup.32 q1, d0[1] + adr r2, L_mldsa_arm32_ntt_zetas_p + ldr r2, [r2] + add r2, r2, #4 + # len = 128 + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0 + add r12, r3, #0x200 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + # len = 64 + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0 + add r12, r3, #0x100 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x200 + add r12, r3, #0x100 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + # len = 32 + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0 + add r12, r3, #0x80 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x100 + add r12, r3, #0x80 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x200 + add r12, r3, #0x80 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x300 + add r12, r3, #0x80 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + # len = 16 + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x80 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x100 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x180 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x200 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x280 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x300 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x380 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + # len = 8 + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x40 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x80 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0xc0 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x100 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x140 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x180 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x1c0 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x200 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x240 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x280 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x2c0 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x300 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x340 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x380 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x3c0 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + # len = 4 + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #32 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x40 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x60 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x80 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0xa0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0xc0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0xe0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x100 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x120 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x140 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x160 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x180 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x1a0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x1c0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x1e0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x200 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x220 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x240 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x260 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x280 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x2a0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x2c0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x2e0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x300 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x320 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x340 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x360 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x380 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x3a0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x3c0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x3e0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + # len = 2 + add r3, r0, #0 + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vmul.i32 d14, d9, d6 + vqrdmulh.s32 d12, d9, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d12, d12, d14 + vshr.s32 d12, d12, #1 + vsub.i32 d9, d8, d12 + vadd.i32 d8, d8, d12 + vst1.8 {q4}, [r3]! + # len = 1 + add r3, r0, #0 + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vmul.i32 q7, q5, q3 + vqrdmulh.s32 q6, q5, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q6, q6, q7 + vshr.s32 q6, q6, #1 + vsub.i32 q5, q4, q6 + vadd.i32 q4, q4, q6 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vpop {d8-d15} + bx lr + .size mldsa_ntt_neon,.-mldsa_ntt_neon +#ifndef __APPLE__ + .text + .type L_mldsa_arm32_invntt_zetas, %object + .size L_mldsa_arm32_invntt_zetas, 1024 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_arm32_invntt_zetas: + .long 0xffe1d632,0x000ce94a,0xffeaa198,0xffc3ea36 + .long 0x0014c921,0x0000bcb2,0xffc430d4,0x000875b0 + .long 0x00361a57,0xffe763d6,0x000ee7fb,0xfffd336d + .long 0x0022e2f7,0x00066c23,0x00221e51,0x002cd89c + .long 0x003a8025,0x0003fa26,0x0010d9cd,0x00197168 + .long 0xffe2d998,0x001b8352,0xffe5b330,0x000682bb + .long 0xfff8cbf2,0xffe5ca19,0x000ee40c,0xffde3b09 + .long 0xfffbe240,0xffc4feeb,0xffca3ac7,0x002e5ec4 + .long 0x001b73c3,0x00385e99,0xffe6c866,0xfff3a35b + .long 0xffd2028f,0xffe755f8,0xfffd83e4,0x00309342 + .long 0x00126c59,0xfffd2b45,0xffcc9768,0xffe22268 + .long 0x00028371,0xffda8c49,0xffdae275,0x001eb9a8 + .long 0x0039a1e1,0xfff6ef28,0x0038d3ee,0x00276ee5 + .long 0x001c2ea9,0x00198008,0x002b35f4,0x000846cc + .long 0xffcc0731,0xffdde218,0xfff42419,0xffe91bfb + .long 0x0014fa53,0x0026da88,0xffe2bf67,0x001386ad + .long 0x001df292,0xffcd8d7d,0xffebf939,0x0006e21c + .long 0x0015d2d1,0x0032a1c2,0xffed1ee5,0x00145742 + .long 0x0010095a,0xffe2f4b5,0xffe37ac1,0x002daf77 + .long 0x00362470,0xffd7c76f,0xffeceb42,0x00397ae8 + .long 0xffe7a5ba,0xffda0faf,0xffecf67c,0xffc21ee4 + .long 0xffecb28f,0x002785c6,0xffd6ee67,0xffd4a11b + .long 0xfffce6dc,0xffe58339,0x0032ffc5,0xffcb8d19 + .long 0xffc14fe5,0x002532bf,0xfffb9ef4,0xfffac6e7 + .long 0x0036de3e,0x000bba6e,0x0008032a,0x00364683 + .long 0xffcf107a,0xffe0ff7c,0x002fa50a,0x0009ffdf + .long 0x0007f904,0x0000a8fc,0x00189d76,0xfff8707d + .long 0xfff380a6,0xfff21f1a,0xffe3a1e6,0xfff241a2 + .long 0x0030ba22,0x001244aa,0x00395d04,0x0035b760 + .long 0xffca64a3,0x0012db10,0xffdada79,0xfffbed0b + .long 0x00365bde,0x00255461,0xffddc205,0x0033008e + .long 0xffc5be08,0xffdca72c,0xffcc00a6,0xffe0156d + .long 0xffc56827,0x00375fa9,0x003b3864,0x002e115e + .long 0x0015f7fe,0x000c66bc,0x00182f20,0xffec61db + .long 0xffeb886e,0xffebecfb,0x0002b520,0x0024c36d + .long 0x001c400a,0xffcfc93e,0x003637f8,0xfffd1b94 + .long 0x001417f8,0xfff4675f,0x00033821,0xffdb8a94 + .long 0x00319640,0xffe6c6b8,0x00002182,0x0038d436 + .long 0xffc398a6,0xfff232bc,0x0010c942,0xffff5300 + .long 0xffd0ba78,0xfff83be9,0xfffbf510,0x00330417 + .long 0x0015d39e,0xffe3ba9d,0xffeb6a2c,0x0005d423 + .long 0x0013f609,0x000059c5,0x0012beed,0x000a3d7e + .long 0x0025cbf7,0x00064593,0x00385bb5,0x002d485d + .long 0xffd69161,0xffdf39c8,0x000f017b,0xffcbef0e + .long 0xfffe1036,0x00376f20,0x00302d52,0x0030ad80 + .long 0x000f430a,0x003e4f8e,0xffe2688e,0x0013308b + .long 0x00183045,0xffdeca39,0xffcaf612,0x001629a3 + .long 0x002e67e7,0x00381e31,0x0017537f,0x003bf91b + .long 0x002ab0d3,0xffe062ac,0x002703d0,0xffc47acc + .long 0xffc4c7ad,0xfff1708a,0xfff7e466,0xfff39c58 + .long 0xffc78c74,0x00186ba4,0x0020a9e9,0xffca7bc1 + .long 0x003a50a7,0xffca81e2,0x0019152a,0x0019edc3 + .long 0x00083aa3,0xffdc2964,0x000495b3,0xffc9fc00 + .long 0x002bc1bf,0xffc9756a,0x002e7184,0x003aea7b + .long 0xffc44151,0x0026b82c,0x0036cfd4,0x00195afd + .long 0xffca213b,0xffd10b33,0xfffe89e0,0xffd6b599 + .long 0xfffbba3b,0xffc2cdff,0x00004bde,0xffe52fcb + .long 0x00320368,0x00155b09,0x003ae519,0x0020522a + .long 0x00202c85,0xffd80698,0x00111560,0x00086270 + .long 0xffc94878,0x00107a5c,0xfff05f90,0xffd669a8 + .long 0xffd71ad9,0xffd085b7,0x002c04f7,0xffd0658b + .long 0x001feb81,0x00057b53,0xffdc16d5,0xffe421d5 + .long 0xfff8e1dc,0x000d5ed8,0x000bdee8,0xfffc61bc + .long 0x0007eafd,0x0027cefe,0xffff9b09,0x0000a3fa +#ifndef __APPLE__ + .text + .type L_mldsa_arm32_invntt_consts, %object + .size L_mldsa_arm32_invntt_consts, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_arm32_invntt_consts: + .long 0x007fe001,0x03802001,0x00000000,0x00000000 +#ifndef __APPLE__ + .text + .type L_mldsa_arm32_invntt_zetas_p, %object + .size L_mldsa_arm32_invntt_zetas_p, 12 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_arm32_invntt_zetas_p: + .long L_mldsa_arm32_invntt_zetas + .text + .align 4 + .globl mldsa_invntt_neon + WC_ASM_ATT_HIDDEN(mldsa_invntt_neon) + .type mldsa_invntt_neon, %function +mldsa_invntt_neon: + vpush {d8-d15} + adr r1, L_mldsa_arm32_invntt_consts + vld1.8 {q0}, [r1] + vdup.32 q1, d0[1] + adr r2, L_mldsa_arm32_invntt_zetas_p + ldr r2, [r2] + # len = 1 + add r3, r0, #0 + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + add r3, r3, #16 + vld1.8 {q5}, [r3] + sub r3, r3, #16 + vuzp.32 q4, q5 + vld1.8 {q2}, [r2]! + vmul.i32 q3, q2, q1 + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vzip.32 q4, q5 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r3]! + # len = 2 + add r3, r0, #0 + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + ldr r1, [r2], #4 + vdup.32 d4, r1 + vmul.i32 d6, d4, d2 + vld1.8 {q4}, [r3] + vsub.i32 d12, d8, d9 + vadd.i32 d8, d8, d9 + vmul.i32 d14, d12, d6 + vqrdmulh.s32 d9, d12, d4 + vqrdmulh.s32 d14, d14, d0[0] + vsub.i32 d9, d9, d14 + vshr.s32 d9, d9, #1 + vst1.8 {q4}, [r3]! + # len = 4 + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #32 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x40 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x60 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x80 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0xa0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0xc0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0xe0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x100 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x120 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x140 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x160 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x180 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x1a0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x1c0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x1e0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x200 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x220 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x240 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x260 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x280 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x2a0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x2c0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x2e0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x300 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x320 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x340 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x360 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x380 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x3a0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x3c0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x3e0 + add r12, r3, #16 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + # len = 8 + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x40 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x80 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0xc0 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x100 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x140 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x180 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x1c0 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x200 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x240 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x280 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x2c0 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x300 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x340 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x380 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x3c0 + add r12, r3, #32 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + # len = 16 + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x80 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x100 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x180 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x200 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x280 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x300 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x380 + add r12, r3, #0x40 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + # len = 32 + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0 + add r12, r3, #0x80 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x100 + add r12, r3, #0x80 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x200 + add r12, r3, #0x80 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x300 + add r12, r3, #0x80 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + # len = 64 + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0 + add r12, r3, #0x100 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0x200 + add r12, r3, #0x100 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + # len = 128 + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0 + add r12, r3, #0x200 + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + vld1.8 {q4}, [r3] + vld1.8 {q5}, [r12] + vsub.i32 q6, q4, q5 + vadd.i32 q4, q4, q5 + vmul.i32 q7, q6, q3 + vqrdmulh.s32 q5, q6, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q4}, [r3]! + vst1.8 {q5}, [r12]! + # scale + ldr r1, [r2], #4 + vdup.32 q2, r1 + vmul.i32 q3, q2, q1 + add r3, r0, #0 + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vld1.8 {q4}, [r3] + vmul.i32 q7, q4, q3 + vqrdmulh.s32 q5, q4, q2 + vqrdmulh.s32 q7, q7, d0[0] + vsub.i32 q5, q5, q7 + vshr.s32 q5, q5, #1 + vst1.8 {q5}, [r3]! + vpop {d8-d15} + bx lr + .size mldsa_invntt_neon,.-mldsa_invntt_neon +#ifndef __APPLE__ + .text + .type L_mldsa_arm32_q88_consts_decompose, %object + .size L_mldsa_arm32_q88_consts_decompose, 96 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_arm32_q88_consts_decompose: + .long 0x0000002c,0x0000002c,0x0000002c,0x0000002c + .long 0x003fefd4,0x003fefd4,0x003fefd4,0x003fefd4 + .long 0x0002e800,0x0002e800,0x0002e800,0x0002e800 + .long 0x00017400,0x00017400,0x00017400,0x00017400 + .long 0x0000002b,0x0000002b,0x0000002b,0x0000002b + .long 0x007fe001,0x007fe001,0x007fe001,0x007fe001 + .text + .align 4 + .globl mldsa_decompose_q88_neon + WC_ASM_ATT_HIDDEN(mldsa_decompose_q88_neon) + .type mldsa_decompose_q88_neon, %function +mldsa_decompose_q88_neon: + vpush {d8-d15} + adr r3, L_mldsa_arm32_q88_consts_decompose + vld1.8 {q0}, [r3]! + vld1.8 {q1}, [r3]! + vld1.8 {q2}, [r3]! + vld1.8 {q3}, [r3]! + vld1.8 {q4}, [r3]! + mov r12, #32 +L_mldsa_decompose_q88_neon_loop: + vld1.8 {q5}, [r0]! + vld1.8 {q6}, [r0]! + vmov q9, q1 + vmla.i32 q9, q5, q0 + vshr.s32 q9, q9, #23 + vmov q7, q5 + vmls.i32 q7, q9, q2 + vsub.i32 q11, q3, q7 + vsra.u32 q9, q11, #31 + vmov q7, q5 + vmls.i32 q7, q9, q2 + vsub.i32 q11, q4, q9 + vshr.u32 q11, q11, #31 + vsub.i32 q7, q7, q11 + vsub.i32 q11, q9, q0 + vshr.u32 q11, q11, #31 + vneg.s32 q11, q11 + vand q9, q9, q11 + vmov q10, q1 + vmla.i32 q10, q6, q0 + vshr.s32 q10, q10, #23 + vmov q8, q6 + vmls.i32 q8, q10, q2 + vsub.i32 q12, q3, q8 + vsra.u32 q10, q12, #31 + vmov q8, q6 + vmls.i32 q8, q10, q2 + vsub.i32 q12, q4, q10 + vshr.u32 q12, q12, #31 + vsub.i32 q8, q8, q12 + vsub.i32 q12, q10, q0 + vshr.u32 q12, q12, #31 + vneg.s32 q12, q12 + vand q10, q10, q12 + vst1.8 {q7}, [r1]! + vst1.8 {q9}, [r2]! + vst1.8 {q8}, [r1]! + vst1.8 {q10}, [r2]! + subs r12, r12, #1 + bne L_mldsa_decompose_q88_neon_loop + vpop {d8-d15} + bx lr + .size mldsa_decompose_q88_neon,.-mldsa_decompose_q88_neon +#ifndef __APPLE__ + .text + .type L_mldsa_arm32_q88_consts_high, %object + .size L_mldsa_arm32_q88_consts_high, 96 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_arm32_q88_consts_high: + .long 0x0000002c,0x0000002c,0x0000002c,0x0000002c + .long 0x003fefd4,0x003fefd4,0x003fefd4,0x003fefd4 + .long 0x0002e800,0x0002e800,0x0002e800,0x0002e800 + .long 0x00017400,0x00017400,0x00017400,0x00017400 + .long 0x0000002b,0x0000002b,0x0000002b,0x0000002b + .long 0x007fe001,0x007fe001,0x007fe001,0x007fe001 + .text + .align 4 + .globl mldsa_use_hint_q88_neon + WC_ASM_ATT_HIDDEN(mldsa_use_hint_q88_neon) + .type mldsa_use_hint_q88_neon, %function +mldsa_use_hint_q88_neon: + vpush {d8-d15} + adr r1, L_mldsa_arm32_q88_consts_high + vld1.8 {q0}, [r1]! + vld1.8 {q1}, [r1]! + vld1.8 {q2}, [r1]! + vld1.8 {q3}, [r1]! + vld1.8 {q4}, [r1]! + vld1.8 {q5}, [r1]! + mov r2, #32 +L_mldsa_use_hint_q88_neon_loop: + vld1.8 {q6}, [r0]! + vld1.8 {q7}, [r0]! + vshr.s32 q12, q6, #31 + vand q12, q12, q5 + vadd.i32 q6, q6, q12 + vshr.s32 q13, q7, #31 + vand q13, q13, q5 + vadd.i32 q7, q7, q13 + vmov q10, q1 + vmla.i32 q10, q6, q0 + vshr.s32 q10, q10, #23 + vmov q8, q6 + vmls.i32 q8, q10, q2 + vsub.i32 q12, q3, q8 + vsra.u32 q10, q12, #31 + vmov q8, q6 + vmls.i32 q8, q10, q2 + vsub.i32 q12, q4, q10 + vshr.u32 q12, q12, #31 + vsub.i32 q8, q8, q12 + vsub.i32 q12, q10, q0 + vshr.u32 q12, q12, #31 + vneg.s32 q12, q12 + vand q10, q10, q12 + vmov q11, q1 + vmla.i32 q11, q7, q0 + vshr.s32 q11, q11, #23 + vmov q9, q7 + vmls.i32 q9, q11, q2 + vsub.i32 q13, q3, q9 + vsra.u32 q11, q13, #31 + vmov q9, q7 + vmls.i32 q9, q11, q2 + vsub.i32 q13, q4, q11 + vshr.u32 q13, q13, #31 + vsub.i32 q9, q9, q13 + vsub.i32 q13, q11, q0 + vshr.u32 q13, q13, #31 + vneg.s32 q13, q13 + vand q11, q11, q13 + sub r0, r0, #32 + vst1.8 {q10}, [r0]! + vst1.8 {q11}, [r0]! + subs r2, r2, #1 + bne L_mldsa_use_hint_q88_neon_loop + vpop {d8-d15} + bx lr + .size mldsa_use_hint_q88_neon,.-mldsa_use_hint_q88_neon +#ifndef __APPLE__ + .text + .type L_mldsa_arm32_q32_consts_decompose, %object + .size L_mldsa_arm32_q32_consts_decompose, 80 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_arm32_q32_consts_decompose: + .long 0x0003feff,0x0003feff,0x0003feff,0x0003feff + .long 0x0003ff00,0x0003ff00,0x0003ff00,0x0003ff00 + .long 0x0000000f,0x0000000f,0x0000000f,0x0000000f + .long 0x0007fe00,0x0007fe00,0x0007fe00,0x0007fe00 + .long 0x007fe001,0x007fe001,0x007fe001,0x007fe001 + .text + .align 4 + .globl mldsa_decompose_q32_neon + WC_ASM_ATT_HIDDEN(mldsa_decompose_q32_neon) + .type mldsa_decompose_q32_neon, %function +mldsa_decompose_q32_neon: + vpush {d8-d15} + adr r3, L_mldsa_arm32_q32_consts_decompose + vld1.8 {q0}, [r3]! + vld1.8 {q1}, [r3]! + vld1.8 {q2}, [r3]! + vld1.8 {q3}, [r3]! + mov r12, #32 +L_mldsa_decompose_q32_neon_loop: + vld1.8 {q4}, [r0]! + vld1.8 {q5}, [r0]! + vadd.i32 q8, q4, q0 + vshr.s32 q8, q8, #19 + vmov q6, q4 + vmls.i32 q6, q8, q3 + vsub.i32 q10, q1, q6 + vsra.u32 q8, q10, #31 + vmov q6, q4 + vmls.i32 q6, q8, q3 + vshr.s32 q10, q8, #4 + vsub.i32 q6, q6, q10 + vand q8, q8, q2 + vadd.i32 q9, q5, q0 + vshr.s32 q9, q9, #19 + vmov q7, q5 + vmls.i32 q7, q9, q3 + vsub.i32 q11, q1, q7 + vsra.u32 q9, q11, #31 + vmov q7, q5 + vmls.i32 q7, q9, q3 + vshr.s32 q11, q9, #4 + vsub.i32 q7, q7, q11 + vand q9, q9, q2 + vst1.8 {q6}, [r1]! + vst1.8 {q8}, [r2]! + vst1.8 {q7}, [r1]! + vst1.8 {q9}, [r2]! + subs r12, r12, #1 + bne L_mldsa_decompose_q32_neon_loop + vpop {d8-d15} + bx lr + .size mldsa_decompose_q32_neon,.-mldsa_decompose_q32_neon +#ifndef __APPLE__ + .text + .type L_mldsa_arm32_q32_consts_high, %object + .size L_mldsa_arm32_q32_consts_high, 80 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_arm32_q32_consts_high: + .long 0x0003feff,0x0003feff,0x0003feff,0x0003feff + .long 0x0003ff00,0x0003ff00,0x0003ff00,0x0003ff00 + .long 0x0000000f,0x0000000f,0x0000000f,0x0000000f + .long 0x0007fe00,0x0007fe00,0x0007fe00,0x0007fe00 + .long 0x007fe001,0x007fe001,0x007fe001,0x007fe001 + .text + .align 4 + .globl mldsa_use_hint_q32_neon + WC_ASM_ATT_HIDDEN(mldsa_use_hint_q32_neon) + .type mldsa_use_hint_q32_neon, %function +mldsa_use_hint_q32_neon: + vpush {d8-d15} + adr r1, L_mldsa_arm32_q32_consts_high + vld1.8 {q0}, [r1]! + vld1.8 {q1}, [r1]! + vld1.8 {q2}, [r1]! + vld1.8 {q3}, [r1]! + vld1.8 {q4}, [r1]! + mov r2, #32 +L_mldsa_use_hint_q32_neon_loop: + vld1.8 {q5}, [r0]! + vld1.8 {q6}, [r0]! + vshr.s32 q11, q5, #31 + vand q11, q11, q4 + vadd.i32 q5, q5, q11 + vshr.s32 q12, q6, #31 + vand q12, q12, q4 + vadd.i32 q6, q6, q12 + vadd.i32 q9, q5, q0 + vshr.s32 q9, q9, #19 + vmov q7, q5 + vmls.i32 q7, q9, q3 + vsub.i32 q11, q1, q7 + vsra.u32 q9, q11, #31 + vmov q7, q5 + vmls.i32 q7, q9, q3 + vshr.s32 q11, q9, #4 + vsub.i32 q7, q7, q11 + vand q9, q9, q2 + vadd.i32 q10, q6, q0 + vshr.s32 q10, q10, #19 + vmov q8, q6 + vmls.i32 q8, q10, q3 + vsub.i32 q12, q1, q8 + vsra.u32 q10, q12, #31 + vmov q8, q6 + vmls.i32 q8, q10, q3 + vshr.s32 q12, q10, #4 + vsub.i32 q8, q8, q12 + vand q10, q10, q2 + sub r0, r0, #32 + vst1.8 {q9}, [r0]! + vst1.8 {q10}, [r0]! + subs r2, r2, #1 + bne L_mldsa_use_hint_q32_neon_loop + vpop {d8-d15} + bx lr + .size mldsa_use_hint_q32_neon,.-mldsa_use_hint_q32_neon + .text + .align 4 + .globl mldsa_poly_add_neon + WC_ASM_ATT_HIDDEN(mldsa_poly_add_neon) + .type mldsa_poly_add_neon, %function +mldsa_poly_add_neon: + vpush {d8-d15} + mov r2, #16 +L_mldsa_poly_add_neon_loop: + vld1.8 {q0}, [r0]! + vld1.8 {q4}, [r1]! + vld1.8 {q1}, [r0]! + vld1.8 {q5}, [r1]! + vld1.8 {q2}, [r0]! + vld1.8 {q6}, [r1]! + vld1.8 {q3}, [r0]! + vld1.8 {q7}, [r1]! + sub r0, r0, #0x40 + vadd.i32 q0, q0, q4 + vadd.i32 q1, q1, q5 + vadd.i32 q2, q2, q6 + vadd.i32 q3, q3, q7 + vst1.8 {q0}, [r0]! + vst1.8 {q1}, [r0]! + vst1.8 {q2}, [r0]! + vst1.8 {q3}, [r0]! + subs r2, r2, #1 + bne L_mldsa_poly_add_neon_loop + vpop {d8-d15} + bx lr + .size mldsa_poly_add_neon,.-mldsa_poly_add_neon + .text + .align 4 + .globl mldsa_poly_sub_neon + WC_ASM_ATT_HIDDEN(mldsa_poly_sub_neon) + .type mldsa_poly_sub_neon, %function +mldsa_poly_sub_neon: + vpush {d8-d15} + mov r2, #16 +L_mldsa_poly_sub_neon_loop: + vld1.8 {q0}, [r0]! + vld1.8 {q4}, [r1]! + vld1.8 {q1}, [r0]! + vld1.8 {q5}, [r1]! + vld1.8 {q2}, [r0]! + vld1.8 {q6}, [r1]! + vld1.8 {q3}, [r0]! + vld1.8 {q7}, [r1]! + sub r0, r0, #0x40 + vsub.i32 q0, q0, q4 + vsub.i32 q1, q1, q5 + vsub.i32 q2, q2, q6 + vsub.i32 q3, q3, q7 + vst1.8 {q0}, [r0]! + vst1.8 {q1}, [r0]! + vst1.8 {q2}, [r0]! + vst1.8 {q3}, [r0]! + subs r2, r2, #1 + bne L_mldsa_poly_sub_neon_loop + vpop {d8-d15} + bx lr + .size mldsa_poly_sub_neon,.-mldsa_poly_sub_neon +#ifndef __APPLE__ + .text + .type L_mldsa_poly_make_pos_neon_q, %object + .size L_mldsa_poly_make_pos_neon_q, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_poly_make_pos_neon_q: + .long 0x007fe001,0x007fe001,0x007fe001,0x007fe001 + .text + .align 4 + .globl mldsa_poly_make_pos_neon + WC_ASM_ATT_HIDDEN(mldsa_poly_make_pos_neon) + .type mldsa_poly_make_pos_neon, %function +mldsa_poly_make_pos_neon: + vpush {d8-d15} + adr r1, L_mldsa_poly_make_pos_neon_q + vld1.8 {q0}, [r1]! + mov r2, #16 +L_mldsa_poly_make_pos_neon_loop: + vld1.8 {q1}, [r0]! + vld1.8 {q2}, [r0]! + vld1.8 {q3}, [r0]! + vld1.8 {q4}, [r0]! + sub r0, r0, #0x40 + vshr.s32 q5, q1, #31 + vmls.i32 q1, q5, q0 + vshr.s32 q6, q2, #31 + vmls.i32 q2, q6, q0 + vshr.s32 q7, q3, #31 + vmls.i32 q3, q7, q0 + vshr.s32 q8, q4, #31 + vmls.i32 q4, q8, q0 + vst1.8 {q1}, [r0]! + vst1.8 {q2}, [r0]! + vst1.8 {q3}, [r0]! + vst1.8 {q4}, [r0]! + subs r2, r2, #1 + bne L_mldsa_poly_make_pos_neon_loop + vpop {d8-d15} + bx lr + .size mldsa_poly_make_pos_neon,.-mldsa_poly_make_pos_neon +#ifndef __APPLE__ + .text + .type L_mldsa_poly_red_neon_q, %object + .size L_mldsa_poly_red_neon_q, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_poly_red_neon_q: + .long 0x007fe001,0x007fe001,0x007fe001,0x007fe001 + .long 0x00400000,0x00400000,0x00400000,0x00400000 + .text + .align 4 + .globl mldsa_poly_red_neon + WC_ASM_ATT_HIDDEN(mldsa_poly_red_neon) + .type mldsa_poly_red_neon, %function +mldsa_poly_red_neon: + vpush {d8-d15} + adr r1, L_mldsa_poly_red_neon_q + vld1.8 {q0}, [r1]! + vld1.8 {q1}, [r1] + mov r2, #16 +L_mldsa_poly_red_neon_loop: + vld1.8 {q2}, [r0]! + vld1.8 {q3}, [r0]! + vld1.8 {q4}, [r0]! + vld1.8 {q5}, [r0]! + sub r0, r0, #0x40 + vadd.i32 q6, q2, q1 + vshr.s32 q6, q6, #23 + vmls.i32 q2, q6, q0 + vadd.i32 q7, q3, q1 + vshr.s32 q7, q7, #23 + vmls.i32 q3, q7, q0 + vadd.i32 q8, q4, q1 + vshr.s32 q8, q8, #23 + vmls.i32 q4, q8, q0 + vadd.i32 q9, q5, q1 + vshr.s32 q9, q9, #23 + vmls.i32 q5, q9, q0 + vst1.8 {q2}, [r0]! + vst1.8 {q3}, [r0]! + vst1.8 {q4}, [r0]! + vst1.8 {q5}, [r0]! + subs r2, r2, #1 + bne L_mldsa_poly_red_neon_loop + vpop {d8-d15} + bx lr + .size mldsa_poly_red_neon,.-mldsa_poly_red_neon + .text + .align 4 + .globl mldsa_vec_check_low_neon + WC_ASM_ATT_HIDDEN(mldsa_vec_check_low_neon) + .type mldsa_vec_check_low_neon, %function +mldsa_vec_check_low_neon: + vpush {d8-d15} + vdup.32 q0, r2 + veor q1, q1, q1 + veor q2, q2, q2 + veor q3, q3, q3 + veor q4, q4, q4 +L_mldsa_arm32_check_low_outer: + mov r3, #16 +L_mldsa_arm32_check_low_inner: + vld1.8 {q5}, [r0]! + vld1.8 {q6}, [r0]! + vld1.8 {q7}, [r0]! + vld1.8 {q8}, [r0]! + vqabs.s32 q9, q5 + vcge.s32 q9, q9, q0 + vorr q1, q1, q9 + vqabs.s32 q10, q6 + vcge.s32 q10, q10, q0 + vorr q2, q2, q10 + vqabs.s32 q11, q7 + vcge.s32 q11, q11, q0 + vorr q3, q3, q11 + vqabs.s32 q12, q8 + vcge.s32 q12, q12, q0 + vorr q4, q4, q12 + subs r3, r3, #1 + bne L_mldsa_arm32_check_low_inner + subs r1, r1, #1 + bne L_mldsa_arm32_check_low_outer + vorr q1, q1, q2 + vorr q3, q3, q4 + vorr q1, q1, q3 + vorr d2, d2, d3 + vpmax.u32 d2, d2, d2 + vmov.32 r12, d2[0] + cmp r12, #0 + moveq r12, #1 + movne r12, #0 + mov r0, r12 + vpop {d8-d15} + bx lr + .size mldsa_vec_check_low_neon,.-mldsa_vec_check_low_neon +#ifndef __APPLE__ + .text + .type L_mldsa_decode_eta_2_neon_idx, %object + .size L_mldsa_decode_eta_2_neon_idx, 192 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_decode_eta_2_neon_idx: + .byte 0x00,0xff,0xff,0xff,0x00,0xff,0xff,0xff + .byte 0x00,0x01,0xff,0xff,0x01,0xff,0xff,0xff + .byte 0x01,0xff,0xff,0xff,0x01,0x02,0xff,0xff + .byte 0x02,0xff,0xff,0xff,0x02,0xff,0xff,0xff + .byte 0x01,0xff,0xff,0xff,0x01,0xff,0xff,0xff + .byte 0x01,0x02,0xff,0xff,0x02,0xff,0xff,0xff + .byte 0x02,0xff,0xff,0xff,0x02,0x03,0xff,0xff + .byte 0x03,0xff,0xff,0xff,0x03,0xff,0xff,0xff + .byte 0x04,0xff,0xff,0xff,0x04,0xff,0xff,0xff + .byte 0x04,0x05,0xff,0xff,0x05,0xff,0xff,0xff + .byte 0x05,0xff,0xff,0xff,0x05,0x06,0xff,0xff + .byte 0x06,0xff,0xff,0xff,0x06,0xff,0xff,0xff + .byte 0x07,0xff,0xff,0xff,0x07,0xff,0xff,0xff + .byte 0x07,0x08,0xff,0xff,0x08,0xff,0xff,0xff + .byte 0x08,0xff,0xff,0xff,0x08,0x09,0xff,0xff + .byte 0x09,0xff,0xff,0xff,0x09,0xff,0xff,0xff + .byte 0x0a,0xff,0xff,0xff,0x0a,0xff,0xff,0xff + .byte 0x0a,0x0b,0xff,0xff,0x0b,0xff,0xff,0xff + .byte 0x0b,0xff,0xff,0xff,0x0b,0x0c,0xff,0xff + .byte 0x0c,0xff,0xff,0xff,0x0c,0xff,0xff,0xff + .byte 0x0d,0xff,0xff,0xff,0x0d,0xff,0xff,0xff + .byte 0x0d,0x0e,0xff,0xff,0x0e,0xff,0xff,0xff + .byte 0x0e,0xff,0xff,0xff,0x0e,0x0f,0xff,0xff + .byte 0x0f,0xff,0xff,0xff,0x0f,0xff,0xff,0xff +#ifndef __APPLE__ + .text + .type L_mldsa_decode_eta_2_neon_aux, %object + .size L_mldsa_decode_eta_2_neon_aux, 48 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_eta_2_neon_aux: + .long 0x00000000,0xfffffffd,0xfffffffa,0xffffffff + .long 0xfffffffc,0xfffffff9,0xfffffffe,0xfffffffb + .long 0x00000007,0x00000007,0x00000002,0x00000002 + .text + .align 4 + .globl mldsa_decode_eta_2_neon + WC_ASM_ATT_HIDDEN(mldsa_decode_eta_2_neon) + .type mldsa_decode_eta_2_neon, %function +mldsa_decode_eta_2_neon: + vpush {d8-d15} + adr r2, L_mldsa_decode_eta_2_neon_idx + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vld1.8 {d4}, [r2]! + vld1.8 {d5}, [r2]! + adr r3, L_mldsa_decode_eta_2_neon_aux + vld1.8 {d6}, [r3]! + vld1.8 {d7}, [r3]! + vld1.8 {d8}, [r3]! + vld1.8 {d9}, [r3]! + vld1.8 {d14}, [r3]! + vld1.8 {d15}, [r3] + mov r12, #27 +L_mldsa_decode_eta_2_neon_loop: + vld1.8 {d0-d1}, [r0] + vtbl.8 d10, {d0-d1}, d2 + vshl.s32 d10, d10, d6 + vand d10, d10, d14 + vsub.i32 d10, d15, d10 + vtbl.8 d11, {d0-d1}, d3 + vshl.s32 d11, d11, d7 + vand d11, d11, d14 + vsub.i32 d11, d15, d11 + vtbl.8 d12, {d0-d1}, d4 + vshl.s32 d12, d12, d8 + vand d12, d12, d14 + vsub.i32 d12, d15, d12 + vtbl.8 d13, {d0-d1}, d5 + vshl.s32 d13, d13, d9 + vand d13, d13, d14 + vsub.i32 d13, d15, d13 + vst1.8 {d10-d11}, [r1]! + vst1.8 {d12-d13}, [r1]! + add r0, r0, #3 + subs r12, r12, #1 + bne L_mldsa_decode_eta_2_neon_loop + sub r0, r0, #1 + vld1.8 {d0-d1}, [r0] + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vld1.8 {d4}, [r2]! + vld1.8 {d5}, [r2]! + vtbl.8 d10, {d0-d1}, d2 + vshl.s32 d10, d10, d6 + vand d10, d10, d14 + vsub.i32 d10, d15, d10 + vtbl.8 d11, {d0-d1}, d3 + vshl.s32 d11, d11, d7 + vand d11, d11, d14 + vsub.i32 d11, d15, d11 + vtbl.8 d12, {d0-d1}, d4 + vshl.s32 d12, d12, d8 + vand d12, d12, d14 + vsub.i32 d12, d15, d12 + vtbl.8 d13, {d0-d1}, d5 + vshl.s32 d13, d13, d9 + vand d13, d13, d14 + vsub.i32 d13, d15, d13 + vst1.8 {d10-d11}, [r1]! + vst1.8 {d12-d13}, [r1]! + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vld1.8 {d4}, [r2]! + vld1.8 {d5}, [r2]! + vtbl.8 d10, {d0-d1}, d2 + vshl.s32 d10, d10, d6 + vand d10, d10, d14 + vsub.i32 d10, d15, d10 + vtbl.8 d11, {d0-d1}, d3 + vshl.s32 d11, d11, d7 + vand d11, d11, d14 + vsub.i32 d11, d15, d11 + vtbl.8 d12, {d0-d1}, d4 + vshl.s32 d12, d12, d8 + vand d12, d12, d14 + vsub.i32 d12, d15, d12 + vtbl.8 d13, {d0-d1}, d5 + vshl.s32 d13, d13, d9 + vand d13, d13, d14 + vsub.i32 d13, d15, d13 + vst1.8 {d10-d11}, [r1]! + vst1.8 {d12-d13}, [r1]! + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vld1.8 {d4}, [r2]! + vld1.8 {d5}, [r2]! + vtbl.8 d10, {d0-d1}, d2 + vshl.s32 d10, d10, d6 + vand d10, d10, d14 + vsub.i32 d10, d15, d10 + vtbl.8 d11, {d0-d1}, d3 + vshl.s32 d11, d11, d7 + vand d11, d11, d14 + vsub.i32 d11, d15, d11 + vtbl.8 d12, {d0-d1}, d4 + vshl.s32 d12, d12, d8 + vand d12, d12, d14 + vsub.i32 d12, d15, d12 + vtbl.8 d13, {d0-d1}, d5 + vshl.s32 d13, d13, d9 + vand d13, d13, d14 + vsub.i32 d13, d15, d13 + vst1.8 {d10-d11}, [r1]! + vst1.8 {d12-d13}, [r1]! + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vld1.8 {d4}, [r2]! + vld1.8 {d5}, [r2]! + vtbl.8 d10, {d0-d1}, d2 + vshl.s32 d10, d10, d6 + vand d10, d10, d14 + vsub.i32 d10, d15, d10 + vtbl.8 d11, {d0-d1}, d3 + vshl.s32 d11, d11, d7 + vand d11, d11, d14 + vsub.i32 d11, d15, d11 + vtbl.8 d12, {d0-d1}, d4 + vshl.s32 d12, d12, d8 + vand d12, d12, d14 + vsub.i32 d12, d15, d12 + vtbl.8 d13, {d0-d1}, d5 + vshl.s32 d13, d13, d9 + vand d13, d13, d14 + vsub.i32 d13, d15, d13 + vst1.8 {d10-d11}, [r1]! + vst1.8 {d12-d13}, [r1]! + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vld1.8 {d4}, [r2]! + vld1.8 {d5}, [r2]! + vtbl.8 d10, {d0-d1}, d2 + vshl.s32 d10, d10, d6 + vand d10, d10, d14 + vsub.i32 d10, d15, d10 + vtbl.8 d11, {d0-d1}, d3 + vshl.s32 d11, d11, d7 + vand d11, d11, d14 + vsub.i32 d11, d15, d11 + vtbl.8 d12, {d0-d1}, d4 + vshl.s32 d12, d12, d8 + vand d12, d12, d14 + vsub.i32 d12, d15, d12 + vtbl.8 d13, {d0-d1}, d5 + vshl.s32 d13, d13, d9 + vand d13, d13, d14 + vsub.i32 d13, d15, d13 + vst1.8 {d10-d11}, [r1]! + vst1.8 {d12-d13}, [r1]! + vpop {d8-d15} + bx lr + .size mldsa_decode_eta_2_neon,.-mldsa_decode_eta_2_neon +#ifndef __APPLE__ + .text + .type L_mldsa_decode_eta_4_neon_idx, %object + .size L_mldsa_decode_eta_4_neon_idx, 128 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_decode_eta_4_neon_idx: + .byte 0x00,0xff,0xff,0xff,0x00,0xff,0xff,0xff + .byte 0x01,0xff,0xff,0xff,0x01,0xff,0xff,0xff + .byte 0x02,0xff,0xff,0xff,0x02,0xff,0xff,0xff + .byte 0x03,0xff,0xff,0xff,0x03,0xff,0xff,0xff + .byte 0x04,0xff,0xff,0xff,0x04,0xff,0xff,0xff + .byte 0x05,0xff,0xff,0xff,0x05,0xff,0xff,0xff + .byte 0x06,0xff,0xff,0xff,0x06,0xff,0xff,0xff + .byte 0x07,0xff,0xff,0xff,0x07,0xff,0xff,0xff + .byte 0x08,0xff,0xff,0xff,0x08,0xff,0xff,0xff + .byte 0x09,0xff,0xff,0xff,0x09,0xff,0xff,0xff + .byte 0x0a,0xff,0xff,0xff,0x0a,0xff,0xff,0xff + .byte 0x0b,0xff,0xff,0xff,0x0b,0xff,0xff,0xff + .byte 0x0c,0xff,0xff,0xff,0x0c,0xff,0xff,0xff + .byte 0x0d,0xff,0xff,0xff,0x0d,0xff,0xff,0xff + .byte 0x0e,0xff,0xff,0xff,0x0e,0xff,0xff,0xff + .byte 0x0f,0xff,0xff,0xff,0x0f,0xff,0xff,0xff +#ifndef __APPLE__ + .text + .type L_mldsa_decode_eta_4_neon_aux, %object + .size L_mldsa_decode_eta_4_neon_aux, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_eta_4_neon_aux: + .long 0x00000000,0xfffffffc,0x00000000,0xfffffffc + .long 0x0000000f,0x0000000f,0x00000004,0x00000004 + .text + .align 4 + .globl mldsa_decode_eta_4_neon + WC_ASM_ATT_HIDDEN(mldsa_decode_eta_4_neon) + .type mldsa_decode_eta_4_neon, %function +mldsa_decode_eta_4_neon: + vpush {d8-d9} + adr r2, L_mldsa_decode_eta_4_neon_idx + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + adr r3, L_mldsa_decode_eta_4_neon_aux + vld1.8 {d4}, [r3]! + vld1.8 {d5}, [r3]! + vld1.8 {d8}, [r3]! + vld1.8 {d9}, [r3] + mov r12, #57 +L_mldsa_decode_eta_4_neon_loop: + vld1.8 {d0-d1}, [r0] + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vsub.i32 d6, d9, d6 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vsub.i32 d7, d9, d7 + vst1.8 {d6-d7}, [r1]! + add r0, r0, #2 + subs r12, r12, #1 + bne L_mldsa_decode_eta_4_neon_loop + sub r0, r0, #2 + vld1.8 {d0-d1}, [r0] + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vsub.i32 d6, d9, d6 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vsub.i32 d7, d9, d7 + vst1.8 {d6-d7}, [r1]! + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vsub.i32 d6, d9, d6 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vsub.i32 d7, d9, d7 + vst1.8 {d6-d7}, [r1]! + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vsub.i32 d6, d9, d6 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vsub.i32 d7, d9, d7 + vst1.8 {d6-d7}, [r1]! + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vsub.i32 d6, d9, d6 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vsub.i32 d7, d9, d7 + vst1.8 {d6-d7}, [r1]! + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vsub.i32 d6, d9, d6 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vsub.i32 d7, d9, d7 + vst1.8 {d6-d7}, [r1]! + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vsub.i32 d6, d9, d6 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vsub.i32 d7, d9, d7 + vst1.8 {d6-d7}, [r1]! + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vsub.i32 d6, d9, d6 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vsub.i32 d7, d9, d7 + vst1.8 {d6-d7}, [r1]! + vpop {d8-d9} + bx lr + .size mldsa_decode_eta_4_neon,.-mldsa_decode_eta_4_neon +#ifndef __APPLE__ + .text + .type L_mldsa_decode_t1_neon_idx, %object + .size L_mldsa_decode_t1_neon_idx, 64 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_decode_t1_neon_idx: + .byte 0x00,0x01,0xff,0xff,0x01,0x02,0xff,0xff + .byte 0x02,0x03,0xff,0xff,0x03,0x04,0xff,0xff + .byte 0x01,0x02,0xff,0xff,0x02,0x03,0xff,0xff + .byte 0x03,0x04,0xff,0xff,0x04,0x05,0xff,0xff + .byte 0x06,0x07,0xff,0xff,0x07,0x08,0xff,0xff + .byte 0x08,0x09,0xff,0xff,0x09,0x0a,0xff,0xff + .byte 0x0b,0x0c,0xff,0xff,0x0c,0x0d,0xff,0xff + .byte 0x0d,0x0e,0xff,0xff,0x0e,0x0f,0xff,0xff +#ifndef __APPLE__ + .text + .type L_mldsa_decode_t1_neon_aux, %object + .size L_mldsa_decode_t1_neon_aux, 24 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_t1_neon_aux: + .long 0x00000000,0xfffffffe,0xfffffffc,0xfffffffa + .long 0x000003ff,0x000003ff + .text + .align 4 + .globl mldsa_decode_t1_neon + WC_ASM_ATT_HIDDEN(mldsa_decode_t1_neon) + .type mldsa_decode_t1_neon, %function +mldsa_decode_t1_neon: + vpush {d8-d9} + adr r2, L_mldsa_decode_t1_neon_idx + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + adr r3, L_mldsa_decode_t1_neon_aux + vld1.8 {d4}, [r3]! + vld1.8 {d5}, [r3]! + vld1.8 {d8}, [r3]! + mov r12, #61 +L_mldsa_decode_t1_neon_loop: + vld1.8 {d0-d1}, [r0] + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vshl.i32 d6, d6, #13 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vshl.i32 d7, d7, #13 + vst1.8 {d6-d7}, [r1]! + add r0, r0, #5 + subs r12, r12, #1 + bne L_mldsa_decode_t1_neon_loop + sub r0, r0, #1 + vld1.8 {d0-d1}, [r0] + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vshl.i32 d6, d6, #13 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vshl.i32 d7, d7, #13 + vst1.8 {d6-d7}, [r1]! + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vshl.i32 d6, d6, #13 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vshl.i32 d7, d7, #13 + vst1.8 {d6-d7}, [r1]! + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vshl.i32 d6, d6, #13 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vshl.i32 d7, d7, #13 + vst1.8 {d6-d7}, [r1]! + vpop {d8-d9} + bx lr + .size mldsa_decode_t1_neon,.-mldsa_decode_t1_neon +#ifndef __APPLE__ + .text + .type L_mldsa_decode_t0_neon_idx, %object + .size L_mldsa_decode_t0_neon_idx, 64 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_decode_t0_neon_idx: + .byte 0x00,0x01,0xff,0xff,0x01,0x02,0x03,0xff + .byte 0x03,0x04,0xff,0xff,0x04,0x05,0x06,0xff + .byte 0x06,0x07,0x08,0xff,0x08,0x09,0xff,0xff + .byte 0x09,0x0a,0x0b,0xff,0x0b,0x0c,0xff,0xff + .byte 0x03,0x04,0xff,0xff,0x04,0x05,0x06,0xff + .byte 0x06,0x07,0xff,0xff,0x07,0x08,0x09,0xff + .byte 0x09,0x0a,0x0b,0xff,0x0b,0x0c,0xff,0xff + .byte 0x0c,0x0d,0x0e,0xff,0x0e,0x0f,0xff,0xff +#ifndef __APPLE__ + .text + .type L_mldsa_decode_t0_neon_aux, %object + .size L_mldsa_decode_t0_neon_aux, 48 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_t0_neon_aux: + .long 0x00000000,0xfffffffb,0xfffffffe,0xfffffff9 + .long 0xfffffffc,0xffffffff,0xfffffffa,0xfffffffd + .long 0x00001fff,0x00001fff,0x00001000,0x00001000 + .text + .align 4 + .globl mldsa_decode_t0_neon + WC_ASM_ATT_HIDDEN(mldsa_decode_t0_neon) + .type mldsa_decode_t0_neon, %function +mldsa_decode_t0_neon: + vpush {d8-d15} + adr r2, L_mldsa_decode_t0_neon_idx + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vld1.8 {d4}, [r2]! + vld1.8 {d5}, [r2]! + adr r3, L_mldsa_decode_t0_neon_aux + vld1.8 {d6}, [r3]! + vld1.8 {d7}, [r3]! + vld1.8 {d8}, [r3]! + vld1.8 {d9}, [r3]! + vld1.8 {d14}, [r3]! + vld1.8 {d15}, [r3] + mov r12, #31 +L_mldsa_decode_t0_neon_loop: + vld1.8 {d0-d1}, [r0] + vtbl.8 d10, {d0-d1}, d2 + vshl.s32 d10, d10, d6 + vand d10, d10, d14 + vsub.i32 d10, d15, d10 + vtbl.8 d11, {d0-d1}, d3 + vshl.s32 d11, d11, d7 + vand d11, d11, d14 + vsub.i32 d11, d15, d11 + vtbl.8 d12, {d0-d1}, d4 + vshl.s32 d12, d12, d8 + vand d12, d12, d14 + vsub.i32 d12, d15, d12 + vtbl.8 d13, {d0-d1}, d5 + vshl.s32 d13, d13, d9 + vand d13, d13, d14 + vsub.i32 d13, d15, d13 + vst1.8 {d10-d11}, [r1]! + vst1.8 {d12-d13}, [r1]! + add r0, r0, #13 + subs r12, r12, #1 + bne L_mldsa_decode_t0_neon_loop + sub r0, r0, #3 + vld1.8 {d0-d1}, [r0] + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vld1.8 {d4}, [r2]! + vld1.8 {d5}, [r2]! + vtbl.8 d10, {d0-d1}, d2 + vshl.s32 d10, d10, d6 + vand d10, d10, d14 + vsub.i32 d10, d15, d10 + vtbl.8 d11, {d0-d1}, d3 + vshl.s32 d11, d11, d7 + vand d11, d11, d14 + vsub.i32 d11, d15, d11 + vtbl.8 d12, {d0-d1}, d4 + vshl.s32 d12, d12, d8 + vand d12, d12, d14 + vsub.i32 d12, d15, d12 + vtbl.8 d13, {d0-d1}, d5 + vshl.s32 d13, d13, d9 + vand d13, d13, d14 + vsub.i32 d13, d15, d13 + vst1.8 {d10-d11}, [r1]! + vst1.8 {d12-d13}, [r1]! + vpop {d8-d15} + bx lr + .size mldsa_decode_t0_neon,.-mldsa_decode_t0_neon +#ifndef __APPLE__ + .text + .type L_mldsa_decode_gamma1_17_neon_idx, %object + .size L_mldsa_decode_gamma1_17_neon_idx, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_decode_gamma1_17_neon_idx: + .byte 0x00,0x01,0x02,0xff,0x02,0x03,0x04,0xff + .byte 0x04,0x05,0x06,0xff,0x06,0x07,0x08,0xff + .byte 0x07,0x08,0x09,0xff,0x09,0x0a,0x0b,0xff + .byte 0x0b,0x0c,0x0d,0xff,0x0d,0x0e,0x0f,0xff +#ifndef __APPLE__ + .text + .type L_mldsa_decode_gamma1_17_neon_aux, %object + .size L_mldsa_decode_gamma1_17_neon_aux, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_gamma1_17_neon_aux: + .long 0x00000000,0xfffffffe,0xfffffffc,0xfffffffa + .long 0x0003ffff,0x0003ffff,0x00020000,0x00020000 + .text + .align 4 + .globl mldsa_decode_gamma1_17_neon + WC_ASM_ATT_HIDDEN(mldsa_decode_gamma1_17_neon) + .type mldsa_decode_gamma1_17_neon, %function +mldsa_decode_gamma1_17_neon: + vpush {d8-d9} + adr r2, L_mldsa_decode_gamma1_17_neon_idx + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + adr r3, L_mldsa_decode_gamma1_17_neon_aux + vld1.8 {d4}, [r3]! + vld1.8 {d5}, [r3]! + vld1.8 {d8}, [r3]! + vld1.8 {d9}, [r3] + mov r12, #63 +L_mldsa_decode_gamma1_17_neon_loop: + vld1.8 {d0-d1}, [r0] + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vsub.i32 d6, d9, d6 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vsub.i32 d7, d9, d7 + vst1.8 {d6-d7}, [r1]! + add r0, r0, #9 + subs r12, r12, #1 + bne L_mldsa_decode_gamma1_17_neon_loop + sub r0, r0, #7 + vld1.8 {d0-d1}, [r0] + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vsub.i32 d6, d9, d6 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vsub.i32 d7, d9, d7 + vst1.8 {d6-d7}, [r1]! + vpop {d8-d9} + bx lr + .size mldsa_decode_gamma1_17_neon,.-mldsa_decode_gamma1_17_neon +#ifndef __APPLE__ + .text + .type L_mldsa_decode_gamma1_19_neon_idx, %object + .size L_mldsa_decode_gamma1_19_neon_idx, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_decode_gamma1_19_neon_idx: + .byte 0x00,0x01,0x02,0xff,0x02,0x03,0x04,0xff + .byte 0x05,0x06,0x07,0xff,0x07,0x08,0x09,0xff + .byte 0x06,0x07,0x08,0xff,0x08,0x09,0x0a,0xff + .byte 0x0b,0x0c,0x0d,0xff,0x0d,0x0e,0x0f,0xff +#ifndef __APPLE__ + .text + .type L_mldsa_decode_gamma1_19_neon_aux, %object + .size L_mldsa_decode_gamma1_19_neon_aux, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_gamma1_19_neon_aux: + .long 0x00000000,0xfffffffc,0x00000000,0xfffffffc + .long 0x000fffff,0x000fffff,0x00080000,0x00080000 + .text + .align 4 + .globl mldsa_decode_gamma1_19_neon + WC_ASM_ATT_HIDDEN(mldsa_decode_gamma1_19_neon) + .type mldsa_decode_gamma1_19_neon, %function +mldsa_decode_gamma1_19_neon: + vpush {d8-d9} + adr r2, L_mldsa_decode_gamma1_19_neon_idx + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + adr r3, L_mldsa_decode_gamma1_19_neon_aux + vld1.8 {d4}, [r3]! + vld1.8 {d5}, [r3]! + vld1.8 {d8}, [r3]! + vld1.8 {d9}, [r3] + mov r12, #63 +L_mldsa_decode_gamma1_19_neon_loop: + vld1.8 {d0-d1}, [r0] + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vsub.i32 d6, d9, d6 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vsub.i32 d7, d9, d7 + vst1.8 {d6-d7}, [r1]! + add r0, r0, #10 + subs r12, r12, #1 + bne L_mldsa_decode_gamma1_19_neon_loop + sub r0, r0, #6 + vld1.8 {d0-d1}, [r0] + vld1.8 {d2}, [r2]! + vld1.8 {d3}, [r2]! + vtbl.8 d6, {d0-d1}, d2 + vshl.s32 d6, d6, d4 + vand d6, d6, d8 + vsub.i32 d6, d9, d6 + vtbl.8 d7, {d0-d1}, d3 + vshl.s32 d7, d7, d5 + vand d7, d7, d8 + vsub.i32 d7, d9, d7 + vst1.8 {d6-d7}, [r1]! + vpop {d8-d9} + bx lr + .size mldsa_decode_gamma1_19_neon,.-mldsa_decode_gamma1_19_neon +#ifndef __APPLE__ + .text + .type L_mldsa_encode_w1_88_neon_idx, %object + .size L_mldsa_encode_w1_88_neon_idx, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_encode_w1_88_neon_idx: + .byte 0x00,0x01,0x02,0x04,0x05,0x06,0x08,0x09 + .byte 0x0a,0x0c,0x0d,0x0e,0xff,0xff,0xff,0xff + .text + .align 4 + .globl mldsa_encode_w1_88_neon + WC_ASM_ATT_HIDDEN(mldsa_encode_w1_88_neon) + .type mldsa_encode_w1_88_neon, %function +mldsa_encode_w1_88_neon: + vpush {d8-d11} + adr r2, L_mldsa_encode_w1_88_neon_idx + vld1.8 {d8}, [r2]! + vld1.8 {d9}, [r2] + mov r3, #16 +L_mldsa_encode_w1_88_neon_loop: + vld1.8 {q0}, [r0]! + vld1.8 {q1}, [r0]! + vld1.8 {q2}, [r0]! + vld1.8 {q3}, [r0]! + vuzp.32 q0, q1 + vsli.32 q0, q1, #6 + vuzp.32 q2, q3 + vsli.32 q2, q3, #6 + vuzp.32 q0, q2 + vsli.32 q0, q2, #12 + vtbl.8 d10, {d0-d1}, d8 + vtbl.8 d11, {d0-d1}, d9 + vst1.8 {d10}, [r1]! + vmov.32 r12, d11[0] + str r12, [r1], #4 + subs r3, r3, #1 + bne L_mldsa_encode_w1_88_neon_loop + vpop {d8-d11} + bx lr + .size mldsa_encode_w1_88_neon,.-mldsa_encode_w1_88_neon +#ifndef __APPLE__ + .text + .type L_mldsa_encode_w1_32_neon_idx, %object + .size L_mldsa_encode_w1_32_neon_idx, 8 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_encode_w1_32_neon_idx: + .byte 0x00,0x04,0x08,0x0c,0x10,0x14,0x18,0x1c + .text + .align 4 + .globl mldsa_encode_w1_32_neon + WC_ASM_ATT_HIDDEN(mldsa_encode_w1_32_neon) + .type mldsa_encode_w1_32_neon, %function +mldsa_encode_w1_32_neon: + vpush {d8-d11} + adr r2, L_mldsa_encode_w1_32_neon_idx + vld1.8 {d8}, [r2]! + mov r3, #16 +L_mldsa_encode_w1_32_neon_loop: + vld1.8 {q0}, [r0]! + vld1.8 {q1}, [r0]! + vld1.8 {q2}, [r0]! + vld1.8 {q3}, [r0]! + vuzp.32 q0, q1 + vsli.32 q0, q1, #4 + vuzp.32 q2, q3 + vsli.32 q2, q3, #4 + vmov q1, q2 + vtbl.8 d10, {d0-d3}, d8 + vst1.8 {d10}, [r1]! + subs r3, r3, #1 + bne L_mldsa_encode_w1_32_neon_loop + vpop {d8-d11} + bx lr + .size mldsa_encode_w1_32_neon,.-mldsa_encode_w1_32_neon +#ifndef __APPLE__ + .text + .type L_mldsa_encode_gamma1_17_neon_idx, %object + .size L_mldsa_encode_gamma1_17_neon_idx, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_encode_gamma1_17_neon_idx: + .byte 0x00,0x01,0x02,0xff,0x08,0x09,0x0a,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0x04,0x05,0x06,0xff,0x0c,0x0d + .byte 0x0e,0xff,0xff,0xff,0xff,0xff,0xff,0xff +#ifndef __APPLE__ + .text + .type L_mldsa_encode_gamma1_17_neon_aux, %object + .size L_mldsa_encode_gamma1_17_neon_aux, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_encode_gamma1_17_neon_aux: + .long 0x00000000,0x00000002,0x00000004,0x00000006 + .long 0x00020000,0x00020000,0x00020000,0x00020000 + .text + .align 4 + .globl mldsa_encode_gamma1_17_neon + WC_ASM_ATT_HIDDEN(mldsa_encode_gamma1_17_neon) + .type mldsa_encode_gamma1_17_neon, %function +mldsa_encode_gamma1_17_neon: + vpush {d8-d13} + adr r2, L_mldsa_encode_gamma1_17_neon_idx + vld1.8 {d6}, [r2]! + vld1.8 {d7}, [r2]! + vld1.8 {d8}, [r2]! + vld1.8 {d9}, [r2]! + adr r2, L_mldsa_encode_gamma1_17_neon_aux + vld1.8 {q1}, [r2]! + vld1.8 {q2}, [r2] + mov r3, #0x40 +L_mldsa_encode_gamma1_17_neon_loop: + vld1.8 {q0}, [r0]! + vsub.i32 q0, q2, q0 + vshl.s32 q0, q0, q1 + vtbl.8 d10, {d0-d1}, d6 + vtbl.8 d11, {d0-d1}, d7 + vtbl.8 d12, {d0-d1}, d8 + vtbl.8 d13, {d0-d1}, d9 + vorr d10, d10, d12 + vorr d11, d11, d13 + vst1.8 {d10}, [r1]! + vmov.32 r12, d11[0] + strb r12, [r1], #1 + subs r3, r3, #1 + bne L_mldsa_encode_gamma1_17_neon_loop + vpop {d8-d13} + bx lr + .size mldsa_encode_gamma1_17_neon,.-mldsa_encode_gamma1_17_neon +#ifndef __APPLE__ + .text + .type L_mldsa_encode_gamma1_19_neon_idx, %object + .size L_mldsa_encode_gamma1_19_neon_idx, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_encode_gamma1_19_neon_idx: + .byte 0x00,0x01,0x02,0xff,0xff,0x08,0x09,0x0a + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0x04,0x05,0x06,0xff,0xff,0x0c + .byte 0x0d,0x0e,0xff,0xff,0xff,0xff,0xff,0xff +#ifndef __APPLE__ + .text + .type L_mldsa_encode_gamma1_19_neon_aux, %object + .size L_mldsa_encode_gamma1_19_neon_aux, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_encode_gamma1_19_neon_aux: + .long 0x00000000,0x00000004,0x00000000,0x00000004 + .long 0x00080000,0x00080000,0x00080000,0x00080000 + .text + .align 4 + .globl mldsa_encode_gamma1_19_neon + WC_ASM_ATT_HIDDEN(mldsa_encode_gamma1_19_neon) + .type mldsa_encode_gamma1_19_neon, %function +mldsa_encode_gamma1_19_neon: + vpush {d8-d13} + adr r2, L_mldsa_encode_gamma1_19_neon_idx + vld1.8 {d6}, [r2]! + vld1.8 {d7}, [r2]! + vld1.8 {d8}, [r2]! + vld1.8 {d9}, [r2]! + adr r2, L_mldsa_encode_gamma1_19_neon_aux + vld1.8 {q1}, [r2]! + vld1.8 {q2}, [r2] + mov r3, #0x40 +L_mldsa_encode_gamma1_19_neon_loop: + vld1.8 {q0}, [r0]! + vsub.i32 q0, q2, q0 + vshl.s32 q0, q0, q1 + vtbl.8 d10, {d0-d1}, d6 + vtbl.8 d11, {d0-d1}, d7 + vtbl.8 d12, {d0-d1}, d8 + vtbl.8 d13, {d0-d1}, d9 + vorr d10, d10, d12 + vorr d11, d11, d13 + vst1.8 {d10}, [r1]! + vmov.32 r12, d11[0] + strh r12, [r1], #2 + subs r3, r3, #1 + bne L_mldsa_encode_gamma1_19_neon_loop + vpop {d8-d13} + bx lr + .size mldsa_encode_gamma1_19_neon,.-mldsa_encode_gamma1_19_neon +#ifndef __APPLE__ + .text + .type L_mldsa_mul_vec_4_neon_q, %object + .size L_mldsa_mul_vec_4_neon_q, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_mul_vec_4_neon_q: + .long 0x007fe001,0x007fe001,0x03802001,0x03802001 + .text + .align 4 + .globl mldsa_mul_vec_4_neon + WC_ASM_ATT_HIDDEN(mldsa_mul_vec_4_neon) + .type mldsa_mul_vec_4_neon, %function +mldsa_mul_vec_4_neon: + push {lr} + vpush {d8-d13} + adr r3, L_mldsa_mul_vec_4_neon_q + vld1.8 {q0}, [r3] + vdup.32 q1, d1[0] + mov lr, #0x40 +L_mldsa_mul_vec_4_neon_loop: + vld1.8 {q2}, [r1] + vld1.8 {q3}, [r2] + vmull.s32 q4, d4, d6 + vmull.s32 q5, d5, d7 + add r12, r1, #0x400 + vld1.8 {q2}, [r12] + add r12, r2, #0x400 + vld1.8 {q3}, [r12] + vmlal.s32 q4, d4, d6 + vmlal.s32 q5, d5, d7 + add r12, r1, #0x800 + vld1.8 {q2}, [r12] + add r12, r2, #0x800 + vld1.8 {q3}, [r12] + vmlal.s32 q4, d4, d6 + vmlal.s32 q5, d5, d7 + add r12, r1, #0xc00 + vld1.8 {q2}, [r12] + add r12, r2, #0xc00 + vld1.8 {q3}, [r12] + vmlal.s32 q4, d4, d6 + vmlal.s32 q5, d5, d7 + vmovn.i64 d12, q4 + vmovn.i64 d13, q5 + vmul.i32 q6, q6, q1 + vmlsl.s32 q4, d12, d0 + vmlsl.s32 q5, d13, d0 + vshrn.i64 d12, q4, #32 + vshrn.i64 d13, q5, #32 + vst1.8 {q6}, [r0]! + add r1, r1, #16 + add r2, r2, #16 + subs lr, lr, #1 + bne L_mldsa_mul_vec_4_neon_loop + vpop {d8-d13} + pop {pc} + .size mldsa_mul_vec_4_neon,.-mldsa_mul_vec_4_neon +#ifndef __APPLE__ + .text + .type L_mldsa_mul_vec_5_neon_q, %object + .size L_mldsa_mul_vec_5_neon_q, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_mul_vec_5_neon_q: + .long 0x007fe001,0x007fe001,0x03802001,0x03802001 + .text + .align 4 + .globl mldsa_mul_vec_5_neon + WC_ASM_ATT_HIDDEN(mldsa_mul_vec_5_neon) + .type mldsa_mul_vec_5_neon, %function +mldsa_mul_vec_5_neon: + push {lr} + vpush {d8-d13} + adr r3, L_mldsa_mul_vec_5_neon_q + vld1.8 {q0}, [r3] + vdup.32 q1, d1[0] + mov lr, #0x40 +L_mldsa_mul_vec_5_neon_loop: + vld1.8 {q2}, [r1] + vld1.8 {q3}, [r2] + vmull.s32 q4, d4, d6 + vmull.s32 q5, d5, d7 + add r12, r1, #0x400 + vld1.8 {q2}, [r12] + add r12, r2, #0x400 + vld1.8 {q3}, [r12] + vmlal.s32 q4, d4, d6 + vmlal.s32 q5, d5, d7 + add r12, r1, #0x800 + vld1.8 {q2}, [r12] + add r12, r2, #0x800 + vld1.8 {q3}, [r12] + vmlal.s32 q4, d4, d6 + vmlal.s32 q5, d5, d7 + add r12, r1, #0xc00 + vld1.8 {q2}, [r12] + add r12, r2, #0xc00 + vld1.8 {q3}, [r12] + vmlal.s32 q4, d4, d6 + vmlal.s32 q5, d5, d7 + add r12, r1, #0x1000 + vld1.8 {q2}, [r12] + add r12, r2, #0x1000 + vld1.8 {q3}, [r12] + vmlal.s32 q4, d4, d6 + vmlal.s32 q5, d5, d7 + vmovn.i64 d12, q4 + vmovn.i64 d13, q5 + vmul.i32 q6, q6, q1 + vmlsl.s32 q4, d12, d0 + vmlsl.s32 q5, d13, d0 + vshrn.i64 d12, q4, #32 + vshrn.i64 d13, q5, #32 + vst1.8 {q6}, [r0]! + add r1, r1, #16 + add r2, r2, #16 + subs lr, lr, #1 + bne L_mldsa_mul_vec_5_neon_loop + vpop {d8-d13} + pop {pc} + .size mldsa_mul_vec_5_neon,.-mldsa_mul_vec_5_neon +#ifndef __APPLE__ + .text + .type L_mldsa_mul_vec_7_neon_q, %object + .size L_mldsa_mul_vec_7_neon_q, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_mul_vec_7_neon_q: + .long 0x007fe001,0x007fe001,0x03802001,0x03802001 + .text + .align 4 + .globl mldsa_mul_vec_7_neon + WC_ASM_ATT_HIDDEN(mldsa_mul_vec_7_neon) + .type mldsa_mul_vec_7_neon, %function +mldsa_mul_vec_7_neon: + push {lr} + vpush {d8-d13} + adr r3, L_mldsa_mul_vec_7_neon_q + vld1.8 {q0}, [r3] + vdup.32 q1, d1[0] + mov lr, #0x40 +L_mldsa_mul_vec_7_neon_loop: + vld1.8 {q2}, [r1] + vld1.8 {q3}, [r2] + vmull.s32 q4, d4, d6 + vmull.s32 q5, d5, d7 + add r12, r1, #0x400 + vld1.8 {q2}, [r12] + add r12, r2, #0x400 + vld1.8 {q3}, [r12] + vmlal.s32 q4, d4, d6 + vmlal.s32 q5, d5, d7 + add r12, r1, #0x800 + vld1.8 {q2}, [r12] + add r12, r2, #0x800 + vld1.8 {q3}, [r12] + vmlal.s32 q4, d4, d6 + vmlal.s32 q5, d5, d7 + add r12, r1, #0xc00 + vld1.8 {q2}, [r12] + add r12, r2, #0xc00 + vld1.8 {q3}, [r12] + vmlal.s32 q4, d4, d6 + vmlal.s32 q5, d5, d7 + add r12, r1, #0x1000 + vld1.8 {q2}, [r12] + add r12, r2, #0x1000 + vld1.8 {q3}, [r12] + vmlal.s32 q4, d4, d6 + vmlal.s32 q5, d5, d7 + add r12, r1, #0x1400 + vld1.8 {q2}, [r12] + add r12, r2, #0x1400 + vld1.8 {q3}, [r12] + vmlal.s32 q4, d4, d6 + vmlal.s32 q5, d5, d7 + add r12, r1, #0x1800 + vld1.8 {q2}, [r12] + add r12, r2, #0x1800 + vld1.8 {q3}, [r12] + vmlal.s32 q4, d4, d6 + vmlal.s32 q5, d5, d7 + vmovn.i64 d12, q4 + vmovn.i64 d13, q5 + vmul.i32 q6, q6, q1 + vmlsl.s32 q4, d12, d0 + vmlsl.s32 q5, d13, d0 + vshrn.i64 d12, q4, #32 + vshrn.i64 d13, q5, #32 + vst1.8 {q6}, [r0]! + add r1, r1, #16 + add r2, r2, #16 + subs lr, lr, #1 + bne L_mldsa_mul_vec_7_neon_loop + vpop {d8-d13} + pop {pc} + .size mldsa_mul_vec_7_neon,.-mldsa_mul_vec_7_neon +#ifndef __APPLE__ + .text + .type L_mldsa_extract_coeffs_eta2_neon_map, %object + .size L_mldsa_extract_coeffs_eta2_neon_map, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta2_neon_map: + .byte 0x02,0x01,0x00,0xff,0xfe,0x02,0x01,0x00 + .byte 0xff,0xfe,0x02,0x01,0x00,0xff,0xfe,0x00 +#ifndef __APPLE__ + .text + .type L_mldsa_extract_coeffs_eta2_neon_aux, %object + .size L_mldsa_extract_coeffs_eta2_neon_aux, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta2_neon_aux: + .byte 0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f + .byte 0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f +#ifndef __APPLE__ + .text + .type L_mldsa_extract_coeffs_eta2_neon_bits, %object + .size L_mldsa_extract_coeffs_eta2_neon_bits, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta2_neon_bits: + .long 0x00000001,0x00000002,0x00000004,0x00000008 +#ifndef __APPLE__ + .text + .type L_mldsa_extract_coeffs_eta2_neon_idx, %object + .size L_mldsa_extract_coeffs_eta2_neon_idx, 256 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta2_neon_idx: + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x08,0x09,0x0a,0x0b,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x08,0x09,0x0a,0x0b + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x08,0x09,0x0a,0x0b,0xff,0xff,0xff,0xff + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x08,0x09,0x0a,0x0b + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f + .text + .align 4 + .globl mldsa_extract_coeffs_eta2_neon + WC_ASM_ATT_HIDDEN(mldsa_extract_coeffs_eta2_neon) + .type mldsa_extract_coeffs_eta2_neon, %function +mldsa_extract_coeffs_eta2_neon: + push {r4, r5, r6, r7, r8, lr} + vpush {d8-d15} + adr r12, L_mldsa_extract_coeffs_eta2_neon_map + vld1.8 {q0}, [r12] + adr lr, L_mldsa_extract_coeffs_eta2_neon_aux + vld1.8 {q1}, [lr] + adr lr, L_mldsa_extract_coeffs_eta2_neon_bits + vld1.8 {q2}, [lr] + adr r4, L_mldsa_extract_coeffs_eta2_neon_idx + ldr r5, [r3] + add r2, r2, r5, lsl #2 +L_mldsa_extract_coeffs_eta2_neon_wide: + cmp r1, #8 + blt L_mldsa_extract_coeffs_eta2_neon_tail + cmp r5, #0xf0 + bgt L_mldsa_extract_coeffs_eta2_neon_tail + vld1.8 {d6}, [r0] + vshr.u8 d7, d6, #4 + vmov.i8 q12, #15 + vand d6, d6, d24 + vzip.8 d6, d7 + vtbl.8 d8, {d0-d1}, d6 + vtbl.8 d9, {d0-d1}, d7 + vcgt.u8 q5, q1, q3 + vmovl.s8 q6, d8 + vmovl.s8 q7, d9 + vmovl.s8 q8, d10 + vmovl.s8 q9, d11 + vmovl.s16 q10, d12 + vmovl.s16 q11, d16 + vshr.u32 q12, q11, #31 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r6, d24[0] + vand q12, q11, q2 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r7, d24[0] + lsl r7, r7, #4 + add r7, r7, r4 + vld1.8 {q13}, [r7] + vtbl.8 d24, {d20-d21}, d26 + vtbl.8 d25, {d20-d21}, d27 + vst1.8 {q12}, [r2] + add r2, r2, r6, lsl #2 + add r5, r5, r6 + vmovl.s16 q10, d13 + vmovl.s16 q11, d17 + vshr.u32 q12, q11, #31 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r6, d24[0] + vand q12, q11, q2 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r7, d24[0] + lsl r7, r7, #4 + add r7, r7, r4 + vld1.8 {q13}, [r7] + vtbl.8 d24, {d20-d21}, d26 + vtbl.8 d25, {d20-d21}, d27 + vst1.8 {q12}, [r2] + add r2, r2, r6, lsl #2 + add r5, r5, r6 + vmovl.s16 q10, d14 + vmovl.s16 q11, d18 + vshr.u32 q12, q11, #31 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r6, d24[0] + vand q12, q11, q2 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r7, d24[0] + lsl r7, r7, #4 + add r7, r7, r4 + vld1.8 {q13}, [r7] + vtbl.8 d24, {d20-d21}, d26 + vtbl.8 d25, {d20-d21}, d27 + vst1.8 {q12}, [r2] + add r2, r2, r6, lsl #2 + add r5, r5, r6 + vmovl.s16 q10, d15 + vmovl.s16 q11, d19 + vshr.u32 q12, q11, #31 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r6, d24[0] + vand q12, q11, q2 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r7, d24[0] + lsl r7, r7, #4 + add r7, r7, r4 + vld1.8 {q13}, [r7] + vtbl.8 d24, {d20-d21}, d26 + vtbl.8 d25, {d20-d21}, d27 + vst1.8 {q12}, [r2] + add r2, r2, r6, lsl #2 + add r5, r5, r6 + add r0, r0, #8 + sub r1, r1, #8 + b L_mldsa_extract_coeffs_eta2_neon_wide +L_mldsa_extract_coeffs_eta2_neon_tail: + cmp r1, #0 + beq L_mldsa_extract_coeffs_eta2_neon_done + cmp r5, #0x100 + beq L_mldsa_extract_coeffs_eta2_neon_done + ldrb r8, [r0], #1 + sub r1, r1, #1 + and r6, r8, #15 + cmp r6, #15 + bcs L_mldsa_extract_coeffs_eta2_neon_skip_lo + add lr, r12, r6 + ldrsb r6, [lr] + str r6, [r2], #4 + add r5, r5, #1 + cmp r5, #0x100 + beq L_mldsa_extract_coeffs_eta2_neon_done +L_mldsa_extract_coeffs_eta2_neon_skip_lo: + lsr r6, r8, #4 + cmp r6, #15 + bcs L_mldsa_extract_coeffs_eta2_neon_skip_hi + add lr, r12, r6 + ldrsb r6, [lr] + str r6, [r2], #4 + add r5, r5, #1 +L_mldsa_extract_coeffs_eta2_neon_skip_hi: + b L_mldsa_extract_coeffs_eta2_neon_tail +L_mldsa_extract_coeffs_eta2_neon_done: + str r5, [r3] + vpop {d8-d15} + pop {r4, r5, r6, r7, r8, pc} + .size mldsa_extract_coeffs_eta2_neon,.-mldsa_extract_coeffs_eta2_neon +#ifndef __APPLE__ + .text + .type L_mldsa_extract_coeffs_eta4_neon_map, %object + .size L_mldsa_extract_coeffs_eta4_neon_map, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta4_neon_map: + .byte 0x04,0x03,0x02,0x01,0x00,0xff,0xfe,0xfd + .byte 0xfc,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +#ifndef __APPLE__ + .text + .type L_mldsa_extract_coeffs_eta4_neon_aux, %object + .size L_mldsa_extract_coeffs_eta4_neon_aux, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta4_neon_aux: + .byte 0x09,0x09,0x09,0x09,0x09,0x09,0x09,0x09 + .byte 0x09,0x09,0x09,0x09,0x09,0x09,0x09,0x09 +#ifndef __APPLE__ + .text + .type L_mldsa_extract_coeffs_eta4_neon_bits, %object + .size L_mldsa_extract_coeffs_eta4_neon_bits, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta4_neon_bits: + .long 0x00000001,0x00000002,0x00000004,0x00000008 +#ifndef __APPLE__ + .text + .type L_mldsa_extract_coeffs_eta4_neon_idx, %object + .size L_mldsa_extract_coeffs_eta4_neon_idx, 256 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta4_neon_idx: + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x08,0x09,0x0a,0x0b,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x08,0x09,0x0a,0x0b + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x08,0x09,0x0a,0x0b,0xff,0xff,0xff,0xff + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x08,0x09,0x0a,0x0b + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f + .text + .align 4 + .globl mldsa_extract_coeffs_eta4_neon + WC_ASM_ATT_HIDDEN(mldsa_extract_coeffs_eta4_neon) + .type mldsa_extract_coeffs_eta4_neon, %function +mldsa_extract_coeffs_eta4_neon: + push {r4, r5, r6, r7, r8, lr} + vpush {d8-d15} + adr r12, L_mldsa_extract_coeffs_eta4_neon_map + vld1.8 {q0}, [r12] + adr lr, L_mldsa_extract_coeffs_eta4_neon_aux + vld1.8 {q1}, [lr] + adr lr, L_mldsa_extract_coeffs_eta4_neon_bits + vld1.8 {q2}, [lr] + adr r4, L_mldsa_extract_coeffs_eta4_neon_idx + ldr r5, [r3] + add r2, r2, r5, lsl #2 +L_mldsa_extract_coeffs_eta4_neon_wide: + cmp r1, #8 + blt L_mldsa_extract_coeffs_eta4_neon_tail + cmp r5, #0xf0 + bgt L_mldsa_extract_coeffs_eta4_neon_tail + vld1.8 {d6}, [r0] + vshr.u8 d7, d6, #4 + vmov.i8 q12, #15 + vand d6, d6, d24 + vzip.8 d6, d7 + vtbl.8 d8, {d0-d1}, d6 + vtbl.8 d9, {d0-d1}, d7 + vcgt.u8 q5, q1, q3 + vmovl.s8 q6, d8 + vmovl.s8 q7, d9 + vmovl.s8 q8, d10 + vmovl.s8 q9, d11 + vmovl.s16 q10, d12 + vmovl.s16 q11, d16 + vshr.u32 q12, q11, #31 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r6, d24[0] + vand q12, q11, q2 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r7, d24[0] + lsl r7, r7, #4 + add r7, r7, r4 + vld1.8 {q13}, [r7] + vtbl.8 d24, {d20-d21}, d26 + vtbl.8 d25, {d20-d21}, d27 + vst1.8 {q12}, [r2] + add r2, r2, r6, lsl #2 + add r5, r5, r6 + vmovl.s16 q10, d13 + vmovl.s16 q11, d17 + vshr.u32 q12, q11, #31 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r6, d24[0] + vand q12, q11, q2 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r7, d24[0] + lsl r7, r7, #4 + add r7, r7, r4 + vld1.8 {q13}, [r7] + vtbl.8 d24, {d20-d21}, d26 + vtbl.8 d25, {d20-d21}, d27 + vst1.8 {q12}, [r2] + add r2, r2, r6, lsl #2 + add r5, r5, r6 + vmovl.s16 q10, d14 + vmovl.s16 q11, d18 + vshr.u32 q12, q11, #31 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r6, d24[0] + vand q12, q11, q2 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r7, d24[0] + lsl r7, r7, #4 + add r7, r7, r4 + vld1.8 {q13}, [r7] + vtbl.8 d24, {d20-d21}, d26 + vtbl.8 d25, {d20-d21}, d27 + vst1.8 {q12}, [r2] + add r2, r2, r6, lsl #2 + add r5, r5, r6 + vmovl.s16 q10, d15 + vmovl.s16 q11, d19 + vshr.u32 q12, q11, #31 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r6, d24[0] + vand q12, q11, q2 + vpadd.u32 d24, d24, d25 + vpadd.u32 d24, d24, d24 + vmov.32 r7, d24[0] + lsl r7, r7, #4 + add r7, r7, r4 + vld1.8 {q13}, [r7] + vtbl.8 d24, {d20-d21}, d26 + vtbl.8 d25, {d20-d21}, d27 + vst1.8 {q12}, [r2] + add r2, r2, r6, lsl #2 + add r5, r5, r6 + add r0, r0, #8 + sub r1, r1, #8 + b L_mldsa_extract_coeffs_eta4_neon_wide +L_mldsa_extract_coeffs_eta4_neon_tail: + cmp r1, #0 + beq L_mldsa_extract_coeffs_eta4_neon_done + cmp r5, #0x100 + beq L_mldsa_extract_coeffs_eta4_neon_done + ldrb r8, [r0], #1 + sub r1, r1, #1 + and r6, r8, #15 + cmp r6, #9 + bcs L_mldsa_extract_coeffs_eta4_neon_skip_lo + add lr, r12, r6 + ldrsb r6, [lr] + str r6, [r2], #4 + add r5, r5, #1 + cmp r5, #0x100 + beq L_mldsa_extract_coeffs_eta4_neon_done +L_mldsa_extract_coeffs_eta4_neon_skip_lo: + lsr r6, r8, #4 + cmp r6, #9 + bcs L_mldsa_extract_coeffs_eta4_neon_skip_hi + add lr, r12, r6 + ldrsb r6, [lr] + str r6, [r2], #4 + add r5, r5, #1 +L_mldsa_extract_coeffs_eta4_neon_skip_hi: + b L_mldsa_extract_coeffs_eta4_neon_tail +L_mldsa_extract_coeffs_eta4_neon_done: + str r5, [r3] + vpop {d8-d15} + pop {r4, r5, r6, r7, r8, pc} + .size mldsa_extract_coeffs_eta4_neon,.-mldsa_extract_coeffs_eta4_neon +#ifndef __APPLE__ + .text + .type L_mldsa_arm32_make_hint_bits, %object + .size L_mldsa_arm32_make_hint_bits, 64 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_arm32_make_hint_bits: + .long 0x00000001,0x00000002,0x00000004,0x00000008 + .long 0x00000010,0x00000020,0x00000040,0x00000080 + .long 0x00000100,0x00000200,0x00000400,0x00000800 + .long 0x00001000,0x00002000,0x00004000,0x00008000 + .text + .align 4 + .globl mldsa_make_hint_neon + WC_ASM_ATT_HIDDEN(mldsa_make_hint_neon) + .type mldsa_make_hint_neon, %function +mldsa_make_hint_neon: + push {r4, r5, r6, r7, lr} + vpush {d8-d15} + ldr r12, [sp, #84] + ldr lr, [sp, #88] + vdup.32 q0, lr + vneg.s32 q1, q0 + adr r4, L_mldsa_arm32_make_hint_bits + vld1.8 {q2}, [r4]! + vld1.8 {q3}, [r4]! + vld1.8 {q4}, [r4]! + vld1.8 {q5}, [r4]! + mov r5, #0 +L_mldsa_arm32_make_hint_loop: + veor q11, q11, q11 + vld1.8 {q6}, [r0]! + vld1.8 {q7}, [r1]! + vcgt.s32 q8, q6, q0 + vcgt.s32 q9, q1, q6 + vceq.i32 q10, q6, q1 + vceq.i32 q7, q7, #0 + vbic q10, q10, q7 + vorr q8, q8, q9 + vorr q8, q8, q10 + vand q8, q8, q2 + vorr q11, q11, q8 + vld1.8 {q6}, [r0]! + vld1.8 {q7}, [r1]! + vcgt.s32 q8, q6, q0 + vcgt.s32 q9, q1, q6 + vceq.i32 q10, q6, q1 + vceq.i32 q7, q7, #0 + vbic q10, q10, q7 + vorr q8, q8, q9 + vorr q8, q8, q10 + vand q8, q8, q3 + vorr q11, q11, q8 + vld1.8 {q6}, [r0]! + vld1.8 {q7}, [r1]! + vcgt.s32 q8, q6, q0 + vcgt.s32 q9, q1, q6 + vceq.i32 q10, q6, q1 + vceq.i32 q7, q7, #0 + vbic q10, q10, q7 + vorr q8, q8, q9 + vorr q8, q8, q10 + vand q8, q8, q4 + vorr q11, q11, q8 + vld1.8 {q6}, [r0]! + vld1.8 {q7}, [r1]! + vcgt.s32 q8, q6, q0 + vcgt.s32 q9, q1, q6 + vceq.i32 q10, q6, q1 + vceq.i32 q7, q7, #0 + vbic q10, q10, q7 + vorr q8, q8, q9 + vorr q8, q8, q10 + vand q8, q8, q5 + vorr q11, q11, q8 + vpadd.u32 d22, d22, d23 + vpadd.u32 d22, d22, d22 + vmov.32 r6, d22[0] + cmp r6, #0 + beq L_mldsa_arm32_make_hint_next +L_mldsa_arm32_make_hint_bit: + rbit r7, r6 + clz r7, r7 + add r7, r7, r5 + strb r7, [r2] + add r2, r2, #1 + add r3, r3, #1 + cmp r3, r12 + bgt L_mldsa_arm32_make_hint_fail + sub r7, r6, #1 + and r6, r6, r7 + cmp r6, #0 + bne L_mldsa_arm32_make_hint_bit +L_mldsa_arm32_make_hint_next: + add r5, r5, #16 + cmp r5, #0x100 + blt L_mldsa_arm32_make_hint_loop + mov r0, r3 + b L_mldsa_arm32_make_hint_done +L_mldsa_arm32_make_hint_fail: + mvn r0, #0 +L_mldsa_arm32_make_hint_done: + vpop {d8-d15} + pop {r4, r5, r6, r7, pc} + .size mldsa_make_hint_neon,.-mldsa_make_hint_neon +#ifndef __APPLE__ + .text + .type L_mldsa_rej_uniform_neon_aux, %object + .size L_mldsa_rej_uniform_neon_aux, 48 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_rej_uniform_neon_aux: + .long 0x007fffff,0x007fffff,0x007fffff,0x007fffff + .long 0x007fe001,0x007fe001,0x007fe001,0x007fe001 + .long 0x00000001,0x00000002,0x00000004,0x00000008 +#ifndef __APPLE__ + .text + .type L_mldsa_rej_uniform_neon_shuffle, %object + .size L_mldsa_rej_uniform_neon_shuffle, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_rej_uniform_neon_shuffle: + .byte 0x00,0x01,0x02,0xff,0x03,0x04,0x05,0xff + .byte 0x06,0x07,0x08,0xff,0x09,0x0a,0x0b,0xff + .byte 0x0c,0x0d,0x0e,0xff,0x0f,0x10,0x11,0xff + .byte 0x12,0x13,0x14,0xff,0x15,0x16,0x17,0xff +#ifndef __APPLE__ + .text + .type L_mldsa_rej_uniform_neon_indices, %object + .size L_mldsa_rej_uniform_neon_indices, 256 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 4-byte aligned, 32-bit aligned +#ifndef __APPLE__ + .align 2 +#else + .p2align 2 +#endif /* __APPLE__ */ +L_mldsa_rej_uniform_neon_indices: + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x08,0x09,0x0a,0x0b,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x08,0x09,0x0a,0x0b + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x08,0x09,0x0a,0x0b,0xff,0xff,0xff,0xff + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x08,0x09,0x0a,0x0b + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f + .text + .align 4 + .globl mldsa_rej_uniform_neon + WC_ASM_ATT_HIDDEN(mldsa_rej_uniform_neon) + .type mldsa_rej_uniform_neon, %function +mldsa_rej_uniform_neon: + push {r4, r5, r6, r7, r8, lr} + vpush {d8-d15} + adr r12, L_mldsa_rej_uniform_neon_aux + vld1.8 {q4}, [r12]! + vld1.8 {q5}, [r12]! + vld1.8 {q6}, [r12] + adr r12, L_mldsa_rej_uniform_neon_shuffle + vld1.8 {d3-d6}, [r12] + adr lr, L_mldsa_rej_uniform_neon_indices + vmov.32 r7, d10[0] + mov r4, #0 + cmp r1, #0 + beq L_mldsa_rej_uniform_neon_done + cmp r1, #16 + blt L_mldsa_rej_uniform_neon_loop_2 +L_mldsa_rej_uniform_neon_loop_8: + vld1.8 {d0-d1}, [r2]! + vld1.8 {d2}, [r2]! + vtbl.8 d16, {d0-d2}, d3 + vtbl.8 d17, {d0-d2}, d4 + vand q7, q8, q4 + vcgt.s32 q8, q5, q7 + vshr.u32 q9, q8, #31 + vpadd.u32 d18, d18, d19 + vpadd.u32 d18, d18, d18 + vmov.32 r5, d18[0] + vand q9, q8, q6 + vpadd.u32 d18, d18, d19 + vpadd.u32 d18, d18, d18 + vmov.32 r6, d18[0] + lsl r6, r6, #4 + add r6, r6, lr + vld1.8 {q10}, [r6] + vtbl.8 d16, {d14-d15}, d20 + vtbl.8 d17, {d14-d15}, d21 + vst1.8 {q8}, [r0] + add r0, r0, r5, lsl #2 + add r4, r4, r5 + vtbl.8 d16, {d0-d2}, d5 + vtbl.8 d17, {d0-d2}, d6 + vand q7, q8, q4 + vcgt.s32 q8, q5, q7 + vshr.u32 q9, q8, #31 + vpadd.u32 d18, d18, d19 + vpadd.u32 d18, d18, d18 + vmov.32 r5, d18[0] + vand q9, q8, q6 + vpadd.u32 d18, d18, d19 + vpadd.u32 d18, d18, d18 + vmov.32 r6, d18[0] + lsl r6, r6, #4 + add r6, r6, lr + vld1.8 {q10}, [r6] + vtbl.8 d16, {d14-d15}, d20 + vtbl.8 d17, {d14-d15}, d21 + vst1.8 {q8}, [r0] + add r0, r0, r5, lsl #2 + add r4, r4, r5 + subs r3, r3, #24 + beq L_mldsa_rej_uniform_neon_done + sub r5, r1, r4 + subs r5, r5, #8 + blt L_mldsa_rej_uniform_neon_loop_2 + b L_mldsa_rej_uniform_neon_loop_8 +L_mldsa_rej_uniform_neon_loop_2: + subs r5, r1, r4 + beq L_mldsa_rej_uniform_neon_done + subs r5, r5, #2 + blt L_mldsa_rej_uniform_neon_loop_lt_2 + ldrb r12, [r2], #1 + ldrb r8, [r2], #1 + orr r12, r12, r8, lsl #8 + ldrb r8, [r2], #1 + orr r12, r12, r8, lsl #16 + bic r12, r12, #0x800000 + ldrb r6, [r2], #1 + ldrb r8, [r2], #1 + orr r6, r6, r8, lsl #8 + ldrb r8, [r2], #1 + orr r6, r6, r8, lsl #16 + bic r6, r6, #0x800000 + str r12, [r0] + add r5, r0, #4 + cmp r12, r7 + movlo r0, r5 + addlo r4, r4, #1 + str r6, [r0] + add r5, r0, #4 + cmp r6, r7 + movlo r0, r5 + addlo r4, r4, #1 + subs r3, r3, #6 + beq L_mldsa_rej_uniform_neon_done + b L_mldsa_rej_uniform_neon_loop_2 +L_mldsa_rej_uniform_neon_loop_lt_2: + ldrb r12, [r2], #1 + ldrb r8, [r2], #1 + orr r12, r12, r8, lsl #8 + ldrb r8, [r2], #1 + orr r12, r12, r8, lsl #16 + bic r12, r12, #0x800000 + ldrb r6, [r2], #1 + ldrb r8, [r2], #1 + orr r6, r6, r8, lsl #8 + ldrb r8, [r2], #1 + orr r6, r6, r8, lsl #16 + bic r6, r6, #0x800000 + str r12, [r0] + add r5, r0, #4 + cmp r12, r7 + movlo r0, r5 + addlo r4, r4, #1 + cmp r1, r4 + beq L_mldsa_rej_uniform_neon_done + str r6, [r0] + add r5, r0, #4 + cmp r6, r7 + movlo r0, r5 + addlo r4, r4, #1 + cmp r1, r4 + beq L_mldsa_rej_uniform_neon_done + subs r3, r3, #6 + beq L_mldsa_rej_uniform_neon_done + b L_mldsa_rej_uniform_neon_loop_lt_2 +L_mldsa_rej_uniform_neon_done: + mov r0, r4 + vpop {d8-d15} + pop {r4, r5, r6, r7, r8, pc} + .size mldsa_rej_uniform_neon,.-mldsa_rej_uniform_neon +#endif /* WOLFSSL_ARMASM_NO_NEON */ +#endif /* WOLFSSL_HAVE_MLDSA */ + +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif +#endif /* !WOLFSSL_ARMASM_INLINE */ +#endif /* !__aarch64__ && !WOLFSSL_ARMASM_THUMB2 */ +#endif /* WOLFSSL_ARMASM */ diff --git a/wolfcrypt/src/port/arm/armv8-32-mldsa-asm_c.c b/wolfcrypt/src/port/arm/armv8-32-mldsa-asm_c.c new file mode 100644 index 00000000000..b2555367297 --- /dev/null +++ b/wolfcrypt/src/port/arm/armv8-32-mldsa-asm_c.c @@ -0,0 +1,11087 @@ +/* armv8-32-mldsa-asm + * + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +/* Generated using (from wolfssl): + * cd ../scripts + * ruby ./mldsa/mldsa.rb arm32 \ + * ../wolfssl/wolfcrypt/src/port/arm/armv8-32-mldsa-asm.c + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_ARMV8_32_MLDSA_ASM_C + +#include +#include + +#ifdef WOLFSSL_ARMASM +#if !defined(__aarch64__) && !defined(WOLFSSL_ARMASM_THUMB2) +#ifdef WOLFSSL_ARMASM_INLINE + +#ifdef __IAR_SYSTEMS_ICC__ +#define __asm__ asm +#define __volatile__ volatile +#define WOLFSSL_NO_VAR_ASSIGN_REG +#endif /* __IAR_SYSTEMS_ICC__ */ +#ifdef __KEIL__ +#define __asm__ __asm +#define __volatile__ volatile +#endif /* __KEIL__ */ +#ifdef __ghs__ +#define __asm__ __asm +#define __volatile__ +#define WOLFSSL_NO_VAR_ASSIGN_REG +#endif /* __ghs__ */ + +#include + +#ifdef WOLFSSL_HAVE_MLDSA +#ifndef WOLFSSL_ARMASM_NO_NEON +XALIGNED(8) static const word32 L_mldsa_arm32_consts[] = { + 0x007fe001, 0x03802001, 0x00000000, 0x00000000, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_mul_neon(sword32* r_p, sword32* a_p, + sword32* b_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_mul_neon(sword32* r, sword32* a, sword32* b) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register sword32* r __asm__ ("r0") = (sword32*)r_p; + register sword32* a __asm__ ("r1") = (sword32*)a_p; + register sword32* b __asm__ ("r2") = (sword32*)b_p; + register word32* L_mldsa_arm32_consts_c __asm__ ("r3") = + (word32*)&L_mldsa_arm32_consts; +#else + register word32* L_mldsa_arm32_consts_c = (word32*)&L_mldsa_arm32_consts; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r3, %[L_mldsa_arm32_consts]\n\t" + "vld1.8 {q0}, [r3]\n\t" + "mov r12, #32\n\t" + "\n" + "L_mldsa_arm32_mul_loop_%=:\n\t" + "vld1.8 {q1-q2}, [%[a]]!\n\t" + "vld1.8 {q3-q4}, [%[b]]!\n\t" + "vmul.i32 q7, q3, d0[1]\n\t" + "vmul.i32 q8, q4, d0[1]\n\t" + "vmul.i32 q9, q1, q7\n\t" + "vmul.i32 q10, q2, q8\n\t" + "vqrdmulh.s32 q5, q1, q3\n\t" + "vqrdmulh.s32 q6, q2, q4\n\t" + "vqrdmulh.s32 q9, q9, d0[0]\n\t" + "vqrdmulh.s32 q10, q10, d0[0]\n\t" + "vsub.i32 q5, q5, q9\n\t" + "vsub.i32 q6, q6, q10\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vst1.8 {q5-q6}, [%[r]]!\n\t" + "subs r12, r12, #1\n\t" + "bne L_mldsa_arm32_mul_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [r] "+r" (r), [a] "+r" (a), [b] "+r" (b), + [L_mldsa_arm32_consts] "+r" (L_mldsa_arm32_consts_c) + : +#else + : + : [r] "r" (r), [a] "r" (a), [b] "r" (b), + [L_mldsa_arm32_consts] "r" (L_mldsa_arm32_consts_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10" + ); +} + +XALIGNED(8) static const word32 L_mldsa_arm32_ntt_zetas[] = { + 0xffff5c06, 0x000064f7, 0xffd83102, 0xfff81503, + 0x00039e44, 0xfff42118, 0xfff2a128, 0x00071e24, + 0x001bde2b, 0x0023e92b, 0xfffa84ad, 0xffe0147f, + 0x002f9a75, 0xffd3fb09, 0x002f7a49, 0x0028e527, + 0x00299658, 0x000fa070, 0xffef85a4, 0x0036b788, + 0xfff79d90, 0xffeeeaa0, 0x0027f968, 0xffdfd37b, + 0xffdfadd6, 0xffc51ae7, 0xffeaa4f7, 0xffcdfc98, + 0x001ad035, 0xffffb422, 0x003d3201, 0x000445c5, + 0x00294a67, 0x00017620, 0x002ef4cd, 0x0035dec5, + 0xffe6a503, 0xffc9302c, 0xffd947d4, 0x003bbeaf, + 0xffc51585, 0xffd18e7c, 0x00368a96, 0xffd43e41, + 0x00360400, 0xfffb6a4d, 0x0023d69c, 0xfff7c55d, + 0xffe6123d, 0xffe6ead6, 0x00357e1e, 0xffc5af59, + 0x0035843f, 0xffdf5617, 0xffe7945c, 0x0038738c, + 0x000c63a8, 0x00081b9a, 0x000e8f76, 0x003b3853, + 0x003b8534, 0xffd8fc30, 0x001f9d54, 0xffd54f2d, + 0xffc406e5, 0xffe8ac81, 0xffc7e1cf, 0xffd19819, + 0xffe9d65d, 0x003509ee, 0x002135c7, 0xffe7cfbb, + 0xffeccf75, 0x001d9772, 0xffc1b072, 0xfff0bcf6, + 0xffcf5280, 0xffcfd2ae, 0xffc890e0, 0x0001efca, + 0x003410f2, 0xfff0fe85, 0x0020c638, 0x00296e9f, + 0xffd2b7a3, 0xffc7a44b, 0xfff9ba6d, 0xffda3409, + 0xfff5c282, 0xffed4113, 0xffffa63b, 0xffec09f7, + 0xfffa2bdd, 0x001495d4, 0x001c4563, 0xffea2c62, + 0xffccfbe9, 0x00040af0, 0x0007c417, 0x002f4588, + 0x0000ad00, 0xffef36be, 0x000dcd44, 0x003c675a, + 0xffc72bca, 0xffffde7e, 0x00193948, 0xffce69c0, + 0x0024756c, 0xfffcc7df, 0x000b98a1, 0xffebe808, + 0x0002e46c, 0xffc9c808, 0x003036c2, 0xffe3bff6, + 0xffdb3c93, 0xfffd4ae0, 0x00141305, 0x00147792, + 0x00139e25, 0xffe7d0e0, 0xfff39944, 0xffea0802, + 0xffd1eea2, 0xffc4c79c, 0xffc8a057, 0x003a97d9, + 0x001fea93, 0x0033ff5a, 0x002358d4, 0x003a41f8, + 0xffccff72, 0x00223dfb, 0xffdaab9f, 0xffc9a422, + 0x000412f5, 0x00252587, 0xffed24f0, 0x00359b5d, + 0xffca48a0, 0xffc6a2fc, 0xffedbb56, 0xffcf45de, + 0x000dbe5e, 0x001c5e1a, 0x000de0e6, 0x000c7f5a, + 0x00078f83, 0xffe7628a, 0xffff5704, 0xfff806fc, + 0xfff60021, 0xffd05af6, 0x001f0084, 0x0030ef86, + 0xffc9b97d, 0xfff7fcd6, 0xfff44592, 0xffc921c2, + 0x00053919, 0x0004610c, 0xffdacd41, 0x003eb01b, + 0x003472e7, 0xffcd003b, 0x001a7cc7, 0x00031924, + 0x002b5ee5, 0x00291199, 0xffd87a3a, 0x00134d71, + 0x003de11c, 0x00130984, 0x0025f051, 0x00185a46, + 0xffc68518, 0x001314be, 0x00283891, 0xffc9db90, + 0xffd25089, 0x001c853f, 0x001d0b4b, 0xffeff6a6, + 0xffeba8be, 0x0012e11b, 0xffcd5e3e, 0xffea2d2f, + 0xfff91de4, 0x001406c7, 0x00327283, 0xffe20d6e, + 0xffec7953, 0x001d4099, 0xffd92578, 0xffeb05ad, + 0x0016e405, 0x000bdbe7, 0x00221de8, 0x0033f8cf, + 0xfff7b934, 0xffd4ca0c, 0xffe67ff8, 0xffe3d157, + 0xffd8911b, 0xffc72c12, 0x000910d8, 0xffc65e1f, + 0xffe14658, 0x00251d8b, 0x002573b7, 0xfffd7c8f, + 0x001ddd98, 0x00336898, 0x0002d4bb, 0xffed93a7, + 0xffcf6cbe, 0x00027c1c, 0x0018aa08, 0x002dfd71, + 0x000c5ca5, 0x0019379a, 0xffc7a167, 0xffe48c3d, + 0xffd1a13c, 0x0035c539, 0x003b0115, 0x00041dc0, + 0x0021c4f7, 0xfff11bf4, 0x001a35e7, 0x0007340e, + 0xfff97d45, 0x001a4cd0, 0xffe47cae, 0x001d2668, + 0xffe68e98, 0xffef2633, 0xfffc05da, 0xffc57fdb, + 0xffd32764, 0xffdde1af, 0xfff993dd, 0xffdd1d09, + 0x0002cc93, 0xfff11805, 0x00189c2a, 0xffc9e5a9, + 0xfff78a50, 0x003bcf2c, 0xffff434e, 0xffeb36df, + 0x003c15ca, 0x00155e68, 0xfff316b6, 0x001e29ce, +}; + +XALIGNED(8) static const word32 L_mldsa_arm32_ntt_consts[] = { + 0x007fe001, 0x03802001, 0x00000000, 0x00000000, +}; + +static const word32* L_mldsa_arm32_ntt_zetas_p = L_mldsa_arm32_ntt_zetas; +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_ntt_neon(sword32* r_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_ntt_neon(sword32* r) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register sword32* r __asm__ ("r0") = (sword32*)r_p; + register word32* L_mldsa_arm32_ntt_consts_c __asm__ ("r1") = + (word32*)&L_mldsa_arm32_ntt_consts; + register word32* L_mldsa_arm32_ntt_zetas_p_c __asm__ ("r2") = + (word32*)L_mldsa_arm32_ntt_zetas_p; +#else + register word32* L_mldsa_arm32_ntt_consts_c = + (word32*)&L_mldsa_arm32_ntt_consts; + register word32* L_mldsa_arm32_ntt_zetas_p_c = + (word32*)L_mldsa_arm32_ntt_zetas_p; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r1, %[L_mldsa_arm32_ntt_consts]\n\t" + "vld1.8 {q0}, [r1]\n\t" + "vdup.32 q1, d0[1]\n\t" + "mov r2, %[L_mldsa_arm32_ntt_zetas_p]\n\t" + "add r2, r2, #4\n\t" + /* len = 128 */ + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0\n\t" + "add r12, r3, #0x200\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + /* len = 64 */ + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0\n\t" + "add r12, r3, #0x100\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x200\n\t" + "add r12, r3, #0x100\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + /* len = 32 */ + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0\n\t" + "add r12, r3, #0x80\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x100\n\t" + "add r12, r3, #0x80\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x200\n\t" + "add r12, r3, #0x80\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x300\n\t" + "add r12, r3, #0x80\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + /* len = 16 */ + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x80\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x100\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x180\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x200\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x280\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x300\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x380\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + /* len = 8 */ + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x40\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x80\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0xc0\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x100\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x140\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x180\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x1c0\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x200\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x240\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x280\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x2c0\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x300\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x340\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x380\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x3c0\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + /* len = 4 */ + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #32\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x40\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x60\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x80\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0xa0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0xc0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0xe0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x100\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x120\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x140\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x160\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x180\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x1a0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x1c0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x1e0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x200\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x220\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x240\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x260\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x280\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x2a0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x2c0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x2e0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x300\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x320\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x340\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x360\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x380\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x3a0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x3c0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x3e0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + /* len = 2 */ + "add r3, %[r], #0\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 d14, d9, d6\n\t" + "vqrdmulh.s32 d12, d9, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d12, d12, d14\n\t" + "vshr.s32 d12, d12, #1\n\t" + "vsub.i32 d9, d8, d12\n\t" + "vadd.i32 d8, d8, d12\n\t" + "vst1.8 {q4}, [r3]!\n\t" + /* len = 1 */ + "add r3, %[r], #0\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vmul.i32 q7, q5, q3\n\t" + "vqrdmulh.s32 q6, q5, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q6, q6, q7\n\t" + "vshr.s32 q6, q6, #1\n\t" + "vsub.i32 q5, q4, q6\n\t" + "vadd.i32 q4, q4, q6\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [r] "+r" (r), + [L_mldsa_arm32_ntt_consts] "+r" (L_mldsa_arm32_ntt_consts_c), + [L_mldsa_arm32_ntt_zetas_p] "+r" (L_mldsa_arm32_ntt_zetas_p_c) + : +#else + : + : [r] "r" (r), + [L_mldsa_arm32_ntt_consts] "r" (L_mldsa_arm32_ntt_consts_c), + [L_mldsa_arm32_ntt_zetas_p] "r" (L_mldsa_arm32_ntt_zetas_p_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r3", "r12", "q0", "q1", "q2", "q3", "q4", "q5", "q6", + "q7" + ); +} + +XALIGNED(8) static const word32 L_mldsa_arm32_invntt_zetas[] = { + 0xffe1d632, 0x000ce94a, 0xffeaa198, 0xffc3ea36, + 0x0014c921, 0x0000bcb2, 0xffc430d4, 0x000875b0, + 0x00361a57, 0xffe763d6, 0x000ee7fb, 0xfffd336d, + 0x0022e2f7, 0x00066c23, 0x00221e51, 0x002cd89c, + 0x003a8025, 0x0003fa26, 0x0010d9cd, 0x00197168, + 0xffe2d998, 0x001b8352, 0xffe5b330, 0x000682bb, + 0xfff8cbf2, 0xffe5ca19, 0x000ee40c, 0xffde3b09, + 0xfffbe240, 0xffc4feeb, 0xffca3ac7, 0x002e5ec4, + 0x001b73c3, 0x00385e99, 0xffe6c866, 0xfff3a35b, + 0xffd2028f, 0xffe755f8, 0xfffd83e4, 0x00309342, + 0x00126c59, 0xfffd2b45, 0xffcc9768, 0xffe22268, + 0x00028371, 0xffda8c49, 0xffdae275, 0x001eb9a8, + 0x0039a1e1, 0xfff6ef28, 0x0038d3ee, 0x00276ee5, + 0x001c2ea9, 0x00198008, 0x002b35f4, 0x000846cc, + 0xffcc0731, 0xffdde218, 0xfff42419, 0xffe91bfb, + 0x0014fa53, 0x0026da88, 0xffe2bf67, 0x001386ad, + 0x001df292, 0xffcd8d7d, 0xffebf939, 0x0006e21c, + 0x0015d2d1, 0x0032a1c2, 0xffed1ee5, 0x00145742, + 0x0010095a, 0xffe2f4b5, 0xffe37ac1, 0x002daf77, + 0x00362470, 0xffd7c76f, 0xffeceb42, 0x00397ae8, + 0xffe7a5ba, 0xffda0faf, 0xffecf67c, 0xffc21ee4, + 0xffecb28f, 0x002785c6, 0xffd6ee67, 0xffd4a11b, + 0xfffce6dc, 0xffe58339, 0x0032ffc5, 0xffcb8d19, + 0xffc14fe5, 0x002532bf, 0xfffb9ef4, 0xfffac6e7, + 0x0036de3e, 0x000bba6e, 0x0008032a, 0x00364683, + 0xffcf107a, 0xffe0ff7c, 0x002fa50a, 0x0009ffdf, + 0x0007f904, 0x0000a8fc, 0x00189d76, 0xfff8707d, + 0xfff380a6, 0xfff21f1a, 0xffe3a1e6, 0xfff241a2, + 0x0030ba22, 0x001244aa, 0x00395d04, 0x0035b760, + 0xffca64a3, 0x0012db10, 0xffdada79, 0xfffbed0b, + 0x00365bde, 0x00255461, 0xffddc205, 0x0033008e, + 0xffc5be08, 0xffdca72c, 0xffcc00a6, 0xffe0156d, + 0xffc56827, 0x00375fa9, 0x003b3864, 0x002e115e, + 0x0015f7fe, 0x000c66bc, 0x00182f20, 0xffec61db, + 0xffeb886e, 0xffebecfb, 0x0002b520, 0x0024c36d, + 0x001c400a, 0xffcfc93e, 0x003637f8, 0xfffd1b94, + 0x001417f8, 0xfff4675f, 0x00033821, 0xffdb8a94, + 0x00319640, 0xffe6c6b8, 0x00002182, 0x0038d436, + 0xffc398a6, 0xfff232bc, 0x0010c942, 0xffff5300, + 0xffd0ba78, 0xfff83be9, 0xfffbf510, 0x00330417, + 0x0015d39e, 0xffe3ba9d, 0xffeb6a2c, 0x0005d423, + 0x0013f609, 0x000059c5, 0x0012beed, 0x000a3d7e, + 0x0025cbf7, 0x00064593, 0x00385bb5, 0x002d485d, + 0xffd69161, 0xffdf39c8, 0x000f017b, 0xffcbef0e, + 0xfffe1036, 0x00376f20, 0x00302d52, 0x0030ad80, + 0x000f430a, 0x003e4f8e, 0xffe2688e, 0x0013308b, + 0x00183045, 0xffdeca39, 0xffcaf612, 0x001629a3, + 0x002e67e7, 0x00381e31, 0x0017537f, 0x003bf91b, + 0x002ab0d3, 0xffe062ac, 0x002703d0, 0xffc47acc, + 0xffc4c7ad, 0xfff1708a, 0xfff7e466, 0xfff39c58, + 0xffc78c74, 0x00186ba4, 0x0020a9e9, 0xffca7bc1, + 0x003a50a7, 0xffca81e2, 0x0019152a, 0x0019edc3, + 0x00083aa3, 0xffdc2964, 0x000495b3, 0xffc9fc00, + 0x002bc1bf, 0xffc9756a, 0x002e7184, 0x003aea7b, + 0xffc44151, 0x0026b82c, 0x0036cfd4, 0x00195afd, + 0xffca213b, 0xffd10b33, 0xfffe89e0, 0xffd6b599, + 0xfffbba3b, 0xffc2cdff, 0x00004bde, 0xffe52fcb, + 0x00320368, 0x00155b09, 0x003ae519, 0x0020522a, + 0x00202c85, 0xffd80698, 0x00111560, 0x00086270, + 0xffc94878, 0x00107a5c, 0xfff05f90, 0xffd669a8, + 0xffd71ad9, 0xffd085b7, 0x002c04f7, 0xffd0658b, + 0x001feb81, 0x00057b53, 0xffdc16d5, 0xffe421d5, + 0xfff8e1dc, 0x000d5ed8, 0x000bdee8, 0xfffc61bc, + 0x0007eafd, 0x0027cefe, 0xffff9b09, 0x0000a3fa, +}; + +XALIGNED(8) static const word32 L_mldsa_arm32_invntt_consts[] = { + 0x007fe001, 0x03802001, 0x00000000, 0x00000000, +}; + +static const word32* L_mldsa_arm32_invntt_zetas_p = L_mldsa_arm32_invntt_zetas; +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_invntt_neon(sword32* r_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_invntt_neon(sword32* r) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register sword32* r __asm__ ("r0") = (sword32*)r_p; + register word32* L_mldsa_arm32_invntt_consts_c __asm__ ("r1") = + (word32*)&L_mldsa_arm32_invntt_consts; + register word32* L_mldsa_arm32_invntt_zetas_p_c __asm__ ("r2") = + (word32*)L_mldsa_arm32_invntt_zetas_p; +#else + register word32* L_mldsa_arm32_invntt_consts_c = + (word32*)&L_mldsa_arm32_invntt_consts; + register word32* L_mldsa_arm32_invntt_zetas_p_c = + (word32*)L_mldsa_arm32_invntt_zetas_p; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r1, %[L_mldsa_arm32_invntt_consts]\n\t" + "vld1.8 {q0}, [r1]\n\t" + "vdup.32 q1, d0[1]\n\t" + "mov r2, %[L_mldsa_arm32_invntt_zetas_p]\n\t" + /* len = 1 */ + "add r3, %[r], #0\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "add r3, r3, #16\n\t" + "vld1.8 {q5}, [r3]\n\t" + "sub r3, r3, #16\n\t" + "vuzp.32 q4, q5\n\t" + "vld1.8 {q2}, [r2]!\n\t" + "vmul.i32 q3, q2, q1\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vzip.32 q4, q5\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r3]!\n\t" + /* len = 2 */ + "add r3, %[r], #0\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 d4, r1\n\t" + "vmul.i32 d6, d4, d2\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vsub.i32 d12, d8, d9\n\t" + "vadd.i32 d8, d8, d9\n\t" + "vmul.i32 d14, d12, d6\n\t" + "vqrdmulh.s32 d9, d12, d4\n\t" + "vqrdmulh.s32 d14, d14, d0[0]\n\t" + "vsub.i32 d9, d9, d14\n\t" + "vshr.s32 d9, d9, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + /* len = 4 */ + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #32\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x40\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x60\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x80\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0xa0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0xc0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0xe0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x100\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x120\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x140\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x160\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x180\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x1a0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x1c0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x1e0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x200\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x220\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x240\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x260\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x280\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x2a0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x2c0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x2e0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x300\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x320\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x340\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x360\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x380\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x3a0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x3c0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x3e0\n\t" + "add r12, r3, #16\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + /* len = 8 */ + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x40\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x80\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0xc0\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x100\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x140\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x180\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x1c0\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x200\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x240\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x280\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x2c0\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x300\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x340\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x380\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x3c0\n\t" + "add r12, r3, #32\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + /* len = 16 */ + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x80\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x100\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x180\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x200\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x280\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x300\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x380\n\t" + "add r12, r3, #0x40\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + /* len = 32 */ + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0\n\t" + "add r12, r3, #0x80\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x100\n\t" + "add r12, r3, #0x80\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x200\n\t" + "add r12, r3, #0x80\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x300\n\t" + "add r12, r3, #0x80\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + /* len = 64 */ + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0\n\t" + "add r12, r3, #0x100\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0x200\n\t" + "add r12, r3, #0x100\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + /* len = 128 */ + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0\n\t" + "add r12, r3, #0x200\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vld1.8 {q5}, [r12]\n\t" + "vsub.i32 q6, q4, q5\n\t" + "vadd.i32 q4, q4, q5\n\t" + "vmul.i32 q7, q6, q3\n\t" + "vqrdmulh.s32 q5, q6, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q4}, [r3]!\n\t" + "vst1.8 {q5}, [r12]!\n\t" + /* scale */ + "ldr r1, [r2], #4\n\t" + "vdup.32 q2, r1\n\t" + "vmul.i32 q3, q2, q1\n\t" + "add r3, %[r], #0\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" + "vld1.8 {q4}, [r3]\n\t" + "vmul.i32 q7, q4, q3\n\t" + "vqrdmulh.s32 q5, q4, q2\n\t" + "vqrdmulh.s32 q7, q7, d0[0]\n\t" + "vsub.i32 q5, q5, q7\n\t" + "vshr.s32 q5, q5, #1\n\t" + "vst1.8 {q5}, [r3]!\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [r] "+r" (r), + [L_mldsa_arm32_invntt_consts] "+r" (L_mldsa_arm32_invntt_consts_c), + [L_mldsa_arm32_invntt_zetas_p] "+r" (L_mldsa_arm32_invntt_zetas_p_c) + : +#else + : + : [r] "r" (r), + [L_mldsa_arm32_invntt_consts] "r" (L_mldsa_arm32_invntt_consts_c), + [L_mldsa_arm32_invntt_zetas_p] "r" (L_mldsa_arm32_invntt_zetas_p_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r3", "r12", "q0", "q1", "q2", "q3", "q4", "q5", "q6", + "q7" + ); +} + +XALIGNED(8) static const word32 L_mldsa_arm32_q88_consts_decompose[] = { + 0x0000002c, 0x0000002c, 0x0000002c, 0x0000002c, + 0x003fefd4, 0x003fefd4, 0x003fefd4, 0x003fefd4, + 0x0002e800, 0x0002e800, 0x0002e800, 0x0002e800, + 0x00017400, 0x00017400, 0x00017400, 0x00017400, + 0x0000002b, 0x0000002b, 0x0000002b, 0x0000002b, + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_decompose_q88_neon(const sword32* r_p, + sword32* r0_p, sword32* r1_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_decompose_q88_neon(const sword32* r, + sword32* r0, sword32* r1) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const sword32* r __asm__ ("r0") = (const sword32*)r_p; + register sword32* r0 __asm__ ("r1") = (sword32*)r0_p; + register sword32* r1 __asm__ ("r2") = (sword32*)r1_p; + register word32* L_mldsa_arm32_q88_consts_decompose_c __asm__ ("r3") = + (word32*)&L_mldsa_arm32_q88_consts_decompose; +#else + register word32* L_mldsa_arm32_q88_consts_decompose_c = + (word32*)&L_mldsa_arm32_q88_consts_decompose; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r3, %[L_mldsa_arm32_q88_consts_decompose]\n\t" + "vld1.8 {q0}, [r3]!\n\t" + "vld1.8 {q1}, [r3]!\n\t" + "vld1.8 {q2}, [r3]!\n\t" + "vld1.8 {q3}, [r3]!\n\t" + "vld1.8 {q4}, [r3]!\n\t" + "mov r12, #32\n\t" + "\n" + "L_mldsa_decompose_q88_neon_loop_%=:\n\t" + "vld1.8 {q5}, [%[r]]!\n\t" + "vld1.8 {q6}, [%[r]]!\n\t" + "vmov q9, q1\n\t" + "vmla.i32 q9, q5, q0\n\t" + "vshr.s32 q9, q9, #23\n\t" + "vmov q7, q5\n\t" + "vmls.i32 q7, q9, q2\n\t" + "vsub.i32 q11, q3, q7\n\t" + "vsra.u32 q9, q11, #31\n\t" + "vmov q7, q5\n\t" + "vmls.i32 q7, q9, q2\n\t" + "vsub.i32 q11, q4, q9\n\t" + "vshr.u32 q11, q11, #31\n\t" + "vsub.i32 q7, q7, q11\n\t" + "vsub.i32 q11, q9, q0\n\t" + "vshr.u32 q11, q11, #31\n\t" + "vneg.s32 q11, q11\n\t" + "vand q9, q9, q11\n\t" + "vmov q10, q1\n\t" + "vmla.i32 q10, q6, q0\n\t" + "vshr.s32 q10, q10, #23\n\t" + "vmov q8, q6\n\t" + "vmls.i32 q8, q10, q2\n\t" + "vsub.i32 q12, q3, q8\n\t" + "vsra.u32 q10, q12, #31\n\t" + "vmov q8, q6\n\t" + "vmls.i32 q8, q10, q2\n\t" + "vsub.i32 q12, q4, q10\n\t" + "vshr.u32 q12, q12, #31\n\t" + "vsub.i32 q8, q8, q12\n\t" + "vsub.i32 q12, q10, q0\n\t" + "vshr.u32 q12, q12, #31\n\t" + "vneg.s32 q12, q12\n\t" + "vand q10, q10, q12\n\t" + "vst1.8 {q7}, [%[r0]]!\n\t" + "vst1.8 {q9}, [%[r1]]!\n\t" + "vst1.8 {q8}, [%[r0]]!\n\t" + "vst1.8 {q10}, [%[r1]]!\n\t" + "subs r12, r12, #1\n\t" + "bne L_mldsa_decompose_q88_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [r] "+r" (r), [r0] "+r" (r0), [r1] "+r" (r1), + [L_mldsa_arm32_q88_consts_decompose] "+r" (L_mldsa_arm32_q88_consts_decompose_c) + : +#else + : + : [r] "r" (r), [r0] "r" (r0), [r1] "r" (r1), + [L_mldsa_arm32_q88_consts_decompose] "r" (L_mldsa_arm32_q88_consts_decompose_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11", "q12" + ); +} + +XALIGNED(8) static const word32 L_mldsa_arm32_q88_consts_high[] = { + 0x0000002c, 0x0000002c, 0x0000002c, 0x0000002c, + 0x003fefd4, 0x003fefd4, 0x003fefd4, 0x003fefd4, + 0x0002e800, 0x0002e800, 0x0002e800, 0x0002e800, + 0x00017400, 0x00017400, 0x00017400, 0x00017400, + 0x0000002b, 0x0000002b, 0x0000002b, 0x0000002b, + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_use_hint_q88_neon(sword32* w1_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_use_hint_q88_neon(sword32* w1) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register sword32* w1 __asm__ ("r0") = (sword32*)w1_p; + register word32* L_mldsa_arm32_q88_consts_high_c __asm__ ("r1") = + (word32*)&L_mldsa_arm32_q88_consts_high; +#else + register word32* L_mldsa_arm32_q88_consts_high_c = + (word32*)&L_mldsa_arm32_q88_consts_high; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r1, %[L_mldsa_arm32_q88_consts_high]\n\t" + "vld1.8 {q0}, [r1]!\n\t" + "vld1.8 {q1}, [r1]!\n\t" + "vld1.8 {q2}, [r1]!\n\t" + "vld1.8 {q3}, [r1]!\n\t" + "vld1.8 {q4}, [r1]!\n\t" + "vld1.8 {q5}, [r1]!\n\t" + "mov r2, #32\n\t" + "\n" + "L_mldsa_use_hint_q88_neon_loop_%=:\n\t" + "vld1.8 {q6}, [%[w1]]!\n\t" + "vld1.8 {q7}, [%[w1]]!\n\t" + "vshr.s32 q12, q6, #31\n\t" + "vand q12, q12, q5\n\t" + "vadd.i32 q6, q6, q12\n\t" + "vshr.s32 q13, q7, #31\n\t" + "vand q13, q13, q5\n\t" + "vadd.i32 q7, q7, q13\n\t" + "vmov q10, q1\n\t" + "vmla.i32 q10, q6, q0\n\t" + "vshr.s32 q10, q10, #23\n\t" + "vmov q8, q6\n\t" + "vmls.i32 q8, q10, q2\n\t" + "vsub.i32 q12, q3, q8\n\t" + "vsra.u32 q10, q12, #31\n\t" + "vmov q8, q6\n\t" + "vmls.i32 q8, q10, q2\n\t" + "vsub.i32 q12, q4, q10\n\t" + "vshr.u32 q12, q12, #31\n\t" + "vsub.i32 q8, q8, q12\n\t" + "vsub.i32 q12, q10, q0\n\t" + "vshr.u32 q12, q12, #31\n\t" + "vneg.s32 q12, q12\n\t" + "vand q10, q10, q12\n\t" + "vmov q11, q1\n\t" + "vmla.i32 q11, q7, q0\n\t" + "vshr.s32 q11, q11, #23\n\t" + "vmov q9, q7\n\t" + "vmls.i32 q9, q11, q2\n\t" + "vsub.i32 q13, q3, q9\n\t" + "vsra.u32 q11, q13, #31\n\t" + "vmov q9, q7\n\t" + "vmls.i32 q9, q11, q2\n\t" + "vsub.i32 q13, q4, q11\n\t" + "vshr.u32 q13, q13, #31\n\t" + "vsub.i32 q9, q9, q13\n\t" + "vsub.i32 q13, q11, q0\n\t" + "vshr.u32 q13, q13, #31\n\t" + "vneg.s32 q13, q13\n\t" + "vand q11, q11, q13\n\t" + "sub %[w1], %[w1], #32\n\t" + "vst1.8 {q10}, [%[w1]]!\n\t" + "vst1.8 {q11}, [%[w1]]!\n\t" + "subs r2, r2, #1\n\t" + "bne L_mldsa_use_hint_q88_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [w1] "+r" (w1), + [L_mldsa_arm32_q88_consts_high] "+r" (L_mldsa_arm32_q88_consts_high_c) + : +#else + : + : [w1] "r" (w1), + [L_mldsa_arm32_q88_consts_high] "r" (L_mldsa_arm32_q88_consts_high_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r2", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11", "q12", "q13" + ); +} + +XALIGNED(8) static const word32 L_mldsa_arm32_q32_consts_decompose[] = { + 0x0003feff, 0x0003feff, 0x0003feff, 0x0003feff, + 0x0003ff00, 0x0003ff00, 0x0003ff00, 0x0003ff00, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0007fe00, 0x0007fe00, 0x0007fe00, 0x0007fe00, + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_decompose_q32_neon(const sword32* r_p, + sword32* r0_p, sword32* r1_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_decompose_q32_neon(const sword32* r, + sword32* r0, sword32* r1) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const sword32* r __asm__ ("r0") = (const sword32*)r_p; + register sword32* r0 __asm__ ("r1") = (sword32*)r0_p; + register sword32* r1 __asm__ ("r2") = (sword32*)r1_p; + register word32* L_mldsa_arm32_q32_consts_decompose_c __asm__ ("r3") = + (word32*)&L_mldsa_arm32_q32_consts_decompose; +#else + register word32* L_mldsa_arm32_q32_consts_decompose_c = + (word32*)&L_mldsa_arm32_q32_consts_decompose; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r3, %[L_mldsa_arm32_q32_consts_decompose]\n\t" + "vld1.8 {q0}, [r3]!\n\t" + "vld1.8 {q1}, [r3]!\n\t" + "vld1.8 {q2}, [r3]!\n\t" + "vld1.8 {q3}, [r3]!\n\t" + "mov r12, #32\n\t" + "\n" + "L_mldsa_decompose_q32_neon_loop_%=:\n\t" + "vld1.8 {q4}, [%[r]]!\n\t" + "vld1.8 {q5}, [%[r]]!\n\t" + "vadd.i32 q8, q4, q0\n\t" + "vshr.s32 q8, q8, #19\n\t" + "vmov q6, q4\n\t" + "vmls.i32 q6, q8, q3\n\t" + "vsub.i32 q10, q1, q6\n\t" + "vsra.u32 q8, q10, #31\n\t" + "vmov q6, q4\n\t" + "vmls.i32 q6, q8, q3\n\t" + "vshr.s32 q10, q8, #4\n\t" + "vsub.i32 q6, q6, q10\n\t" + "vand q8, q8, q2\n\t" + "vadd.i32 q9, q5, q0\n\t" + "vshr.s32 q9, q9, #19\n\t" + "vmov q7, q5\n\t" + "vmls.i32 q7, q9, q3\n\t" + "vsub.i32 q11, q1, q7\n\t" + "vsra.u32 q9, q11, #31\n\t" + "vmov q7, q5\n\t" + "vmls.i32 q7, q9, q3\n\t" + "vshr.s32 q11, q9, #4\n\t" + "vsub.i32 q7, q7, q11\n\t" + "vand q9, q9, q2\n\t" + "vst1.8 {q6}, [%[r0]]!\n\t" + "vst1.8 {q8}, [%[r1]]!\n\t" + "vst1.8 {q7}, [%[r0]]!\n\t" + "vst1.8 {q9}, [%[r1]]!\n\t" + "subs r12, r12, #1\n\t" + "bne L_mldsa_decompose_q32_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [r] "+r" (r), [r0] "+r" (r0), [r1] "+r" (r1), + [L_mldsa_arm32_q32_consts_decompose] "+r" (L_mldsa_arm32_q32_consts_decompose_c) + : +#else + : + : [r] "r" (r), [r0] "r" (r0), [r1] "r" (r1), + [L_mldsa_arm32_q32_consts_decompose] "r" (L_mldsa_arm32_q32_consts_decompose_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11" + ); +} + +XALIGNED(8) static const word32 L_mldsa_arm32_q32_consts_high[] = { + 0x0003feff, 0x0003feff, 0x0003feff, 0x0003feff, + 0x0003ff00, 0x0003ff00, 0x0003ff00, 0x0003ff00, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0007fe00, 0x0007fe00, 0x0007fe00, 0x0007fe00, + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_use_hint_q32_neon(sword32* w1_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_use_hint_q32_neon(sword32* w1) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register sword32* w1 __asm__ ("r0") = (sword32*)w1_p; + register word32* L_mldsa_arm32_q32_consts_high_c __asm__ ("r1") = + (word32*)&L_mldsa_arm32_q32_consts_high; +#else + register word32* L_mldsa_arm32_q32_consts_high_c = + (word32*)&L_mldsa_arm32_q32_consts_high; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r1, %[L_mldsa_arm32_q32_consts_high]\n\t" + "vld1.8 {q0}, [r1]!\n\t" + "vld1.8 {q1}, [r1]!\n\t" + "vld1.8 {q2}, [r1]!\n\t" + "vld1.8 {q3}, [r1]!\n\t" + "vld1.8 {q4}, [r1]!\n\t" + "mov r2, #32\n\t" + "\n" + "L_mldsa_use_hint_q32_neon_loop_%=:\n\t" + "vld1.8 {q5}, [%[w1]]!\n\t" + "vld1.8 {q6}, [%[w1]]!\n\t" + "vshr.s32 q11, q5, #31\n\t" + "vand q11, q11, q4\n\t" + "vadd.i32 q5, q5, q11\n\t" + "vshr.s32 q12, q6, #31\n\t" + "vand q12, q12, q4\n\t" + "vadd.i32 q6, q6, q12\n\t" + "vadd.i32 q9, q5, q0\n\t" + "vshr.s32 q9, q9, #19\n\t" + "vmov q7, q5\n\t" + "vmls.i32 q7, q9, q3\n\t" + "vsub.i32 q11, q1, q7\n\t" + "vsra.u32 q9, q11, #31\n\t" + "vmov q7, q5\n\t" + "vmls.i32 q7, q9, q3\n\t" + "vshr.s32 q11, q9, #4\n\t" + "vsub.i32 q7, q7, q11\n\t" + "vand q9, q9, q2\n\t" + "vadd.i32 q10, q6, q0\n\t" + "vshr.s32 q10, q10, #19\n\t" + "vmov q8, q6\n\t" + "vmls.i32 q8, q10, q3\n\t" + "vsub.i32 q12, q1, q8\n\t" + "vsra.u32 q10, q12, #31\n\t" + "vmov q8, q6\n\t" + "vmls.i32 q8, q10, q3\n\t" + "vshr.s32 q12, q10, #4\n\t" + "vsub.i32 q8, q8, q12\n\t" + "vand q10, q10, q2\n\t" + "sub %[w1], %[w1], #32\n\t" + "vst1.8 {q9}, [%[w1]]!\n\t" + "vst1.8 {q10}, [%[w1]]!\n\t" + "subs r2, r2, #1\n\t" + "bne L_mldsa_use_hint_q32_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [w1] "+r" (w1), + [L_mldsa_arm32_q32_consts_high] "+r" (L_mldsa_arm32_q32_consts_high_c) + : +#else + : + : [w1] "r" (w1), + [L_mldsa_arm32_q32_consts_high] "r" (L_mldsa_arm32_q32_consts_high_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r2", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9", "q10", "q11", "q12" + ); +} + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_poly_add_neon(sword32* r_p, const sword32* a_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_poly_add_neon(sword32* r, const sword32* a) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register sword32* r __asm__ ("r0") = (sword32*)r_p; + register const sword32* a __asm__ ("r1") = (const sword32*)a_p; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r2, #16\n\t" + "\n" + "L_mldsa_poly_add_neon_loop_%=:\n\t" + "vld1.8 {q0}, [%[r]]!\n\t" + "vld1.8 {q4}, [%[a]]!\n\t" + "vld1.8 {q1}, [%[r]]!\n\t" + "vld1.8 {q5}, [%[a]]!\n\t" + "vld1.8 {q2}, [%[r]]!\n\t" + "vld1.8 {q6}, [%[a]]!\n\t" + "vld1.8 {q3}, [%[r]]!\n\t" + "vld1.8 {q7}, [%[a]]!\n\t" + "sub %[r], %[r], #0x40\n\t" + "vadd.i32 q0, q0, q4\n\t" + "vadd.i32 q1, q1, q5\n\t" + "vadd.i32 q2, q2, q6\n\t" + "vadd.i32 q3, q3, q7\n\t" + "vst1.8 {q0}, [%[r]]!\n\t" + "vst1.8 {q1}, [%[r]]!\n\t" + "vst1.8 {q2}, [%[r]]!\n\t" + "vst1.8 {q3}, [%[r]]!\n\t" + "subs r2, r2, #1\n\t" + "bne L_mldsa_poly_add_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [r] "+r" (r), [a] "+r" (a) + : +#else + : + : [r] "r" (r), [a] "r" (a) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r2", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7" + ); +} + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_poly_sub_neon(sword32* r_p, const sword32* a_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_poly_sub_neon(sword32* r, const sword32* a) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register sword32* r __asm__ ("r0") = (sword32*)r_p; + register const sword32* a __asm__ ("r1") = (const sword32*)a_p; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r2, #16\n\t" + "\n" + "L_mldsa_poly_sub_neon_loop_%=:\n\t" + "vld1.8 {q0}, [%[r]]!\n\t" + "vld1.8 {q4}, [%[a]]!\n\t" + "vld1.8 {q1}, [%[r]]!\n\t" + "vld1.8 {q5}, [%[a]]!\n\t" + "vld1.8 {q2}, [%[r]]!\n\t" + "vld1.8 {q6}, [%[a]]!\n\t" + "vld1.8 {q3}, [%[r]]!\n\t" + "vld1.8 {q7}, [%[a]]!\n\t" + "sub %[r], %[r], #0x40\n\t" + "vsub.i32 q0, q0, q4\n\t" + "vsub.i32 q1, q1, q5\n\t" + "vsub.i32 q2, q2, q6\n\t" + "vsub.i32 q3, q3, q7\n\t" + "vst1.8 {q0}, [%[r]]!\n\t" + "vst1.8 {q1}, [%[r]]!\n\t" + "vst1.8 {q2}, [%[r]]!\n\t" + "vst1.8 {q3}, [%[r]]!\n\t" + "subs r2, r2, #1\n\t" + "bne L_mldsa_poly_sub_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [r] "+r" (r), [a] "+r" (a) + : +#else + : + : [r] "r" (r), [a] "r" (a) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r2", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7" + ); +} + +XALIGNED(8) static const word32 L_mldsa_poly_make_pos_neon_q[] = { + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_poly_make_pos_neon(sword32* a_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_poly_make_pos_neon(sword32* a) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register sword32* a __asm__ ("r0") = (sword32*)a_p; + register word32* L_mldsa_poly_make_pos_neon_q_c __asm__ ("r1") = + (word32*)&L_mldsa_poly_make_pos_neon_q; +#else + register word32* L_mldsa_poly_make_pos_neon_q_c = + (word32*)&L_mldsa_poly_make_pos_neon_q; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r1, %[L_mldsa_poly_make_pos_neon_q]\n\t" + "vld1.8 {q0}, [r1]!\n\t" + "mov r2, #16\n\t" + "\n" + "L_mldsa_poly_make_pos_neon_loop_%=:\n\t" + "vld1.8 {q1}, [%[a]]!\n\t" + "vld1.8 {q2}, [%[a]]!\n\t" + "vld1.8 {q3}, [%[a]]!\n\t" + "vld1.8 {q4}, [%[a]]!\n\t" + "sub %[a], %[a], #0x40\n\t" + "vshr.s32 q5, q1, #31\n\t" + "vmls.i32 q1, q5, q0\n\t" + "vshr.s32 q6, q2, #31\n\t" + "vmls.i32 q2, q6, q0\n\t" + "vshr.s32 q7, q3, #31\n\t" + "vmls.i32 q3, q7, q0\n\t" + "vshr.s32 q8, q4, #31\n\t" + "vmls.i32 q4, q8, q0\n\t" + "vst1.8 {q1}, [%[a]]!\n\t" + "vst1.8 {q2}, [%[a]]!\n\t" + "vst1.8 {q3}, [%[a]]!\n\t" + "vst1.8 {q4}, [%[a]]!\n\t" + "subs r2, r2, #1\n\t" + "bne L_mldsa_poly_make_pos_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [a] "+r" (a), + [L_mldsa_poly_make_pos_neon_q] "+r" (L_mldsa_poly_make_pos_neon_q_c) + : +#else + : + : [a] "r" (a), + [L_mldsa_poly_make_pos_neon_q] "r" (L_mldsa_poly_make_pos_neon_q_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r2", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8" + ); +} + +XALIGNED(8) static const word32 L_mldsa_poly_red_neon_q[] = { + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, + 0x00400000, 0x00400000, 0x00400000, 0x00400000, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_poly_red_neon(sword32* a_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_poly_red_neon(sword32* a) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register sword32* a __asm__ ("r0") = (sword32*)a_p; + register word32* L_mldsa_poly_red_neon_q_c __asm__ ("r1") = + (word32*)&L_mldsa_poly_red_neon_q; +#else + register word32* L_mldsa_poly_red_neon_q_c = + (word32*)&L_mldsa_poly_red_neon_q; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r1, %[L_mldsa_poly_red_neon_q]\n\t" + "vld1.8 {q0}, [r1]!\n\t" + "vld1.8 {q1}, [r1]\n\t" + "mov r2, #16\n\t" + "\n" + "L_mldsa_poly_red_neon_loop_%=:\n\t" + "vld1.8 {q2}, [%[a]]!\n\t" + "vld1.8 {q3}, [%[a]]!\n\t" + "vld1.8 {q4}, [%[a]]!\n\t" + "vld1.8 {q5}, [%[a]]!\n\t" + "sub %[a], %[a], #0x40\n\t" + "vadd.i32 q6, q2, q1\n\t" + "vshr.s32 q6, q6, #23\n\t" + "vmls.i32 q2, q6, q0\n\t" + "vadd.i32 q7, q3, q1\n\t" + "vshr.s32 q7, q7, #23\n\t" + "vmls.i32 q3, q7, q0\n\t" + "vadd.i32 q8, q4, q1\n\t" + "vshr.s32 q8, q8, #23\n\t" + "vmls.i32 q4, q8, q0\n\t" + "vadd.i32 q9, q5, q1\n\t" + "vshr.s32 q9, q9, #23\n\t" + "vmls.i32 q5, q9, q0\n\t" + "vst1.8 {q2}, [%[a]]!\n\t" + "vst1.8 {q3}, [%[a]]!\n\t" + "vst1.8 {q4}, [%[a]]!\n\t" + "vst1.8 {q5}, [%[a]]!\n\t" + "subs r2, r2, #1\n\t" + "bne L_mldsa_poly_red_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [a] "+r" (a), + [L_mldsa_poly_red_neon_q] "+r" (L_mldsa_poly_red_neon_q_c) + : +#else + : + : [a] "r" (a), + [L_mldsa_poly_red_neon_q] "r" (L_mldsa_poly_red_neon_q_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r2", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q7", + "q8", "q9" + ); +} + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER int mldsa_vec_check_low_neon(const sword32* a_p, byte l_p, + sword32 hi_p) +#else +WC_OMIT_FRAME_POINTER int mldsa_vec_check_low_neon(const sword32* a, byte l, + sword32 hi) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const sword32* a __asm__ ("r0") = (const sword32*)a_p; + register byte l __asm__ ("r1") = (byte)l_p; + register sword32 hi __asm__ ("r2") = (sword32)hi_p; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "vdup.32 q0, %[hi]\n\t" + "veor q1, q1, q1\n\t" + "veor q2, q2, q2\n\t" + "veor q3, q3, q3\n\t" + "veor q4, q4, q4\n\t" + "\n" + "L_mldsa_arm32_check_low_outer_%=:\n\t" + "mov r3, #16\n\t" + "\n" + "L_mldsa_arm32_check_low_inner_%=:\n\t" + "vld1.8 {q5}, [%[a]]!\n\t" + "vld1.8 {q6}, [%[a]]!\n\t" + "vld1.8 {q7}, [%[a]]!\n\t" + "vld1.8 {q8}, [%[a]]!\n\t" + "vqabs.s32 q9, q5\n\t" + "vcge.s32 q9, q9, q0\n\t" + "vorr q1, q1, q9\n\t" + "vqabs.s32 q10, q6\n\t" + "vcge.s32 q10, q10, q0\n\t" + "vorr q2, q2, q10\n\t" + "vqabs.s32 q11, q7\n\t" + "vcge.s32 q11, q11, q0\n\t" + "vorr q3, q3, q11\n\t" + "vqabs.s32 q12, q8\n\t" + "vcge.s32 q12, q12, q0\n\t" + "vorr q4, q4, q12\n\t" + "subs r3, r3, #1\n\t" + "bne L_mldsa_arm32_check_low_inner_%=\n\t" + "subs %[l], %[l], #1\n\t" + "bne L_mldsa_arm32_check_low_outer_%=\n\t" + "vorr q1, q1, q2\n\t" + "vorr q3, q3, q4\n\t" + "vorr q1, q1, q3\n\t" + "vorr d2, d2, d3\n\t" + "vpmax.u32 d2, d2, d2\n\t" + "vmov.32 r12, d2[0]\n\t" + "cmp r12, #0\n\t" + "moveq r12, #1\n\t" + "movne r12, #0\n\t" + "mov r0, r12\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [a] "+r" (a), [l] "+r" (l), [hi] "+r" (hi) + : +#else + : + : [a] "r" (a), [l] "r" (l), [hi] "r" (hi) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r3", "r12", "q0", "q1", "q2", "q3", "q4", "q5", "q6", + "q7", "q8", "q9", "q10", "q11", "q12" + ); + return (word32)(size_t)a; +} + +XALIGNED(4) static const word8 L_mldsa_decode_eta_2_neon_idx[] = { + 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, + 0x00, 0x01, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff, + 0x01, 0xff, 0xff, 0xff, 0x01, 0x02, 0xff, 0xff, + 0x02, 0xff, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, + 0x01, 0xff, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff, + 0x01, 0x02, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, + 0x02, 0xff, 0xff, 0xff, 0x02, 0x03, 0xff, 0xff, + 0x03, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, + 0x04, 0xff, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff, + 0x04, 0x05, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff, + 0x05, 0xff, 0xff, 0xff, 0x05, 0x06, 0xff, 0xff, + 0x06, 0xff, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff, + 0x07, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, + 0x07, 0x08, 0xff, 0xff, 0x08, 0xff, 0xff, 0xff, + 0x08, 0xff, 0xff, 0xff, 0x08, 0x09, 0xff, 0xff, + 0x09, 0xff, 0xff, 0xff, 0x09, 0xff, 0xff, 0xff, + 0x0a, 0xff, 0xff, 0xff, 0x0a, 0xff, 0xff, 0xff, + 0x0a, 0x0b, 0xff, 0xff, 0x0b, 0xff, 0xff, 0xff, + 0x0b, 0xff, 0xff, 0xff, 0x0b, 0x0c, 0xff, 0xff, + 0x0c, 0xff, 0xff, 0xff, 0x0c, 0xff, 0xff, 0xff, + 0x0d, 0xff, 0xff, 0xff, 0x0d, 0xff, 0xff, 0xff, + 0x0d, 0x0e, 0xff, 0xff, 0x0e, 0xff, 0xff, 0xff, + 0x0e, 0xff, 0xff, 0xff, 0x0e, 0x0f, 0xff, 0xff, + 0x0f, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_eta_2_neon_aux[] = { + 0x00000000, 0xfffffffd, 0xfffffffa, 0xffffffff, + 0xfffffffc, 0xfffffff9, 0xfffffffe, 0xfffffffb, + 0x00000007, 0x00000007, 0x00000002, 0x00000002, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_decode_eta_2_neon(const byte* p_p, + sword32* z_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_decode_eta_2_neon(const byte* p, sword32* z) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* p __asm__ ("r0") = (const byte*)p_p; + register sword32* z __asm__ ("r1") = (sword32*)z_p; + register word8* L_mldsa_decode_eta_2_neon_idx_c __asm__ ("r2") = + (word8*)&L_mldsa_decode_eta_2_neon_idx; + register word32* L_mldsa_decode_eta_2_neon_aux_c __asm__ ("r3") = + (word32*)&L_mldsa_decode_eta_2_neon_aux; +#else + register word8* L_mldsa_decode_eta_2_neon_idx_c = + (word8*)&L_mldsa_decode_eta_2_neon_idx; + register word32* L_mldsa_decode_eta_2_neon_aux_c = + (word32*)&L_mldsa_decode_eta_2_neon_aux; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r2, %[L_mldsa_decode_eta_2_neon_idx]\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vld1.8 {d4}, [r2]!\n\t" + "vld1.8 {d5}, [r2]!\n\t" + "mov r3, %[L_mldsa_decode_eta_2_neon_aux]\n\t" + "vld1.8 {d6}, [r3]!\n\t" + "vld1.8 {d7}, [r3]!\n\t" + "vld1.8 {d8}, [r3]!\n\t" + "vld1.8 {d9}, [r3]!\n\t" + "vld1.8 {d14}, [r3]!\n\t" + "vld1.8 {d15}, [r3]\n\t" + "mov r12, #27\n\t" + "\n" + "L_mldsa_decode_eta_2_neon_loop_%=:\n\t" + "vld1.8 {d0-d1}, [%[p]]\n\t" + "vtbl.8 d10, {d0-d1}, d2\n\t" + "vshl.s32 d10, d10, d6\n\t" + "vand d10, d10, d14\n\t" + "vsub.i32 d10, d15, d10\n\t" + "vtbl.8 d11, {d0-d1}, d3\n\t" + "vshl.s32 d11, d11, d7\n\t" + "vand d11, d11, d14\n\t" + "vsub.i32 d11, d15, d11\n\t" + "vtbl.8 d12, {d0-d1}, d4\n\t" + "vshl.s32 d12, d12, d8\n\t" + "vand d12, d12, d14\n\t" + "vsub.i32 d12, d15, d12\n\t" + "vtbl.8 d13, {d0-d1}, d5\n\t" + "vshl.s32 d13, d13, d9\n\t" + "vand d13, d13, d14\n\t" + "vsub.i32 d13, d15, d13\n\t" + "vst1.8 {d10-d11}, [%[z]]!\n\t" + "vst1.8 {d12-d13}, [%[z]]!\n\t" + "add %[p], %[p], #3\n\t" + "subs r12, r12, #1\n\t" + "bne L_mldsa_decode_eta_2_neon_loop_%=\n\t" + "sub %[p], %[p], #1\n\t" + "vld1.8 {d0-d1}, [%[p]]\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vld1.8 {d4}, [r2]!\n\t" + "vld1.8 {d5}, [r2]!\n\t" + "vtbl.8 d10, {d0-d1}, d2\n\t" + "vshl.s32 d10, d10, d6\n\t" + "vand d10, d10, d14\n\t" + "vsub.i32 d10, d15, d10\n\t" + "vtbl.8 d11, {d0-d1}, d3\n\t" + "vshl.s32 d11, d11, d7\n\t" + "vand d11, d11, d14\n\t" + "vsub.i32 d11, d15, d11\n\t" + "vtbl.8 d12, {d0-d1}, d4\n\t" + "vshl.s32 d12, d12, d8\n\t" + "vand d12, d12, d14\n\t" + "vsub.i32 d12, d15, d12\n\t" + "vtbl.8 d13, {d0-d1}, d5\n\t" + "vshl.s32 d13, d13, d9\n\t" + "vand d13, d13, d14\n\t" + "vsub.i32 d13, d15, d13\n\t" + "vst1.8 {d10-d11}, [%[z]]!\n\t" + "vst1.8 {d12-d13}, [%[z]]!\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vld1.8 {d4}, [r2]!\n\t" + "vld1.8 {d5}, [r2]!\n\t" + "vtbl.8 d10, {d0-d1}, d2\n\t" + "vshl.s32 d10, d10, d6\n\t" + "vand d10, d10, d14\n\t" + "vsub.i32 d10, d15, d10\n\t" + "vtbl.8 d11, {d0-d1}, d3\n\t" + "vshl.s32 d11, d11, d7\n\t" + "vand d11, d11, d14\n\t" + "vsub.i32 d11, d15, d11\n\t" + "vtbl.8 d12, {d0-d1}, d4\n\t" + "vshl.s32 d12, d12, d8\n\t" + "vand d12, d12, d14\n\t" + "vsub.i32 d12, d15, d12\n\t" + "vtbl.8 d13, {d0-d1}, d5\n\t" + "vshl.s32 d13, d13, d9\n\t" + "vand d13, d13, d14\n\t" + "vsub.i32 d13, d15, d13\n\t" + "vst1.8 {d10-d11}, [%[z]]!\n\t" + "vst1.8 {d12-d13}, [%[z]]!\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vld1.8 {d4}, [r2]!\n\t" + "vld1.8 {d5}, [r2]!\n\t" + "vtbl.8 d10, {d0-d1}, d2\n\t" + "vshl.s32 d10, d10, d6\n\t" + "vand d10, d10, d14\n\t" + "vsub.i32 d10, d15, d10\n\t" + "vtbl.8 d11, {d0-d1}, d3\n\t" + "vshl.s32 d11, d11, d7\n\t" + "vand d11, d11, d14\n\t" + "vsub.i32 d11, d15, d11\n\t" + "vtbl.8 d12, {d0-d1}, d4\n\t" + "vshl.s32 d12, d12, d8\n\t" + "vand d12, d12, d14\n\t" + "vsub.i32 d12, d15, d12\n\t" + "vtbl.8 d13, {d0-d1}, d5\n\t" + "vshl.s32 d13, d13, d9\n\t" + "vand d13, d13, d14\n\t" + "vsub.i32 d13, d15, d13\n\t" + "vst1.8 {d10-d11}, [%[z]]!\n\t" + "vst1.8 {d12-d13}, [%[z]]!\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vld1.8 {d4}, [r2]!\n\t" + "vld1.8 {d5}, [r2]!\n\t" + "vtbl.8 d10, {d0-d1}, d2\n\t" + "vshl.s32 d10, d10, d6\n\t" + "vand d10, d10, d14\n\t" + "vsub.i32 d10, d15, d10\n\t" + "vtbl.8 d11, {d0-d1}, d3\n\t" + "vshl.s32 d11, d11, d7\n\t" + "vand d11, d11, d14\n\t" + "vsub.i32 d11, d15, d11\n\t" + "vtbl.8 d12, {d0-d1}, d4\n\t" + "vshl.s32 d12, d12, d8\n\t" + "vand d12, d12, d14\n\t" + "vsub.i32 d12, d15, d12\n\t" + "vtbl.8 d13, {d0-d1}, d5\n\t" + "vshl.s32 d13, d13, d9\n\t" + "vand d13, d13, d14\n\t" + "vsub.i32 d13, d15, d13\n\t" + "vst1.8 {d10-d11}, [%[z]]!\n\t" + "vst1.8 {d12-d13}, [%[z]]!\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vld1.8 {d4}, [r2]!\n\t" + "vld1.8 {d5}, [r2]!\n\t" + "vtbl.8 d10, {d0-d1}, d2\n\t" + "vshl.s32 d10, d10, d6\n\t" + "vand d10, d10, d14\n\t" + "vsub.i32 d10, d15, d10\n\t" + "vtbl.8 d11, {d0-d1}, d3\n\t" + "vshl.s32 d11, d11, d7\n\t" + "vand d11, d11, d14\n\t" + "vsub.i32 d11, d15, d11\n\t" + "vtbl.8 d12, {d0-d1}, d4\n\t" + "vshl.s32 d12, d12, d8\n\t" + "vand d12, d12, d14\n\t" + "vsub.i32 d12, d15, d12\n\t" + "vtbl.8 d13, {d0-d1}, d5\n\t" + "vshl.s32 d13, d13, d9\n\t" + "vand d13, d13, d14\n\t" + "vsub.i32 d13, d15, d13\n\t" + "vst1.8 {d10-d11}, [%[z]]!\n\t" + "vst1.8 {d12-d13}, [%[z]]!\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [p] "+r" (p), [z] "+r" (z), + [L_mldsa_decode_eta_2_neon_idx] "+r" (L_mldsa_decode_eta_2_neon_idx_c), + [L_mldsa_decode_eta_2_neon_aux] "+r" (L_mldsa_decode_eta_2_neon_aux_c) + : +#else + : + : [p] "r" (p), [z] "r" (z), + [L_mldsa_decode_eta_2_neon_idx] "r" (L_mldsa_decode_eta_2_neon_idx_c), + [L_mldsa_decode_eta_2_neon_aux] "r" (L_mldsa_decode_eta_2_neon_aux_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", + "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15" + ); +} + +XALIGNED(4) static const word8 L_mldsa_decode_eta_4_neon_idx[] = { + 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, + 0x01, 0xff, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff, + 0x02, 0xff, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, + 0x03, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, + 0x04, 0xff, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff, + 0x05, 0xff, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff, + 0x06, 0xff, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff, + 0x07, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, + 0x08, 0xff, 0xff, 0xff, 0x08, 0xff, 0xff, 0xff, + 0x09, 0xff, 0xff, 0xff, 0x09, 0xff, 0xff, 0xff, + 0x0a, 0xff, 0xff, 0xff, 0x0a, 0xff, 0xff, 0xff, + 0x0b, 0xff, 0xff, 0xff, 0x0b, 0xff, 0xff, 0xff, + 0x0c, 0xff, 0xff, 0xff, 0x0c, 0xff, 0xff, 0xff, + 0x0d, 0xff, 0xff, 0xff, 0x0d, 0xff, 0xff, 0xff, + 0x0e, 0xff, 0xff, 0xff, 0x0e, 0xff, 0xff, 0xff, + 0x0f, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_eta_4_neon_aux[] = { + 0x00000000, 0xfffffffc, 0x00000000, 0xfffffffc, + 0x0000000f, 0x0000000f, 0x00000004, 0x00000004, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_decode_eta_4_neon(const byte* p_p, + sword32* z_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_decode_eta_4_neon(const byte* p, sword32* z) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* p __asm__ ("r0") = (const byte*)p_p; + register sword32* z __asm__ ("r1") = (sword32*)z_p; + register word8* L_mldsa_decode_eta_4_neon_idx_c __asm__ ("r2") = + (word8*)&L_mldsa_decode_eta_4_neon_idx; + register word32* L_mldsa_decode_eta_4_neon_aux_c __asm__ ("r3") = + (word32*)&L_mldsa_decode_eta_4_neon_aux; +#else + register word8* L_mldsa_decode_eta_4_neon_idx_c = + (word8*)&L_mldsa_decode_eta_4_neon_idx; + register word32* L_mldsa_decode_eta_4_neon_aux_c = + (word32*)&L_mldsa_decode_eta_4_neon_aux; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r2, %[L_mldsa_decode_eta_4_neon_idx]\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "mov r3, %[L_mldsa_decode_eta_4_neon_aux]\n\t" + "vld1.8 {d4}, [r3]!\n\t" + "vld1.8 {d5}, [r3]!\n\t" + "vld1.8 {d8}, [r3]!\n\t" + "vld1.8 {d9}, [r3]\n\t" + "mov r12, #57\n\t" + "\n" + "L_mldsa_decode_eta_4_neon_loop_%=:\n\t" + "vld1.8 {d0-d1}, [%[p]]\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vsub.i32 d6, d9, d6\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vsub.i32 d7, d9, d7\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" + "add %[p], %[p], #2\n\t" + "subs r12, r12, #1\n\t" + "bne L_mldsa_decode_eta_4_neon_loop_%=\n\t" + "sub %[p], %[p], #2\n\t" + "vld1.8 {d0-d1}, [%[p]]\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vsub.i32 d6, d9, d6\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vsub.i32 d7, d9, d7\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vsub.i32 d6, d9, d6\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vsub.i32 d7, d9, d7\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vsub.i32 d6, d9, d6\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vsub.i32 d7, d9, d7\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vsub.i32 d6, d9, d6\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vsub.i32 d7, d9, d7\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vsub.i32 d6, d9, d6\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vsub.i32 d7, d9, d7\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vsub.i32 d6, d9, d6\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vsub.i32 d7, d9, d7\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vsub.i32 d6, d9, d6\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vsub.i32 d7, d9, d7\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [p] "+r" (p), [z] "+r" (z), + [L_mldsa_decode_eta_4_neon_idx] "+r" (L_mldsa_decode_eta_4_neon_idx_c), + [L_mldsa_decode_eta_4_neon_aux] "+r" (L_mldsa_decode_eta_4_neon_aux_c) + : +#else + : + : [p] "r" (p), [z] "r" (z), + [L_mldsa_decode_eta_4_neon_idx] "r" (L_mldsa_decode_eta_4_neon_idx_c), + [L_mldsa_decode_eta_4_neon_aux] "r" (L_mldsa_decode_eta_4_neon_aux_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", + "d8", "d9" + ); +} + +XALIGNED(4) static const word8 L_mldsa_decode_t1_neon_idx[] = { + 0x00, 0x01, 0xff, 0xff, 0x01, 0x02, 0xff, 0xff, + 0x02, 0x03, 0xff, 0xff, 0x03, 0x04, 0xff, 0xff, + 0x01, 0x02, 0xff, 0xff, 0x02, 0x03, 0xff, 0xff, + 0x03, 0x04, 0xff, 0xff, 0x04, 0x05, 0xff, 0xff, + 0x06, 0x07, 0xff, 0xff, 0x07, 0x08, 0xff, 0xff, + 0x08, 0x09, 0xff, 0xff, 0x09, 0x0a, 0xff, 0xff, + 0x0b, 0x0c, 0xff, 0xff, 0x0c, 0x0d, 0xff, 0xff, + 0x0d, 0x0e, 0xff, 0xff, 0x0e, 0x0f, 0xff, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_t1_neon_aux[] = { + 0x00000000, 0xfffffffe, 0xfffffffc, 0xfffffffa, + 0x000003ff, 0x000003ff +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_decode_t1_neon(const byte* p_p, sword32* z_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_decode_t1_neon(const byte* p, sword32* z) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* p __asm__ ("r0") = (const byte*)p_p; + register sword32* z __asm__ ("r1") = (sword32*)z_p; + register word8* L_mldsa_decode_t1_neon_idx_c __asm__ ("r2") = + (word8*)&L_mldsa_decode_t1_neon_idx; + register word32* L_mldsa_decode_t1_neon_aux_c __asm__ ("r3") = + (word32*)&L_mldsa_decode_t1_neon_aux; +#else + register word8* L_mldsa_decode_t1_neon_idx_c = + (word8*)&L_mldsa_decode_t1_neon_idx; + register word32* L_mldsa_decode_t1_neon_aux_c = + (word32*)&L_mldsa_decode_t1_neon_aux; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r2, %[L_mldsa_decode_t1_neon_idx]\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "mov r3, %[L_mldsa_decode_t1_neon_aux]\n\t" + "vld1.8 {d4}, [r3]!\n\t" + "vld1.8 {d5}, [r3]!\n\t" + "vld1.8 {d8}, [r3]!\n\t" + "mov r12, #61\n\t" + "\n" + "L_mldsa_decode_t1_neon_loop_%=:\n\t" + "vld1.8 {d0-d1}, [%[p]]\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vshl.i32 d6, d6, #13\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vshl.i32 d7, d7, #13\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" + "add %[p], %[p], #5\n\t" + "subs r12, r12, #1\n\t" + "bne L_mldsa_decode_t1_neon_loop_%=\n\t" + "sub %[p], %[p], #1\n\t" + "vld1.8 {d0-d1}, [%[p]]\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vshl.i32 d6, d6, #13\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vshl.i32 d7, d7, #13\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vshl.i32 d6, d6, #13\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vshl.i32 d7, d7, #13\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vshl.i32 d6, d6, #13\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vshl.i32 d7, d7, #13\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [p] "+r" (p), [z] "+r" (z), + [L_mldsa_decode_t1_neon_idx] "+r" (L_mldsa_decode_t1_neon_idx_c), + [L_mldsa_decode_t1_neon_aux] "+r" (L_mldsa_decode_t1_neon_aux_c) + : +#else + : + : [p] "r" (p), [z] "r" (z), + [L_mldsa_decode_t1_neon_idx] "r" (L_mldsa_decode_t1_neon_idx_c), + [L_mldsa_decode_t1_neon_aux] "r" (L_mldsa_decode_t1_neon_aux_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", + "d8" + ); +} + +XALIGNED(4) static const word8 L_mldsa_decode_t0_neon_idx[] = { + 0x00, 0x01, 0xff, 0xff, 0x01, 0x02, 0x03, 0xff, + 0x03, 0x04, 0xff, 0xff, 0x04, 0x05, 0x06, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x08, 0x09, 0xff, 0xff, + 0x09, 0x0a, 0x0b, 0xff, 0x0b, 0x0c, 0xff, 0xff, + 0x03, 0x04, 0xff, 0xff, 0x04, 0x05, 0x06, 0xff, + 0x06, 0x07, 0xff, 0xff, 0x07, 0x08, 0x09, 0xff, + 0x09, 0x0a, 0x0b, 0xff, 0x0b, 0x0c, 0xff, 0xff, + 0x0c, 0x0d, 0x0e, 0xff, 0x0e, 0x0f, 0xff, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_t0_neon_aux[] = { + 0x00000000, 0xfffffffb, 0xfffffffe, 0xfffffff9, + 0xfffffffc, 0xffffffff, 0xfffffffa, 0xfffffffd, + 0x00001fff, 0x00001fff, 0x00001000, 0x00001000, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_decode_t0_neon(const byte* p_p, sword32* z_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_decode_t0_neon(const byte* p, sword32* z) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* p __asm__ ("r0") = (const byte*)p_p; + register sword32* z __asm__ ("r1") = (sword32*)z_p; + register word8* L_mldsa_decode_t0_neon_idx_c __asm__ ("r2") = + (word8*)&L_mldsa_decode_t0_neon_idx; + register word32* L_mldsa_decode_t0_neon_aux_c __asm__ ("r3") = + (word32*)&L_mldsa_decode_t0_neon_aux; +#else + register word8* L_mldsa_decode_t0_neon_idx_c = + (word8*)&L_mldsa_decode_t0_neon_idx; + register word32* L_mldsa_decode_t0_neon_aux_c = + (word32*)&L_mldsa_decode_t0_neon_aux; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r2, %[L_mldsa_decode_t0_neon_idx]\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vld1.8 {d4}, [r2]!\n\t" + "vld1.8 {d5}, [r2]!\n\t" + "mov r3, %[L_mldsa_decode_t0_neon_aux]\n\t" + "vld1.8 {d6}, [r3]!\n\t" + "vld1.8 {d7}, [r3]!\n\t" + "vld1.8 {d8}, [r3]!\n\t" + "vld1.8 {d9}, [r3]!\n\t" + "vld1.8 {d14}, [r3]!\n\t" + "vld1.8 {d15}, [r3]\n\t" + "mov r12, #31\n\t" + "\n" + "L_mldsa_decode_t0_neon_loop_%=:\n\t" + "vld1.8 {d0-d1}, [%[p]]\n\t" + "vtbl.8 d10, {d0-d1}, d2\n\t" + "vshl.s32 d10, d10, d6\n\t" + "vand d10, d10, d14\n\t" + "vsub.i32 d10, d15, d10\n\t" + "vtbl.8 d11, {d0-d1}, d3\n\t" + "vshl.s32 d11, d11, d7\n\t" + "vand d11, d11, d14\n\t" + "vsub.i32 d11, d15, d11\n\t" + "vtbl.8 d12, {d0-d1}, d4\n\t" + "vshl.s32 d12, d12, d8\n\t" + "vand d12, d12, d14\n\t" + "vsub.i32 d12, d15, d12\n\t" + "vtbl.8 d13, {d0-d1}, d5\n\t" + "vshl.s32 d13, d13, d9\n\t" + "vand d13, d13, d14\n\t" + "vsub.i32 d13, d15, d13\n\t" + "vst1.8 {d10-d11}, [%[z]]!\n\t" + "vst1.8 {d12-d13}, [%[z]]!\n\t" + "add %[p], %[p], #13\n\t" + "subs r12, r12, #1\n\t" + "bne L_mldsa_decode_t0_neon_loop_%=\n\t" + "sub %[p], %[p], #3\n\t" + "vld1.8 {d0-d1}, [%[p]]\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vld1.8 {d4}, [r2]!\n\t" + "vld1.8 {d5}, [r2]!\n\t" + "vtbl.8 d10, {d0-d1}, d2\n\t" + "vshl.s32 d10, d10, d6\n\t" + "vand d10, d10, d14\n\t" + "vsub.i32 d10, d15, d10\n\t" + "vtbl.8 d11, {d0-d1}, d3\n\t" + "vshl.s32 d11, d11, d7\n\t" + "vand d11, d11, d14\n\t" + "vsub.i32 d11, d15, d11\n\t" + "vtbl.8 d12, {d0-d1}, d4\n\t" + "vshl.s32 d12, d12, d8\n\t" + "vand d12, d12, d14\n\t" + "vsub.i32 d12, d15, d12\n\t" + "vtbl.8 d13, {d0-d1}, d5\n\t" + "vshl.s32 d13, d13, d9\n\t" + "vand d13, d13, d14\n\t" + "vsub.i32 d13, d15, d13\n\t" + "vst1.8 {d10-d11}, [%[z]]!\n\t" + "vst1.8 {d12-d13}, [%[z]]!\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [p] "+r" (p), [z] "+r" (z), + [L_mldsa_decode_t0_neon_idx] "+r" (L_mldsa_decode_t0_neon_idx_c), + [L_mldsa_decode_t0_neon_aux] "+r" (L_mldsa_decode_t0_neon_aux_c) + : +#else + : + : [p] "r" (p), [z] "r" (z), + [L_mldsa_decode_t0_neon_idx] "r" (L_mldsa_decode_t0_neon_idx_c), + [L_mldsa_decode_t0_neon_aux] "r" (L_mldsa_decode_t0_neon_aux_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", + "d8", "d9", "d10", "d11", "d12", "d13", "d14", "d15" + ); +} + +XALIGNED(4) static const word8 L_mldsa_decode_gamma1_17_neon_idx[] = { + 0x00, 0x01, 0x02, 0xff, 0x02, 0x03, 0x04, 0xff, + 0x04, 0x05, 0x06, 0xff, 0x06, 0x07, 0x08, 0xff, + 0x07, 0x08, 0x09, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x0b, 0x0c, 0x0d, 0xff, 0x0d, 0x0e, 0x0f, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_gamma1_17_neon_aux[] = { + 0x00000000, 0xfffffffe, 0xfffffffc, 0xfffffffa, + 0x0003ffff, 0x0003ffff, 0x00020000, 0x00020000, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_decode_gamma1_17_neon(const byte* p_p, + sword32* z_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_decode_gamma1_17_neon(const byte* p, + sword32* z) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* p __asm__ ("r0") = (const byte*)p_p; + register sword32* z __asm__ ("r1") = (sword32*)z_p; + register word8* L_mldsa_decode_gamma1_17_neon_idx_c __asm__ ("r2") = + (word8*)&L_mldsa_decode_gamma1_17_neon_idx; + register word32* L_mldsa_decode_gamma1_17_neon_aux_c __asm__ ("r3") = + (word32*)&L_mldsa_decode_gamma1_17_neon_aux; +#else + register word8* L_mldsa_decode_gamma1_17_neon_idx_c = + (word8*)&L_mldsa_decode_gamma1_17_neon_idx; + register word32* L_mldsa_decode_gamma1_17_neon_aux_c = + (word32*)&L_mldsa_decode_gamma1_17_neon_aux; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r2, %[L_mldsa_decode_gamma1_17_neon_idx]\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "mov r3, %[L_mldsa_decode_gamma1_17_neon_aux]\n\t" + "vld1.8 {d4}, [r3]!\n\t" + "vld1.8 {d5}, [r3]!\n\t" + "vld1.8 {d8}, [r3]!\n\t" + "vld1.8 {d9}, [r3]\n\t" + "mov r12, #63\n\t" + "\n" + "L_mldsa_decode_gamma1_17_neon_loop_%=:\n\t" + "vld1.8 {d0-d1}, [%[p]]\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vsub.i32 d6, d9, d6\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vsub.i32 d7, d9, d7\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" + "add %[p], %[p], #9\n\t" + "subs r12, r12, #1\n\t" + "bne L_mldsa_decode_gamma1_17_neon_loop_%=\n\t" + "sub %[p], %[p], #7\n\t" + "vld1.8 {d0-d1}, [%[p]]\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vsub.i32 d6, d9, d6\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vsub.i32 d7, d9, d7\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [p] "+r" (p), [z] "+r" (z), + [L_mldsa_decode_gamma1_17_neon_idx] "+r" (L_mldsa_decode_gamma1_17_neon_idx_c), + [L_mldsa_decode_gamma1_17_neon_aux] "+r" (L_mldsa_decode_gamma1_17_neon_aux_c) + : +#else + : + : [p] "r" (p), [z] "r" (z), + [L_mldsa_decode_gamma1_17_neon_idx] "r" (L_mldsa_decode_gamma1_17_neon_idx_c), + [L_mldsa_decode_gamma1_17_neon_aux] "r" (L_mldsa_decode_gamma1_17_neon_aux_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", + "d8", "d9" + ); +} + +XALIGNED(4) static const word8 L_mldsa_decode_gamma1_19_neon_idx[] = { + 0x00, 0x01, 0x02, 0xff, 0x02, 0x03, 0x04, 0xff, + 0x05, 0x06, 0x07, 0xff, 0x07, 0x08, 0x09, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x08, 0x09, 0x0a, 0xff, + 0x0b, 0x0c, 0x0d, 0xff, 0x0d, 0x0e, 0x0f, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_gamma1_19_neon_aux[] = { + 0x00000000, 0xfffffffc, 0x00000000, 0xfffffffc, + 0x000fffff, 0x000fffff, 0x00080000, 0x00080000, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_decode_gamma1_19_neon(const byte* p_p, + sword32* z_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_decode_gamma1_19_neon(const byte* p, + sword32* z) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* p __asm__ ("r0") = (const byte*)p_p; + register sword32* z __asm__ ("r1") = (sword32*)z_p; + register word8* L_mldsa_decode_gamma1_19_neon_idx_c __asm__ ("r2") = + (word8*)&L_mldsa_decode_gamma1_19_neon_idx; + register word32* L_mldsa_decode_gamma1_19_neon_aux_c __asm__ ("r3") = + (word32*)&L_mldsa_decode_gamma1_19_neon_aux; +#else + register word8* L_mldsa_decode_gamma1_19_neon_idx_c = + (word8*)&L_mldsa_decode_gamma1_19_neon_idx; + register word32* L_mldsa_decode_gamma1_19_neon_aux_c = + (word32*)&L_mldsa_decode_gamma1_19_neon_aux; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r2, %[L_mldsa_decode_gamma1_19_neon_idx]\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "mov r3, %[L_mldsa_decode_gamma1_19_neon_aux]\n\t" + "vld1.8 {d4}, [r3]!\n\t" + "vld1.8 {d5}, [r3]!\n\t" + "vld1.8 {d8}, [r3]!\n\t" + "vld1.8 {d9}, [r3]\n\t" + "mov r12, #63\n\t" + "\n" + "L_mldsa_decode_gamma1_19_neon_loop_%=:\n\t" + "vld1.8 {d0-d1}, [%[p]]\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vsub.i32 d6, d9, d6\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vsub.i32 d7, d9, d7\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" + "add %[p], %[p], #10\n\t" + "subs r12, r12, #1\n\t" + "bne L_mldsa_decode_gamma1_19_neon_loop_%=\n\t" + "sub %[p], %[p], #6\n\t" + "vld1.8 {d0-d1}, [%[p]]\n\t" + "vld1.8 {d2}, [r2]!\n\t" + "vld1.8 {d3}, [r2]!\n\t" + "vtbl.8 d6, {d0-d1}, d2\n\t" + "vshl.s32 d6, d6, d4\n\t" + "vand d6, d6, d8\n\t" + "vsub.i32 d6, d9, d6\n\t" + "vtbl.8 d7, {d0-d1}, d3\n\t" + "vshl.s32 d7, d7, d5\n\t" + "vand d7, d7, d8\n\t" + "vsub.i32 d7, d9, d7\n\t" + "vst1.8 {d6-d7}, [%[z]]!\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [p] "+r" (p), [z] "+r" (z), + [L_mldsa_decode_gamma1_19_neon_idx] "+r" (L_mldsa_decode_gamma1_19_neon_idx_c), + [L_mldsa_decode_gamma1_19_neon_aux] "+r" (L_mldsa_decode_gamma1_19_neon_aux_c) + : +#else + : + : [p] "r" (p), [z] "r" (z), + [L_mldsa_decode_gamma1_19_neon_idx] "r" (L_mldsa_decode_gamma1_19_neon_idx_c), + [L_mldsa_decode_gamma1_19_neon_aux] "r" (L_mldsa_decode_gamma1_19_neon_aux_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "d0", "d1", "d2", "d3", "d4", "d5", "d6", "d7", + "d8", "d9" + ); +} + +XALIGNED(4) static const word8 L_mldsa_encode_w1_88_neon_idx[] = { + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, + 0x0a, 0x0c, 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_encode_w1_88_neon(const sword32* w1_p, + byte* w1e_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_encode_w1_88_neon(const sword32* w1, byte* w1e) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const sword32* w1 __asm__ ("r0") = (const sword32*)w1_p; + register byte* w1e __asm__ ("r1") = (byte*)w1e_p; + register word8* L_mldsa_encode_w1_88_neon_idx_c __asm__ ("r2") = + (word8*)&L_mldsa_encode_w1_88_neon_idx; +#else + register word8* L_mldsa_encode_w1_88_neon_idx_c = + (word8*)&L_mldsa_encode_w1_88_neon_idx; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r2, %[L_mldsa_encode_w1_88_neon_idx]\n\t" + "vld1.8 {d8}, [r2]!\n\t" + "vld1.8 {d9}, [r2]\n\t" + "mov r3, #16\n\t" + "\n" + "L_mldsa_encode_w1_88_neon_loop_%=:\n\t" + "vld1.8 {q0}, [%[w1]]!\n\t" + "vld1.8 {q1}, [%[w1]]!\n\t" + "vld1.8 {q2}, [%[w1]]!\n\t" + "vld1.8 {q3}, [%[w1]]!\n\t" + "vuzp.32 q0, q1\n\t" + "vsli.32 q0, q1, #6\n\t" + "vuzp.32 q2, q3\n\t" + "vsli.32 q2, q3, #6\n\t" + "vuzp.32 q0, q2\n\t" + "vsli.32 q0, q2, #12\n\t" + "vtbl.8 d10, {d0-d1}, d8\n\t" + "vtbl.8 d11, {d0-d1}, d9\n\t" + "vst1.8 {d10}, [%[w1e]]!\n\t" + "vmov.32 r12, d11[0]\n\t" + "str r12, [%[w1e]], #4\n\t" + "subs r3, r3, #1\n\t" + "bne L_mldsa_encode_w1_88_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [w1] "+r" (w1), [w1e] "+r" (w1e), + [L_mldsa_encode_w1_88_neon_idx] "+r" (L_mldsa_encode_w1_88_neon_idx_c) + : +#else + : + : [w1] "r" (w1), [w1e] "r" (w1e), + [L_mldsa_encode_w1_88_neon_idx] "r" (L_mldsa_encode_w1_88_neon_idx_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r3", "r12", "q0", "q1", "q2", "q3", "d8", "d9", + "d10", "d11" + ); +} + +XALIGNED(4) static const word8 L_mldsa_encode_w1_32_neon_idx[] = { + 0x00, 0x04, 0x08, 0x0c, 0x10, 0x14, 0x18, 0x1c, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_encode_w1_32_neon(const sword32* w1_p, + byte* w1e_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_encode_w1_32_neon(const sword32* w1, byte* w1e) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const sword32* w1 __asm__ ("r0") = (const sword32*)w1_p; + register byte* w1e __asm__ ("r1") = (byte*)w1e_p; + register word8* L_mldsa_encode_w1_32_neon_idx_c __asm__ ("r2") = + (word8*)&L_mldsa_encode_w1_32_neon_idx; +#else + register word8* L_mldsa_encode_w1_32_neon_idx_c = + (word8*)&L_mldsa_encode_w1_32_neon_idx; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r2, %[L_mldsa_encode_w1_32_neon_idx]\n\t" + "vld1.8 {d8}, [r2]!\n\t" + "mov r3, #16\n\t" + "\n" + "L_mldsa_encode_w1_32_neon_loop_%=:\n\t" + "vld1.8 {q0}, [%[w1]]!\n\t" + "vld1.8 {q1}, [%[w1]]!\n\t" + "vld1.8 {q2}, [%[w1]]!\n\t" + "vld1.8 {q3}, [%[w1]]!\n\t" + "vuzp.32 q0, q1\n\t" + "vsli.32 q0, q1, #4\n\t" + "vuzp.32 q2, q3\n\t" + "vsli.32 q2, q3, #4\n\t" + "vmov q1, q2\n\t" + "vtbl.8 d10, {d0-d3}, d8\n\t" + "vst1.8 {d10}, [%[w1e]]!\n\t" + "subs r3, r3, #1\n\t" + "bne L_mldsa_encode_w1_32_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [w1] "+r" (w1), [w1e] "+r" (w1e), + [L_mldsa_encode_w1_32_neon_idx] "+r" (L_mldsa_encode_w1_32_neon_idx_c) + : +#else + : + : [w1] "r" (w1), [w1e] "r" (w1e), + [L_mldsa_encode_w1_32_neon_idx] "r" (L_mldsa_encode_w1_32_neon_idx_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r3", "r12", "q0", "q1", "q2", "q3", "d8", "d9", + "d10", "d11" + ); +} + +XALIGNED(4) static const word8 L_mldsa_encode_gamma1_17_neon_idx[] = { + 0x00, 0x01, 0x02, 0xff, 0x08, 0x09, 0x0a, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x04, 0x05, 0x06, 0xff, 0x0c, 0x0d, + 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_encode_gamma1_17_neon_aux[] = { + 0x00000000, 0x00000002, 0x00000004, 0x00000006, + 0x00020000, 0x00020000, 0x00020000, 0x00020000, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_encode_gamma1_17_neon(const sword32* z_p, + byte* s_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_encode_gamma1_17_neon(const sword32* z, + byte* s) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const sword32* z __asm__ ("r0") = (const sword32*)z_p; + register byte* s __asm__ ("r1") = (byte*)s_p; + register word8* L_mldsa_encode_gamma1_17_neon_idx_c __asm__ ("r2") = + (word8*)&L_mldsa_encode_gamma1_17_neon_idx; + register word32* L_mldsa_encode_gamma1_17_neon_aux_c __asm__ ("r3") = + (word32*)&L_mldsa_encode_gamma1_17_neon_aux; +#else + register word8* L_mldsa_encode_gamma1_17_neon_idx_c = + (word8*)&L_mldsa_encode_gamma1_17_neon_idx; + register word32* L_mldsa_encode_gamma1_17_neon_aux_c = + (word32*)&L_mldsa_encode_gamma1_17_neon_aux; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r2, %[L_mldsa_encode_gamma1_17_neon_idx]\n\t" + "vld1.8 {d6}, [r2]!\n\t" + "vld1.8 {d7}, [r2]!\n\t" + "vld1.8 {d8}, [r2]!\n\t" + "vld1.8 {d9}, [r2]!\n\t" + "mov r2, %[L_mldsa_encode_gamma1_17_neon_aux]\n\t" + "vld1.8 {q1}, [r2]!\n\t" + "vld1.8 {q2}, [r2]\n\t" + "mov r3, #0x40\n\t" + "\n" + "L_mldsa_encode_gamma1_17_neon_loop_%=:\n\t" + "vld1.8 {q0}, [%[z]]!\n\t" + "vsub.i32 q0, q2, q0\n\t" + "vshl.s32 q0, q0, q1\n\t" + "vtbl.8 d10, {d0-d1}, d6\n\t" + "vtbl.8 d11, {d0-d1}, d7\n\t" + "vtbl.8 d12, {d0-d1}, d8\n\t" + "vtbl.8 d13, {d0-d1}, d9\n\t" + "vorr d10, d10, d12\n\t" + "vorr d11, d11, d13\n\t" + "vst1.8 {d10}, [%[s]]!\n\t" + "vmov.32 r12, d11[0]\n\t" + "strb r12, [%[s]], #1\n\t" + "subs r3, r3, #1\n\t" + "bne L_mldsa_encode_gamma1_17_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [z] "+r" (z), [s] "+r" (s), + [L_mldsa_encode_gamma1_17_neon_idx] "+r" (L_mldsa_encode_gamma1_17_neon_idx_c), + [L_mldsa_encode_gamma1_17_neon_aux] "+r" (L_mldsa_encode_gamma1_17_neon_aux_c) + : +#else + : + : [z] "r" (z), [s] "r" (s), + [L_mldsa_encode_gamma1_17_neon_idx] "r" (L_mldsa_encode_gamma1_17_neon_idx_c), + [L_mldsa_encode_gamma1_17_neon_aux] "r" (L_mldsa_encode_gamma1_17_neon_aux_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "q0", "q1", "q2", "d6", "d7", "d8", "d9", + "d10", "d11", "d12", "d13" + ); +} + +XALIGNED(4) static const word8 L_mldsa_encode_gamma1_19_neon_idx[] = { + 0x00, 0x01, 0x02, 0xff, 0xff, 0x08, 0x09, 0x0a, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x04, 0x05, 0x06, 0xff, 0xff, 0x0c, + 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_encode_gamma1_19_neon_aux[] = { + 0x00000000, 0x00000004, 0x00000000, 0x00000004, + 0x00080000, 0x00080000, 0x00080000, 0x00080000, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_encode_gamma1_19_neon(const sword32* z_p, + byte* s_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_encode_gamma1_19_neon(const sword32* z, + byte* s) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const sword32* z __asm__ ("r0") = (const sword32*)z_p; + register byte* s __asm__ ("r1") = (byte*)s_p; + register word8* L_mldsa_encode_gamma1_19_neon_idx_c __asm__ ("r2") = + (word8*)&L_mldsa_encode_gamma1_19_neon_idx; + register word32* L_mldsa_encode_gamma1_19_neon_aux_c __asm__ ("r3") = + (word32*)&L_mldsa_encode_gamma1_19_neon_aux; +#else + register word8* L_mldsa_encode_gamma1_19_neon_idx_c = + (word8*)&L_mldsa_encode_gamma1_19_neon_idx; + register word32* L_mldsa_encode_gamma1_19_neon_aux_c = + (word32*)&L_mldsa_encode_gamma1_19_neon_aux; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r2, %[L_mldsa_encode_gamma1_19_neon_idx]\n\t" + "vld1.8 {d6}, [r2]!\n\t" + "vld1.8 {d7}, [r2]!\n\t" + "vld1.8 {d8}, [r2]!\n\t" + "vld1.8 {d9}, [r2]!\n\t" + "mov r2, %[L_mldsa_encode_gamma1_19_neon_aux]\n\t" + "vld1.8 {q1}, [r2]!\n\t" + "vld1.8 {q2}, [r2]\n\t" + "mov r3, #0x40\n\t" + "\n" + "L_mldsa_encode_gamma1_19_neon_loop_%=:\n\t" + "vld1.8 {q0}, [%[z]]!\n\t" + "vsub.i32 q0, q2, q0\n\t" + "vshl.s32 q0, q0, q1\n\t" + "vtbl.8 d10, {d0-d1}, d6\n\t" + "vtbl.8 d11, {d0-d1}, d7\n\t" + "vtbl.8 d12, {d0-d1}, d8\n\t" + "vtbl.8 d13, {d0-d1}, d9\n\t" + "vorr d10, d10, d12\n\t" + "vorr d11, d11, d13\n\t" + "vst1.8 {d10}, [%[s]]!\n\t" + "vmov.32 r12, d11[0]\n\t" + "strh r12, [%[s]], #2\n\t" + "subs r3, r3, #1\n\t" + "bne L_mldsa_encode_gamma1_19_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [z] "+r" (z), [s] "+r" (s), + [L_mldsa_encode_gamma1_19_neon_idx] "+r" (L_mldsa_encode_gamma1_19_neon_idx_c), + [L_mldsa_encode_gamma1_19_neon_aux] "+r" (L_mldsa_encode_gamma1_19_neon_aux_c) + : +#else + : + : [z] "r" (z), [s] "r" (s), + [L_mldsa_encode_gamma1_19_neon_idx] "r" (L_mldsa_encode_gamma1_19_neon_idx_c), + [L_mldsa_encode_gamma1_19_neon_aux] "r" (L_mldsa_encode_gamma1_19_neon_aux_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "q0", "q1", "q2", "d6", "d7", "d8", "d9", + "d10", "d11", "d12", "d13" + ); +} + +XALIGNED(8) static const word32 L_mldsa_mul_vec_4_neon_q[] = { + 0x007fe001, 0x007fe001, 0x03802001, 0x03802001, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_mul_vec_4_neon(sword32* r_p, + const sword32* m_p, const sword32* v_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_mul_vec_4_neon(sword32* r, const sword32* m, + const sword32* v) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register sword32* r __asm__ ("r0") = (sword32*)r_p; + register const sword32* m __asm__ ("r1") = (const sword32*)m_p; + register const sword32* v __asm__ ("r2") = (const sword32*)v_p; + register word32* L_mldsa_mul_vec_4_neon_q_c __asm__ ("r3") = + (word32*)&L_mldsa_mul_vec_4_neon_q; +#else + register word32* L_mldsa_mul_vec_4_neon_q_c = + (word32*)&L_mldsa_mul_vec_4_neon_q; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r3, %[L_mldsa_mul_vec_4_neon_q]\n\t" + "vld1.8 {q0}, [r3]\n\t" + "vdup.32 q1, d1[0]\n\t" + "mov lr, #0x40\n\t" + "\n" + "L_mldsa_mul_vec_4_neon_loop_%=:\n\t" + "vld1.8 {q2}, [%[m]]\n\t" + "vld1.8 {q3}, [%[v]]\n\t" + "vmull.s32 q4, d4, d6\n\t" + "vmull.s32 q5, d5, d7\n\t" + "add r12, %[m], #0x400\n\t" + "vld1.8 {q2}, [r12]\n\t" + "add r12, %[v], #0x400\n\t" + "vld1.8 {q3}, [r12]\n\t" + "vmlal.s32 q4, d4, d6\n\t" + "vmlal.s32 q5, d5, d7\n\t" + "add r12, %[m], #0x800\n\t" + "vld1.8 {q2}, [r12]\n\t" + "add r12, %[v], #0x800\n\t" + "vld1.8 {q3}, [r12]\n\t" + "vmlal.s32 q4, d4, d6\n\t" + "vmlal.s32 q5, d5, d7\n\t" + "add r12, %[m], #0xc00\n\t" + "vld1.8 {q2}, [r12]\n\t" + "add r12, %[v], #0xc00\n\t" + "vld1.8 {q3}, [r12]\n\t" + "vmlal.s32 q4, d4, d6\n\t" + "vmlal.s32 q5, d5, d7\n\t" + "vmovn.i64 d12, q4\n\t" + "vmovn.i64 d13, q5\n\t" + "vmul.i32 q6, q6, q1\n\t" + "vmlsl.s32 q4, d12, d0\n\t" + "vmlsl.s32 q5, d13, d0\n\t" + "vshrn.i64 d12, q4, #32\n\t" + "vshrn.i64 d13, q5, #32\n\t" + "vst1.8 {q6}, [%[r]]!\n\t" + "add %[m], %[m], #16\n\t" + "add %[v], %[v], #16\n\t" + "subs lr, lr, #1\n\t" + "bne L_mldsa_mul_vec_4_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [r] "+r" (r), [m] "+r" (m), [v] "+r" (v), + [L_mldsa_mul_vec_4_neon_q] "+r" (L_mldsa_mul_vec_4_neon_q_c) + : +#else + : + : [r] "r" (r), [m] "r" (m), [v] "r" (v), + [L_mldsa_mul_vec_4_neon_q] "r" (L_mldsa_mul_vec_4_neon_q_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "lr", "q0", "q1", "q2", "q3", "q4", "q5", "q6" + ); +} + +XALIGNED(8) static const word32 L_mldsa_mul_vec_5_neon_q[] = { + 0x007fe001, 0x007fe001, 0x03802001, 0x03802001, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_mul_vec_5_neon(sword32* r_p, + const sword32* m_p, const sword32* v_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_mul_vec_5_neon(sword32* r, const sword32* m, + const sword32* v) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register sword32* r __asm__ ("r0") = (sword32*)r_p; + register const sword32* m __asm__ ("r1") = (const sword32*)m_p; + register const sword32* v __asm__ ("r2") = (const sword32*)v_p; + register word32* L_mldsa_mul_vec_5_neon_q_c __asm__ ("r3") = + (word32*)&L_mldsa_mul_vec_5_neon_q; +#else + register word32* L_mldsa_mul_vec_5_neon_q_c = + (word32*)&L_mldsa_mul_vec_5_neon_q; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r3, %[L_mldsa_mul_vec_5_neon_q]\n\t" + "vld1.8 {q0}, [r3]\n\t" + "vdup.32 q1, d1[0]\n\t" + "mov lr, #0x40\n\t" + "\n" + "L_mldsa_mul_vec_5_neon_loop_%=:\n\t" + "vld1.8 {q2}, [%[m]]\n\t" + "vld1.8 {q3}, [%[v]]\n\t" + "vmull.s32 q4, d4, d6\n\t" + "vmull.s32 q5, d5, d7\n\t" + "add r12, %[m], #0x400\n\t" + "vld1.8 {q2}, [r12]\n\t" + "add r12, %[v], #0x400\n\t" + "vld1.8 {q3}, [r12]\n\t" + "vmlal.s32 q4, d4, d6\n\t" + "vmlal.s32 q5, d5, d7\n\t" + "add r12, %[m], #0x800\n\t" + "vld1.8 {q2}, [r12]\n\t" + "add r12, %[v], #0x800\n\t" + "vld1.8 {q3}, [r12]\n\t" + "vmlal.s32 q4, d4, d6\n\t" + "vmlal.s32 q5, d5, d7\n\t" + "add r12, %[m], #0xc00\n\t" + "vld1.8 {q2}, [r12]\n\t" + "add r12, %[v], #0xc00\n\t" + "vld1.8 {q3}, [r12]\n\t" + "vmlal.s32 q4, d4, d6\n\t" + "vmlal.s32 q5, d5, d7\n\t" + "add r12, %[m], #0x1000\n\t" + "vld1.8 {q2}, [r12]\n\t" + "add r12, %[v], #0x1000\n\t" + "vld1.8 {q3}, [r12]\n\t" + "vmlal.s32 q4, d4, d6\n\t" + "vmlal.s32 q5, d5, d7\n\t" + "vmovn.i64 d12, q4\n\t" + "vmovn.i64 d13, q5\n\t" + "vmul.i32 q6, q6, q1\n\t" + "vmlsl.s32 q4, d12, d0\n\t" + "vmlsl.s32 q5, d13, d0\n\t" + "vshrn.i64 d12, q4, #32\n\t" + "vshrn.i64 d13, q5, #32\n\t" + "vst1.8 {q6}, [%[r]]!\n\t" + "add %[m], %[m], #16\n\t" + "add %[v], %[v], #16\n\t" + "subs lr, lr, #1\n\t" + "bne L_mldsa_mul_vec_5_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [r] "+r" (r), [m] "+r" (m), [v] "+r" (v), + [L_mldsa_mul_vec_5_neon_q] "+r" (L_mldsa_mul_vec_5_neon_q_c) + : +#else + : + : [r] "r" (r), [m] "r" (m), [v] "r" (v), + [L_mldsa_mul_vec_5_neon_q] "r" (L_mldsa_mul_vec_5_neon_q_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "lr", "q0", "q1", "q2", "q3", "q4", "q5", "q6" + ); +} + +XALIGNED(8) static const word32 L_mldsa_mul_vec_7_neon_q[] = { + 0x007fe001, 0x007fe001, 0x03802001, 0x03802001, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_mul_vec_7_neon(sword32* r_p, + const sword32* m_p, const sword32* v_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_mul_vec_7_neon(sword32* r, const sword32* m, + const sword32* v) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register sword32* r __asm__ ("r0") = (sword32*)r_p; + register const sword32* m __asm__ ("r1") = (const sword32*)m_p; + register const sword32* v __asm__ ("r2") = (const sword32*)v_p; + register word32* L_mldsa_mul_vec_7_neon_q_c __asm__ ("r3") = + (word32*)&L_mldsa_mul_vec_7_neon_q; +#else + register word32* L_mldsa_mul_vec_7_neon_q_c = + (word32*)&L_mldsa_mul_vec_7_neon_q; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "mov r3, %[L_mldsa_mul_vec_7_neon_q]\n\t" + "vld1.8 {q0}, [r3]\n\t" + "vdup.32 q1, d1[0]\n\t" + "mov lr, #0x40\n\t" + "\n" + "L_mldsa_mul_vec_7_neon_loop_%=:\n\t" + "vld1.8 {q2}, [%[m]]\n\t" + "vld1.8 {q3}, [%[v]]\n\t" + "vmull.s32 q4, d4, d6\n\t" + "vmull.s32 q5, d5, d7\n\t" + "add r12, %[m], #0x400\n\t" + "vld1.8 {q2}, [r12]\n\t" + "add r12, %[v], #0x400\n\t" + "vld1.8 {q3}, [r12]\n\t" + "vmlal.s32 q4, d4, d6\n\t" + "vmlal.s32 q5, d5, d7\n\t" + "add r12, %[m], #0x800\n\t" + "vld1.8 {q2}, [r12]\n\t" + "add r12, %[v], #0x800\n\t" + "vld1.8 {q3}, [r12]\n\t" + "vmlal.s32 q4, d4, d6\n\t" + "vmlal.s32 q5, d5, d7\n\t" + "add r12, %[m], #0xc00\n\t" + "vld1.8 {q2}, [r12]\n\t" + "add r12, %[v], #0xc00\n\t" + "vld1.8 {q3}, [r12]\n\t" + "vmlal.s32 q4, d4, d6\n\t" + "vmlal.s32 q5, d5, d7\n\t" + "add r12, %[m], #0x1000\n\t" + "vld1.8 {q2}, [r12]\n\t" + "add r12, %[v], #0x1000\n\t" + "vld1.8 {q3}, [r12]\n\t" + "vmlal.s32 q4, d4, d6\n\t" + "vmlal.s32 q5, d5, d7\n\t" + "add r12, %[m], #0x1400\n\t" + "vld1.8 {q2}, [r12]\n\t" + "add r12, %[v], #0x1400\n\t" + "vld1.8 {q3}, [r12]\n\t" + "vmlal.s32 q4, d4, d6\n\t" + "vmlal.s32 q5, d5, d7\n\t" + "add r12, %[m], #0x1800\n\t" + "vld1.8 {q2}, [r12]\n\t" + "add r12, %[v], #0x1800\n\t" + "vld1.8 {q3}, [r12]\n\t" + "vmlal.s32 q4, d4, d6\n\t" + "vmlal.s32 q5, d5, d7\n\t" + "vmovn.i64 d12, q4\n\t" + "vmovn.i64 d13, q5\n\t" + "vmul.i32 q6, q6, q1\n\t" + "vmlsl.s32 q4, d12, d0\n\t" + "vmlsl.s32 q5, d13, d0\n\t" + "vshrn.i64 d12, q4, #32\n\t" + "vshrn.i64 d13, q5, #32\n\t" + "vst1.8 {q6}, [%[r]]!\n\t" + "add %[m], %[m], #16\n\t" + "add %[v], %[v], #16\n\t" + "subs lr, lr, #1\n\t" + "bne L_mldsa_mul_vec_7_neon_loop_%=\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [r] "+r" (r), [m] "+r" (m), [v] "+r" (v), + [L_mldsa_mul_vec_7_neon_q] "+r" (L_mldsa_mul_vec_7_neon_q_c) + : +#else + : + : [r] "r" (r), [m] "r" (m), [v] "r" (v), + [L_mldsa_mul_vec_7_neon_q] "r" (L_mldsa_mul_vec_7_neon_q_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r12", "lr", "q0", "q1", "q2", "q3", "q4", "q5", "q6" + ); +} + +XALIGNED(4) static const word8 L_mldsa_extract_coeffs_eta2_neon_map[] = { + 0x02, 0x01, 0x00, 0xff, 0xfe, 0x02, 0x01, 0x00, + 0xff, 0xfe, 0x02, 0x01, 0x00, 0xff, 0xfe, 0x00, +}; + +XALIGNED(4) static const word8 L_mldsa_extract_coeffs_eta2_neon_aux[] = { + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, +}; + +XALIGNED(8) static const word32 L_mldsa_extract_coeffs_eta2_neon_bits[] = { + 0x00000001, 0x00000002, 0x00000004, 0x00000008, +}; + +XALIGNED(4) static const word8 L_mldsa_extract_coeffs_eta2_neon_idx[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_extract_coeffs_eta2_neon(const byte* z_p, + unsigned int zLen_p, sword32* s_p, unsigned int* cnt_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_extract_coeffs_eta2_neon(const byte* z, + unsigned int zLen, sword32* s, unsigned int* cnt) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* z __asm__ ("r0") = (const byte*)z_p; + register unsigned int zLen __asm__ ("r1") = (unsigned int)zLen_p; + register sword32* s __asm__ ("r2") = (sword32*)s_p; + register unsigned int* cnt __asm__ ("r3") = (unsigned int*)cnt_p; + register word8* L_mldsa_extract_coeffs_eta2_neon_map_c __asm__ ("r12") = + (word8*)&L_mldsa_extract_coeffs_eta2_neon_map; + register word8* L_mldsa_extract_coeffs_eta2_neon_aux_c __asm__ ("lr") = + (word8*)&L_mldsa_extract_coeffs_eta2_neon_aux; + register word32* L_mldsa_extract_coeffs_eta2_neon_bits_c __asm__ ("r4") = + (word32*)&L_mldsa_extract_coeffs_eta2_neon_bits; + register word8* L_mldsa_extract_coeffs_eta2_neon_idx_c __asm__ ("r5") = + (word8*)&L_mldsa_extract_coeffs_eta2_neon_idx; +#else + register word8* L_mldsa_extract_coeffs_eta2_neon_map_c = + (word8*)&L_mldsa_extract_coeffs_eta2_neon_map; + register word8* L_mldsa_extract_coeffs_eta2_neon_aux_c = + (word8*)&L_mldsa_extract_coeffs_eta2_neon_aux; + register word32* L_mldsa_extract_coeffs_eta2_neon_bits_c = + (word32*)&L_mldsa_extract_coeffs_eta2_neon_bits; + register word8* L_mldsa_extract_coeffs_eta2_neon_idx_c = + (word8*)&L_mldsa_extract_coeffs_eta2_neon_idx; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_mldsa_extract_coeffs_eta2_neon_bits], " + "%[L_mldsa_extract_coeffs_eta2_neon_idx]}\n\t" + "push {%[L_mldsa_extract_coeffs_eta2_neon_map], " + "%[L_mldsa_extract_coeffs_eta2_neon_aux]}\n\t" + "mov r12, %[L_mldsa_extract_coeffs_eta2_neon_map]\n\t" + "vld1.8 {q0}, [r12]\n\t" + "mov lr, %[L_mldsa_extract_coeffs_eta2_neon_aux]\n\t" + "vld1.8 {q1}, [lr]\n\t" + "mov lr, %[L_mldsa_extract_coeffs_eta2_neon_bits]\n\t" + "vld1.8 {q2}, [lr]\n\t" + "mov r4, %[L_mldsa_extract_coeffs_eta2_neon_idx]\n\t" + "ldr r5, [%[cnt]]\n\t" + "add %[s], %[s], r5, lsl #2\n\t" + "\n" + "L_mldsa_extract_coeffs_eta2_neon_wide_%=:\n\t" + "cmp %[zLen], #8\n\t" + "blt L_mldsa_extract_coeffs_eta2_neon_tail_%=\n\t" + "cmp r5, #0xf0\n\t" + "bgt L_mldsa_extract_coeffs_eta2_neon_tail_%=\n\t" + "vld1.8 {d6}, [%[z]]\n\t" + "vshr.u8 d7, d6, #4\n\t" + "vmov.i8 q12, #15\n\t" + "vand d6, d6, d24\n\t" + "vzip.8 d6, d7\n\t" + "vtbl.8 d8, {d0-d1}, d6\n\t" + "vtbl.8 d9, {d0-d1}, d7\n\t" + "vcgt.u8 q5, q1, q3\n\t" + "vmovl.s8 q6, d8\n\t" + "vmovl.s8 q7, d9\n\t" + "vmovl.s8 q8, d10\n\t" + "vmovl.s8 q9, d11\n\t" + "vmovl.s16 q10, d12\n\t" + "vmovl.s16 q11, d16\n\t" + "vshr.u32 q12, q11, #31\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r6, d24[0]\n\t" + "vand q12, q11, q2\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r7, d24[0]\n\t" + "lsl r7, r7, #4\n\t" + "add r7, r7, r4\n\t" + "vld1.8 {q13}, [r7]\n\t" + "vtbl.8 d24, {d20-d21}, d26\n\t" + "vtbl.8 d25, {d20-d21}, d27\n\t" + "vst1.8 {q12}, [%[s]]\n\t" + "add %[s], %[s], r6, lsl #2\n\t" + "add r5, r5, r6\n\t" + "vmovl.s16 q10, d13\n\t" + "vmovl.s16 q11, d17\n\t" + "vshr.u32 q12, q11, #31\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r6, d24[0]\n\t" + "vand q12, q11, q2\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r7, d24[0]\n\t" + "lsl r7, r7, #4\n\t" + "add r7, r7, r4\n\t" + "vld1.8 {q13}, [r7]\n\t" + "vtbl.8 d24, {d20-d21}, d26\n\t" + "vtbl.8 d25, {d20-d21}, d27\n\t" + "vst1.8 {q12}, [%[s]]\n\t" + "add %[s], %[s], r6, lsl #2\n\t" + "add r5, r5, r6\n\t" + "vmovl.s16 q10, d14\n\t" + "vmovl.s16 q11, d18\n\t" + "vshr.u32 q12, q11, #31\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r6, d24[0]\n\t" + "vand q12, q11, q2\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r7, d24[0]\n\t" + "lsl r7, r7, #4\n\t" + "add r7, r7, r4\n\t" + "vld1.8 {q13}, [r7]\n\t" + "vtbl.8 d24, {d20-d21}, d26\n\t" + "vtbl.8 d25, {d20-d21}, d27\n\t" + "vst1.8 {q12}, [%[s]]\n\t" + "add %[s], %[s], r6, lsl #2\n\t" + "add r5, r5, r6\n\t" + "vmovl.s16 q10, d15\n\t" + "vmovl.s16 q11, d19\n\t" + "vshr.u32 q12, q11, #31\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r6, d24[0]\n\t" + "vand q12, q11, q2\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r7, d24[0]\n\t" + "lsl r7, r7, #4\n\t" + "add r7, r7, r4\n\t" + "vld1.8 {q13}, [r7]\n\t" + "vtbl.8 d24, {d20-d21}, d26\n\t" + "vtbl.8 d25, {d20-d21}, d27\n\t" + "vst1.8 {q12}, [%[s]]\n\t" + "add %[s], %[s], r6, lsl #2\n\t" + "add r5, r5, r6\n\t" + "add %[z], %[z], #8\n\t" + "sub %[zLen], %[zLen], #8\n\t" + "b L_mldsa_extract_coeffs_eta2_neon_wide_%=\n\t" + "\n" + "L_mldsa_extract_coeffs_eta2_neon_tail_%=:\n\t" + "cmp %[zLen], #0\n\t" + "beq L_mldsa_extract_coeffs_eta2_neon_done_%=\n\t" + "cmp r5, #0x100\n\t" + "beq L_mldsa_extract_coeffs_eta2_neon_done_%=\n\t" + "ldrb r8, [%[z]], #1\n\t" + "sub %[zLen], %[zLen], #1\n\t" + "and r6, r8, #15\n\t" + "cmp r6, #15\n\t" + "bcs L_mldsa_extract_coeffs_eta2_neon_skip_lo_%=\n\t" + "add lr, r12, r6\n\t" + "ldrsb r6, [lr]\n\t" + "str r6, [%[s]], #4\n\t" + "add r5, r5, #1\n\t" + "cmp r5, #0x100\n\t" + "beq L_mldsa_extract_coeffs_eta2_neon_done_%=\n\t" + "\n" + "L_mldsa_extract_coeffs_eta2_neon_skip_lo_%=:\n\t" + "lsr r6, r8, #4\n\t" + "cmp r6, #15\n\t" + "bcs L_mldsa_extract_coeffs_eta2_neon_skip_hi_%=\n\t" + "add lr, r12, r6\n\t" + "ldrsb r6, [lr]\n\t" + "str r6, [%[s]], #4\n\t" + "add r5, r5, #1\n\t" + "\n" + "L_mldsa_extract_coeffs_eta2_neon_skip_hi_%=:\n\t" + "b L_mldsa_extract_coeffs_eta2_neon_tail_%=\n\t" + "\n" + "L_mldsa_extract_coeffs_eta2_neon_done_%=:\n\t" + "str r5, [%[cnt]]\n\t" + "pop {%[L_mldsa_extract_coeffs_eta2_neon_map], " + "%[L_mldsa_extract_coeffs_eta2_neon_aux]}\n\t" + "pop {%[L_mldsa_extract_coeffs_eta2_neon_bits], " + "%[L_mldsa_extract_coeffs_eta2_neon_idx]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [z] "+r" (z), [zLen] "+r" (zLen), [s] "+r" (s), [cnt] "+r" (cnt), + [L_mldsa_extract_coeffs_eta2_neon_map] "+r" (L_mldsa_extract_coeffs_eta2_neon_map_c), + [L_mldsa_extract_coeffs_eta2_neon_aux] "+r" (L_mldsa_extract_coeffs_eta2_neon_aux_c), + [L_mldsa_extract_coeffs_eta2_neon_bits] "+r" (L_mldsa_extract_coeffs_eta2_neon_bits_c), + [L_mldsa_extract_coeffs_eta2_neon_idx] "+r" (L_mldsa_extract_coeffs_eta2_neon_idx_c) + : +#else + : + : [z] "r" (z), [zLen] "r" (zLen), [s] "r" (s), [cnt] "r" (cnt), + [L_mldsa_extract_coeffs_eta2_neon_map] "r" (L_mldsa_extract_coeffs_eta2_neon_map_c), + [L_mldsa_extract_coeffs_eta2_neon_aux] "r" (L_mldsa_extract_coeffs_eta2_neon_aux_c), + [L_mldsa_extract_coeffs_eta2_neon_bits] "r" (L_mldsa_extract_coeffs_eta2_neon_bits_c), + [L_mldsa_extract_coeffs_eta2_neon_idx] "r" (L_mldsa_extract_coeffs_eta2_neon_idx_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r6", "r7", "r8", "q0", "q1", "q2", "q3", "q4", "q5", + "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13" + ); +} + +XALIGNED(4) static const word8 L_mldsa_extract_coeffs_eta4_neon_map[] = { + 0x04, 0x03, 0x02, 0x01, 0x00, 0xff, 0xfe, 0xfd, + 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +XALIGNED(4) static const word8 L_mldsa_extract_coeffs_eta4_neon_aux[] = { + 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, + 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, +}; + +XALIGNED(8) static const word32 L_mldsa_extract_coeffs_eta4_neon_bits[] = { + 0x00000001, 0x00000002, 0x00000004, 0x00000008, +}; + +XALIGNED(4) static const word8 L_mldsa_extract_coeffs_eta4_neon_idx[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void mldsa_extract_coeffs_eta4_neon(const byte* z_p, + unsigned int zLen_p, sword32* s_p, unsigned int* cnt_p) +#else +WC_OMIT_FRAME_POINTER void mldsa_extract_coeffs_eta4_neon(const byte* z, + unsigned int zLen, sword32* s, unsigned int* cnt) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const byte* z __asm__ ("r0") = (const byte*)z_p; + register unsigned int zLen __asm__ ("r1") = (unsigned int)zLen_p; + register sword32* s __asm__ ("r2") = (sword32*)s_p; + register unsigned int* cnt __asm__ ("r3") = (unsigned int*)cnt_p; + register word8* L_mldsa_extract_coeffs_eta4_neon_map_c __asm__ ("r12") = + (word8*)&L_mldsa_extract_coeffs_eta4_neon_map; + register word8* L_mldsa_extract_coeffs_eta4_neon_aux_c __asm__ ("lr") = + (word8*)&L_mldsa_extract_coeffs_eta4_neon_aux; + register word32* L_mldsa_extract_coeffs_eta4_neon_bits_c __asm__ ("r4") = + (word32*)&L_mldsa_extract_coeffs_eta4_neon_bits; + register word8* L_mldsa_extract_coeffs_eta4_neon_idx_c __asm__ ("r5") = + (word8*)&L_mldsa_extract_coeffs_eta4_neon_idx; +#else + register word8* L_mldsa_extract_coeffs_eta4_neon_map_c = + (word8*)&L_mldsa_extract_coeffs_eta4_neon_map; + register word8* L_mldsa_extract_coeffs_eta4_neon_aux_c = + (word8*)&L_mldsa_extract_coeffs_eta4_neon_aux; + register word32* L_mldsa_extract_coeffs_eta4_neon_bits_c = + (word32*)&L_mldsa_extract_coeffs_eta4_neon_bits; + register word8* L_mldsa_extract_coeffs_eta4_neon_idx_c = + (word8*)&L_mldsa_extract_coeffs_eta4_neon_idx; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_mldsa_extract_coeffs_eta4_neon_bits], " + "%[L_mldsa_extract_coeffs_eta4_neon_idx]}\n\t" + "push {%[L_mldsa_extract_coeffs_eta4_neon_map], " + "%[L_mldsa_extract_coeffs_eta4_neon_aux]}\n\t" + "mov r12, %[L_mldsa_extract_coeffs_eta4_neon_map]\n\t" + "vld1.8 {q0}, [r12]\n\t" + "mov lr, %[L_mldsa_extract_coeffs_eta4_neon_aux]\n\t" + "vld1.8 {q1}, [lr]\n\t" + "mov lr, %[L_mldsa_extract_coeffs_eta4_neon_bits]\n\t" + "vld1.8 {q2}, [lr]\n\t" + "mov r4, %[L_mldsa_extract_coeffs_eta4_neon_idx]\n\t" + "ldr r5, [%[cnt]]\n\t" + "add %[s], %[s], r5, lsl #2\n\t" + "\n" + "L_mldsa_extract_coeffs_eta4_neon_wide_%=:\n\t" + "cmp %[zLen], #8\n\t" + "blt L_mldsa_extract_coeffs_eta4_neon_tail_%=\n\t" + "cmp r5, #0xf0\n\t" + "bgt L_mldsa_extract_coeffs_eta4_neon_tail_%=\n\t" + "vld1.8 {d6}, [%[z]]\n\t" + "vshr.u8 d7, d6, #4\n\t" + "vmov.i8 q12, #15\n\t" + "vand d6, d6, d24\n\t" + "vzip.8 d6, d7\n\t" + "vtbl.8 d8, {d0-d1}, d6\n\t" + "vtbl.8 d9, {d0-d1}, d7\n\t" + "vcgt.u8 q5, q1, q3\n\t" + "vmovl.s8 q6, d8\n\t" + "vmovl.s8 q7, d9\n\t" + "vmovl.s8 q8, d10\n\t" + "vmovl.s8 q9, d11\n\t" + "vmovl.s16 q10, d12\n\t" + "vmovl.s16 q11, d16\n\t" + "vshr.u32 q12, q11, #31\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r6, d24[0]\n\t" + "vand q12, q11, q2\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r7, d24[0]\n\t" + "lsl r7, r7, #4\n\t" + "add r7, r7, r4\n\t" + "vld1.8 {q13}, [r7]\n\t" + "vtbl.8 d24, {d20-d21}, d26\n\t" + "vtbl.8 d25, {d20-d21}, d27\n\t" + "vst1.8 {q12}, [%[s]]\n\t" + "add %[s], %[s], r6, lsl #2\n\t" + "add r5, r5, r6\n\t" + "vmovl.s16 q10, d13\n\t" + "vmovl.s16 q11, d17\n\t" + "vshr.u32 q12, q11, #31\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r6, d24[0]\n\t" + "vand q12, q11, q2\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r7, d24[0]\n\t" + "lsl r7, r7, #4\n\t" + "add r7, r7, r4\n\t" + "vld1.8 {q13}, [r7]\n\t" + "vtbl.8 d24, {d20-d21}, d26\n\t" + "vtbl.8 d25, {d20-d21}, d27\n\t" + "vst1.8 {q12}, [%[s]]\n\t" + "add %[s], %[s], r6, lsl #2\n\t" + "add r5, r5, r6\n\t" + "vmovl.s16 q10, d14\n\t" + "vmovl.s16 q11, d18\n\t" + "vshr.u32 q12, q11, #31\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r6, d24[0]\n\t" + "vand q12, q11, q2\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r7, d24[0]\n\t" + "lsl r7, r7, #4\n\t" + "add r7, r7, r4\n\t" + "vld1.8 {q13}, [r7]\n\t" + "vtbl.8 d24, {d20-d21}, d26\n\t" + "vtbl.8 d25, {d20-d21}, d27\n\t" + "vst1.8 {q12}, [%[s]]\n\t" + "add %[s], %[s], r6, lsl #2\n\t" + "add r5, r5, r6\n\t" + "vmovl.s16 q10, d15\n\t" + "vmovl.s16 q11, d19\n\t" + "vshr.u32 q12, q11, #31\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r6, d24[0]\n\t" + "vand q12, q11, q2\n\t" + "vpadd.u32 d24, d24, d25\n\t" + "vpadd.u32 d24, d24, d24\n\t" + "vmov.32 r7, d24[0]\n\t" + "lsl r7, r7, #4\n\t" + "add r7, r7, r4\n\t" + "vld1.8 {q13}, [r7]\n\t" + "vtbl.8 d24, {d20-d21}, d26\n\t" + "vtbl.8 d25, {d20-d21}, d27\n\t" + "vst1.8 {q12}, [%[s]]\n\t" + "add %[s], %[s], r6, lsl #2\n\t" + "add r5, r5, r6\n\t" + "add %[z], %[z], #8\n\t" + "sub %[zLen], %[zLen], #8\n\t" + "b L_mldsa_extract_coeffs_eta4_neon_wide_%=\n\t" + "\n" + "L_mldsa_extract_coeffs_eta4_neon_tail_%=:\n\t" + "cmp %[zLen], #0\n\t" + "beq L_mldsa_extract_coeffs_eta4_neon_done_%=\n\t" + "cmp r5, #0x100\n\t" + "beq L_mldsa_extract_coeffs_eta4_neon_done_%=\n\t" + "ldrb r8, [%[z]], #1\n\t" + "sub %[zLen], %[zLen], #1\n\t" + "and r6, r8, #15\n\t" + "cmp r6, #9\n\t" + "bcs L_mldsa_extract_coeffs_eta4_neon_skip_lo_%=\n\t" + "add lr, r12, r6\n\t" + "ldrsb r6, [lr]\n\t" + "str r6, [%[s]], #4\n\t" + "add r5, r5, #1\n\t" + "cmp r5, #0x100\n\t" + "beq L_mldsa_extract_coeffs_eta4_neon_done_%=\n\t" + "\n" + "L_mldsa_extract_coeffs_eta4_neon_skip_lo_%=:\n\t" + "lsr r6, r8, #4\n\t" + "cmp r6, #9\n\t" + "bcs L_mldsa_extract_coeffs_eta4_neon_skip_hi_%=\n\t" + "add lr, r12, r6\n\t" + "ldrsb r6, [lr]\n\t" + "str r6, [%[s]], #4\n\t" + "add r5, r5, #1\n\t" + "\n" + "L_mldsa_extract_coeffs_eta4_neon_skip_hi_%=:\n\t" + "b L_mldsa_extract_coeffs_eta4_neon_tail_%=\n\t" + "\n" + "L_mldsa_extract_coeffs_eta4_neon_done_%=:\n\t" + "str r5, [%[cnt]]\n\t" + "pop {%[L_mldsa_extract_coeffs_eta4_neon_map], " + "%[L_mldsa_extract_coeffs_eta4_neon_aux]}\n\t" + "pop {%[L_mldsa_extract_coeffs_eta4_neon_bits], " + "%[L_mldsa_extract_coeffs_eta4_neon_idx]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [z] "+r" (z), [zLen] "+r" (zLen), [s] "+r" (s), [cnt] "+r" (cnt), + [L_mldsa_extract_coeffs_eta4_neon_map] "+r" (L_mldsa_extract_coeffs_eta4_neon_map_c), + [L_mldsa_extract_coeffs_eta4_neon_aux] "+r" (L_mldsa_extract_coeffs_eta4_neon_aux_c), + [L_mldsa_extract_coeffs_eta4_neon_bits] "+r" (L_mldsa_extract_coeffs_eta4_neon_bits_c), + [L_mldsa_extract_coeffs_eta4_neon_idx] "+r" (L_mldsa_extract_coeffs_eta4_neon_idx_c) + : +#else + : + : [z] "r" (z), [zLen] "r" (zLen), [s] "r" (s), [cnt] "r" (cnt), + [L_mldsa_extract_coeffs_eta4_neon_map] "r" (L_mldsa_extract_coeffs_eta4_neon_map_c), + [L_mldsa_extract_coeffs_eta4_neon_aux] "r" (L_mldsa_extract_coeffs_eta4_neon_aux_c), + [L_mldsa_extract_coeffs_eta4_neon_bits] "r" (L_mldsa_extract_coeffs_eta4_neon_bits_c), + [L_mldsa_extract_coeffs_eta4_neon_idx] "r" (L_mldsa_extract_coeffs_eta4_neon_idx_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r6", "r7", "r8", "q0", "q1", "q2", "q3", "q4", "q5", + "q6", "q7", "q8", "q9", "q10", "q11", "q12", "q13" + ); +} + +XALIGNED(8) static const word32 L_mldsa_arm32_make_hint_bits[] = { + 0x00000001, 0x00000002, 0x00000004, 0x00000008, + 0x00000010, 0x00000020, 0x00000040, 0x00000080, + 0x00000100, 0x00000200, 0x00000400, 0x00000800, + 0x00001000, 0x00002000, 0x00004000, 0x00008000, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER int mldsa_make_hint_neon(const sword32* s_p, + const sword32* w1_p, byte* h_p, int idx_p, int omega_p, sword32 gamma2_p) +#else +WC_OMIT_FRAME_POINTER int mldsa_make_hint_neon(const sword32* s, + const sword32* w1, byte* h, int idx, int omega, sword32 gamma2) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register const sword32* s __asm__ ("r0") = (const sword32*)s_p; + register const sword32* w1 __asm__ ("r1") = (const sword32*)w1_p; + register byte* h __asm__ ("r2") = (byte*)h_p; + register int idx __asm__ ("r3") = (int)idx_p; + register int omega __asm__ ("r12") = (int)omega_p; + register sword32 gamma2 __asm__ ("lr") = (sword32)gamma2_p; + register word32* L_mldsa_arm32_make_hint_bits_c __asm__ ("r4") = + (word32*)&L_mldsa_arm32_make_hint_bits; +#else + register word32* L_mldsa_arm32_make_hint_bits_c = + (word32*)&L_mldsa_arm32_make_hint_bits; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_mldsa_arm32_make_hint_bits]}\n\t" + "push {%[omega], %[gamma2]}\n\t" + "vdup.32 q0, %[gamma2]\n\t" + "vneg.s32 q1, q0\n\t" + "mov r4, %[L_mldsa_arm32_make_hint_bits]\n\t" + "vld1.8 {q2}, [r4]!\n\t" + "vld1.8 {q3}, [r4]!\n\t" + "vld1.8 {q4}, [r4]!\n\t" + "vld1.8 {q5}, [r4]!\n\t" + "mov r5, #0\n\t" + "\n" + "L_mldsa_arm32_make_hint_loop_%=:\n\t" + "veor q11, q11, q11\n\t" + "vld1.8 {q6}, [%[s]]!\n\t" + "vld1.8 {q7}, [%[w1]]!\n\t" + "vcgt.s32 q8, q6, q0\n\t" + "vcgt.s32 q9, q1, q6\n\t" + "vceq.i32 q10, q6, q1\n\t" + "vceq.i32 q7, q7, #0\n\t" + "vbic q10, q10, q7\n\t" + "vorr q8, q8, q9\n\t" + "vorr q8, q8, q10\n\t" + "vand q8, q8, q2\n\t" + "vorr q11, q11, q8\n\t" + "vld1.8 {q6}, [%[s]]!\n\t" + "vld1.8 {q7}, [%[w1]]!\n\t" + "vcgt.s32 q8, q6, q0\n\t" + "vcgt.s32 q9, q1, q6\n\t" + "vceq.i32 q10, q6, q1\n\t" + "vceq.i32 q7, q7, #0\n\t" + "vbic q10, q10, q7\n\t" + "vorr q8, q8, q9\n\t" + "vorr q8, q8, q10\n\t" + "vand q8, q8, q3\n\t" + "vorr q11, q11, q8\n\t" + "vld1.8 {q6}, [%[s]]!\n\t" + "vld1.8 {q7}, [%[w1]]!\n\t" + "vcgt.s32 q8, q6, q0\n\t" + "vcgt.s32 q9, q1, q6\n\t" + "vceq.i32 q10, q6, q1\n\t" + "vceq.i32 q7, q7, #0\n\t" + "vbic q10, q10, q7\n\t" + "vorr q8, q8, q9\n\t" + "vorr q8, q8, q10\n\t" + "vand q8, q8, q4\n\t" + "vorr q11, q11, q8\n\t" + "vld1.8 {q6}, [%[s]]!\n\t" + "vld1.8 {q7}, [%[w1]]!\n\t" + "vcgt.s32 q8, q6, q0\n\t" + "vcgt.s32 q9, q1, q6\n\t" + "vceq.i32 q10, q6, q1\n\t" + "vceq.i32 q7, q7, #0\n\t" + "vbic q10, q10, q7\n\t" + "vorr q8, q8, q9\n\t" + "vorr q8, q8, q10\n\t" + "vand q8, q8, q5\n\t" + "vorr q11, q11, q8\n\t" + "vpadd.u32 d22, d22, d23\n\t" + "vpadd.u32 d22, d22, d22\n\t" + "vmov.32 r6, d22[0]\n\t" + "cmp r6, #0\n\t" + "beq L_mldsa_arm32_make_hint_next_%=\n\t" + "\n" + "L_mldsa_arm32_make_hint_bit_%=:\n\t" + "rbit r7, r6\n\t" + "clz r7, r7\n\t" + "add r7, r7, r5\n\t" + "strb r7, [%[h]]\n\t" + "add %[h], %[h], #1\n\t" + "add %[idx], %[idx], #1\n\t" + "cmp %[idx], %[omega]\n\t" + "bgt L_mldsa_arm32_make_hint_fail_%=\n\t" + "sub r7, r6, #1\n\t" + "and r6, r6, r7\n\t" + "cmp r6, #0\n\t" + "bne L_mldsa_arm32_make_hint_bit_%=\n\t" + "\n" + "L_mldsa_arm32_make_hint_next_%=:\n\t" + "add r5, r5, #16\n\t" + "cmp r5, #0x100\n\t" + "blt L_mldsa_arm32_make_hint_loop_%=\n\t" + "mov r0, %[idx]\n\t" + "b L_mldsa_arm32_make_hint_done_%=\n\t" + "\n" + "L_mldsa_arm32_make_hint_fail_%=:\n\t" + "mvn r0, #0\n\t" + "\n" + "L_mldsa_arm32_make_hint_done_%=:\n\t" + "pop {%[omega], %[gamma2]}\n\t" + "pop {%[L_mldsa_arm32_make_hint_bits]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [s] "+r" (s), [w1] "+r" (w1), [h] "+r" (h), [idx] "+r" (idx), + [omega] "+r" (omega), [gamma2] "+r" (gamma2), + [L_mldsa_arm32_make_hint_bits] "+r" (L_mldsa_arm32_make_hint_bits_c) + : +#else + : + : [s] "r" (s), [w1] "r" (w1), [h] "r" (h), [idx] "r" (idx), + [omega] "r" (omega), [gamma2] "r" (gamma2), + [L_mldsa_arm32_make_hint_bits] "r" (L_mldsa_arm32_make_hint_bits_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r5", "r6", "r7", "q0", "q1", "q2", "q3", "q4", "q5", + "q6", "q7", "q8", "q9", "q10", "q11" + ); + return (word32)(size_t)s; +} + +XALIGNED(8) static const word32 L_mldsa_rej_uniform_neon_aux[] = { + 0x007fffff, 0x007fffff, 0x007fffff, 0x007fffff, + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, + 0x00000001, 0x00000002, 0x00000004, 0x00000008, +}; + +XALIGNED(4) static const word8 L_mldsa_rej_uniform_neon_shuffle[] = { + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x0c, 0x0d, 0x0e, 0xff, 0x0f, 0x10, 0x11, 0xff, + 0x12, 0x13, 0x14, 0xff, 0x15, 0x16, 0x17, 0xff, +}; + +XALIGNED(4) static const word8 L_mldsa_rej_uniform_neon_indices[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER unsigned int mldsa_rej_uniform_neon(sword32* p_p, + unsigned int len_p, const byte* r_p, unsigned int rLen_p) +#else +WC_OMIT_FRAME_POINTER unsigned int mldsa_rej_uniform_neon(sword32* p, + unsigned int len, const byte* r, unsigned int rLen) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register sword32* p __asm__ ("r0") = (sword32*)p_p; + register unsigned int len __asm__ ("r1") = (unsigned int)len_p; + register const byte* r __asm__ ("r2") = (const byte*)r_p; + register unsigned int rLen __asm__ ("r3") = (unsigned int)rLen_p; + register word32* L_mldsa_rej_uniform_neon_aux_c __asm__ ("r12") = + (word32*)&L_mldsa_rej_uniform_neon_aux; + register word8* L_mldsa_rej_uniform_neon_shuffle_c __asm__ ("lr") = + (word8*)&L_mldsa_rej_uniform_neon_shuffle; + register word8* L_mldsa_rej_uniform_neon_indices_c __asm__ ("r4") = + (word8*)&L_mldsa_rej_uniform_neon_indices; +#else + register word32* L_mldsa_rej_uniform_neon_aux_c = + (word32*)&L_mldsa_rej_uniform_neon_aux; + register word8* L_mldsa_rej_uniform_neon_shuffle_c = + (word8*)&L_mldsa_rej_uniform_neon_shuffle; + register word8* L_mldsa_rej_uniform_neon_indices_c = + (word8*)&L_mldsa_rej_uniform_neon_indices; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "push {%[L_mldsa_rej_uniform_neon_indices]}\n\t" + "push {%[L_mldsa_rej_uniform_neon_aux], " + "%[L_mldsa_rej_uniform_neon_shuffle]}\n\t" + "mov r12, %[L_mldsa_rej_uniform_neon_aux]\n\t" + "vld1.8 {q4}, [r12]!\n\t" + "vld1.8 {q5}, [r12]!\n\t" + "vld1.8 {q6}, [r12]\n\t" + "mov r12, %[L_mldsa_rej_uniform_neon_shuffle]\n\t" + "vld1.8 {d3-d6}, [r12]\n\t" + "mov lr, %[L_mldsa_rej_uniform_neon_indices]\n\t" + "vmov.32 r7, d10[0]\n\t" + "mov r4, #0\n\t" + "cmp %[len], #0\n\t" + "beq L_mldsa_rej_uniform_neon_done_%=\n\t" + "cmp %[len], #16\n\t" + "blt L_mldsa_rej_uniform_neon_loop_2_%=\n\t" + "\n" + "L_mldsa_rej_uniform_neon_loop_8_%=:\n\t" + "vld1.8 {d0-d1}, [%[r]]!\n\t" + "vld1.8 {d2}, [%[r]]!\n\t" + "vtbl.8 d16, {d0-d2}, d3\n\t" + "vtbl.8 d17, {d0-d2}, d4\n\t" + "vand q7, q8, q4\n\t" + "vcgt.s32 q8, q5, q7\n\t" + "vshr.u32 q9, q8, #31\n\t" + "vpadd.u32 d18, d18, d19\n\t" + "vpadd.u32 d18, d18, d18\n\t" + "vmov.32 r5, d18[0]\n\t" + "vand q9, q8, q6\n\t" + "vpadd.u32 d18, d18, d19\n\t" + "vpadd.u32 d18, d18, d18\n\t" + "vmov.32 r6, d18[0]\n\t" + "lsl r6, r6, #4\n\t" + "add r6, r6, lr\n\t" + "vld1.8 {q10}, [r6]\n\t" + "vtbl.8 d16, {d14-d15}, d20\n\t" + "vtbl.8 d17, {d14-d15}, d21\n\t" + "vst1.8 {q8}, [%[p]]\n\t" + "add %[p], %[p], r5, lsl #2\n\t" + "add r4, r4, r5\n\t" + "vtbl.8 d16, {d0-d2}, d5\n\t" + "vtbl.8 d17, {d0-d2}, d6\n\t" + "vand q7, q8, q4\n\t" + "vcgt.s32 q8, q5, q7\n\t" + "vshr.u32 q9, q8, #31\n\t" + "vpadd.u32 d18, d18, d19\n\t" + "vpadd.u32 d18, d18, d18\n\t" + "vmov.32 r5, d18[0]\n\t" + "vand q9, q8, q6\n\t" + "vpadd.u32 d18, d18, d19\n\t" + "vpadd.u32 d18, d18, d18\n\t" + "vmov.32 r6, d18[0]\n\t" + "lsl r6, r6, #4\n\t" + "add r6, r6, lr\n\t" + "vld1.8 {q10}, [r6]\n\t" + "vtbl.8 d16, {d14-d15}, d20\n\t" + "vtbl.8 d17, {d14-d15}, d21\n\t" + "vst1.8 {q8}, [%[p]]\n\t" + "add %[p], %[p], r5, lsl #2\n\t" + "add r4, r4, r5\n\t" + "subs %[rLen], %[rLen], #24\n\t" + "beq L_mldsa_rej_uniform_neon_done_%=\n\t" + "sub r5, %[len], r4\n\t" + "subs r5, r5, #8\n\t" + "blt L_mldsa_rej_uniform_neon_loop_2_%=\n\t" + "b L_mldsa_rej_uniform_neon_loop_8_%=\n\t" + "\n" + "L_mldsa_rej_uniform_neon_loop_2_%=:\n\t" + "subs r5, %[len], r4\n\t" + "beq L_mldsa_rej_uniform_neon_done_%=\n\t" + "subs r5, r5, #2\n\t" + "blt L_mldsa_rej_uniform_neon_loop_lt_2_%=\n\t" + "ldrb r12, [%[r]], #1\n\t" + "ldrb r8, [%[r]], #1\n\t" + "orr r12, r12, r8, lsl #8\n\t" + "ldrb r8, [%[r]], #1\n\t" + "orr r12, r12, r8, lsl #16\n\t" + "bic r12, r12, #0x800000\n\t" + "ldrb r6, [%[r]], #1\n\t" + "ldrb r8, [%[r]], #1\n\t" + "orr r6, r6, r8, lsl #8\n\t" + "ldrb r8, [%[r]], #1\n\t" + "orr r6, r6, r8, lsl #16\n\t" + "bic r6, r6, #0x800000\n\t" + "str r12, [%[p]]\n\t" + "add r5, %[p], #4\n\t" + "cmp r12, r7\n\t" + "movlo %[p], r5\n\t" + "addlo r4, r4, #1\n\t" + "str r6, [%[p]]\n\t" + "add r5, %[p], #4\n\t" + "cmp r6, r7\n\t" + "movlo %[p], r5\n\t" + "addlo r4, r4, #1\n\t" + "subs %[rLen], %[rLen], #6\n\t" + "beq L_mldsa_rej_uniform_neon_done_%=\n\t" + "b L_mldsa_rej_uniform_neon_loop_2_%=\n\t" + "\n" + "L_mldsa_rej_uniform_neon_loop_lt_2_%=:\n\t" + "ldrb r12, [%[r]], #1\n\t" + "ldrb r8, [%[r]], #1\n\t" + "orr r12, r12, r8, lsl #8\n\t" + "ldrb r8, [%[r]], #1\n\t" + "orr r12, r12, r8, lsl #16\n\t" + "bic r12, r12, #0x800000\n\t" + "ldrb r6, [%[r]], #1\n\t" + "ldrb r8, [%[r]], #1\n\t" + "orr r6, r6, r8, lsl #8\n\t" + "ldrb r8, [%[r]], #1\n\t" + "orr r6, r6, r8, lsl #16\n\t" + "bic r6, r6, #0x800000\n\t" + "str r12, [%[p]]\n\t" + "add r5, %[p], #4\n\t" + "cmp r12, r7\n\t" + "movlo %[p], r5\n\t" + "addlo r4, r4, #1\n\t" + "cmp %[len], r4\n\t" + "beq L_mldsa_rej_uniform_neon_done_%=\n\t" + "str r6, [%[p]]\n\t" + "add r5, %[p], #4\n\t" + "cmp r6, r7\n\t" + "movlo %[p], r5\n\t" + "addlo r4, r4, #1\n\t" + "cmp %[len], r4\n\t" + "beq L_mldsa_rej_uniform_neon_done_%=\n\t" + "subs %[rLen], %[rLen], #6\n\t" + "beq L_mldsa_rej_uniform_neon_done_%=\n\t" + "b L_mldsa_rej_uniform_neon_loop_lt_2_%=\n\t" + "\n" + "L_mldsa_rej_uniform_neon_done_%=:\n\t" + "mov r0, r4\n\t" + "pop {%[L_mldsa_rej_uniform_neon_aux], " + "%[L_mldsa_rej_uniform_neon_shuffle]}\n\t" + "pop {%[L_mldsa_rej_uniform_neon_indices]}\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [p] "+r" (p), [len] "+r" (len), [r] "+r" (r), [rLen] "+r" (rLen), + [L_mldsa_rej_uniform_neon_aux] "+r" (L_mldsa_rej_uniform_neon_aux_c), + [L_mldsa_rej_uniform_neon_shuffle] "+r" (L_mldsa_rej_uniform_neon_shuffle_c), + [L_mldsa_rej_uniform_neon_indices] "+r" (L_mldsa_rej_uniform_neon_indices_c) + : +#else + : + : [p] "r" (p), [len] "r" (len), [r] "r" (r), [rLen] "r" (rLen), + [L_mldsa_rej_uniform_neon_aux] "r" (L_mldsa_rej_uniform_neon_aux_c), + [L_mldsa_rej_uniform_neon_shuffle] "r" (L_mldsa_rej_uniform_neon_shuffle_c), + [L_mldsa_rej_uniform_neon_indices] "r" (L_mldsa_rej_uniform_neon_indices_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r5", "r6", "r7", "r8", "d0", "d1", "d2", "d3", "d4", + "d5", "d6", "q4", "q5", "q6", "q7", "q8", "q9", "q10" + ); + return (word32)(size_t)p; +} + +#endif /* WOLFSSL_ARMASM_NO_NEON */ +#endif /* WOLFSSL_HAVE_MLDSA */ + +#endif /* WOLFSSL_ARMASM_INLINE */ +#endif /* !__aarch64__ && !WOLFSSL_ARMASM_THUMB2 */ +#endif /* WOLFSSL_ARMASM */ diff --git a/wolfcrypt/src/port/arm/armv8-32-sha3-asm.S b/wolfcrypt/src/port/arm/armv8-32-sha3-asm.S index aa7192c67ac..14da06b4ff5 100644 --- a/wolfcrypt/src/port/arm/armv8-32-sha3-asm.S +++ b/wolfcrypt/src/port/arm/armv8-32-sha3-asm.S @@ -30,6 +30,12 @@ #include +/* Honor WC_SHA3_NO_ASM as sha3.c does: suppress this NEON BlockSha3 so it + * doesn't multiply-define against sha3.c's C BlockSha3 on arm32. */ +#ifdef WC_SHA3_NO_ASM + #undef WOLFSSL_ARMASM +#endif + #ifdef WOLFSSL_ARMASM #if !defined(__aarch64__) && !defined(WOLFSSL_ARMASM_THUMB2) #ifndef WOLFSSL_ARMASM_INLINE @@ -328,6 +334,1253 @@ L_sha3_arm32_neon_begin: vpop {d8-d15} bx lr .size BlockSha3,.-BlockSha3 +#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) +#ifndef __APPLE__ + .text + .type L_sha3_arm32_neon_x3_rt, %object + .size L_sha3_arm32_neon_x3_rt, 192 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 16-byte aligned, 128-bit aligned +#ifndef __APPLE__ + .align 4 +#else + .p2align 4 +#endif /* __APPLE__ */ +L_sha3_arm32_neon_x3_rt: + .quad 0x0000000000000001,0x0000000000008082 + .quad 0x800000000000808a,0x8000000080008000 + .quad 0x000000000000808b,0x0000000080000001 + .quad 0x8000000080008081,0x8000000000008009 + .quad 0x000000000000008a,0x0000000000000088 + .quad 0x0000000080008009,0x000000008000000a + .quad 0x000000008000808b,0x800000000000008b + .quad 0x8000000000008089,0x8000000000008003 + .quad 0x8000000000008002,0x8000000000000080 + .quad 0x000000000000800a,0x800000008000000a + .quad 0x8000000080008081,0x8000000000008080 + .quad 0x0000000080000001,0x8000000080008008 + .text + .align 4 + .globl sha3_blocksx3_neon + WC_ASM_ATT_HIDDEN(sha3_blocksx3_neon) + .type sha3_blocksx3_neon, %function +sha3_blocksx3_neon: + push {lr} + vpush {d8-d15} + sub sp, sp, #16 + adr r1, L_sha3_arm32_neon_x3_rt + mov lr, #3 +L_sha3_arm32_neon_x3_state: + vld1.8 {d0-d3}, [r0]! + vld1.8 {d4-d7}, [r0]! + vld1.8 {d8-d11}, [r0]! + vld1.8 {d12-d15}, [r0]! + vld1.8 {d16-d19}, [r0]! + vld1.8 {d20-d23}, [r0]! + vld1.8 {d24}, [r0] + sub r0, r0, #0xc0 + mov r2, r1 + mov r3, #24 + mov r12, sp +L_sha3_arm32_neon_x3_begin: + # Calc b[0..4] + veor d26, d0, d5 + veor d27, d1, d6 + veor d28, d2, d7 + veor d29, d3, d8 + veor d25, d4, d9 + veor d26, d26, d10 + veor d27, d27, d11 + veor d28, d28, d12 + veor d29, d29, d13 + veor d25, d25, d14 + veor d26, d26, d15 + veor d27, d27, d16 + veor d28, d28, d17 + veor d29, d29, d18 + veor d25, d25, d19 + veor d26, d26, d20 + veor d27, d27, d21 + veor d28, d28, d22 + veor d29, d29, d23 + veor d25, d25, d24 + vst1.8 {d25-d26}, [r12] + # Calc t[0..4] and XOR into s[i*5..i*5+4] + # t[0] + vshr.u64 d30, d27, #63 + vshl.u64 d31, d27, #1 + veor d25, d25, d30 + veor d25, d25, d31 + # t[1] + vshr.u64 d30, d28, #63 + vshl.u64 d31, d28, #1 + veor d26, d26, d30 + veor d26, d26, d31 + # t[2] + vshr.u64 d30, d29, #63 + vshl.u64 d31, d29, #1 + veor d27, d27, d30 + veor d27, d27, d31 + # t[3] + vldr.8 d31, [r12] + vshr.u64 d30, d31, #63 + vshl.u64 d31, d31, #1 + veor d28, d28, d30 + veor d28, d28, d31 + # t[4] + vldr.8 d31, [r12, #8] + vshr.u64 d30, d31, #63 + vshl.u64 d31, d31, #1 + veor d29, d29, d30 + veor d29, d29, d31 + sub r12, r12, #16 + veor d0, d0, d25 + # s[1] => s[10] (tmp) + veor d30, d1, d26 + vshr.u64 d31, d30, #63 + vshl.u64 d30, d30, #1 + veor d30, d30, d31 + # s[6] => s[1] + veor d1, d6, d26 + vshr.u64 d31, d1, #20 + vshl.u64 d1, d1, #44 + veor d1, d1, d31 + # s[9] => s[6] + veor d6, d9, d29 + vshr.u64 d31, d6, #44 + vshl.u64 d6, d6, #20 + veor d6, d6, d31 + # s[22] => s[9] + veor d9, d22, d27 + vshr.u64 d31, d9, #3 + vshl.u64 d9, d9, #61 + veor d9, d9, d31 + # s[14] => s[22] + veor d22, d14, d29 + vshr.u64 d31, d22, #25 + vshl.u64 d22, d22, #39 + veor d22, d22, d31 + # s[20] => s[14] + veor d14, d20, d25 + vshr.u64 d31, d14, #46 + vshl.u64 d14, d14, #18 + veor d14, d14, d31 + # s[2] => s[20] + veor d20, d2, d27 + vshr.u64 d31, d20, #2 + vshl.u64 d20, d20, #62 + veor d20, d20, d31 + # s[12] => s[2] + veor d2, d12, d27 + vshr.u64 d31, d2, #21 + vshl.u64 d2, d2, #43 + veor d2, d2, d31 + # s[13] => s[12] + veor d12, d13, d28 + vshr.u64 d31, d12, #39 + vshl.u64 d12, d12, #25 + veor d12, d12, d31 + # s[19] => s[13] + veor d13, d19, d29 + vshr.u64 d31, d13, #56 + vshl.u64 d13, d13, #8 + veor d13, d13, d31 + # s[23] => s[19] + veor d19, d23, d28 + vshr.u64 d31, d19, #8 + vshl.u64 d19, d19, #56 + veor d19, d19, d31 + # s[15] => s[23] + veor d23, d15, d25 + vshr.u64 d31, d23, #23 + vshl.u64 d23, d23, #41 + veor d23, d23, d31 + # s[4] => s[15] + veor d15, d4, d29 + vshr.u64 d31, d15, #37 + vshl.u64 d15, d15, #27 + veor d15, d15, d31 + # s[24] => s[4] + veor d4, d24, d29 + vshr.u64 d31, d4, #50 + vshl.u64 d4, d4, #14 + veor d4, d4, d31 + # s[21] => s[24] + veor d24, d21, d26 + vshr.u64 d31, d24, #62 + vshl.u64 d24, d24, #2 + veor d24, d24, d31 + # s[8] => s[21] + veor d21, d8, d28 + vshr.u64 d31, d21, #9 + vshl.u64 d21, d21, #55 + veor d21, d21, d31 + # s[16] => s[8] + veor d8, d16, d26 + vshr.u64 d31, d8, #19 + vshl.u64 d8, d8, #45 + veor d8, d8, d31 + # s[5] => s[16] + veor d16, d5, d25 + vshr.u64 d31, d16, #28 + vshl.u64 d16, d16, #36 + veor d16, d16, d31 + # s[3] => s[5] + veor d5, d3, d28 + vshr.u64 d31, d5, #36 + vshl.u64 d5, d5, #28 + veor d5, d5, d31 + # s[18] => s[3] + veor d3, d18, d28 + vshr.u64 d31, d3, #43 + vshl.u64 d3, d3, #21 + veor d3, d3, d31 + # s[17] => s[18] + veor d18, d17, d27 + vshr.u64 d31, d18, #49 + vshl.u64 d18, d18, #15 + veor d18, d18, d31 + # s[11] => s[17] + veor d17, d11, d26 + vshr.u64 d31, d17, #54 + vshl.u64 d17, d17, #10 + veor d17, d17, d31 + # s[7] => s[11] + veor d11, d7, d27 + vshr.u64 d31, d11, #58 + vshl.u64 d11, d11, #6 + veor d11, d11, d31 + # s[10] => s[7] + veor d7, d10, d25 + vshr.u64 d31, d7, #61 + vshl.u64 d7, d7, #3 + veor d7, d7, d31 + # Row Mix + vmov d25, d0 + vmov d26, d1 + vbic d31, d2, d26 + veor d0, d25, d31 + vbic d31, d3, d2 + veor d1, d26, d31 + vbic d31, d4, d3 + veor d2, d2, d31 + vbic d31, d25, d4 + veor d3, d3, d31 + vbic d31, d26, d25 + veor d4, d4, d31 + vmov d25, d5 + vmov d26, d6 + vbic d31, d7, d26 + veor d5, d25, d31 + vbic d31, d8, d7 + veor d6, d26, d31 + vbic d31, d9, d8 + veor d7, d7, d31 + vbic d31, d25, d9 + veor d8, d8, d31 + vbic d31, d26, d25 + veor d9, d9, d31 + vmov d26, d11 + vbic d31, d12, d26 + veor d10, d30, d31 + vbic d31, d13, d12 + veor d11, d26, d31 + vbic d31, d14, d13 + veor d12, d12, d31 + vbic d31, d30, d14 + veor d13, d13, d31 + vbic d31, d26, d30 + veor d14, d14, d31 + vmov d25, d15 + vmov d26, d16 + vbic d31, d17, d26 + veor d15, d25, d31 + vbic d31, d18, d17 + veor d16, d26, d31 + vbic d31, d19, d18 + veor d17, d17, d31 + vbic d31, d25, d19 + veor d18, d18, d31 + vbic d31, d26, d25 + veor d19, d19, d31 + vmov d25, d20 + vmov d26, d21 + vbic d31, d22, d26 + veor d20, d25, d31 + vbic d31, d23, d22 + veor d21, d26, d31 + vbic d31, d24, d23 + veor d22, d22, d31 + vbic d31, d25, d24 + veor d23, d23, d31 + vbic d31, d26, d25 + veor d24, d24, d31 + vld1.8 {d30}, [r2]! + subs r3, r3, #1 + veor d0, d0, d30 + bne L_sha3_arm32_neon_x3_begin + vst1.8 {d0-d3}, [r0]! + vst1.8 {d4-d7}, [r0]! + vst1.8 {d8-d11}, [r0]! + vst1.8 {d12-d15}, [r0]! + vst1.8 {d16-d19}, [r0]! + vst1.8 {d20-d23}, [r0]! + vst1.8 {d24}, [r0] + add r0, r0, #8 + subs lr, lr, #1 + bne L_sha3_arm32_neon_x3_state + add sp, sp, #16 + vpop {d8-d15} + pop {pc} + .size sha3_blocksx3_neon,.-sha3_blocksx3_neon +#ifndef __APPLE__ + .text + .type L_sha3_128_blocksx3_seed_neon_rt, %object + .size L_sha3_128_blocksx3_seed_neon_rt, 200 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 16-byte aligned, 128-bit aligned +#ifndef __APPLE__ + .align 4 +#else + .p2align 4 +#endif /* __APPLE__ */ +L_sha3_128_blocksx3_seed_neon_rt: + .quad 0x0000000000000001,0x0000000000008082 + .quad 0x800000000000808a,0x8000000080008000 + .quad 0x000000000000808b,0x0000000080000001 + .quad 0x8000000080008081,0x8000000000008009 + .quad 0x000000000000008a,0x0000000000000088 + .quad 0x0000000080008009,0x000000008000000a + .quad 0x000000008000808b,0x800000000000008b + .quad 0x8000000000008089,0x8000000000008003 + .quad 0x8000000000008002,0x8000000000000080 + .quad 0x000000000000800a,0x800000008000000a + .quad 0x8000000080008081,0x8000000000008080 + .quad 0x0000000080000001,0x8000000080008008 + .quad 0x8000000000000000 + .text + .align 4 + .globl sha3_128_blocksx3_seed_neon + WC_ASM_ATT_HIDDEN(sha3_128_blocksx3_seed_neon) + .type sha3_128_blocksx3_seed_neon, %function +sha3_128_blocksx3_seed_neon: + push {r4, lr} + vpush {d8-d15} + sub sp, sp, #16 + adr r2, L_sha3_128_blocksx3_seed_neon_rt + mov r4, #3 +L_sha3_128_blocksx3_seed_neon_state: + vld1.8 {d0-d3}, [r1]! + sub r1, r1, #32 + vldr.8 d4, [r0, #32] + veor d5, d5, d5 + veor d6, d6, d6 + veor d7, d7, d7 + veor d8, d8, d8 + veor d9, d9, d9 + veor d10, d10, d10 + veor d11, d11, d11 + veor d12, d12, d12 + veor d13, d13, d13 + veor d14, d14, d14 + veor d15, d15, d15 + veor d16, d16, d16 + veor d17, d17, d17 + veor d18, d18, d18 + veor d19, d19, d19 + vldr.8 d20, [r2, #192] + veor d21, d21, d21 + veor d22, d22, d22 + veor d23, d23, d23 + veor d24, d24, d24 + mov r3, r2 + mov r12, #24 + mov lr, sp +L_sha3_128_blocksx3_seed_neon_begin: + # Calc b[0..4] + veor d26, d0, d5 + veor d27, d1, d6 + veor d28, d2, d7 + veor d29, d3, d8 + veor d25, d4, d9 + veor d26, d26, d10 + veor d27, d27, d11 + veor d28, d28, d12 + veor d29, d29, d13 + veor d25, d25, d14 + veor d26, d26, d15 + veor d27, d27, d16 + veor d28, d28, d17 + veor d29, d29, d18 + veor d25, d25, d19 + veor d26, d26, d20 + veor d27, d27, d21 + veor d28, d28, d22 + veor d29, d29, d23 + veor d25, d25, d24 + vst1.8 {d25-d26}, [lr] + # Calc t[0..4] and XOR into s[i*5..i*5+4] + # t[0] + vshr.u64 d30, d27, #63 + vshl.u64 d31, d27, #1 + veor d25, d25, d30 + veor d25, d25, d31 + # t[1] + vshr.u64 d30, d28, #63 + vshl.u64 d31, d28, #1 + veor d26, d26, d30 + veor d26, d26, d31 + # t[2] + vshr.u64 d30, d29, #63 + vshl.u64 d31, d29, #1 + veor d27, d27, d30 + veor d27, d27, d31 + # t[3] + vldr.8 d31, [lr] + vshr.u64 d30, d31, #63 + vshl.u64 d31, d31, #1 + veor d28, d28, d30 + veor d28, d28, d31 + # t[4] + vldr.8 d31, [lr, #8] + vshr.u64 d30, d31, #63 + vshl.u64 d31, d31, #1 + veor d29, d29, d30 + veor d29, d29, d31 + sub lr, lr, #16 + veor d0, d0, d25 + # s[1] => s[10] (tmp) + veor d30, d1, d26 + vshr.u64 d31, d30, #63 + vshl.u64 d30, d30, #1 + veor d30, d30, d31 + # s[6] => s[1] + veor d1, d6, d26 + vshr.u64 d31, d1, #20 + vshl.u64 d1, d1, #44 + veor d1, d1, d31 + # s[9] => s[6] + veor d6, d9, d29 + vshr.u64 d31, d6, #44 + vshl.u64 d6, d6, #20 + veor d6, d6, d31 + # s[22] => s[9] + veor d9, d22, d27 + vshr.u64 d31, d9, #3 + vshl.u64 d9, d9, #61 + veor d9, d9, d31 + # s[14] => s[22] + veor d22, d14, d29 + vshr.u64 d31, d22, #25 + vshl.u64 d22, d22, #39 + veor d22, d22, d31 + # s[20] => s[14] + veor d14, d20, d25 + vshr.u64 d31, d14, #46 + vshl.u64 d14, d14, #18 + veor d14, d14, d31 + # s[2] => s[20] + veor d20, d2, d27 + vshr.u64 d31, d20, #2 + vshl.u64 d20, d20, #62 + veor d20, d20, d31 + # s[12] => s[2] + veor d2, d12, d27 + vshr.u64 d31, d2, #21 + vshl.u64 d2, d2, #43 + veor d2, d2, d31 + # s[13] => s[12] + veor d12, d13, d28 + vshr.u64 d31, d12, #39 + vshl.u64 d12, d12, #25 + veor d12, d12, d31 + # s[19] => s[13] + veor d13, d19, d29 + vshr.u64 d31, d13, #56 + vshl.u64 d13, d13, #8 + veor d13, d13, d31 + # s[23] => s[19] + veor d19, d23, d28 + vshr.u64 d31, d19, #8 + vshl.u64 d19, d19, #56 + veor d19, d19, d31 + # s[15] => s[23] + veor d23, d15, d25 + vshr.u64 d31, d23, #23 + vshl.u64 d23, d23, #41 + veor d23, d23, d31 + # s[4] => s[15] + veor d15, d4, d29 + vshr.u64 d31, d15, #37 + vshl.u64 d15, d15, #27 + veor d15, d15, d31 + # s[24] => s[4] + veor d4, d24, d29 + vshr.u64 d31, d4, #50 + vshl.u64 d4, d4, #14 + veor d4, d4, d31 + # s[21] => s[24] + veor d24, d21, d26 + vshr.u64 d31, d24, #62 + vshl.u64 d24, d24, #2 + veor d24, d24, d31 + # s[8] => s[21] + veor d21, d8, d28 + vshr.u64 d31, d21, #9 + vshl.u64 d21, d21, #55 + veor d21, d21, d31 + # s[16] => s[8] + veor d8, d16, d26 + vshr.u64 d31, d8, #19 + vshl.u64 d8, d8, #45 + veor d8, d8, d31 + # s[5] => s[16] + veor d16, d5, d25 + vshr.u64 d31, d16, #28 + vshl.u64 d16, d16, #36 + veor d16, d16, d31 + # s[3] => s[5] + veor d5, d3, d28 + vshr.u64 d31, d5, #36 + vshl.u64 d5, d5, #28 + veor d5, d5, d31 + # s[18] => s[3] + veor d3, d18, d28 + vshr.u64 d31, d3, #43 + vshl.u64 d3, d3, #21 + veor d3, d3, d31 + # s[17] => s[18] + veor d18, d17, d27 + vshr.u64 d31, d18, #49 + vshl.u64 d18, d18, #15 + veor d18, d18, d31 + # s[11] => s[17] + veor d17, d11, d26 + vshr.u64 d31, d17, #54 + vshl.u64 d17, d17, #10 + veor d17, d17, d31 + # s[7] => s[11] + veor d11, d7, d27 + vshr.u64 d31, d11, #58 + vshl.u64 d11, d11, #6 + veor d11, d11, d31 + # s[10] => s[7] + veor d7, d10, d25 + vshr.u64 d31, d7, #61 + vshl.u64 d7, d7, #3 + veor d7, d7, d31 + # Row Mix + vmov d25, d0 + vmov d26, d1 + vbic d31, d2, d26 + veor d0, d25, d31 + vbic d31, d3, d2 + veor d1, d26, d31 + vbic d31, d4, d3 + veor d2, d2, d31 + vbic d31, d25, d4 + veor d3, d3, d31 + vbic d31, d26, d25 + veor d4, d4, d31 + vmov d25, d5 + vmov d26, d6 + vbic d31, d7, d26 + veor d5, d25, d31 + vbic d31, d8, d7 + veor d6, d26, d31 + vbic d31, d9, d8 + veor d7, d7, d31 + vbic d31, d25, d9 + veor d8, d8, d31 + vbic d31, d26, d25 + veor d9, d9, d31 + vmov d26, d11 + vbic d31, d12, d26 + veor d10, d30, d31 + vbic d31, d13, d12 + veor d11, d26, d31 + vbic d31, d14, d13 + veor d12, d12, d31 + vbic d31, d30, d14 + veor d13, d13, d31 + vbic d31, d26, d30 + veor d14, d14, d31 + vmov d25, d15 + vmov d26, d16 + vbic d31, d17, d26 + veor d15, d25, d31 + vbic d31, d18, d17 + veor d16, d26, d31 + vbic d31, d19, d18 + veor d17, d17, d31 + vbic d31, d25, d19 + veor d18, d18, d31 + vbic d31, d26, d25 + veor d19, d19, d31 + vmov d25, d20 + vmov d26, d21 + vbic d31, d22, d26 + veor d20, d25, d31 + vbic d31, d23, d22 + veor d21, d26, d31 + vbic d31, d24, d23 + veor d22, d22, d31 + vbic d31, d25, d24 + veor d23, d23, d31 + vbic d31, d26, d25 + veor d24, d24, d31 + vld1.8 {d30}, [r3]! + subs r12, r12, #1 + veor d0, d0, d30 + bne L_sha3_128_blocksx3_seed_neon_begin + vst1.8 {d0-d3}, [r0]! + vst1.8 {d4-d7}, [r0]! + vst1.8 {d8-d11}, [r0]! + vst1.8 {d12-d15}, [r0]! + vst1.8 {d16-d19}, [r0]! + vst1.8 {d20-d23}, [r0]! + vst1.8 {d24}, [r0] + add r0, r0, #8 + subs r4, r4, #1 + bne L_sha3_128_blocksx3_seed_neon_state + add sp, sp, #16 + vpop {d8-d15} + pop {r4, pc} + .size sha3_128_blocksx3_seed_neon,.-sha3_128_blocksx3_seed_neon +#ifndef __APPLE__ + .text + .type L_sha3_256_blocksx3_seed_neon_rt, %object + .size L_sha3_256_blocksx3_seed_neon_rt, 200 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 16-byte aligned, 128-bit aligned +#ifndef __APPLE__ + .align 4 +#else + .p2align 4 +#endif /* __APPLE__ */ +L_sha3_256_blocksx3_seed_neon_rt: + .quad 0x0000000000000001,0x0000000000008082 + .quad 0x800000000000808a,0x8000000080008000 + .quad 0x000000000000808b,0x0000000080000001 + .quad 0x8000000080008081,0x8000000000008009 + .quad 0x000000000000008a,0x0000000000000088 + .quad 0x0000000080008009,0x000000008000000a + .quad 0x000000008000808b,0x800000000000008b + .quad 0x8000000000008089,0x8000000000008003 + .quad 0x8000000000008002,0x8000000000000080 + .quad 0x000000000000800a,0x800000008000000a + .quad 0x8000000080008081,0x8000000000008080 + .quad 0x0000000080000001,0x8000000080008008 + .quad 0x8000000000000000 + .text + .align 4 + .globl sha3_256_blocksx3_seed_neon + WC_ASM_ATT_HIDDEN(sha3_256_blocksx3_seed_neon) + .type sha3_256_blocksx3_seed_neon, %function +sha3_256_blocksx3_seed_neon: + push {r4, lr} + vpush {d8-d15} + sub sp, sp, #16 + adr r2, L_sha3_256_blocksx3_seed_neon_rt + mov r4, #3 +L_sha3_256_blocksx3_seed_neon_state: + vld1.8 {d0-d3}, [r1]! + sub r1, r1, #32 + vldr.8 d4, [r0, #32] + veor d5, d5, d5 + veor d6, d6, d6 + veor d7, d7, d7 + veor d8, d8, d8 + veor d9, d9, d9 + veor d10, d10, d10 + veor d11, d11, d11 + veor d12, d12, d12 + veor d13, d13, d13 + veor d14, d14, d14 + veor d15, d15, d15 + vldr.8 d16, [r2, #192] + veor d17, d17, d17 + veor d18, d18, d18 + veor d19, d19, d19 + veor d20, d20, d20 + veor d21, d21, d21 + veor d22, d22, d22 + veor d23, d23, d23 + veor d24, d24, d24 + mov r3, r2 + mov r12, #24 + mov lr, sp +L_sha3_256_blocksx3_seed_neon_begin: + # Calc b[0..4] + veor d26, d0, d5 + veor d27, d1, d6 + veor d28, d2, d7 + veor d29, d3, d8 + veor d25, d4, d9 + veor d26, d26, d10 + veor d27, d27, d11 + veor d28, d28, d12 + veor d29, d29, d13 + veor d25, d25, d14 + veor d26, d26, d15 + veor d27, d27, d16 + veor d28, d28, d17 + veor d29, d29, d18 + veor d25, d25, d19 + veor d26, d26, d20 + veor d27, d27, d21 + veor d28, d28, d22 + veor d29, d29, d23 + veor d25, d25, d24 + vst1.8 {d25-d26}, [lr] + # Calc t[0..4] and XOR into s[i*5..i*5+4] + # t[0] + vshr.u64 d30, d27, #63 + vshl.u64 d31, d27, #1 + veor d25, d25, d30 + veor d25, d25, d31 + # t[1] + vshr.u64 d30, d28, #63 + vshl.u64 d31, d28, #1 + veor d26, d26, d30 + veor d26, d26, d31 + # t[2] + vshr.u64 d30, d29, #63 + vshl.u64 d31, d29, #1 + veor d27, d27, d30 + veor d27, d27, d31 + # t[3] + vldr.8 d31, [lr] + vshr.u64 d30, d31, #63 + vshl.u64 d31, d31, #1 + veor d28, d28, d30 + veor d28, d28, d31 + # t[4] + vldr.8 d31, [lr, #8] + vshr.u64 d30, d31, #63 + vshl.u64 d31, d31, #1 + veor d29, d29, d30 + veor d29, d29, d31 + sub lr, lr, #16 + veor d0, d0, d25 + # s[1] => s[10] (tmp) + veor d30, d1, d26 + vshr.u64 d31, d30, #63 + vshl.u64 d30, d30, #1 + veor d30, d30, d31 + # s[6] => s[1] + veor d1, d6, d26 + vshr.u64 d31, d1, #20 + vshl.u64 d1, d1, #44 + veor d1, d1, d31 + # s[9] => s[6] + veor d6, d9, d29 + vshr.u64 d31, d6, #44 + vshl.u64 d6, d6, #20 + veor d6, d6, d31 + # s[22] => s[9] + veor d9, d22, d27 + vshr.u64 d31, d9, #3 + vshl.u64 d9, d9, #61 + veor d9, d9, d31 + # s[14] => s[22] + veor d22, d14, d29 + vshr.u64 d31, d22, #25 + vshl.u64 d22, d22, #39 + veor d22, d22, d31 + # s[20] => s[14] + veor d14, d20, d25 + vshr.u64 d31, d14, #46 + vshl.u64 d14, d14, #18 + veor d14, d14, d31 + # s[2] => s[20] + veor d20, d2, d27 + vshr.u64 d31, d20, #2 + vshl.u64 d20, d20, #62 + veor d20, d20, d31 + # s[12] => s[2] + veor d2, d12, d27 + vshr.u64 d31, d2, #21 + vshl.u64 d2, d2, #43 + veor d2, d2, d31 + # s[13] => s[12] + veor d12, d13, d28 + vshr.u64 d31, d12, #39 + vshl.u64 d12, d12, #25 + veor d12, d12, d31 + # s[19] => s[13] + veor d13, d19, d29 + vshr.u64 d31, d13, #56 + vshl.u64 d13, d13, #8 + veor d13, d13, d31 + # s[23] => s[19] + veor d19, d23, d28 + vshr.u64 d31, d19, #8 + vshl.u64 d19, d19, #56 + veor d19, d19, d31 + # s[15] => s[23] + veor d23, d15, d25 + vshr.u64 d31, d23, #23 + vshl.u64 d23, d23, #41 + veor d23, d23, d31 + # s[4] => s[15] + veor d15, d4, d29 + vshr.u64 d31, d15, #37 + vshl.u64 d15, d15, #27 + veor d15, d15, d31 + # s[24] => s[4] + veor d4, d24, d29 + vshr.u64 d31, d4, #50 + vshl.u64 d4, d4, #14 + veor d4, d4, d31 + # s[21] => s[24] + veor d24, d21, d26 + vshr.u64 d31, d24, #62 + vshl.u64 d24, d24, #2 + veor d24, d24, d31 + # s[8] => s[21] + veor d21, d8, d28 + vshr.u64 d31, d21, #9 + vshl.u64 d21, d21, #55 + veor d21, d21, d31 + # s[16] => s[8] + veor d8, d16, d26 + vshr.u64 d31, d8, #19 + vshl.u64 d8, d8, #45 + veor d8, d8, d31 + # s[5] => s[16] + veor d16, d5, d25 + vshr.u64 d31, d16, #28 + vshl.u64 d16, d16, #36 + veor d16, d16, d31 + # s[3] => s[5] + veor d5, d3, d28 + vshr.u64 d31, d5, #36 + vshl.u64 d5, d5, #28 + veor d5, d5, d31 + # s[18] => s[3] + veor d3, d18, d28 + vshr.u64 d31, d3, #43 + vshl.u64 d3, d3, #21 + veor d3, d3, d31 + # s[17] => s[18] + veor d18, d17, d27 + vshr.u64 d31, d18, #49 + vshl.u64 d18, d18, #15 + veor d18, d18, d31 + # s[11] => s[17] + veor d17, d11, d26 + vshr.u64 d31, d17, #54 + vshl.u64 d17, d17, #10 + veor d17, d17, d31 + # s[7] => s[11] + veor d11, d7, d27 + vshr.u64 d31, d11, #58 + vshl.u64 d11, d11, #6 + veor d11, d11, d31 + # s[10] => s[7] + veor d7, d10, d25 + vshr.u64 d31, d7, #61 + vshl.u64 d7, d7, #3 + veor d7, d7, d31 + # Row Mix + vmov d25, d0 + vmov d26, d1 + vbic d31, d2, d26 + veor d0, d25, d31 + vbic d31, d3, d2 + veor d1, d26, d31 + vbic d31, d4, d3 + veor d2, d2, d31 + vbic d31, d25, d4 + veor d3, d3, d31 + vbic d31, d26, d25 + veor d4, d4, d31 + vmov d25, d5 + vmov d26, d6 + vbic d31, d7, d26 + veor d5, d25, d31 + vbic d31, d8, d7 + veor d6, d26, d31 + vbic d31, d9, d8 + veor d7, d7, d31 + vbic d31, d25, d9 + veor d8, d8, d31 + vbic d31, d26, d25 + veor d9, d9, d31 + vmov d26, d11 + vbic d31, d12, d26 + veor d10, d30, d31 + vbic d31, d13, d12 + veor d11, d26, d31 + vbic d31, d14, d13 + veor d12, d12, d31 + vbic d31, d30, d14 + veor d13, d13, d31 + vbic d31, d26, d30 + veor d14, d14, d31 + vmov d25, d15 + vmov d26, d16 + vbic d31, d17, d26 + veor d15, d25, d31 + vbic d31, d18, d17 + veor d16, d26, d31 + vbic d31, d19, d18 + veor d17, d17, d31 + vbic d31, d25, d19 + veor d18, d18, d31 + vbic d31, d26, d25 + veor d19, d19, d31 + vmov d25, d20 + vmov d26, d21 + vbic d31, d22, d26 + veor d20, d25, d31 + vbic d31, d23, d22 + veor d21, d26, d31 + vbic d31, d24, d23 + veor d22, d22, d31 + vbic d31, d25, d24 + veor d23, d23, d31 + vbic d31, d26, d25 + veor d24, d24, d31 + vld1.8 {d30}, [r3]! + subs r12, r12, #1 + veor d0, d0, d30 + bne L_sha3_256_blocksx3_seed_neon_begin + vst1.8 {d0-d3}, [r0]! + vst1.8 {d4-d7}, [r0]! + vst1.8 {d8-d11}, [r0]! + vst1.8 {d12-d15}, [r0]! + vst1.8 {d16-d19}, [r0]! + vst1.8 {d20-d23}, [r0]! + vst1.8 {d24}, [r0] + add r0, r0, #8 + subs r4, r4, #1 + bne L_sha3_256_blocksx3_seed_neon_state + add sp, sp, #16 + vpop {d8-d15} + pop {r4, pc} + .size sha3_256_blocksx3_seed_neon,.-sha3_256_blocksx3_seed_neon +#ifndef __APPLE__ + .text + .type L_sha3_256_blocksx3_seed_64_neon_rt, %object + .size L_sha3_256_blocksx3_seed_64_neon_rt, 200 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 16-byte aligned, 128-bit aligned +#ifndef __APPLE__ + .align 4 +#else + .p2align 4 +#endif /* __APPLE__ */ +L_sha3_256_blocksx3_seed_64_neon_rt: + .quad 0x0000000000000001,0x0000000000008082 + .quad 0x800000000000808a,0x8000000080008000 + .quad 0x000000000000808b,0x0000000080000001 + .quad 0x8000000080008081,0x8000000000008009 + .quad 0x000000000000008a,0x0000000000000088 + .quad 0x0000000080008009,0x000000008000000a + .quad 0x000000008000808b,0x800000000000008b + .quad 0x8000000000008089,0x8000000000008003 + .quad 0x8000000000008002,0x8000000000000080 + .quad 0x000000000000800a,0x800000008000000a + .quad 0x8000000080008081,0x8000000000008080 + .quad 0x0000000080000001,0x8000000080008008 + .quad 0x8000000000000000 + .text + .align 4 + .globl sha3_256_blocksx3_seed_64_neon + WC_ASM_ATT_HIDDEN(sha3_256_blocksx3_seed_64_neon) + .type sha3_256_blocksx3_seed_64_neon, %function +sha3_256_blocksx3_seed_64_neon: + push {r4, lr} + vpush {d8-d15} + sub sp, sp, #16 + adr r2, L_sha3_256_blocksx3_seed_64_neon_rt + mov r4, #3 +L_sha3_256_blocksx3_seed_64_neon_state: + vld1.8 {d0-d3}, [r1]! + vld1.8 {d4-d7}, [r1]! + sub r1, r1, #0x40 + vldr.8 d8, [r0, #64] + veor d9, d9, d9 + veor d10, d10, d10 + veor d11, d11, d11 + veor d12, d12, d12 + veor d13, d13, d13 + veor d14, d14, d14 + veor d15, d15, d15 + vldr.8 d16, [r2, #192] + veor d17, d17, d17 + veor d18, d18, d18 + veor d19, d19, d19 + veor d20, d20, d20 + veor d21, d21, d21 + veor d22, d22, d22 + veor d23, d23, d23 + veor d24, d24, d24 + mov r3, r2 + mov r12, #24 + mov lr, sp +L_sha3_256_blocksx3_seed_64_neon_begin: + # Calc b[0..4] + veor d26, d0, d5 + veor d27, d1, d6 + veor d28, d2, d7 + veor d29, d3, d8 + veor d25, d4, d9 + veor d26, d26, d10 + veor d27, d27, d11 + veor d28, d28, d12 + veor d29, d29, d13 + veor d25, d25, d14 + veor d26, d26, d15 + veor d27, d27, d16 + veor d28, d28, d17 + veor d29, d29, d18 + veor d25, d25, d19 + veor d26, d26, d20 + veor d27, d27, d21 + veor d28, d28, d22 + veor d29, d29, d23 + veor d25, d25, d24 + vst1.8 {d25-d26}, [lr] + # Calc t[0..4] and XOR into s[i*5..i*5+4] + # t[0] + vshr.u64 d30, d27, #63 + vshl.u64 d31, d27, #1 + veor d25, d25, d30 + veor d25, d25, d31 + # t[1] + vshr.u64 d30, d28, #63 + vshl.u64 d31, d28, #1 + veor d26, d26, d30 + veor d26, d26, d31 + # t[2] + vshr.u64 d30, d29, #63 + vshl.u64 d31, d29, #1 + veor d27, d27, d30 + veor d27, d27, d31 + # t[3] + vldr.8 d31, [lr] + vshr.u64 d30, d31, #63 + vshl.u64 d31, d31, #1 + veor d28, d28, d30 + veor d28, d28, d31 + # t[4] + vldr.8 d31, [lr, #8] + vshr.u64 d30, d31, #63 + vshl.u64 d31, d31, #1 + veor d29, d29, d30 + veor d29, d29, d31 + sub lr, lr, #16 + veor d0, d0, d25 + # s[1] => s[10] (tmp) + veor d30, d1, d26 + vshr.u64 d31, d30, #63 + vshl.u64 d30, d30, #1 + veor d30, d30, d31 + # s[6] => s[1] + veor d1, d6, d26 + vshr.u64 d31, d1, #20 + vshl.u64 d1, d1, #44 + veor d1, d1, d31 + # s[9] => s[6] + veor d6, d9, d29 + vshr.u64 d31, d6, #44 + vshl.u64 d6, d6, #20 + veor d6, d6, d31 + # s[22] => s[9] + veor d9, d22, d27 + vshr.u64 d31, d9, #3 + vshl.u64 d9, d9, #61 + veor d9, d9, d31 + # s[14] => s[22] + veor d22, d14, d29 + vshr.u64 d31, d22, #25 + vshl.u64 d22, d22, #39 + veor d22, d22, d31 + # s[20] => s[14] + veor d14, d20, d25 + vshr.u64 d31, d14, #46 + vshl.u64 d14, d14, #18 + veor d14, d14, d31 + # s[2] => s[20] + veor d20, d2, d27 + vshr.u64 d31, d20, #2 + vshl.u64 d20, d20, #62 + veor d20, d20, d31 + # s[12] => s[2] + veor d2, d12, d27 + vshr.u64 d31, d2, #21 + vshl.u64 d2, d2, #43 + veor d2, d2, d31 + # s[13] => s[12] + veor d12, d13, d28 + vshr.u64 d31, d12, #39 + vshl.u64 d12, d12, #25 + veor d12, d12, d31 + # s[19] => s[13] + veor d13, d19, d29 + vshr.u64 d31, d13, #56 + vshl.u64 d13, d13, #8 + veor d13, d13, d31 + # s[23] => s[19] + veor d19, d23, d28 + vshr.u64 d31, d19, #8 + vshl.u64 d19, d19, #56 + veor d19, d19, d31 + # s[15] => s[23] + veor d23, d15, d25 + vshr.u64 d31, d23, #23 + vshl.u64 d23, d23, #41 + veor d23, d23, d31 + # s[4] => s[15] + veor d15, d4, d29 + vshr.u64 d31, d15, #37 + vshl.u64 d15, d15, #27 + veor d15, d15, d31 + # s[24] => s[4] + veor d4, d24, d29 + vshr.u64 d31, d4, #50 + vshl.u64 d4, d4, #14 + veor d4, d4, d31 + # s[21] => s[24] + veor d24, d21, d26 + vshr.u64 d31, d24, #62 + vshl.u64 d24, d24, #2 + veor d24, d24, d31 + # s[8] => s[21] + veor d21, d8, d28 + vshr.u64 d31, d21, #9 + vshl.u64 d21, d21, #55 + veor d21, d21, d31 + # s[16] => s[8] + veor d8, d16, d26 + vshr.u64 d31, d8, #19 + vshl.u64 d8, d8, #45 + veor d8, d8, d31 + # s[5] => s[16] + veor d16, d5, d25 + vshr.u64 d31, d16, #28 + vshl.u64 d16, d16, #36 + veor d16, d16, d31 + # s[3] => s[5] + veor d5, d3, d28 + vshr.u64 d31, d5, #36 + vshl.u64 d5, d5, #28 + veor d5, d5, d31 + # s[18] => s[3] + veor d3, d18, d28 + vshr.u64 d31, d3, #43 + vshl.u64 d3, d3, #21 + veor d3, d3, d31 + # s[17] => s[18] + veor d18, d17, d27 + vshr.u64 d31, d18, #49 + vshl.u64 d18, d18, #15 + veor d18, d18, d31 + # s[11] => s[17] + veor d17, d11, d26 + vshr.u64 d31, d17, #54 + vshl.u64 d17, d17, #10 + veor d17, d17, d31 + # s[7] => s[11] + veor d11, d7, d27 + vshr.u64 d31, d11, #58 + vshl.u64 d11, d11, #6 + veor d11, d11, d31 + # s[10] => s[7] + veor d7, d10, d25 + vshr.u64 d31, d7, #61 + vshl.u64 d7, d7, #3 + veor d7, d7, d31 + # Row Mix + vmov d25, d0 + vmov d26, d1 + vbic d31, d2, d26 + veor d0, d25, d31 + vbic d31, d3, d2 + veor d1, d26, d31 + vbic d31, d4, d3 + veor d2, d2, d31 + vbic d31, d25, d4 + veor d3, d3, d31 + vbic d31, d26, d25 + veor d4, d4, d31 + vmov d25, d5 + vmov d26, d6 + vbic d31, d7, d26 + veor d5, d25, d31 + vbic d31, d8, d7 + veor d6, d26, d31 + vbic d31, d9, d8 + veor d7, d7, d31 + vbic d31, d25, d9 + veor d8, d8, d31 + vbic d31, d26, d25 + veor d9, d9, d31 + vmov d26, d11 + vbic d31, d12, d26 + veor d10, d30, d31 + vbic d31, d13, d12 + veor d11, d26, d31 + vbic d31, d14, d13 + veor d12, d12, d31 + vbic d31, d30, d14 + veor d13, d13, d31 + vbic d31, d26, d30 + veor d14, d14, d31 + vmov d25, d15 + vmov d26, d16 + vbic d31, d17, d26 + veor d15, d25, d31 + vbic d31, d18, d17 + veor d16, d26, d31 + vbic d31, d19, d18 + veor d17, d17, d31 + vbic d31, d25, d19 + veor d18, d18, d31 + vbic d31, d26, d25 + veor d19, d19, d31 + vmov d25, d20 + vmov d26, d21 + vbic d31, d22, d26 + veor d20, d25, d31 + vbic d31, d23, d22 + veor d21, d26, d31 + vbic d31, d24, d23 + veor d22, d22, d31 + vbic d31, d25, d24 + veor d23, d23, d31 + vbic d31, d26, d25 + veor d24, d24, d31 + vld1.8 {d30}, [r3]! + subs r12, r12, #1 + veor d0, d0, d30 + bne L_sha3_256_blocksx3_seed_64_neon_begin + vst1.8 {d0-d3}, [r0]! + vst1.8 {d4-d7}, [r0]! + vst1.8 {d8-d11}, [r0]! + vst1.8 {d12-d15}, [r0]! + vst1.8 {d16-d19}, [r0]! + vst1.8 {d20-d23}, [r0]! + vst1.8 {d24}, [r0] + add r0, r0, #8 + subs r4, r4, #1 + bne L_sha3_256_blocksx3_seed_64_neon_state + add sp, sp, #16 + vpop {d8-d15} + pop {r4, pc} + .size sha3_256_blocksx3_seed_64_neon,.-sha3_256_blocksx3_seed_64_neon +#endif /* defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) */ #endif /* WOLFSSL_ARMASM_NO_NEON */ #ifdef WOLFSSL_ARMASM_NO_NEON #ifndef __APPLE__ diff --git a/wolfcrypt/src/port/arm/armv8-32-sha3-asm_c.c b/wolfcrypt/src/port/arm/armv8-32-sha3-asm_c.c index abc1a9fe09e..226a5157b38 100644 --- a/wolfcrypt/src/port/arm/armv8-32-sha3-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-32-sha3-asm_c.c @@ -31,6 +31,12 @@ #include #include +/* Honor WC_SHA3_NO_ASM as sha3.c does: suppress this NEON BlockSha3 so it + * doesn't multiply-define against sha3.c's C BlockSha3 on arm32. */ +#ifdef WC_SHA3_NO_ASM + #undef WOLFSSL_ARMASM +#endif + #ifdef WOLFSSL_ARMASM #if !defined(__aarch64__) && !defined(WOLFSSL_ARMASM_THUMB2) #ifdef WOLFSSL_ARMASM_INLINE @@ -358,6 +364,1310 @@ WC_OMIT_FRAME_POINTER void BlockSha3(word64* state) ); } +#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) +XALIGNED(16) static const word64 L_sha3_arm32_neon_x3_rt[] = { + 0x0000000000000001UL, 0x0000000000008082UL, + 0x800000000000808aUL, 0x8000000080008000UL, + 0x000000000000808bUL, 0x0000000080000001UL, + 0x8000000080008081UL, 0x8000000000008009UL, + 0x000000000000008aUL, 0x0000000000000088UL, + 0x0000000080008009UL, 0x000000008000000aUL, + 0x000000008000808bUL, 0x800000000000008bUL, + 0x8000000000008089UL, 0x8000000000008003UL, + 0x8000000000008002UL, 0x8000000000000080UL, + 0x000000000000800aUL, 0x800000008000000aUL, + 0x8000000080008081UL, 0x8000000000008080UL, + 0x0000000080000001UL, 0x8000000080008008UL, +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void sha3_blocksx3_neon(word64* state_p) +#else +WC_OMIT_FRAME_POINTER void sha3_blocksx3_neon(word64* state) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register word64* state __asm__ ("r0") = (word64*)state_p; + register word64* L_sha3_arm32_neon_x3_rt_c __asm__ ("r1") = + (word64*)&L_sha3_arm32_neon_x3_rt; +#else + register word64* L_sha3_arm32_neon_x3_rt_c = + (word64*)&L_sha3_arm32_neon_x3_rt; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "sub sp, sp, #16\n\t" + "mov r1, %[L_sha3_arm32_neon_x3_rt]\n\t" + "mov lr, #3\n\t" + "\n" + "L_sha3_arm32_neon_x3_state_%=:\n\t" + "vld1.8 {d0-d3}, [%[state]]!\n\t" + "vld1.8 {d4-d7}, [%[state]]!\n\t" + "vld1.8 {d8-d11}, [%[state]]!\n\t" + "vld1.8 {d12-d15}, [%[state]]!\n\t" + "vld1.8 {d16-d19}, [%[state]]!\n\t" + "vld1.8 {d20-d23}, [%[state]]!\n\t" + "vld1.8 {d24}, [%[state]]\n\t" + "sub %[state], %[state], #0xc0\n\t" + "mov r2, r1\n\t" + "mov r3, #24\n\t" + "mov r12, sp\n\t" + "\n" + "L_sha3_arm32_neon_x3_begin_%=:\n\t" + /* Calc b[0..4] */ + "veor d26, d0, d5\n\t" + "veor d27, d1, d6\n\t" + "veor d28, d2, d7\n\t" + "veor d29, d3, d8\n\t" + "veor d25, d4, d9\n\t" + "veor d26, d26, d10\n\t" + "veor d27, d27, d11\n\t" + "veor d28, d28, d12\n\t" + "veor d29, d29, d13\n\t" + "veor d25, d25, d14\n\t" + "veor d26, d26, d15\n\t" + "veor d27, d27, d16\n\t" + "veor d28, d28, d17\n\t" + "veor d29, d29, d18\n\t" + "veor d25, d25, d19\n\t" + "veor d26, d26, d20\n\t" + "veor d27, d27, d21\n\t" + "veor d28, d28, d22\n\t" + "veor d29, d29, d23\n\t" + "veor d25, d25, d24\n\t" + "vst1.8 {d25-d26}, [r12]\n\t" + /* Calc t[0..4] and XOR into s[i*5..i*5+4] */ + /* t[0] */ + "vshr.u64 d30, d27, #63\n\t" + "vshl.u64 d31, d27, #1\n\t" + "veor d25, d25, d30\n\t" + "veor d25, d25, d31\n\t" + /* t[1] */ + "vshr.u64 d30, d28, #63\n\t" + "vshl.u64 d31, d28, #1\n\t" + "veor d26, d26, d30\n\t" + "veor d26, d26, d31\n\t" + /* t[2] */ + "vshr.u64 d30, d29, #63\n\t" + "vshl.u64 d31, d29, #1\n\t" + "veor d27, d27, d30\n\t" + "veor d27, d27, d31\n\t" + /* t[3] */ + "vldr.8 d31, [r12]\n\t" + "vshr.u64 d30, d31, #63\n\t" + "vshl.u64 d31, d31, #1\n\t" + "veor d28, d28, d30\n\t" + "veor d28, d28, d31\n\t" + /* t[4] */ + "vldr.8 d31, [r12, #8]\n\t" + "vshr.u64 d30, d31, #63\n\t" + "vshl.u64 d31, d31, #1\n\t" + "veor d29, d29, d30\n\t" + "veor d29, d29, d31\n\t" + "sub r12, r12, #16\n\t" + "veor d0, d0, d25\n\t" + /* s[1] => s[10] (tmp) */ + "veor d30, d1, d26\n\t" + "vshr.u64 d31, d30, #63\n\t" + "vshl.u64 d30, d30, #1\n\t" + "veor d30, d30, d31\n\t" + /* s[6] => s[1] */ + "veor d1, d6, d26\n\t" + "vshr.u64 d31, d1, #20\n\t" + "vshl.u64 d1, d1, #44\n\t" + "veor d1, d1, d31\n\t" + /* s[9] => s[6] */ + "veor d6, d9, d29\n\t" + "vshr.u64 d31, d6, #44\n\t" + "vshl.u64 d6, d6, #20\n\t" + "veor d6, d6, d31\n\t" + /* s[22] => s[9] */ + "veor d9, d22, d27\n\t" + "vshr.u64 d31, d9, #3\n\t" + "vshl.u64 d9, d9, #61\n\t" + "veor d9, d9, d31\n\t" + /* s[14] => s[22] */ + "veor d22, d14, d29\n\t" + "vshr.u64 d31, d22, #25\n\t" + "vshl.u64 d22, d22, #39\n\t" + "veor d22, d22, d31\n\t" + /* s[20] => s[14] */ + "veor d14, d20, d25\n\t" + "vshr.u64 d31, d14, #46\n\t" + "vshl.u64 d14, d14, #18\n\t" + "veor d14, d14, d31\n\t" + /* s[2] => s[20] */ + "veor d20, d2, d27\n\t" + "vshr.u64 d31, d20, #2\n\t" + "vshl.u64 d20, d20, #62\n\t" + "veor d20, d20, d31\n\t" + /* s[12] => s[2] */ + "veor d2, d12, d27\n\t" + "vshr.u64 d31, d2, #21\n\t" + "vshl.u64 d2, d2, #43\n\t" + "veor d2, d2, d31\n\t" + /* s[13] => s[12] */ + "veor d12, d13, d28\n\t" + "vshr.u64 d31, d12, #39\n\t" + "vshl.u64 d12, d12, #25\n\t" + "veor d12, d12, d31\n\t" + /* s[19] => s[13] */ + "veor d13, d19, d29\n\t" + "vshr.u64 d31, d13, #56\n\t" + "vshl.u64 d13, d13, #8\n\t" + "veor d13, d13, d31\n\t" + /* s[23] => s[19] */ + "veor d19, d23, d28\n\t" + "vshr.u64 d31, d19, #8\n\t" + "vshl.u64 d19, d19, #56\n\t" + "veor d19, d19, d31\n\t" + /* s[15] => s[23] */ + "veor d23, d15, d25\n\t" + "vshr.u64 d31, d23, #23\n\t" + "vshl.u64 d23, d23, #41\n\t" + "veor d23, d23, d31\n\t" + /* s[4] => s[15] */ + "veor d15, d4, d29\n\t" + "vshr.u64 d31, d15, #37\n\t" + "vshl.u64 d15, d15, #27\n\t" + "veor d15, d15, d31\n\t" + /* s[24] => s[4] */ + "veor d4, d24, d29\n\t" + "vshr.u64 d31, d4, #50\n\t" + "vshl.u64 d4, d4, #14\n\t" + "veor d4, d4, d31\n\t" + /* s[21] => s[24] */ + "veor d24, d21, d26\n\t" + "vshr.u64 d31, d24, #62\n\t" + "vshl.u64 d24, d24, #2\n\t" + "veor d24, d24, d31\n\t" + /* s[8] => s[21] */ + "veor d21, d8, d28\n\t" + "vshr.u64 d31, d21, #9\n\t" + "vshl.u64 d21, d21, #55\n\t" + "veor d21, d21, d31\n\t" + /* s[16] => s[8] */ + "veor d8, d16, d26\n\t" + "vshr.u64 d31, d8, #19\n\t" + "vshl.u64 d8, d8, #45\n\t" + "veor d8, d8, d31\n\t" + /* s[5] => s[16] */ + "veor d16, d5, d25\n\t" + "vshr.u64 d31, d16, #28\n\t" + "vshl.u64 d16, d16, #36\n\t" + "veor d16, d16, d31\n\t" + /* s[3] => s[5] */ + "veor d5, d3, d28\n\t" + "vshr.u64 d31, d5, #36\n\t" + "vshl.u64 d5, d5, #28\n\t" + "veor d5, d5, d31\n\t" + /* s[18] => s[3] */ + "veor d3, d18, d28\n\t" + "vshr.u64 d31, d3, #43\n\t" + "vshl.u64 d3, d3, #21\n\t" + "veor d3, d3, d31\n\t" + /* s[17] => s[18] */ + "veor d18, d17, d27\n\t" + "vshr.u64 d31, d18, #49\n\t" + "vshl.u64 d18, d18, #15\n\t" + "veor d18, d18, d31\n\t" + /* s[11] => s[17] */ + "veor d17, d11, d26\n\t" + "vshr.u64 d31, d17, #54\n\t" + "vshl.u64 d17, d17, #10\n\t" + "veor d17, d17, d31\n\t" + /* s[7] => s[11] */ + "veor d11, d7, d27\n\t" + "vshr.u64 d31, d11, #58\n\t" + "vshl.u64 d11, d11, #6\n\t" + "veor d11, d11, d31\n\t" + /* s[10] => s[7] */ + "veor d7, d10, d25\n\t" + "vshr.u64 d31, d7, #61\n\t" + "vshl.u64 d7, d7, #3\n\t" + "veor d7, d7, d31\n\t" + /* Row Mix */ + "vmov d25, d0\n\t" + "vmov d26, d1\n\t" + "vbic d31, d2, d26\n\t" + "veor d0, d25, d31\n\t" + "vbic d31, d3, d2\n\t" + "veor d1, d26, d31\n\t" + "vbic d31, d4, d3\n\t" + "veor d2, d2, d31\n\t" + "vbic d31, d25, d4\n\t" + "veor d3, d3, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d4, d4, d31\n\t" + "vmov d25, d5\n\t" + "vmov d26, d6\n\t" + "vbic d31, d7, d26\n\t" + "veor d5, d25, d31\n\t" + "vbic d31, d8, d7\n\t" + "veor d6, d26, d31\n\t" + "vbic d31, d9, d8\n\t" + "veor d7, d7, d31\n\t" + "vbic d31, d25, d9\n\t" + "veor d8, d8, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d9, d9, d31\n\t" + "vmov d26, d11\n\t" + "vbic d31, d12, d26\n\t" + "veor d10, d30, d31\n\t" + "vbic d31, d13, d12\n\t" + "veor d11, d26, d31\n\t" + "vbic d31, d14, d13\n\t" + "veor d12, d12, d31\n\t" + "vbic d31, d30, d14\n\t" + "veor d13, d13, d31\n\t" + "vbic d31, d26, d30\n\t" + "veor d14, d14, d31\n\t" + "vmov d25, d15\n\t" + "vmov d26, d16\n\t" + "vbic d31, d17, d26\n\t" + "veor d15, d25, d31\n\t" + "vbic d31, d18, d17\n\t" + "veor d16, d26, d31\n\t" + "vbic d31, d19, d18\n\t" + "veor d17, d17, d31\n\t" + "vbic d31, d25, d19\n\t" + "veor d18, d18, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d19, d19, d31\n\t" + "vmov d25, d20\n\t" + "vmov d26, d21\n\t" + "vbic d31, d22, d26\n\t" + "veor d20, d25, d31\n\t" + "vbic d31, d23, d22\n\t" + "veor d21, d26, d31\n\t" + "vbic d31, d24, d23\n\t" + "veor d22, d22, d31\n\t" + "vbic d31, d25, d24\n\t" + "veor d23, d23, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d24, d24, d31\n\t" + "vld1.8 {d30}, [r2]!\n\t" + "subs r3, r3, #1\n\t" + "veor d0, d0, d30\n\t" + "bne L_sha3_arm32_neon_x3_begin_%=\n\t" + "vst1.8 {d0-d3}, [%[state]]!\n\t" + "vst1.8 {d4-d7}, [%[state]]!\n\t" + "vst1.8 {d8-d11}, [%[state]]!\n\t" + "vst1.8 {d12-d15}, [%[state]]!\n\t" + "vst1.8 {d16-d19}, [%[state]]!\n\t" + "vst1.8 {d20-d23}, [%[state]]!\n\t" + "vst1.8 {d24}, [%[state]]\n\t" + "add %[state], %[state], #8\n\t" + "subs lr, lr, #1\n\t" + "bne L_sha3_arm32_neon_x3_state_%=\n\t" + "add sp, sp, #16\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [state] "+r" (state), + [L_sha3_arm32_neon_x3_rt] "+r" (L_sha3_arm32_neon_x3_rt_c) + : +#else + : + : [state] "r" (state), + [L_sha3_arm32_neon_x3_rt] "r" (L_sha3_arm32_neon_x3_rt_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r2", "r3", "r12", "lr", "d0", "d1", "d2", "d3", "d4", + "d5", "d6", "d7", "d8", "d9", "d10", "d11", "d12", "d13", "d14", + "d15", "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23", + "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31" + ); +} + +XALIGNED(16) static const word64 L_sha3_128_blocksx3_seed_neon_rt[] = { + 0x0000000000000001UL, 0x0000000000008082UL, + 0x800000000000808aUL, 0x8000000080008000UL, + 0x000000000000808bUL, 0x0000000080000001UL, + 0x8000000080008081UL, 0x8000000000008009UL, + 0x000000000000008aUL, 0x0000000000000088UL, + 0x0000000080008009UL, 0x000000008000000aUL, + 0x000000008000808bUL, 0x800000000000008bUL, + 0x8000000000008089UL, 0x8000000000008003UL, + 0x8000000000008002UL, 0x8000000000000080UL, + 0x000000000000800aUL, 0x800000008000000aUL, + 0x8000000080008081UL, 0x8000000000008080UL, + 0x0000000080000001UL, 0x8000000080008008UL, + 0x8000000000000000UL +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void sha3_128_blocksx3_seed_neon(word64* state_p, + byte* seed_p) +#else +WC_OMIT_FRAME_POINTER void sha3_128_blocksx3_seed_neon(word64* state, + byte* seed) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register word64* state __asm__ ("r0") = (word64*)state_p; + register byte* seed __asm__ ("r1") = (byte*)seed_p; + register word64* L_sha3_128_blocksx3_seed_neon_rt_c __asm__ ("r2") = + (word64*)&L_sha3_128_blocksx3_seed_neon_rt; +#else + register word64* L_sha3_128_blocksx3_seed_neon_rt_c = + (word64*)&L_sha3_128_blocksx3_seed_neon_rt; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "sub sp, sp, #16\n\t" + "mov r2, %[L_sha3_128_blocksx3_seed_neon_rt]\n\t" + "mov r4, #3\n\t" + "\n" + "L_sha3_128_blocksx3_seed_neon_state_%=:\n\t" + "vld1.8 {d0-d3}, [%[seed]]!\n\t" + "sub %[seed], %[seed], #32\n\t" + "vldr.8 d4, [%[state], #32]\n\t" + "veor d5, d5, d5\n\t" + "veor d6, d6, d6\n\t" + "veor d7, d7, d7\n\t" + "veor d8, d8, d8\n\t" + "veor d9, d9, d9\n\t" + "veor d10, d10, d10\n\t" + "veor d11, d11, d11\n\t" + "veor d12, d12, d12\n\t" + "veor d13, d13, d13\n\t" + "veor d14, d14, d14\n\t" + "veor d15, d15, d15\n\t" + "veor d16, d16, d16\n\t" + "veor d17, d17, d17\n\t" + "veor d18, d18, d18\n\t" + "veor d19, d19, d19\n\t" + "vldr.8 d20, [r2, #192]\n\t" + "veor d21, d21, d21\n\t" + "veor d22, d22, d22\n\t" + "veor d23, d23, d23\n\t" + "veor d24, d24, d24\n\t" + "mov r3, r2\n\t" + "mov r12, #24\n\t" + "mov lr, sp\n\t" + "\n" + "L_sha3_128_blocksx3_seed_neon_begin_%=:\n\t" + /* Calc b[0..4] */ + "veor d26, d0, d5\n\t" + "veor d27, d1, d6\n\t" + "veor d28, d2, d7\n\t" + "veor d29, d3, d8\n\t" + "veor d25, d4, d9\n\t" + "veor d26, d26, d10\n\t" + "veor d27, d27, d11\n\t" + "veor d28, d28, d12\n\t" + "veor d29, d29, d13\n\t" + "veor d25, d25, d14\n\t" + "veor d26, d26, d15\n\t" + "veor d27, d27, d16\n\t" + "veor d28, d28, d17\n\t" + "veor d29, d29, d18\n\t" + "veor d25, d25, d19\n\t" + "veor d26, d26, d20\n\t" + "veor d27, d27, d21\n\t" + "veor d28, d28, d22\n\t" + "veor d29, d29, d23\n\t" + "veor d25, d25, d24\n\t" + "vst1.8 {d25-d26}, [lr]\n\t" + /* Calc t[0..4] and XOR into s[i*5..i*5+4] */ + /* t[0] */ + "vshr.u64 d30, d27, #63\n\t" + "vshl.u64 d31, d27, #1\n\t" + "veor d25, d25, d30\n\t" + "veor d25, d25, d31\n\t" + /* t[1] */ + "vshr.u64 d30, d28, #63\n\t" + "vshl.u64 d31, d28, #1\n\t" + "veor d26, d26, d30\n\t" + "veor d26, d26, d31\n\t" + /* t[2] */ + "vshr.u64 d30, d29, #63\n\t" + "vshl.u64 d31, d29, #1\n\t" + "veor d27, d27, d30\n\t" + "veor d27, d27, d31\n\t" + /* t[3] */ + "vldr.8 d31, [lr]\n\t" + "vshr.u64 d30, d31, #63\n\t" + "vshl.u64 d31, d31, #1\n\t" + "veor d28, d28, d30\n\t" + "veor d28, d28, d31\n\t" + /* t[4] */ + "vldr.8 d31, [lr, #8]\n\t" + "vshr.u64 d30, d31, #63\n\t" + "vshl.u64 d31, d31, #1\n\t" + "veor d29, d29, d30\n\t" + "veor d29, d29, d31\n\t" + "sub lr, lr, #16\n\t" + "veor d0, d0, d25\n\t" + /* s[1] => s[10] (tmp) */ + "veor d30, d1, d26\n\t" + "vshr.u64 d31, d30, #63\n\t" + "vshl.u64 d30, d30, #1\n\t" + "veor d30, d30, d31\n\t" + /* s[6] => s[1] */ + "veor d1, d6, d26\n\t" + "vshr.u64 d31, d1, #20\n\t" + "vshl.u64 d1, d1, #44\n\t" + "veor d1, d1, d31\n\t" + /* s[9] => s[6] */ + "veor d6, d9, d29\n\t" + "vshr.u64 d31, d6, #44\n\t" + "vshl.u64 d6, d6, #20\n\t" + "veor d6, d6, d31\n\t" + /* s[22] => s[9] */ + "veor d9, d22, d27\n\t" + "vshr.u64 d31, d9, #3\n\t" + "vshl.u64 d9, d9, #61\n\t" + "veor d9, d9, d31\n\t" + /* s[14] => s[22] */ + "veor d22, d14, d29\n\t" + "vshr.u64 d31, d22, #25\n\t" + "vshl.u64 d22, d22, #39\n\t" + "veor d22, d22, d31\n\t" + /* s[20] => s[14] */ + "veor d14, d20, d25\n\t" + "vshr.u64 d31, d14, #46\n\t" + "vshl.u64 d14, d14, #18\n\t" + "veor d14, d14, d31\n\t" + /* s[2] => s[20] */ + "veor d20, d2, d27\n\t" + "vshr.u64 d31, d20, #2\n\t" + "vshl.u64 d20, d20, #62\n\t" + "veor d20, d20, d31\n\t" + /* s[12] => s[2] */ + "veor d2, d12, d27\n\t" + "vshr.u64 d31, d2, #21\n\t" + "vshl.u64 d2, d2, #43\n\t" + "veor d2, d2, d31\n\t" + /* s[13] => s[12] */ + "veor d12, d13, d28\n\t" + "vshr.u64 d31, d12, #39\n\t" + "vshl.u64 d12, d12, #25\n\t" + "veor d12, d12, d31\n\t" + /* s[19] => s[13] */ + "veor d13, d19, d29\n\t" + "vshr.u64 d31, d13, #56\n\t" + "vshl.u64 d13, d13, #8\n\t" + "veor d13, d13, d31\n\t" + /* s[23] => s[19] */ + "veor d19, d23, d28\n\t" + "vshr.u64 d31, d19, #8\n\t" + "vshl.u64 d19, d19, #56\n\t" + "veor d19, d19, d31\n\t" + /* s[15] => s[23] */ + "veor d23, d15, d25\n\t" + "vshr.u64 d31, d23, #23\n\t" + "vshl.u64 d23, d23, #41\n\t" + "veor d23, d23, d31\n\t" + /* s[4] => s[15] */ + "veor d15, d4, d29\n\t" + "vshr.u64 d31, d15, #37\n\t" + "vshl.u64 d15, d15, #27\n\t" + "veor d15, d15, d31\n\t" + /* s[24] => s[4] */ + "veor d4, d24, d29\n\t" + "vshr.u64 d31, d4, #50\n\t" + "vshl.u64 d4, d4, #14\n\t" + "veor d4, d4, d31\n\t" + /* s[21] => s[24] */ + "veor d24, d21, d26\n\t" + "vshr.u64 d31, d24, #62\n\t" + "vshl.u64 d24, d24, #2\n\t" + "veor d24, d24, d31\n\t" + /* s[8] => s[21] */ + "veor d21, d8, d28\n\t" + "vshr.u64 d31, d21, #9\n\t" + "vshl.u64 d21, d21, #55\n\t" + "veor d21, d21, d31\n\t" + /* s[16] => s[8] */ + "veor d8, d16, d26\n\t" + "vshr.u64 d31, d8, #19\n\t" + "vshl.u64 d8, d8, #45\n\t" + "veor d8, d8, d31\n\t" + /* s[5] => s[16] */ + "veor d16, d5, d25\n\t" + "vshr.u64 d31, d16, #28\n\t" + "vshl.u64 d16, d16, #36\n\t" + "veor d16, d16, d31\n\t" + /* s[3] => s[5] */ + "veor d5, d3, d28\n\t" + "vshr.u64 d31, d5, #36\n\t" + "vshl.u64 d5, d5, #28\n\t" + "veor d5, d5, d31\n\t" + /* s[18] => s[3] */ + "veor d3, d18, d28\n\t" + "vshr.u64 d31, d3, #43\n\t" + "vshl.u64 d3, d3, #21\n\t" + "veor d3, d3, d31\n\t" + /* s[17] => s[18] */ + "veor d18, d17, d27\n\t" + "vshr.u64 d31, d18, #49\n\t" + "vshl.u64 d18, d18, #15\n\t" + "veor d18, d18, d31\n\t" + /* s[11] => s[17] */ + "veor d17, d11, d26\n\t" + "vshr.u64 d31, d17, #54\n\t" + "vshl.u64 d17, d17, #10\n\t" + "veor d17, d17, d31\n\t" + /* s[7] => s[11] */ + "veor d11, d7, d27\n\t" + "vshr.u64 d31, d11, #58\n\t" + "vshl.u64 d11, d11, #6\n\t" + "veor d11, d11, d31\n\t" + /* s[10] => s[7] */ + "veor d7, d10, d25\n\t" + "vshr.u64 d31, d7, #61\n\t" + "vshl.u64 d7, d7, #3\n\t" + "veor d7, d7, d31\n\t" + /* Row Mix */ + "vmov d25, d0\n\t" + "vmov d26, d1\n\t" + "vbic d31, d2, d26\n\t" + "veor d0, d25, d31\n\t" + "vbic d31, d3, d2\n\t" + "veor d1, d26, d31\n\t" + "vbic d31, d4, d3\n\t" + "veor d2, d2, d31\n\t" + "vbic d31, d25, d4\n\t" + "veor d3, d3, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d4, d4, d31\n\t" + "vmov d25, d5\n\t" + "vmov d26, d6\n\t" + "vbic d31, d7, d26\n\t" + "veor d5, d25, d31\n\t" + "vbic d31, d8, d7\n\t" + "veor d6, d26, d31\n\t" + "vbic d31, d9, d8\n\t" + "veor d7, d7, d31\n\t" + "vbic d31, d25, d9\n\t" + "veor d8, d8, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d9, d9, d31\n\t" + "vmov d26, d11\n\t" + "vbic d31, d12, d26\n\t" + "veor d10, d30, d31\n\t" + "vbic d31, d13, d12\n\t" + "veor d11, d26, d31\n\t" + "vbic d31, d14, d13\n\t" + "veor d12, d12, d31\n\t" + "vbic d31, d30, d14\n\t" + "veor d13, d13, d31\n\t" + "vbic d31, d26, d30\n\t" + "veor d14, d14, d31\n\t" + "vmov d25, d15\n\t" + "vmov d26, d16\n\t" + "vbic d31, d17, d26\n\t" + "veor d15, d25, d31\n\t" + "vbic d31, d18, d17\n\t" + "veor d16, d26, d31\n\t" + "vbic d31, d19, d18\n\t" + "veor d17, d17, d31\n\t" + "vbic d31, d25, d19\n\t" + "veor d18, d18, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d19, d19, d31\n\t" + "vmov d25, d20\n\t" + "vmov d26, d21\n\t" + "vbic d31, d22, d26\n\t" + "veor d20, d25, d31\n\t" + "vbic d31, d23, d22\n\t" + "veor d21, d26, d31\n\t" + "vbic d31, d24, d23\n\t" + "veor d22, d22, d31\n\t" + "vbic d31, d25, d24\n\t" + "veor d23, d23, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d24, d24, d31\n\t" + "vld1.8 {d30}, [r3]!\n\t" + "subs r12, r12, #1\n\t" + "veor d0, d0, d30\n\t" + "bne L_sha3_128_blocksx3_seed_neon_begin_%=\n\t" + "vst1.8 {d0-d3}, [%[state]]!\n\t" + "vst1.8 {d4-d7}, [%[state]]!\n\t" + "vst1.8 {d8-d11}, [%[state]]!\n\t" + "vst1.8 {d12-d15}, [%[state]]!\n\t" + "vst1.8 {d16-d19}, [%[state]]!\n\t" + "vst1.8 {d20-d23}, [%[state]]!\n\t" + "vst1.8 {d24}, [%[state]]\n\t" + "add %[state], %[state], #8\n\t" + "subs r4, r4, #1\n\t" + "bne L_sha3_128_blocksx3_seed_neon_state_%=\n\t" + "add sp, sp, #16\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [state] "+r" (state), [seed] "+r" (seed), + [L_sha3_128_blocksx3_seed_neon_rt] "+r" (L_sha3_128_blocksx3_seed_neon_rt_c) + : +#else + : + : [state] "r" (state), [seed] "r" (seed), + [L_sha3_128_blocksx3_seed_neon_rt] "r" (L_sha3_128_blocksx3_seed_neon_rt_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r3", "r12", "lr", "r4", "d0", "d1", "d2", "d3", "d4", + "d5", "d6", "d7", "d8", "d9", "d10", "d11", "d12", "d13", "d14", + "d15", "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23", + "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31" + ); +} + +XALIGNED(16) static const word64 L_sha3_256_blocksx3_seed_neon_rt[] = { + 0x0000000000000001UL, 0x0000000000008082UL, + 0x800000000000808aUL, 0x8000000080008000UL, + 0x000000000000808bUL, 0x0000000080000001UL, + 0x8000000080008081UL, 0x8000000000008009UL, + 0x000000000000008aUL, 0x0000000000000088UL, + 0x0000000080008009UL, 0x000000008000000aUL, + 0x000000008000808bUL, 0x800000000000008bUL, + 0x8000000000008089UL, 0x8000000000008003UL, + 0x8000000000008002UL, 0x8000000000000080UL, + 0x000000000000800aUL, 0x800000008000000aUL, + 0x8000000080008081UL, 0x8000000000008080UL, + 0x0000000080000001UL, 0x8000000080008008UL, + 0x8000000000000000UL +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void sha3_256_blocksx3_seed_neon(word64* state_p, + byte* seed_p) +#else +WC_OMIT_FRAME_POINTER void sha3_256_blocksx3_seed_neon(word64* state, + byte* seed) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register word64* state __asm__ ("r0") = (word64*)state_p; + register byte* seed __asm__ ("r1") = (byte*)seed_p; + register word64* L_sha3_256_blocksx3_seed_neon_rt_c __asm__ ("r2") = + (word64*)&L_sha3_256_blocksx3_seed_neon_rt; +#else + register word64* L_sha3_256_blocksx3_seed_neon_rt_c = + (word64*)&L_sha3_256_blocksx3_seed_neon_rt; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "sub sp, sp, #16\n\t" + "mov r2, %[L_sha3_256_blocksx3_seed_neon_rt]\n\t" + "mov r4, #3\n\t" + "\n" + "L_sha3_256_blocksx3_seed_neon_state_%=:\n\t" + "vld1.8 {d0-d3}, [%[seed]]!\n\t" + "sub %[seed], %[seed], #32\n\t" + "vldr.8 d4, [%[state], #32]\n\t" + "veor d5, d5, d5\n\t" + "veor d6, d6, d6\n\t" + "veor d7, d7, d7\n\t" + "veor d8, d8, d8\n\t" + "veor d9, d9, d9\n\t" + "veor d10, d10, d10\n\t" + "veor d11, d11, d11\n\t" + "veor d12, d12, d12\n\t" + "veor d13, d13, d13\n\t" + "veor d14, d14, d14\n\t" + "veor d15, d15, d15\n\t" + "vldr.8 d16, [r2, #192]\n\t" + "veor d17, d17, d17\n\t" + "veor d18, d18, d18\n\t" + "veor d19, d19, d19\n\t" + "veor d20, d20, d20\n\t" + "veor d21, d21, d21\n\t" + "veor d22, d22, d22\n\t" + "veor d23, d23, d23\n\t" + "veor d24, d24, d24\n\t" + "mov r3, r2\n\t" + "mov r12, #24\n\t" + "mov lr, sp\n\t" + "\n" + "L_sha3_256_blocksx3_seed_neon_begin_%=:\n\t" + /* Calc b[0..4] */ + "veor d26, d0, d5\n\t" + "veor d27, d1, d6\n\t" + "veor d28, d2, d7\n\t" + "veor d29, d3, d8\n\t" + "veor d25, d4, d9\n\t" + "veor d26, d26, d10\n\t" + "veor d27, d27, d11\n\t" + "veor d28, d28, d12\n\t" + "veor d29, d29, d13\n\t" + "veor d25, d25, d14\n\t" + "veor d26, d26, d15\n\t" + "veor d27, d27, d16\n\t" + "veor d28, d28, d17\n\t" + "veor d29, d29, d18\n\t" + "veor d25, d25, d19\n\t" + "veor d26, d26, d20\n\t" + "veor d27, d27, d21\n\t" + "veor d28, d28, d22\n\t" + "veor d29, d29, d23\n\t" + "veor d25, d25, d24\n\t" + "vst1.8 {d25-d26}, [lr]\n\t" + /* Calc t[0..4] and XOR into s[i*5..i*5+4] */ + /* t[0] */ + "vshr.u64 d30, d27, #63\n\t" + "vshl.u64 d31, d27, #1\n\t" + "veor d25, d25, d30\n\t" + "veor d25, d25, d31\n\t" + /* t[1] */ + "vshr.u64 d30, d28, #63\n\t" + "vshl.u64 d31, d28, #1\n\t" + "veor d26, d26, d30\n\t" + "veor d26, d26, d31\n\t" + /* t[2] */ + "vshr.u64 d30, d29, #63\n\t" + "vshl.u64 d31, d29, #1\n\t" + "veor d27, d27, d30\n\t" + "veor d27, d27, d31\n\t" + /* t[3] */ + "vldr.8 d31, [lr]\n\t" + "vshr.u64 d30, d31, #63\n\t" + "vshl.u64 d31, d31, #1\n\t" + "veor d28, d28, d30\n\t" + "veor d28, d28, d31\n\t" + /* t[4] */ + "vldr.8 d31, [lr, #8]\n\t" + "vshr.u64 d30, d31, #63\n\t" + "vshl.u64 d31, d31, #1\n\t" + "veor d29, d29, d30\n\t" + "veor d29, d29, d31\n\t" + "sub lr, lr, #16\n\t" + "veor d0, d0, d25\n\t" + /* s[1] => s[10] (tmp) */ + "veor d30, d1, d26\n\t" + "vshr.u64 d31, d30, #63\n\t" + "vshl.u64 d30, d30, #1\n\t" + "veor d30, d30, d31\n\t" + /* s[6] => s[1] */ + "veor d1, d6, d26\n\t" + "vshr.u64 d31, d1, #20\n\t" + "vshl.u64 d1, d1, #44\n\t" + "veor d1, d1, d31\n\t" + /* s[9] => s[6] */ + "veor d6, d9, d29\n\t" + "vshr.u64 d31, d6, #44\n\t" + "vshl.u64 d6, d6, #20\n\t" + "veor d6, d6, d31\n\t" + /* s[22] => s[9] */ + "veor d9, d22, d27\n\t" + "vshr.u64 d31, d9, #3\n\t" + "vshl.u64 d9, d9, #61\n\t" + "veor d9, d9, d31\n\t" + /* s[14] => s[22] */ + "veor d22, d14, d29\n\t" + "vshr.u64 d31, d22, #25\n\t" + "vshl.u64 d22, d22, #39\n\t" + "veor d22, d22, d31\n\t" + /* s[20] => s[14] */ + "veor d14, d20, d25\n\t" + "vshr.u64 d31, d14, #46\n\t" + "vshl.u64 d14, d14, #18\n\t" + "veor d14, d14, d31\n\t" + /* s[2] => s[20] */ + "veor d20, d2, d27\n\t" + "vshr.u64 d31, d20, #2\n\t" + "vshl.u64 d20, d20, #62\n\t" + "veor d20, d20, d31\n\t" + /* s[12] => s[2] */ + "veor d2, d12, d27\n\t" + "vshr.u64 d31, d2, #21\n\t" + "vshl.u64 d2, d2, #43\n\t" + "veor d2, d2, d31\n\t" + /* s[13] => s[12] */ + "veor d12, d13, d28\n\t" + "vshr.u64 d31, d12, #39\n\t" + "vshl.u64 d12, d12, #25\n\t" + "veor d12, d12, d31\n\t" + /* s[19] => s[13] */ + "veor d13, d19, d29\n\t" + "vshr.u64 d31, d13, #56\n\t" + "vshl.u64 d13, d13, #8\n\t" + "veor d13, d13, d31\n\t" + /* s[23] => s[19] */ + "veor d19, d23, d28\n\t" + "vshr.u64 d31, d19, #8\n\t" + "vshl.u64 d19, d19, #56\n\t" + "veor d19, d19, d31\n\t" + /* s[15] => s[23] */ + "veor d23, d15, d25\n\t" + "vshr.u64 d31, d23, #23\n\t" + "vshl.u64 d23, d23, #41\n\t" + "veor d23, d23, d31\n\t" + /* s[4] => s[15] */ + "veor d15, d4, d29\n\t" + "vshr.u64 d31, d15, #37\n\t" + "vshl.u64 d15, d15, #27\n\t" + "veor d15, d15, d31\n\t" + /* s[24] => s[4] */ + "veor d4, d24, d29\n\t" + "vshr.u64 d31, d4, #50\n\t" + "vshl.u64 d4, d4, #14\n\t" + "veor d4, d4, d31\n\t" + /* s[21] => s[24] */ + "veor d24, d21, d26\n\t" + "vshr.u64 d31, d24, #62\n\t" + "vshl.u64 d24, d24, #2\n\t" + "veor d24, d24, d31\n\t" + /* s[8] => s[21] */ + "veor d21, d8, d28\n\t" + "vshr.u64 d31, d21, #9\n\t" + "vshl.u64 d21, d21, #55\n\t" + "veor d21, d21, d31\n\t" + /* s[16] => s[8] */ + "veor d8, d16, d26\n\t" + "vshr.u64 d31, d8, #19\n\t" + "vshl.u64 d8, d8, #45\n\t" + "veor d8, d8, d31\n\t" + /* s[5] => s[16] */ + "veor d16, d5, d25\n\t" + "vshr.u64 d31, d16, #28\n\t" + "vshl.u64 d16, d16, #36\n\t" + "veor d16, d16, d31\n\t" + /* s[3] => s[5] */ + "veor d5, d3, d28\n\t" + "vshr.u64 d31, d5, #36\n\t" + "vshl.u64 d5, d5, #28\n\t" + "veor d5, d5, d31\n\t" + /* s[18] => s[3] */ + "veor d3, d18, d28\n\t" + "vshr.u64 d31, d3, #43\n\t" + "vshl.u64 d3, d3, #21\n\t" + "veor d3, d3, d31\n\t" + /* s[17] => s[18] */ + "veor d18, d17, d27\n\t" + "vshr.u64 d31, d18, #49\n\t" + "vshl.u64 d18, d18, #15\n\t" + "veor d18, d18, d31\n\t" + /* s[11] => s[17] */ + "veor d17, d11, d26\n\t" + "vshr.u64 d31, d17, #54\n\t" + "vshl.u64 d17, d17, #10\n\t" + "veor d17, d17, d31\n\t" + /* s[7] => s[11] */ + "veor d11, d7, d27\n\t" + "vshr.u64 d31, d11, #58\n\t" + "vshl.u64 d11, d11, #6\n\t" + "veor d11, d11, d31\n\t" + /* s[10] => s[7] */ + "veor d7, d10, d25\n\t" + "vshr.u64 d31, d7, #61\n\t" + "vshl.u64 d7, d7, #3\n\t" + "veor d7, d7, d31\n\t" + /* Row Mix */ + "vmov d25, d0\n\t" + "vmov d26, d1\n\t" + "vbic d31, d2, d26\n\t" + "veor d0, d25, d31\n\t" + "vbic d31, d3, d2\n\t" + "veor d1, d26, d31\n\t" + "vbic d31, d4, d3\n\t" + "veor d2, d2, d31\n\t" + "vbic d31, d25, d4\n\t" + "veor d3, d3, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d4, d4, d31\n\t" + "vmov d25, d5\n\t" + "vmov d26, d6\n\t" + "vbic d31, d7, d26\n\t" + "veor d5, d25, d31\n\t" + "vbic d31, d8, d7\n\t" + "veor d6, d26, d31\n\t" + "vbic d31, d9, d8\n\t" + "veor d7, d7, d31\n\t" + "vbic d31, d25, d9\n\t" + "veor d8, d8, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d9, d9, d31\n\t" + "vmov d26, d11\n\t" + "vbic d31, d12, d26\n\t" + "veor d10, d30, d31\n\t" + "vbic d31, d13, d12\n\t" + "veor d11, d26, d31\n\t" + "vbic d31, d14, d13\n\t" + "veor d12, d12, d31\n\t" + "vbic d31, d30, d14\n\t" + "veor d13, d13, d31\n\t" + "vbic d31, d26, d30\n\t" + "veor d14, d14, d31\n\t" + "vmov d25, d15\n\t" + "vmov d26, d16\n\t" + "vbic d31, d17, d26\n\t" + "veor d15, d25, d31\n\t" + "vbic d31, d18, d17\n\t" + "veor d16, d26, d31\n\t" + "vbic d31, d19, d18\n\t" + "veor d17, d17, d31\n\t" + "vbic d31, d25, d19\n\t" + "veor d18, d18, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d19, d19, d31\n\t" + "vmov d25, d20\n\t" + "vmov d26, d21\n\t" + "vbic d31, d22, d26\n\t" + "veor d20, d25, d31\n\t" + "vbic d31, d23, d22\n\t" + "veor d21, d26, d31\n\t" + "vbic d31, d24, d23\n\t" + "veor d22, d22, d31\n\t" + "vbic d31, d25, d24\n\t" + "veor d23, d23, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d24, d24, d31\n\t" + "vld1.8 {d30}, [r3]!\n\t" + "subs r12, r12, #1\n\t" + "veor d0, d0, d30\n\t" + "bne L_sha3_256_blocksx3_seed_neon_begin_%=\n\t" + "vst1.8 {d0-d3}, [%[state]]!\n\t" + "vst1.8 {d4-d7}, [%[state]]!\n\t" + "vst1.8 {d8-d11}, [%[state]]!\n\t" + "vst1.8 {d12-d15}, [%[state]]!\n\t" + "vst1.8 {d16-d19}, [%[state]]!\n\t" + "vst1.8 {d20-d23}, [%[state]]!\n\t" + "vst1.8 {d24}, [%[state]]\n\t" + "add %[state], %[state], #8\n\t" + "subs r4, r4, #1\n\t" + "bne L_sha3_256_blocksx3_seed_neon_state_%=\n\t" + "add sp, sp, #16\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [state] "+r" (state), [seed] "+r" (seed), + [L_sha3_256_blocksx3_seed_neon_rt] "+r" (L_sha3_256_blocksx3_seed_neon_rt_c) + : +#else + : + : [state] "r" (state), [seed] "r" (seed), + [L_sha3_256_blocksx3_seed_neon_rt] "r" (L_sha3_256_blocksx3_seed_neon_rt_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r3", "r12", "lr", "r4", "d0", "d1", "d2", "d3", "d4", + "d5", "d6", "d7", "d8", "d9", "d10", "d11", "d12", "d13", "d14", + "d15", "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23", + "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31" + ); +} + +XALIGNED(16) static const word64 L_sha3_256_blocksx3_seed_64_neon_rt[] = { + 0x0000000000000001UL, 0x0000000000008082UL, + 0x800000000000808aUL, 0x8000000080008000UL, + 0x000000000000808bUL, 0x0000000080000001UL, + 0x8000000080008081UL, 0x8000000000008009UL, + 0x000000000000008aUL, 0x0000000000000088UL, + 0x0000000080008009UL, 0x000000008000000aUL, + 0x000000008000808bUL, 0x800000000000008bUL, + 0x8000000000008089UL, 0x8000000000008003UL, + 0x8000000000008002UL, 0x8000000000000080UL, + 0x000000000000800aUL, 0x800000008000000aUL, + 0x8000000080008081UL, 0x8000000000008080UL, + 0x0000000080000001UL, 0x8000000080008008UL, + 0x8000000000000000UL +}; + +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG +WC_OMIT_FRAME_POINTER void sha3_256_blocksx3_seed_64_neon(word64* state_p, + byte* seed_p) +#else +WC_OMIT_FRAME_POINTER void sha3_256_blocksx3_seed_64_neon(word64* state, + byte* seed) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ +{ +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + register word64* state __asm__ ("r0") = (word64*)state_p; + register byte* seed __asm__ ("r1") = (byte*)seed_p; + register word64* L_sha3_256_blocksx3_seed_64_neon_rt_c __asm__ ("r2") = + (word64*)&L_sha3_256_blocksx3_seed_64_neon_rt; +#else + register word64* L_sha3_256_blocksx3_seed_64_neon_rt_c = + (word64*)&L_sha3_256_blocksx3_seed_64_neon_rt; +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + + __asm__ __volatile__ ( + "sub sp, sp, #16\n\t" + "mov r2, %[L_sha3_256_blocksx3_seed_64_neon_rt]\n\t" + "mov r4, #3\n\t" + "\n" + "L_sha3_256_blocksx3_seed_64_neon_state_%=:\n\t" + "vld1.8 {d0-d3}, [%[seed]]!\n\t" + "vld1.8 {d4-d7}, [%[seed]]!\n\t" + "sub %[seed], %[seed], #0x40\n\t" + "vldr.8 d8, [%[state], #64]\n\t" + "veor d9, d9, d9\n\t" + "veor d10, d10, d10\n\t" + "veor d11, d11, d11\n\t" + "veor d12, d12, d12\n\t" + "veor d13, d13, d13\n\t" + "veor d14, d14, d14\n\t" + "veor d15, d15, d15\n\t" + "vldr.8 d16, [r2, #192]\n\t" + "veor d17, d17, d17\n\t" + "veor d18, d18, d18\n\t" + "veor d19, d19, d19\n\t" + "veor d20, d20, d20\n\t" + "veor d21, d21, d21\n\t" + "veor d22, d22, d22\n\t" + "veor d23, d23, d23\n\t" + "veor d24, d24, d24\n\t" + "mov r3, r2\n\t" + "mov r12, #24\n\t" + "mov lr, sp\n\t" + "\n" + "L_sha3_256_blocksx3_seed_64_neon_begin_%=:\n\t" + /* Calc b[0..4] */ + "veor d26, d0, d5\n\t" + "veor d27, d1, d6\n\t" + "veor d28, d2, d7\n\t" + "veor d29, d3, d8\n\t" + "veor d25, d4, d9\n\t" + "veor d26, d26, d10\n\t" + "veor d27, d27, d11\n\t" + "veor d28, d28, d12\n\t" + "veor d29, d29, d13\n\t" + "veor d25, d25, d14\n\t" + "veor d26, d26, d15\n\t" + "veor d27, d27, d16\n\t" + "veor d28, d28, d17\n\t" + "veor d29, d29, d18\n\t" + "veor d25, d25, d19\n\t" + "veor d26, d26, d20\n\t" + "veor d27, d27, d21\n\t" + "veor d28, d28, d22\n\t" + "veor d29, d29, d23\n\t" + "veor d25, d25, d24\n\t" + "vst1.8 {d25-d26}, [lr]\n\t" + /* Calc t[0..4] and XOR into s[i*5..i*5+4] */ + /* t[0] */ + "vshr.u64 d30, d27, #63\n\t" + "vshl.u64 d31, d27, #1\n\t" + "veor d25, d25, d30\n\t" + "veor d25, d25, d31\n\t" + /* t[1] */ + "vshr.u64 d30, d28, #63\n\t" + "vshl.u64 d31, d28, #1\n\t" + "veor d26, d26, d30\n\t" + "veor d26, d26, d31\n\t" + /* t[2] */ + "vshr.u64 d30, d29, #63\n\t" + "vshl.u64 d31, d29, #1\n\t" + "veor d27, d27, d30\n\t" + "veor d27, d27, d31\n\t" + /* t[3] */ + "vldr.8 d31, [lr]\n\t" + "vshr.u64 d30, d31, #63\n\t" + "vshl.u64 d31, d31, #1\n\t" + "veor d28, d28, d30\n\t" + "veor d28, d28, d31\n\t" + /* t[4] */ + "vldr.8 d31, [lr, #8]\n\t" + "vshr.u64 d30, d31, #63\n\t" + "vshl.u64 d31, d31, #1\n\t" + "veor d29, d29, d30\n\t" + "veor d29, d29, d31\n\t" + "sub lr, lr, #16\n\t" + "veor d0, d0, d25\n\t" + /* s[1] => s[10] (tmp) */ + "veor d30, d1, d26\n\t" + "vshr.u64 d31, d30, #63\n\t" + "vshl.u64 d30, d30, #1\n\t" + "veor d30, d30, d31\n\t" + /* s[6] => s[1] */ + "veor d1, d6, d26\n\t" + "vshr.u64 d31, d1, #20\n\t" + "vshl.u64 d1, d1, #44\n\t" + "veor d1, d1, d31\n\t" + /* s[9] => s[6] */ + "veor d6, d9, d29\n\t" + "vshr.u64 d31, d6, #44\n\t" + "vshl.u64 d6, d6, #20\n\t" + "veor d6, d6, d31\n\t" + /* s[22] => s[9] */ + "veor d9, d22, d27\n\t" + "vshr.u64 d31, d9, #3\n\t" + "vshl.u64 d9, d9, #61\n\t" + "veor d9, d9, d31\n\t" + /* s[14] => s[22] */ + "veor d22, d14, d29\n\t" + "vshr.u64 d31, d22, #25\n\t" + "vshl.u64 d22, d22, #39\n\t" + "veor d22, d22, d31\n\t" + /* s[20] => s[14] */ + "veor d14, d20, d25\n\t" + "vshr.u64 d31, d14, #46\n\t" + "vshl.u64 d14, d14, #18\n\t" + "veor d14, d14, d31\n\t" + /* s[2] => s[20] */ + "veor d20, d2, d27\n\t" + "vshr.u64 d31, d20, #2\n\t" + "vshl.u64 d20, d20, #62\n\t" + "veor d20, d20, d31\n\t" + /* s[12] => s[2] */ + "veor d2, d12, d27\n\t" + "vshr.u64 d31, d2, #21\n\t" + "vshl.u64 d2, d2, #43\n\t" + "veor d2, d2, d31\n\t" + /* s[13] => s[12] */ + "veor d12, d13, d28\n\t" + "vshr.u64 d31, d12, #39\n\t" + "vshl.u64 d12, d12, #25\n\t" + "veor d12, d12, d31\n\t" + /* s[19] => s[13] */ + "veor d13, d19, d29\n\t" + "vshr.u64 d31, d13, #56\n\t" + "vshl.u64 d13, d13, #8\n\t" + "veor d13, d13, d31\n\t" + /* s[23] => s[19] */ + "veor d19, d23, d28\n\t" + "vshr.u64 d31, d19, #8\n\t" + "vshl.u64 d19, d19, #56\n\t" + "veor d19, d19, d31\n\t" + /* s[15] => s[23] */ + "veor d23, d15, d25\n\t" + "vshr.u64 d31, d23, #23\n\t" + "vshl.u64 d23, d23, #41\n\t" + "veor d23, d23, d31\n\t" + /* s[4] => s[15] */ + "veor d15, d4, d29\n\t" + "vshr.u64 d31, d15, #37\n\t" + "vshl.u64 d15, d15, #27\n\t" + "veor d15, d15, d31\n\t" + /* s[24] => s[4] */ + "veor d4, d24, d29\n\t" + "vshr.u64 d31, d4, #50\n\t" + "vshl.u64 d4, d4, #14\n\t" + "veor d4, d4, d31\n\t" + /* s[21] => s[24] */ + "veor d24, d21, d26\n\t" + "vshr.u64 d31, d24, #62\n\t" + "vshl.u64 d24, d24, #2\n\t" + "veor d24, d24, d31\n\t" + /* s[8] => s[21] */ + "veor d21, d8, d28\n\t" + "vshr.u64 d31, d21, #9\n\t" + "vshl.u64 d21, d21, #55\n\t" + "veor d21, d21, d31\n\t" + /* s[16] => s[8] */ + "veor d8, d16, d26\n\t" + "vshr.u64 d31, d8, #19\n\t" + "vshl.u64 d8, d8, #45\n\t" + "veor d8, d8, d31\n\t" + /* s[5] => s[16] */ + "veor d16, d5, d25\n\t" + "vshr.u64 d31, d16, #28\n\t" + "vshl.u64 d16, d16, #36\n\t" + "veor d16, d16, d31\n\t" + /* s[3] => s[5] */ + "veor d5, d3, d28\n\t" + "vshr.u64 d31, d5, #36\n\t" + "vshl.u64 d5, d5, #28\n\t" + "veor d5, d5, d31\n\t" + /* s[18] => s[3] */ + "veor d3, d18, d28\n\t" + "vshr.u64 d31, d3, #43\n\t" + "vshl.u64 d3, d3, #21\n\t" + "veor d3, d3, d31\n\t" + /* s[17] => s[18] */ + "veor d18, d17, d27\n\t" + "vshr.u64 d31, d18, #49\n\t" + "vshl.u64 d18, d18, #15\n\t" + "veor d18, d18, d31\n\t" + /* s[11] => s[17] */ + "veor d17, d11, d26\n\t" + "vshr.u64 d31, d17, #54\n\t" + "vshl.u64 d17, d17, #10\n\t" + "veor d17, d17, d31\n\t" + /* s[7] => s[11] */ + "veor d11, d7, d27\n\t" + "vshr.u64 d31, d11, #58\n\t" + "vshl.u64 d11, d11, #6\n\t" + "veor d11, d11, d31\n\t" + /* s[10] => s[7] */ + "veor d7, d10, d25\n\t" + "vshr.u64 d31, d7, #61\n\t" + "vshl.u64 d7, d7, #3\n\t" + "veor d7, d7, d31\n\t" + /* Row Mix */ + "vmov d25, d0\n\t" + "vmov d26, d1\n\t" + "vbic d31, d2, d26\n\t" + "veor d0, d25, d31\n\t" + "vbic d31, d3, d2\n\t" + "veor d1, d26, d31\n\t" + "vbic d31, d4, d3\n\t" + "veor d2, d2, d31\n\t" + "vbic d31, d25, d4\n\t" + "veor d3, d3, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d4, d4, d31\n\t" + "vmov d25, d5\n\t" + "vmov d26, d6\n\t" + "vbic d31, d7, d26\n\t" + "veor d5, d25, d31\n\t" + "vbic d31, d8, d7\n\t" + "veor d6, d26, d31\n\t" + "vbic d31, d9, d8\n\t" + "veor d7, d7, d31\n\t" + "vbic d31, d25, d9\n\t" + "veor d8, d8, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d9, d9, d31\n\t" + "vmov d26, d11\n\t" + "vbic d31, d12, d26\n\t" + "veor d10, d30, d31\n\t" + "vbic d31, d13, d12\n\t" + "veor d11, d26, d31\n\t" + "vbic d31, d14, d13\n\t" + "veor d12, d12, d31\n\t" + "vbic d31, d30, d14\n\t" + "veor d13, d13, d31\n\t" + "vbic d31, d26, d30\n\t" + "veor d14, d14, d31\n\t" + "vmov d25, d15\n\t" + "vmov d26, d16\n\t" + "vbic d31, d17, d26\n\t" + "veor d15, d25, d31\n\t" + "vbic d31, d18, d17\n\t" + "veor d16, d26, d31\n\t" + "vbic d31, d19, d18\n\t" + "veor d17, d17, d31\n\t" + "vbic d31, d25, d19\n\t" + "veor d18, d18, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d19, d19, d31\n\t" + "vmov d25, d20\n\t" + "vmov d26, d21\n\t" + "vbic d31, d22, d26\n\t" + "veor d20, d25, d31\n\t" + "vbic d31, d23, d22\n\t" + "veor d21, d26, d31\n\t" + "vbic d31, d24, d23\n\t" + "veor d22, d22, d31\n\t" + "vbic d31, d25, d24\n\t" + "veor d23, d23, d31\n\t" + "vbic d31, d26, d25\n\t" + "veor d24, d24, d31\n\t" + "vld1.8 {d30}, [r3]!\n\t" + "subs r12, r12, #1\n\t" + "veor d0, d0, d30\n\t" + "bne L_sha3_256_blocksx3_seed_64_neon_begin_%=\n\t" + "vst1.8 {d0-d3}, [%[state]]!\n\t" + "vst1.8 {d4-d7}, [%[state]]!\n\t" + "vst1.8 {d8-d11}, [%[state]]!\n\t" + "vst1.8 {d12-d15}, [%[state]]!\n\t" + "vst1.8 {d16-d19}, [%[state]]!\n\t" + "vst1.8 {d20-d23}, [%[state]]!\n\t" + "vst1.8 {d24}, [%[state]]\n\t" + "add %[state], %[state], #8\n\t" + "subs r4, r4, #1\n\t" + "bne L_sha3_256_blocksx3_seed_64_neon_state_%=\n\t" + "add sp, sp, #16\n\t" +#ifndef WOLFSSL_NO_VAR_ASSIGN_REG + : [state] "+r" (state), [seed] "+r" (seed), + [L_sha3_256_blocksx3_seed_64_neon_rt] "+r" (L_sha3_256_blocksx3_seed_64_neon_rt_c) + : +#else + : + : [state] "r" (state), [seed] "r" (seed), + [L_sha3_256_blocksx3_seed_64_neon_rt] "r" (L_sha3_256_blocksx3_seed_64_neon_rt_c) +#endif /* !WOLFSSL_NO_VAR_ASSIGN_REG */ + : "memory", "cc", "r3", "r12", "lr", "r4", "d0", "d1", "d2", "d3", "d4", + "d5", "d6", "d7", "d8", "d9", "d10", "d11", "d12", "d13", "d14", + "d15", "d16", "d17", "d18", "d19", "d20", "d21", "d22", "d23", + "d24", "d25", "d26", "d27", "d28", "d29", "d30", "d31" + ); +} + +#endif /* defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) */ #endif /* WOLFSSL_ARMASM_NO_NEON */ #ifdef WOLFSSL_ARMASM_NO_NEON XALIGNED(16) static const word64 L_sha3_arm32_rt[] = { diff --git a/wolfcrypt/src/port/arm/armv8-aes-asm.S b/wolfcrypt/src/port/arm/armv8-aes-asm.S index 6895ef04e8f..f27e8b25445 100644 --- a/wolfcrypt/src/port/arm/armv8-aes-asm.S +++ b/wolfcrypt/src/port/arm/armv8-aes-asm.S @@ -14934,6 +14934,9 @@ _AES_GCM_encrypt_AARCH64_EOR3: stp d10, d11, [x29, #96] stp d12, d13, [x29, #112] stp d14, d15, [x29, #128] +#ifdef __APPLE__ +.arch_extension sha3 +#endif /* __APPLE__ */ ldr w8, [x29, #144] ldr x9, [x29, #152] ldr x10, [x29, #160] @@ -19628,6 +19631,9 @@ _AES_GCM_decrypt_AARCH64_EOR3: stp d10, d11, [x29, #96] stp d12, d13, [x29, #112] stp d14, d15, [x29, #128] +#ifdef __APPLE__ +.arch_extension sha3 +#endif /* __APPLE__ */ ldr w8, [x29, #144] ldr x9, [x29, #152] ldr x10, [x29, #160] @@ -32824,6 +32830,9 @@ _AES_GCM_init_AARCH64_EOR3: add x29, sp, #0 stp d8, d9, [x29, #16] stp d10, d11, [x29, #32] +#ifdef __APPLE__ +.arch_extension sha3 +#endif /* __APPLE__ */ movi v6.16b, #0x87 ld1 {v5.2d}, [x4] ushr v6.2d, v6.2d, #56 @@ -33000,6 +33009,9 @@ _AES_GCM_ghash_block_AARCH64_EOR3: stp x29, x30, [sp, #-32]! add x29, sp, #0 stp d8, d9, [x29, #16] +#ifdef __APPLE__ +.arch_extension sha3 +#endif /* __APPLE__ */ ld1 {v6.2d}, [x1] movi v7.16b, #0x87 ld1 {v5.2d}, [x2] @@ -33049,6 +33061,9 @@ _AES_GCM_aad_update_AARCH64_EOR3: stp d10, d11, [x29, #32] stp d12, d13, [x29, #48] stp d14, d15, [x29, #64] +#ifdef __APPLE__ +.arch_extension sha3 +#endif /* __APPLE__ */ ld1 {v20.2d}, [x2] movi v21.16b, #0x87 ld1 {v12.2d}, [x3] @@ -33380,6 +33395,9 @@ AES_GCM_encrypt_block_AARCH64_EOR3: WC_ASM_ATT_HIDDEN(_AES_GCM_encrypt_block_AARCH64_EOR3) .p2align 2 _AES_GCM_encrypt_block_AARCH64_EOR3: +#endif /* __APPLE__ */ +#ifdef __APPLE__ +.arch_extension sha3 #endif /* __APPLE__ */ ld1 {v5.2d}, [x4] ld1 {v4.2d}, [x3] @@ -33455,6 +33473,9 @@ _AES_GCM_encrypt_update_AARCH64_EOR3: stp d10, d11, [x29, #48] stp d12, d13, [x29, #64] stp d14, d15, [x29, #80] +#ifdef __APPLE__ +.arch_extension sha3 +#endif /* __APPLE__ */ ld1 {v13.2d}, [x7] movi v27.16b, #0x87 ld1 {v26.2d}, [x5] @@ -37175,6 +37196,9 @@ AES_GCM_encrypt_final_AARCH64_EOR3: WC_ASM_ATT_HIDDEN(_AES_GCM_encrypt_final_AARCH64_EOR3) .p2align 2 _AES_GCM_encrypt_final_AARCH64_EOR3: +#endif /* __APPLE__ */ +#ifdef __APPLE__ +.arch_extension sha3 #endif /* __APPLE__ */ ld1 {v5.2d}, [x0] movi v6.16b, #0x87 @@ -37260,6 +37284,9 @@ _AES_GCM_decrypt_update_AARCH64_EOR3: stp d10, d11, [x29, #48] stp d12, d13, [x29, #64] stp d14, d15, [x29, #80] +#ifdef __APPLE__ +.arch_extension sha3 +#endif /* __APPLE__ */ ld1 {v13.2d}, [x7] movi v27.16b, #0x87 ld1 {v26.2d}, [x5] @@ -40981,6 +41008,9 @@ AES_GCM_decrypt_final_AARCH64_EOR3: WC_ASM_ATT_HIDDEN(_AES_GCM_decrypt_final_AARCH64_EOR3) .p2align 2 _AES_GCM_decrypt_final_AARCH64_EOR3: +#endif /* __APPLE__ */ +#ifdef __APPLE__ +.arch_extension sha3 #endif /* __APPLE__ */ ld1 {v5.2d}, [x0] movi v6.16b, #0x87 diff --git a/wolfcrypt/src/port/arm/armv8-aes-asm.asm b/wolfcrypt/src/port/arm/armv8-aes-asm.asm index 92b4a0aa4a7..f1ad914a3d0 100644 --- a/wolfcrypt/src/port/arm/armv8-aes-asm.asm +++ b/wolfcrypt/src/port/arm/armv8-aes-asm.asm @@ -14782,6 +14782,7 @@ AES_GCM_encrypt_AARCH64_EOR3 PROC stp D10, D11, [x29, #96] stp D12, D13, [x29, #112] stp D14, D15, [x29, #128] +; .arch_extension sha3 ldr w8, [x29, #144] ldr x9, [x29, #152] ldr x10, [x29, #160] @@ -19464,6 +19465,7 @@ AES_GCM_decrypt_AARCH64_EOR3 PROC stp D10, D11, [x29, #96] stp D12, D13, [x29, #112] stp D14, D15, [x29, #128] +; .arch_extension sha3 ldr w8, [x29, #144] ldr x9, [x29, #152] ldr x10, [x29, #160] @@ -32552,6 +32554,7 @@ AES_GCM_init_AARCH64_EOR3 PROC add x29, sp, #0 stp D8, D9, [x29, #16] stp D10, D11, [x29, #32] +; .arch_extension sha3 movi V6.16B, #0x87 ld1 {V5.2D}, [x4] ushr V6.2D, V6.2D, #56 @@ -32716,6 +32719,7 @@ AES_GCM_ghash_block_AARCH64_EOR3 PROC stp x29, x30, [sp, #-32]! add x29, sp, #0 stp D8, D9, [x29, #16] +; .arch_extension sha3 ld1 {V6.2D}, [x1] movi V7.16B, #0x87 ld1 {V5.2D}, [x2] @@ -32753,6 +32757,7 @@ AES_GCM_aad_update_AARCH64_EOR3 PROC stp D10, D11, [x29, #32] stp D12, D13, [x29, #48] stp D14, D15, [x29, #64] +; .arch_extension sha3 ld1 {V20.2D}, [x2] movi V21.16B, #0x87 ld1 {V12.2D}, [x3] @@ -33073,6 +33078,7 @@ L_aes_gcm_aad_update_arm64_crypto_eor3_done ALIGN 4 EXPORT AES_GCM_encrypt_block_AARCH64_EOR3 AES_GCM_encrypt_block_AARCH64_EOR3 PROC +; .arch_extension sha3 ld1 {V5.2D}, [x4] ld1 {V4.2D}, [x3] mov w5, V5.S[3] @@ -33135,6 +33141,7 @@ AES_GCM_encrypt_update_AARCH64_EOR3 PROC stp D10, D11, [x29, #48] stp D12, D13, [x29, #64] stp D14, D15, [x29, #80] +; .arch_extension sha3 ld1 {V13.2D}, [x7] movi V27.16B, #0x87 ld1 {V26.2D}, [x5] @@ -36844,6 +36851,7 @@ L_aes_gcm_encrypt_update_arm64_crypto_eor3_done ALIGN 4 EXPORT AES_GCM_encrypt_final_AARCH64_EOR3 AES_GCM_encrypt_final_AARCH64_EOR3 PROC +; .arch_extension sha3 ld1 {V5.2D}, [x0] movi V6.16B, #0x87 ld1 {V4.2D}, [x5] @@ -36916,6 +36924,7 @@ AES_GCM_decrypt_update_AARCH64_EOR3 PROC stp D10, D11, [x29, #48] stp D12, D13, [x29, #64] stp D14, D15, [x29, #80] +; .arch_extension sha3 ld1 {V13.2D}, [x7] movi V27.16B, #0x87 ld1 {V26.2D}, [x5] @@ -40626,6 +40635,7 @@ L_aes_gcm_decrypt_update_arm64_crypto_eor3_done ALIGN 4 EXPORT AES_GCM_decrypt_final_AARCH64_EOR3 AES_GCM_decrypt_final_AARCH64_EOR3 PROC +; .arch_extension sha3 ld1 {V5.2D}, [x0] movi V6.16B, #0x87 ld1 {V4.2D}, [x5] diff --git a/wolfcrypt/src/port/arm/armv8-aes-asm_c.c b/wolfcrypt/src/port/arm/armv8-aes-asm_c.c index a39acf46bae..35c21a5c5fe 100644 --- a/wolfcrypt/src/port/arm/armv8-aes-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-aes-asm_c.c @@ -15068,6 +15068,9 @@ void AES_GCM_encrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, word32 aadSz, byte* key, byte* gcm_h, byte* tmp, byte* reg, int nr) { __asm__ __volatile__ ( +#ifdef __APPLE__ + ".arch_extension sha3\n\t" +#endif /* __APPLE__ */ "movi v27.16b, #0x87\n\t" "eor v26.16b, v26.16b, v26.16b\n\t" "ushr v27.2d, v27.2d, #56\n\t" @@ -19837,6 +19840,9 @@ int AES_GCM_decrypt_AARCH64_EOR3(const byte* in, byte* out, word32 sz, int nr) { __asm__ __volatile__ ( +#ifdef __APPLE__ + ".arch_extension sha3\n\t" +#endif /* __APPLE__ */ "movi v27.16b, #0x87\n\t" "eor v26.16b, v26.16b, v26.16b\n\t" "ushr v27.2d, v27.2d, #56\n\t" @@ -33115,6 +33121,9 @@ void AES_GCM_init_AARCH64_EOR3(byte* key, int nr, const byte* nonce, word32 nonceSz, byte* gcm_h, byte* counter, byte* initCtr) { __asm__ __volatile__ ( +#ifdef __APPLE__ + ".arch_extension sha3\n\t" +#endif /* __APPLE__ */ "movi v6.16b, #0x87\n\t" "ld1 {v5.2d}, [%x[gcm_h]]\n\t" "ushr v6.2d, v6.2d, #56\n\t" @@ -33289,6 +33298,9 @@ void AES_GCM_init_AARCH64_EOR3(byte* key, int nr, const byte* nonce, void AES_GCM_ghash_block_AARCH64_EOR3(const byte* data, byte* tag, byte* gcm_h) { __asm__ __volatile__ ( +#ifdef __APPLE__ + ".arch_extension sha3\n\t" +#endif /* __APPLE__ */ "ld1 {v6.2d}, [%x[tag]]\n\t" "movi v7.16b, #0x87\n\t" "ld1 {v5.2d}, [%x[gcm_h]]\n\t" @@ -33322,6 +33334,9 @@ void AES_GCM_aad_update_AARCH64_EOR3(const byte* aadt, word32 abytes, byte* tag, byte* gcm_h) { __asm__ __volatile__ ( +#ifdef __APPLE__ + ".arch_extension sha3\n\t" +#endif /* __APPLE__ */ "ld1 {v20.2d}, [%x[tag]]\n\t" "movi v21.16b, #0x87\n\t" "ld1 {v12.2d}, [%x[gcm_h]]\n\t" @@ -33650,6 +33665,9 @@ void AES_GCM_encrypt_block_AARCH64_EOR3(const byte* key, int nr, byte* out, const byte* in, byte* counter) { __asm__ __volatile__ ( +#ifdef __APPLE__ + ".arch_extension sha3\n\t" +#endif /* __APPLE__ */ "ld1 {v5.2d}, [%x[counter]]\n\t" "ld1 {v4.2d}, [%x[in]]\n\t" "mov w5, v5.s[3]\n\t" @@ -33710,6 +33728,9 @@ void AES_GCM_encrypt_update_AARCH64_EOR3(const byte* key, int nr, byte* out, const byte* in, word32 nbytes, byte* tag, byte* h, byte* counter) { __asm__ __volatile__ ( +#ifdef __APPLE__ + ".arch_extension sha3\n\t" +#endif /* __APPLE__ */ "ld1 {v13.2d}, [%x[counter]]\n\t" "movi v27.16b, #0x87\n\t" "ld1 {v26.2d}, [%x[tag]]\n\t" @@ -37453,6 +37474,9 @@ void AES_GCM_encrypt_final_AARCH64_EOR3(byte* tag, byte* authTag, word32 tbytes, word32 nbytes, word32 abytes, byte* h, byte* initCtr) { __asm__ __volatile__ ( +#ifdef __APPLE__ + ".arch_extension sha3\n\t" +#endif /* __APPLE__ */ "ld1 {v5.2d}, [%x[tag]]\n\t" "movi v6.16b, #0x87\n\t" "ld1 {v4.2d}, [%x[h]]\n\t" @@ -37532,6 +37556,9 @@ void AES_GCM_decrypt_update_AARCH64_EOR3(const byte* key, int nr, byte* out, const byte* in, word32 nbytes, byte* tag, byte* h, byte* counter) { __asm__ __volatile__ ( +#ifdef __APPLE__ + ".arch_extension sha3\n\t" +#endif /* __APPLE__ */ "ld1 {v13.2d}, [%x[counter]]\n\t" "movi v27.16b, #0x87\n\t" "ld1 {v26.2d}, [%x[tag]]\n\t" @@ -41277,6 +41304,9 @@ void AES_GCM_decrypt_final_AARCH64_EOR3(byte* tag, const byte* authTag, int* res) { __asm__ __volatile__ ( +#ifdef __APPLE__ + ".arch_extension sha3\n\t" +#endif /* __APPLE__ */ "ld1 {v5.2d}, [%x[tag]]\n\t" "movi v6.16b, #0x87\n\t" "ld1 {v4.2d}, [%x[h]]\n\t" diff --git a/wolfcrypt/src/port/arm/armv8-mldsa-asm.S b/wolfcrypt/src/port/arm/armv8-mldsa-asm.S new file mode 100644 index 00000000000..7221f509a55 --- /dev/null +++ b/wolfcrypt/src/port/arm/armv8-mldsa-asm.S @@ -0,0 +1,29288 @@ +/* armv8-mldsa-asm + * + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +/* Generated using (from wolfssl): + * cd ../scripts + * ruby ./mldsa/mldsa.rb arm64 \ + * ../wolfssl/wolfcrypt/src/port/arm/armv8-mldsa-asm.S + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_ARMV8_MLDSA_ASM_S + +#include + +#ifdef WOLFSSL_ARMASM +#ifdef __aarch64__ +#ifndef WOLFSSL_ARMASM_INLINE +#ifdef WOLFSSL_HAVE_MLDSA +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_aarch64_consts, %object + .size L_mldsa_aarch64_consts, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_aarch64_consts: + .long 0x007fe001,0x03802001,0x00400000,0x00000000 +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_aarch64_zetas, %object + .size L_mldsa_aarch64_zetas, 1280 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_aarch64_zetas: + .long 0xffff5c06,0x000064f7,0xffd83102,0xfff81503 + .long 0x00039e44,0xfff42118,0xfff2a128,0x00071e24 + .long 0x001bde2b,0x0023e92b,0xfffa84ad,0xffe0147f + .long 0x002f9a75,0xffd3fb09,0x002f7a49,0x0028e527 + .long 0x00299658,0x000fa070,0xffef85a4,0x0036b788 + .long 0xfff79d90,0xffeeeaa0,0x0027f968,0xffdfd37b + .long 0xffdfadd6,0xffc51ae7,0xffeaa4f7,0xffcdfc98 + .long 0x001ad035,0xffffb422,0x003d3201,0x000445c5 + .long 0x00294a67,0x00017620,0x002ef4cd,0x0035dec5 + .long 0xffe6a503,0xffc9302c,0xffd947d4,0x003bbeaf + .long 0xffc51585,0xffd18e7c,0x00368a96,0xffd43e41 + .long 0x00360400,0xfffb6a4d,0x0023d69c,0xfff7c55d + .long 0xffe6123d,0xffe6ead6,0x00357e1e,0xffc5af59 + .long 0x0035843f,0xffdf5617,0xffe7945c,0x0038738c + .long 0x000c63a8,0x00081b9a,0x000e8f76,0x003b3853 + .long 0x003b8534,0xffd8fc30,0x001f9d54,0xffd54f2d + .long 0xffc406e5,0xffc406e5,0xffe8ac81,0xffe8ac81 + .long 0xffc7e1cf,0xffc7e1cf,0xffd19819,0xffd19819 + .long 0xffe9d65d,0xffe9d65d,0x003509ee,0x003509ee + .long 0x002135c7,0x002135c7,0xffe7cfbb,0xffe7cfbb + .long 0xffeccf75,0xffeccf75,0x001d9772,0x001d9772 + .long 0xffc1b072,0xffc1b072,0xfff0bcf6,0xfff0bcf6 + .long 0xffcf5280,0xffcf5280,0xffcfd2ae,0xffcfd2ae + .long 0xffc890e0,0xffc890e0,0x0001efca,0x0001efca + .long 0x003410f2,0x003410f2,0xfff0fe85,0xfff0fe85 + .long 0x0020c638,0x0020c638,0x00296e9f,0x00296e9f + .long 0xffd2b7a3,0xffd2b7a3,0xffc7a44b,0xffc7a44b + .long 0xfff9ba6d,0xfff9ba6d,0xffda3409,0xffda3409 + .long 0xfff5c282,0xfff5c282,0xffed4113,0xffed4113 + .long 0xffffa63b,0xffffa63b,0xffec09f7,0xffec09f7 + .long 0xfffa2bdd,0xfffa2bdd,0x001495d4,0x001495d4 + .long 0x001c4563,0x001c4563,0xffea2c62,0xffea2c62 + .long 0xffccfbe9,0xffccfbe9,0x00040af0,0x00040af0 + .long 0x0007c417,0x0007c417,0x002f4588,0x002f4588 + .long 0x0000ad00,0x0000ad00,0xffef36be,0xffef36be + .long 0x000dcd44,0x000dcd44,0x003c675a,0x003c675a + .long 0xffc72bca,0xffc72bca,0xffffde7e,0xffffde7e + .long 0x00193948,0x00193948,0xffce69c0,0xffce69c0 + .long 0x0024756c,0x0024756c,0xfffcc7df,0xfffcc7df + .long 0x000b98a1,0x000b98a1,0xffebe808,0xffebe808 + .long 0x0002e46c,0x0002e46c,0xffc9c808,0xffc9c808 + .long 0x003036c2,0x003036c2,0xffe3bff6,0xffe3bff6 + .long 0xffdb3c93,0xffdb3c93,0xfffd4ae0,0xfffd4ae0 + .long 0x00141305,0x00141305,0x00147792,0x00147792 + .long 0x00139e25,0x00139e25,0xffe7d0e0,0xffe7d0e0 + .long 0xfff39944,0xfff39944,0xffea0802,0xffea0802 + .long 0xffd1eea2,0xffd1eea2,0xffc4c79c,0xffc4c79c + .long 0xffc8a057,0xffc8a057,0x003a97d9,0x003a97d9 + .long 0x001fea93,0x0033ff5a,0x002358d4,0x003a41f8 + .long 0xffccff72,0x00223dfb,0xffdaab9f,0xffc9a422 + .long 0x000412f5,0x00252587,0xffed24f0,0x00359b5d + .long 0xffca48a0,0xffc6a2fc,0xffedbb56,0xffcf45de + .long 0x000dbe5e,0x001c5e1a,0x000de0e6,0x000c7f5a + .long 0x00078f83,0xffe7628a,0xffff5704,0xfff806fc + .long 0xfff60021,0xffd05af6,0x001f0084,0x0030ef86 + .long 0xffc9b97d,0xfff7fcd6,0xfff44592,0xffc921c2 + .long 0x00053919,0x0004610c,0xffdacd41,0x003eb01b + .long 0x003472e7,0xffcd003b,0x001a7cc7,0x00031924 + .long 0x002b5ee5,0x00291199,0xffd87a3a,0x00134d71 + .long 0x003de11c,0x00130984,0x0025f051,0x00185a46 + .long 0xffc68518,0x001314be,0x00283891,0xffc9db90 + .long 0xffd25089,0x001c853f,0x001d0b4b,0xffeff6a6 + .long 0xffeba8be,0x0012e11b,0xffcd5e3e,0xffea2d2f + .long 0xfff91de4,0x001406c7,0x00327283,0xffe20d6e + .long 0xffec7953,0x001d4099,0xffd92578,0xffeb05ad + .long 0x0016e405,0x000bdbe7,0x00221de8,0x0033f8cf + .long 0xfff7b934,0xffd4ca0c,0xffe67ff8,0xffe3d157 + .long 0xffd8911b,0xffc72c12,0x000910d8,0xffc65e1f + .long 0xffe14658,0x00251d8b,0x002573b7,0xfffd7c8f + .long 0x001ddd98,0x00336898,0x0002d4bb,0xffed93a7 + .long 0xffcf6cbe,0x00027c1c,0x0018aa08,0x002dfd71 + .long 0x000c5ca5,0x0019379a,0xffc7a167,0xffe48c3d + .long 0xffd1a13c,0x0035c539,0x003b0115,0x00041dc0 + .long 0x0021c4f7,0xfff11bf4,0x001a35e7,0x0007340e + .long 0xfff97d45,0x001a4cd0,0xffe47cae,0x001d2668 + .long 0xffe68e98,0xffef2633,0xfffc05da,0xffc57fdb + .long 0xffd32764,0xffdde1af,0xfff993dd,0xffdd1d09 + .long 0x0002cc93,0xfff11805,0x00189c2a,0xffc9e5a9 + .long 0xfff78a50,0x003bcf2c,0xffff434e,0xffeb36df + .long 0x003c15ca,0x00155e68,0xfff316b6,0x001e29ce +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_aarch64_zetas_qinv, %object + .size L_mldsa_aarch64_zetas_qinv, 1280 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_aarch64_zetas_qinv: + .long 0x00801c06,0x6d1f44f7,0x8cf87102,0x8d187503 + .long 0x61cc1e44,0x58172118,0x6017a128,0x61cb9e24 + .long 0x12613e2b,0x93c9492b,0xae1024ad,0xbeeff47f + .long 0x8cfe3a75,0x1eb51b09,0xeef89a49,0x254dc527 + .long 0x66f49658,0x7c1da070,0xaea405a4,0x3327b788 + .long 0x6ba99d90,0x0d42eaa0,0xeb54f968,0x28cf337b + .long 0x629a6dd6,0xcba1fae7,0xb50984f7,0xd360fc98 + .long 0x13a17035,0x6d83f422,0xa9fd5201,0xba3ce5c5 + .long 0x91f62a67,0x9ec57620,0xac4894cd,0x6d8e7ec5 + .long 0x5f070503,0xbfceb02c,0x8ed3c7d4,0xdc919eaf + .long 0xf3f5b585,0xe3a10e7c,0xde894a96,0x6b1c5e41 + .long 0xc0b60400,0x7ac50a4d,0x9cf7569c,0xbe23655d + .long 0x97adb23d,0xca41aad6,0x18f93e1e,0x6d30cf59 + .long 0x8d3d643f,0x3b223617,0x3473145c,0x78a9f38c + .long 0x588163a8,0x9e7b5b9a,0xeefd4f76,0x89c59853 + .long 0xa6e20534,0xc75efc30,0x99ca1d54,0xc73aef2d + .long 0xa220a6e5,0xa220a6e5,0xd8f8cc81,0xd8f8cc81 + .long 0x5081c1cf,0x5081c1cf,0x8a54b819,0x8a54b819 + .long 0x8035765d,0x8035765d,0x6272c9ee,0x6272c9ee + .long 0x5f5a15c7,0x5f5a15c7,0x085f2fbb,0x085f2fbb + .long 0xb35b6f75,0xb35b6f75,0xc20bd772,0xc20bd772 + .long 0xc4cff072,0xc4cff072,0x748f7cf6,0x748f7cf6 + .long 0xaa1f5280,0xaa1f5280,0x5b2592ae,0x5b2592ae + .long 0x21e490e0,0x21e490e0,0x80fb2fca,0x80fb2fca + .long 0xd15250f2,0xd15250f2,0xf1419e85,0xf1419e85 + .long 0xdce7c638,0xdce7c638,0x5a7d4e9f,0x5a7d4e9f + .long 0x114717a3,0x114717a3,0xfad1044b,0xfad1044b + .long 0xb4c75a6d,0xb4c75a6d,0x65db5409,0x65db5409 + .long 0x7f460282,0x7f460282,0x6a8fa113,0x6a8fa113 + .long 0xc347063b,0xc347063b,0x61aae9f7,0x61aae9f7 + .long 0xcaf5cbdd,0xcaf5cbdd,0xf8cf15d4,0xf8cf15d4 + .long 0x6348a563,0x6348a563,0x9c766c62,0x9c766c62 + .long 0x4eca1be9,0x4eca1be9,0xc9620af0,0xc9620af0 + .long 0x490aa417,0x490aa417,0x44e04588,0x44e04588 + .long 0x95a0ad00,0x95a0ad00,0x7fc6f6be,0x7fc6f6be + .long 0x27b64d44,0x27b64d44,0x4827a75a,0x4827a75a + .long 0x28406bca,0x28406bca,0xb4cf9e7e,0xb4cf9e7e + .long 0xa3423948,0xa3423948,0xed0669c0,0xed0669c0 + .long 0x88d1f56c,0x88d1f56c,0x2578a7df,0x2578a7df + .long 0xa69fb8a1,0xa69fb8a1,0x98ece808,0x98ece808 + .long 0xd690646c,0xd690646c,0x54cac808,0x54cac808 + .long 0xae0876c2,0xae0876c2,0x54e27ff6,0x54e27ff6 + .long 0x69ed9c93,0x69ed9c93,0xb9594ae0,0xb9594ae0 + .long 0x13f4b305,0x13f4b305,0x0e06b792,0x0e06b792 + .long 0xf5583e25,0xf5583e25,0x0a03d0e0,0x0a03d0e0 + .long 0xe11c1944,0xe11c1944,0x47ea4802,0x47ea4802 + .long 0x74a62ea2,0x74a62ea2,0x3ab8479c,0x3ab8479c + .long 0x44538057,0x44538057,0xcab5b7d9,0xcab5b7d9 + .long 0xfff24a93,0x3b1f3f5a,0x513dd8d4,0x2c7941f8 + .long 0xaebb3f72,0x36619dfb,0x01ce8b9f,0xab4de422 + .long 0xdbe2b2f5,0xfd560587,0xec8b24f0,0x79213b5d + .long 0x78de48a0,0x462622fc,0x64587b56,0x718b05de + .long 0x00d97e5e,0xe6df9e1a,0xe12aa0e6,0x4af7bf5a + .long 0x3c77ef83,0xcf38a28a,0x78dfd704,0x72d786fc + .long 0x337a2021,0x682f1af6,0xae2f8084,0x7321af86 + .long 0x6c79597d,0xec92bcd6,0x07a68592,0x4b0161c2 + .long 0x7ea85919,0x3625e10c,0xbd02ed41,0x34c2101b + .long 0xb71152e7,0x6e54603b,0x08335cc7,0x61279924 + .long 0x8d87fee5,0xb9dc3199,0xda1fba3a,0x75416d71 + .long 0x9e61611c,0xaf438984,0xd9b01051,0x00611a46 + .long 0xa4698518,0xfbaad4be,0x02ba5891,0xb33bdb90 + .long 0x29637089,0xed44653f,0x28066b4b,0x43c4b6a6 + .long 0x0e0368be,0x3ab6411b,0x84951e3e,0x6a100d2f + .long 0xc1b59de4,0x396ce6c7,0x1902d283,0x428fcd6e + .long 0x3196d953,0xbfb06099,0x48882578,0x3e20a5ad + .long 0xee178405,0x2408bbe7,0xefdf1de8,0x53cdd8cf + .long 0x2d1e3934,0xc3164a0c,0xb3e57ff8,0x2a8eb157 + .long 0xf07bf11b,0x24496c12,0x162410d8,0x380a3e1f + .long 0x5cac4658,0x0a567d8b,0xaf1c53b7,0xa40f5c8f + .long 0x4fd0dd98,0x81466898,0xe91a34bb,0x7ae273a7 + .long 0x86672cbe,0xb185fc1c,0x3159aa08,0xcb5c1d71 + .long 0xcd20fca5,0xc20c779a,0xdc748167,0x66ec2c3d + .long 0x85f9213c,0x005ce539,0x29dda115,0xa3bc1dc0 + .long 0x9940a4f7,0xf96f9bf4,0xef5715e7,0x1788f40e + .long 0xa1221d45,0x21b44cd0,0xf07a3cae,0x10ea2668 + .long 0xe5b98e98,0x97358633,0xfbb745da,0x2e40dfdb + .long 0x42bfa764,0xa093c1af,0xb7f533dd,0x42fe3d09 + .long 0x5c152c93,0x3471b805,0xa69ddc2a,0x0bff05a9 + .long 0x09418a50,0x94214f2c,0x7969034e,0x734716df + .long 0xc5f555ca,0x17e25e68,0xdfc9d6b6,0x1657e9ce +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_aarch64_zetas_inv, %object + .size L_mldsa_aarch64_zetas_inv, 1280 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_aarch64_zetas_inv: + .long 0xffe1d632,0x000ce94a,0xffeaa198,0xffc3ea36 + .long 0x0014c921,0x0000bcb2,0xffc430d4,0x000875b0 + .long 0x00361a57,0xffe763d6,0x000ee7fb,0xfffd336d + .long 0x0022e2f7,0x00066c23,0x00221e51,0x002cd89c + .long 0x003a8025,0x0003fa26,0x0010d9cd,0x00197168 + .long 0xffe2d998,0x001b8352,0xffe5b330,0x000682bb + .long 0xfff8cbf2,0xffe5ca19,0x000ee40c,0xffde3b09 + .long 0xfffbe240,0xffc4feeb,0xffca3ac7,0x002e5ec4 + .long 0x001b73c3,0x00385e99,0xffe6c866,0xfff3a35b + .long 0xffd2028f,0xffe755f8,0xfffd83e4,0x00309342 + .long 0x00126c59,0xfffd2b45,0xffcc9768,0xffe22268 + .long 0x00028371,0xffda8c49,0xffdae275,0x001eb9a8 + .long 0x0039a1e1,0xfff6ef28,0x0038d3ee,0x00276ee5 + .long 0x001c2ea9,0x00198008,0x002b35f4,0x000846cc + .long 0xffcc0731,0xffdde218,0xfff42419,0xffe91bfb + .long 0x0014fa53,0x0026da88,0xffe2bf67,0x001386ad + .long 0x001df292,0xffcd8d7d,0xffebf939,0x0006e21c + .long 0x0015d2d1,0x0032a1c2,0xffed1ee5,0x00145742 + .long 0x0010095a,0xffe2f4b5,0xffe37ac1,0x002daf77 + .long 0x00362470,0xffd7c76f,0xffeceb42,0x00397ae8 + .long 0xffe7a5ba,0xffda0faf,0xffecf67c,0xffc21ee4 + .long 0xffecb28f,0x002785c6,0xffd6ee67,0xffd4a11b + .long 0xfffce6dc,0xffe58339,0x0032ffc5,0xffcb8d19 + .long 0xffc14fe5,0x002532bf,0xfffb9ef4,0xfffac6e7 + .long 0x0036de3e,0x000bba6e,0x0008032a,0x00364683 + .long 0xffcf107a,0xffe0ff7c,0x002fa50a,0x0009ffdf + .long 0x0007f904,0x0000a8fc,0x00189d76,0xfff8707d + .long 0xfff380a6,0xfff21f1a,0xffe3a1e6,0xfff241a2 + .long 0x0030ba22,0x001244aa,0x00395d04,0x0035b760 + .long 0xffca64a3,0x0012db10,0xffdada79,0xfffbed0b + .long 0x00365bde,0x00255461,0xffddc205,0x0033008e + .long 0xffc5be08,0xffdca72c,0xffcc00a6,0xffe0156d + .long 0xffc56827,0xffc56827,0x00375fa9,0x00375fa9 + .long 0x003b3864,0x003b3864,0x002e115e,0x002e115e + .long 0x0015f7fe,0x0015f7fe,0x000c66bc,0x000c66bc + .long 0x00182f20,0x00182f20,0xffec61db,0xffec61db + .long 0xffeb886e,0xffeb886e,0xffebecfb,0xffebecfb + .long 0x0002b520,0x0002b520,0x0024c36d,0x0024c36d + .long 0x001c400a,0x001c400a,0xffcfc93e,0xffcfc93e + .long 0x003637f8,0x003637f8,0xfffd1b94,0xfffd1b94 + .long 0x001417f8,0x001417f8,0xfff4675f,0xfff4675f + .long 0x00033821,0x00033821,0xffdb8a94,0xffdb8a94 + .long 0x00319640,0x00319640,0xffe6c6b8,0xffe6c6b8 + .long 0x00002182,0x00002182,0x0038d436,0x0038d436 + .long 0xffc398a6,0xffc398a6,0xfff232bc,0xfff232bc + .long 0x0010c942,0x0010c942,0xffff5300,0xffff5300 + .long 0xffd0ba78,0xffd0ba78,0xfff83be9,0xfff83be9 + .long 0xfffbf510,0xfffbf510,0x00330417,0x00330417 + .long 0x0015d39e,0x0015d39e,0xffe3ba9d,0xffe3ba9d + .long 0xffeb6a2c,0xffeb6a2c,0x0005d423,0x0005d423 + .long 0x0013f609,0x0013f609,0x000059c5,0x000059c5 + .long 0x0012beed,0x0012beed,0x000a3d7e,0x000a3d7e + .long 0x0025cbf7,0x0025cbf7,0x00064593,0x00064593 + .long 0x00385bb5,0x00385bb5,0x002d485d,0x002d485d + .long 0xffd69161,0xffd69161,0xffdf39c8,0xffdf39c8 + .long 0x000f017b,0x000f017b,0xffcbef0e,0xffcbef0e + .long 0xfffe1036,0xfffe1036,0x00376f20,0x00376f20 + .long 0x00302d52,0x00302d52,0x0030ad80,0x0030ad80 + .long 0x000f430a,0x000f430a,0x003e4f8e,0x003e4f8e + .long 0xffe2688e,0xffe2688e,0x0013308b,0x0013308b + .long 0x00183045,0x00183045,0xffdeca39,0xffdeca39 + .long 0xffcaf612,0xffcaf612,0x001629a3,0x001629a3 + .long 0x002e67e7,0x002e67e7,0x00381e31,0x00381e31 + .long 0x0017537f,0x0017537f,0x003bf91b,0x003bf91b + .long 0x002ab0d3,0xffe062ac,0x002703d0,0xffc47acc + .long 0xffc4c7ad,0xfff1708a,0xfff7e466,0xfff39c58 + .long 0xffc78c74,0x00186ba4,0x0020a9e9,0xffca7bc1 + .long 0x003a50a7,0xffca81e2,0x0019152a,0x0019edc3 + .long 0x00083aa3,0xffdc2964,0x000495b3,0xffc9fc00 + .long 0x002bc1bf,0xffc9756a,0x002e7184,0x003aea7b + .long 0xffc44151,0x0026b82c,0x0036cfd4,0x00195afd + .long 0xffca213b,0xffd10b33,0xfffe89e0,0xffd6b599 + .long 0xfffbba3b,0xffc2cdff,0x00004bde,0xffe52fcb + .long 0x00320368,0x00155b09,0x003ae519,0x0020522a + .long 0x00202c85,0xffd80698,0x00111560,0x00086270 + .long 0xffc94878,0x00107a5c,0xfff05f90,0xffd669a8 + .long 0xffd71ad9,0xffd085b7,0x002c04f7,0xffd0658b + .long 0x001feb81,0x00057b53,0xffdc16d5,0xffe421d5 + .long 0xfff8e1dc,0x000d5ed8,0x000bdee8,0xfffc61bc + .long 0x0007eafd,0x0027cefe,0x003caa21,0x0000a3fa +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_aarch64_zetas_inv_qinv, %object + .size L_mldsa_aarch64_zetas_inv_qinv, 1280 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_aarch64_zetas_inv_qinv: + .long 0xe9a81632,0x2036294a,0xe81da198,0x3a0aaa36 + .long 0x8cb8e921,0x8696fcb2,0x6bdeb0d4,0xf6be75b0 + .long 0xf400fa57,0x596223d6,0xcb8e47fb,0xa3ead36d + .long 0xbd01c2f7,0x480acc23,0x5f6c3e51,0xbd40589c + .long 0xd1bf2025,0x0448ba26,0x68ca79cd,0x1a467168 + .long 0xef15d998,0x0f85c352,0xde4bb330,0x5edde2bb + .long 0xe8770bf2,0x10a8ea19,0x0690640c,0x66bf5b09 + .long 0x5c43e240,0xd6225eeb,0xffa31ac7,0x7a06dec4 + .long 0x9913d3c3,0x238b7e99,0x3df38866,0x32df035b + .long 0x34a3e28f,0xcea655f8,0x4e7a03e4,0x7998d342 + .long 0x851d8c59,0x16e5cb45,0x7eb99768,0xb02f2268 + .long 0x5bf0a371,0x50e3ac49,0xf5a98275,0xa353b9a8 + .long 0xc7f5c1e1,0xe9dbef28,0xdbb693ee,0x0f840ee5 + .long 0xd5714ea9,0x4c1a8008,0x3ce9b5f4,0xd2e1c6cc + .long 0xac322731,0x1020e218,0xdbf74419,0x11e87bfb + .long 0xc1df5a53,0xb777da88,0x404f9f67,0xce6926ad + .long 0xbd703292,0xe6fd2d7d,0xc6931939,0x3e4a621c + .long 0x95eff2d1,0x7b6ae1c2,0xc549bee5,0xf1fc9742 + .long 0xbc3b495a,0xd7f994b5,0x12bb9ac1,0xd69c8f77 + .long 0x4cc42470,0xfd45a76f,0x04552b42,0x5b967ae8 + .long 0xff9ee5ba,0x264fefaf,0x50bc767c,0x619e9ee4 + .long 0x8abe928f,0x25e045c6,0x4623ce67,0x7278011b + .long 0x9ed866dc,0xf7cca339,0x91ab9fc5,0x48eead19 + .long 0xcb3defe5,0x42fd12bf,0xc9da1ef4,0x8157a6e7 + .long 0xb4fe9e3e,0xf8597a6e,0x136d432a,0x9386a683 + .long 0x8cde507a,0x51d07f7c,0x97d0e50a,0xcc85dfdf + .long 0x8d287904,0x872028fc,0x30c75d76,0xc388107d + .long 0xb50840a6,0x1ed55f1a,0x192061e6,0xff2681a2 + .long 0x8e74fa22,0x9ba784aa,0xb9d9dd04,0x8721b760 + .long 0x86dec4a3,0x1374db10,0x02a9fa79,0x241d4d0b + .long 0x54b21bde,0xfe317461,0xc99e6205,0x5144c08e + .long 0xd386be08,0xaec2272c,0xc4e0c0a6,0x000db56d + .long 0x354a4827,0x354a4827,0xbbac7fa9,0xbbac7fa9 + .long 0xc547b864,0xc547b864,0x8b59d15e,0x8b59d15e + .long 0xb815b7fe,0xb815b7fe,0x1ee3e6bc,0x1ee3e6bc + .long 0xf5fc2f20,0xf5fc2f20,0x0aa7c1db,0x0aa7c1db + .long 0xf1f9486e,0xf1f9486e,0xec0b4cfb,0xec0b4cfb + .long 0x46a6b520,0x46a6b520,0x9612636d,0x9612636d + .long 0xab1d800a,0xab1d800a,0x51f7893e,0x51f7893e + .long 0xab3537f8,0xab3537f8,0x296f9b94,0x296f9b94 + .long 0x671317f8,0x671317f8,0x5960475f,0x5960475f + .long 0xda875821,0xda875821,0x772e0a94,0x772e0a94 + .long 0x12f99640,0x12f99640,0x5cbdc6b8,0x5cbdc6b8 + .long 0x4b306182,0x4b306182,0xd7bf9436,0xd7bf9436 + .long 0xb7d858a6,0xb7d858a6,0xd849b2bc,0xd849b2bc + .long 0x80390942,0x80390942,0x6a5f5300,0x6a5f5300 + .long 0xbb1fba78,0xbb1fba78,0xb6f55be9,0xb6f55be9 + .long 0x369df510,0x369df510,0xb135e417,0xb135e417 + .long 0x6389939e,0x6389939e,0x9cb75a9d,0x9cb75a9d + .long 0x0730ea2c,0x0730ea2c,0x350a3423,0x350a3423 + .long 0x9e551609,0x9e551609,0x3cb8f9c5,0x3cb8f9c5 + .long 0x95705eed,0x95705eed,0x80b9fd7e,0x80b9fd7e + .long 0x9a24abf7,0x9a24abf7,0x4b38a593,0x4b38a593 + .long 0x052efbb5,0x052efbb5,0xeeb8e85d,0xeeb8e85d + .long 0xa582b161,0xa582b161,0x231839c8,0x231839c8 + .long 0x0ebe617b,0x0ebe617b,0x2eadaf0e,0x2eadaf0e + .long 0x7f04d036,0x7f04d036,0xde1b6f20,0xde1b6f20 + .long 0xa4da6d52,0xa4da6d52,0x55e0ad80,0x55e0ad80 + .long 0x8b70830a,0x8b70830a,0x3b300f8e,0x3b300f8e + .long 0x3df4288e,0x3df4288e,0x4ca4908b,0x4ca4908b + .long 0xf7a0d045,0xf7a0d045,0xa0a5ea39,0xa0a5ea39 + .long 0x9d8d3612,0x9d8d3612,0x7fca89a3,0x7fca89a3 + .long 0x75ab47e7,0x75ab47e7,0xaf7e3e31,0xaf7e3e31 + .long 0x2707337f,0x2707337f,0x5ddf591b,0x5ddf591b + .long 0x38c510d3,0x6635e2ac,0x38a103d0,0x591dfacc + .long 0x763a67ad,0x1102b08a,0x6184a466,0xa77e9c58 + .long 0x87560c74,0xcb8ceba4,0xc4ddc9e9,0x72c29bc1 + .long 0x92cf30a7,0xe706c1e2,0x35be552a,0x68524dc3 + .long 0x41dc9aa3,0x6308a964,0x853af5b3,0x3f49fc00 + .long 0x94e3a1bf,0x2176b56a,0x1c5ef184,0x0c0a4a7b + .long 0x236e6151,0x712c382c,0x40314fd4,0xa0f8fafd + .long 0x9271813b,0x53b76b33,0x613a89e0,0x6e09d599 + .long 0x45c31a3b,0x5602adff,0x927c0bde,0xec5e8fcb + .long 0x2c9f0368,0x4af67b09,0x345e0519,0x9d65922a + .long 0xd730cc85,0x14ab0698,0xf2bd1560,0x94566270 + .long 0xccd84878,0x515bfa5c,0x83e25f90,0x990b69a8 + .long 0xdab23ad9,0x110765b7,0xe14ae4f7,0x7301c58b + .long 0x41100b81,0x51efdb53,0x6c36b6d5,0xed9ec1d5 + .long 0x9e3461dc,0x9fe85ed8,0xa7e8dee8,0x9e33e1bc + .long 0x72e78afd,0x73078efe,0x0900ca21,0xff7fe3fa +#ifndef __APPLE__ +.text +.globl mldsa_ntt_neon +WC_ASM_ATT_HIDDEN(mldsa_ntt_neon) +.type mldsa_ntt_neon,@function +.align 2 +mldsa_ntt_neon: +#else +.section __TEXT,__text +.globl _mldsa_ntt_neon +WC_ASM_ATT_HIDDEN(_mldsa_ntt_neon) +.p2align 2 +_mldsa_ntt_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x1, L_mldsa_aarch64_zetas + add x1, x1, :lo12:L_mldsa_aarch64_zetas +#else + adrp x1, L_mldsa_aarch64_zetas@PAGE + add x1, x1, L_mldsa_aarch64_zetas@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x2, L_mldsa_aarch64_zetas_qinv + add x2, x2, :lo12:L_mldsa_aarch64_zetas_qinv +#else + adrp x2, L_mldsa_aarch64_zetas_qinv@PAGE + add x2, x2, L_mldsa_aarch64_zetas_qinv@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_aarch64_consts + add x3, x3, :lo12:L_mldsa_aarch64_consts +#else + adrp x3, L_mldsa_aarch64_consts@PAGE + add x3, x3, L_mldsa_aarch64_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q4, [x3] + ldr q5, [x0] + ldr q6, [x0, #64] + ldr q7, [x0, #128] + ldr q8, [x0, #192] + ldr q9, [x0, #256] + ldr q10, [x0, #320] + ldr q11, [x0, #384] + ldr q12, [x0, #448] + ldr q13, [x0, #512] + ldr q14, [x0, #576] + ldr q15, [x0, #640] + ldr q16, [x0, #704] + ldr q17, [x0, #768] + ldr q18, [x0, #832] + ldr q19, [x0, #896] + ldr q20, [x0, #960] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0] + str q6, [x0, #64] + str q7, [x0, #128] + str q8, [x0, #192] + str q9, [x0, #256] + str q10, [x0, #320] + str q11, [x0, #384] + str q12, [x0, #448] + str q13, [x0, #512] + str q14, [x0, #576] + str q15, [x0, #640] + str q16, [x0, #704] + str q17, [x0, #768] + str q18, [x0, #832] + str q19, [x0, #896] + str q20, [x0, #960] + ldr q5, [x0, #16] + ldr q6, [x0, #80] + ldr q7, [x0, #144] + ldr q8, [x0, #208] + ldr q9, [x0, #272] + ldr q10, [x0, #336] + ldr q11, [x0, #400] + ldr q12, [x0, #464] + ldr q13, [x0, #528] + ldr q14, [x0, #592] + ldr q15, [x0, #656] + ldr q16, [x0, #720] + ldr q17, [x0, #784] + ldr q18, [x0, #848] + ldr q19, [x0, #912] + ldr q20, [x0, #976] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0, #16] + str q6, [x0, #80] + str q7, [x0, #144] + str q8, [x0, #208] + str q9, [x0, #272] + str q10, [x0, #336] + str q11, [x0, #400] + str q12, [x0, #464] + str q13, [x0, #528] + str q14, [x0, #592] + str q15, [x0, #656] + str q16, [x0, #720] + str q17, [x0, #784] + str q18, [x0, #848] + str q19, [x0, #912] + str q20, [x0, #976] + ldr q5, [x0, #32] + ldr q6, [x0, #96] + ldr q7, [x0, #160] + ldr q8, [x0, #224] + ldr q9, [x0, #288] + ldr q10, [x0, #352] + ldr q11, [x0, #416] + ldr q12, [x0, #480] + ldr q13, [x0, #544] + ldr q14, [x0, #608] + ldr q15, [x0, #672] + ldr q16, [x0, #736] + ldr q17, [x0, #800] + ldr q18, [x0, #864] + ldr q19, [x0, #928] + ldr q20, [x0, #992] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0, #32] + str q6, [x0, #96] + str q7, [x0, #160] + str q8, [x0, #224] + str q9, [x0, #288] + str q10, [x0, #352] + str q11, [x0, #416] + str q12, [x0, #480] + str q13, [x0, #544] + str q14, [x0, #608] + str q15, [x0, #672] + str q16, [x0, #736] + str q17, [x0, #800] + str q18, [x0, #864] + str q19, [x0, #928] + str q20, [x0, #992] + ldr q5, [x0, #48] + ldr q6, [x0, #112] + ldr q7, [x0, #176] + ldr q8, [x0, #240] + ldr q9, [x0, #304] + ldr q10, [x0, #368] + ldr q11, [x0, #432] + ldr q12, [x0, #496] + ldr q13, [x0, #560] + ldr q14, [x0, #624] + ldr q15, [x0, #688] + ldr q16, [x0, #752] + ldr q17, [x0, #816] + ldr q18, [x0, #880] + ldr q19, [x0, #944] + ldr q20, [x0, #1008] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0, #48] + str q6, [x0, #112] + str q7, [x0, #176] + str q8, [x0, #240] + str q9, [x0, #304] + str q10, [x0, #368] + str q11, [x0, #432] + str q12, [x0, #496] + str q13, [x0, #560] + str q14, [x0, #624] + str q15, [x0, #688] + str q16, [x0, #752] + str q17, [x0, #816] + str q18, [x0, #880] + str q19, [x0, #944] + str q20, [x0, #1008] + ldp q5, q6, [x0] + ldp q7, q8, [x0, #32] + ldp q9, q10, [x0, #64] + ldp q11, q12, [x0, #96] + ldp q13, q14, [x0, #128] + ldp q15, q16, [x0, #160] + ldp q17, q18, [x0, #192] + ldp q19, q20, [x0, #224] + ldr q0, [x1, #64] + ldr q1, [x2, #64] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #128] + ldp q1, q3, [x2, #128] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #256] + ldp q1, q3, [x2, #256] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #288] + ldp q1, q3, [x2, #288] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #320] + ldp q1, q3, [x2, #320] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #352] + ldp q1, q3, [x2, #352] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #768] + ldp q1, q3, [x2, #768] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #800] + ldp q1, q3, [x2, #800] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #832] + ldp q1, q3, [x2, #832] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #864] + ldp q1, q3, [x2, #864] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + stp q5, q6, [x0] + stp q7, q8, [x0, #32] + stp q9, q10, [x0, #64] + stp q11, q12, [x0, #96] + stp q13, q14, [x0, #128] + stp q15, q16, [x0, #160] + stp q17, q18, [x0, #192] + stp q19, q20, [x0, #224] + ldp q5, q6, [x0, #256] + ldp q7, q8, [x0, #288] + ldp q9, q10, [x0, #320] + ldp q11, q12, [x0, #352] + ldp q13, q14, [x0, #384] + ldp q15, q16, [x0, #416] + ldp q17, q18, [x0, #448] + ldp q19, q20, [x0, #480] + ldr q0, [x1, #80] + ldr q1, [x2, #80] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #160] + ldp q1, q3, [x2, #160] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #384] + ldp q1, q3, [x2, #384] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #416] + ldp q1, q3, [x2, #416] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #448] + ldp q1, q3, [x2, #448] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #480] + ldp q1, q3, [x2, #480] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #896] + ldp q1, q3, [x2, #896] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #928] + ldp q1, q3, [x2, #928] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #960] + ldp q1, q3, [x2, #960] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #992] + ldp q1, q3, [x2, #992] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + stp q5, q6, [x0, #256] + stp q7, q8, [x0, #288] + stp q9, q10, [x0, #320] + stp q11, q12, [x0, #352] + stp q13, q14, [x0, #384] + stp q15, q16, [x0, #416] + stp q17, q18, [x0, #448] + stp q19, q20, [x0, #480] + ldp q5, q6, [x0, #512] + ldp q7, q8, [x0, #544] + ldp q9, q10, [x0, #576] + ldp q11, q12, [x0, #608] + ldp q13, q14, [x0, #640] + ldp q15, q16, [x0, #672] + ldp q17, q18, [x0, #704] + ldp q19, q20, [x0, #736] + ldr q0, [x1, #96] + ldr q1, [x2, #96] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #192] + ldp q1, q3, [x2, #192] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #512] + ldp q1, q3, [x2, #512] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #544] + ldp q1, q3, [x2, #544] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #576] + ldp q1, q3, [x2, #576] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #608] + ldp q1, q3, [x2, #608] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldr q0, [x1, #1024] + ldr q2, [x1, #1040] + ldr q1, [x2, #1024] + ldr q3, [x2, #1040] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldr q0, [x1, #1056] + ldr q2, [x1, #1072] + ldr q1, [x2, #1056] + ldr q3, [x2, #1072] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldr q0, [x1, #1088] + ldr q2, [x1, #1104] + ldr q1, [x2, #1088] + ldr q3, [x2, #1104] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldr q0, [x1, #1120] + ldr q2, [x1, #1136] + ldr q1, [x2, #1120] + ldr q3, [x2, #1136] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + stp q5, q6, [x0, #512] + stp q7, q8, [x0, #544] + stp q9, q10, [x0, #576] + stp q11, q12, [x0, #608] + stp q13, q14, [x0, #640] + stp q15, q16, [x0, #672] + stp q17, q18, [x0, #704] + stp q19, q20, [x0, #736] + ldp q5, q6, [x0, #768] + ldp q7, q8, [x0, #800] + ldp q9, q10, [x0, #832] + ldp q11, q12, [x0, #864] + ldp q13, q14, [x0, #896] + ldp q15, q16, [x0, #928] + ldp q17, q18, [x0, #960] + ldp q19, q20, [x0, #992] + ldr q0, [x1, #112] + ldr q1, [x2, #112] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #224] + ldp q1, q3, [x2, #224] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #640] + ldp q1, q3, [x2, #640] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #672] + ldp q1, q3, [x2, #672] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #704] + ldp q1, q3, [x2, #704] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #736] + ldp q1, q3, [x2, #736] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldr q0, [x1, #1152] + ldr q2, [x1, #1168] + ldr q1, [x2, #1152] + ldr q3, [x2, #1168] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldr q0, [x1, #1184] + ldr q2, [x1, #1200] + ldr q1, [x2, #1184] + ldr q3, [x2, #1200] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldr q0, [x1, #1216] + ldr q2, [x1, #1232] + ldr q1, [x2, #1216] + ldr q3, [x2, #1232] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x1, #1264] + ldr q1, [x2, #1248] + ldr q3, [x2, #1264] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + stp q5, q6, [x0, #768] + stp q7, q8, [x0, #800] + stp q9, q10, [x0, #832] + stp q11, q12, [x0, #864] + stp q13, q14, [x0, #896] + stp q15, q16, [x0, #928] + stp q17, q18, [x0, #960] + stp q19, q20, [x0, #992] + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_ntt_neon,.-mldsa_ntt_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mldsa_ntt_full_neon +WC_ASM_ATT_HIDDEN(mldsa_ntt_full_neon) +.type mldsa_ntt_full_neon,@function +.align 2 +mldsa_ntt_full_neon: +#else +.section __TEXT,__text +.globl _mldsa_ntt_full_neon +WC_ASM_ATT_HIDDEN(_mldsa_ntt_full_neon) +.p2align 2 +_mldsa_ntt_full_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x1, L_mldsa_aarch64_zetas + add x1, x1, :lo12:L_mldsa_aarch64_zetas +#else + adrp x1, L_mldsa_aarch64_zetas@PAGE + add x1, x1, L_mldsa_aarch64_zetas@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x2, L_mldsa_aarch64_zetas_qinv + add x2, x2, :lo12:L_mldsa_aarch64_zetas_qinv +#else + adrp x2, L_mldsa_aarch64_zetas_qinv@PAGE + add x2, x2, L_mldsa_aarch64_zetas_qinv@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_aarch64_consts + add x3, x3, :lo12:L_mldsa_aarch64_consts +#else + adrp x3, L_mldsa_aarch64_consts@PAGE + add x3, x3, L_mldsa_aarch64_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q4, [x3] + ldr q5, [x0] + ldr q6, [x0, #64] + ldr q7, [x0, #128] + ldr q8, [x0, #192] + ldr q9, [x0, #256] + ldr q10, [x0, #320] + ldr q11, [x0, #384] + ldr q12, [x0, #448] + ldr q13, [x0, #512] + ldr q14, [x0, #576] + ldr q15, [x0, #640] + ldr q16, [x0, #704] + ldr q17, [x0, #768] + ldr q18, [x0, #832] + ldr q19, [x0, #896] + ldr q20, [x0, #960] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0] + str q6, [x0, #64] + str q7, [x0, #128] + str q8, [x0, #192] + str q9, [x0, #256] + str q10, [x0, #320] + str q11, [x0, #384] + str q12, [x0, #448] + str q13, [x0, #512] + str q14, [x0, #576] + str q15, [x0, #640] + str q16, [x0, #704] + str q17, [x0, #768] + str q18, [x0, #832] + str q19, [x0, #896] + str q20, [x0, #960] + ldr q5, [x0, #16] + ldr q6, [x0, #80] + ldr q7, [x0, #144] + ldr q8, [x0, #208] + ldr q9, [x0, #272] + ldr q10, [x0, #336] + ldr q11, [x0, #400] + ldr q12, [x0, #464] + ldr q13, [x0, #528] + ldr q14, [x0, #592] + ldr q15, [x0, #656] + ldr q16, [x0, #720] + ldr q17, [x0, #784] + ldr q18, [x0, #848] + ldr q19, [x0, #912] + ldr q20, [x0, #976] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0, #16] + str q6, [x0, #80] + str q7, [x0, #144] + str q8, [x0, #208] + str q9, [x0, #272] + str q10, [x0, #336] + str q11, [x0, #400] + str q12, [x0, #464] + str q13, [x0, #528] + str q14, [x0, #592] + str q15, [x0, #656] + str q16, [x0, #720] + str q17, [x0, #784] + str q18, [x0, #848] + str q19, [x0, #912] + str q20, [x0, #976] + ldr q5, [x0, #32] + ldr q6, [x0, #96] + ldr q7, [x0, #160] + ldr q8, [x0, #224] + ldr q9, [x0, #288] + ldr q10, [x0, #352] + ldr q11, [x0, #416] + ldr q12, [x0, #480] + ldr q13, [x0, #544] + ldr q14, [x0, #608] + ldr q15, [x0, #672] + ldr q16, [x0, #736] + ldr q17, [x0, #800] + ldr q18, [x0, #864] + ldr q19, [x0, #928] + ldr q20, [x0, #992] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0, #32] + str q6, [x0, #96] + str q7, [x0, #160] + str q8, [x0, #224] + str q9, [x0, #288] + str q10, [x0, #352] + str q11, [x0, #416] + str q12, [x0, #480] + str q13, [x0, #544] + str q14, [x0, #608] + str q15, [x0, #672] + str q16, [x0, #736] + str q17, [x0, #800] + str q18, [x0, #864] + str q19, [x0, #928] + str q20, [x0, #992] + ldr q5, [x0, #48] + ldr q6, [x0, #112] + ldr q7, [x0, #176] + ldr q8, [x0, #240] + ldr q9, [x0, #304] + ldr q10, [x0, #368] + ldr q11, [x0, #432] + ldr q12, [x0, #496] + ldr q13, [x0, #560] + ldr q14, [x0, #624] + ldr q15, [x0, #688] + ldr q16, [x0, #752] + ldr q17, [x0, #816] + ldr q18, [x0, #880] + ldr q19, [x0, #944] + ldr q20, [x0, #1008] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0, #48] + str q6, [x0, #112] + str q7, [x0, #176] + str q8, [x0, #240] + str q9, [x0, #304] + str q10, [x0, #368] + str q11, [x0, #432] + str q12, [x0, #496] + str q13, [x0, #560] + str q14, [x0, #624] + str q15, [x0, #688] + str q16, [x0, #752] + str q17, [x0, #816] + str q18, [x0, #880] + str q19, [x0, #944] + str q20, [x0, #1008] + ldp q5, q6, [x0] + ldp q7, q8, [x0, #32] + ldp q9, q10, [x0, #64] + ldp q11, q12, [x0, #96] + ldp q13, q14, [x0, #128] + ldp q15, q16, [x0, #160] + ldp q17, q18, [x0, #192] + ldp q19, q20, [x0, #224] + ldr q0, [x1, #64] + ldr q1, [x2, #64] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #128] + ldp q1, q3, [x2, #128] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #256] + ldp q1, q3, [x2, #256] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #288] + ldp q1, q3, [x2, #288] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #320] + ldp q1, q3, [x2, #320] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #352] + ldp q1, q3, [x2, #352] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #768] + ldp q1, q3, [x2, #768] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #800] + ldp q1, q3, [x2, #800] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #832] + ldp q1, q3, [x2, #832] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #864] + ldp q1, q3, [x2, #864] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + trn1 v29.4s, v5.4s, v6.4s + trn2 v30.4s, v5.4s, v6.4s + trn1 v5.2d, v29.2d, v30.2d + trn2 v6.2d, v29.2d, v30.2d + trn1 v29.4s, v7.4s, v8.4s + trn2 v30.4s, v7.4s, v8.4s + trn1 v7.2d, v29.2d, v30.2d + trn2 v8.2d, v29.2d, v30.2d + trn1 v29.4s, v9.4s, v10.4s + trn2 v30.4s, v9.4s, v10.4s + trn1 v9.2d, v29.2d, v30.2d + trn2 v10.2d, v29.2d, v30.2d + trn1 v29.4s, v11.4s, v12.4s + trn2 v30.4s, v11.4s, v12.4s + trn1 v11.2d, v29.2d, v30.2d + trn2 v12.2d, v29.2d, v30.2d + trn1 v29.4s, v13.4s, v14.4s + trn2 v30.4s, v13.4s, v14.4s + trn1 v13.2d, v29.2d, v30.2d + trn2 v14.2d, v29.2d, v30.2d + trn1 v29.4s, v15.4s, v16.4s + trn2 v30.4s, v15.4s, v16.4s + trn1 v15.2d, v29.2d, v30.2d + trn2 v16.2d, v29.2d, v30.2d + trn1 v29.4s, v17.4s, v18.4s + trn2 v30.4s, v17.4s, v18.4s + trn1 v17.2d, v29.2d, v30.2d + trn2 v18.2d, v29.2d, v30.2d + trn1 v29.4s, v19.4s, v20.4s + trn2 v30.4s, v19.4s, v20.4s + trn1 v19.2d, v29.2d, v30.2d + trn2 v20.2d, v29.2d, v30.2d + stp q5, q6, [x0] + stp q7, q8, [x0, #32] + stp q9, q10, [x0, #64] + stp q11, q12, [x0, #96] + stp q13, q14, [x0, #128] + stp q15, q16, [x0, #160] + stp q17, q18, [x0, #192] + stp q19, q20, [x0, #224] + ldp q5, q6, [x0, #256] + ldp q7, q8, [x0, #288] + ldp q9, q10, [x0, #320] + ldp q11, q12, [x0, #352] + ldp q13, q14, [x0, #384] + ldp q15, q16, [x0, #416] + ldp q17, q18, [x0, #448] + ldp q19, q20, [x0, #480] + ldr q0, [x1, #80] + ldr q1, [x2, #80] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #160] + ldp q1, q3, [x2, #160] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #384] + ldp q1, q3, [x2, #384] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #416] + ldp q1, q3, [x2, #416] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #448] + ldp q1, q3, [x2, #448] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #480] + ldp q1, q3, [x2, #480] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #896] + ldp q1, q3, [x2, #896] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #928] + ldp q1, q3, [x2, #928] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #960] + ldp q1, q3, [x2, #960] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #992] + ldp q1, q3, [x2, #992] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + trn1 v29.4s, v5.4s, v6.4s + trn2 v30.4s, v5.4s, v6.4s + trn1 v5.2d, v29.2d, v30.2d + trn2 v6.2d, v29.2d, v30.2d + trn1 v29.4s, v7.4s, v8.4s + trn2 v30.4s, v7.4s, v8.4s + trn1 v7.2d, v29.2d, v30.2d + trn2 v8.2d, v29.2d, v30.2d + trn1 v29.4s, v9.4s, v10.4s + trn2 v30.4s, v9.4s, v10.4s + trn1 v9.2d, v29.2d, v30.2d + trn2 v10.2d, v29.2d, v30.2d + trn1 v29.4s, v11.4s, v12.4s + trn2 v30.4s, v11.4s, v12.4s + trn1 v11.2d, v29.2d, v30.2d + trn2 v12.2d, v29.2d, v30.2d + trn1 v29.4s, v13.4s, v14.4s + trn2 v30.4s, v13.4s, v14.4s + trn1 v13.2d, v29.2d, v30.2d + trn2 v14.2d, v29.2d, v30.2d + trn1 v29.4s, v15.4s, v16.4s + trn2 v30.4s, v15.4s, v16.4s + trn1 v15.2d, v29.2d, v30.2d + trn2 v16.2d, v29.2d, v30.2d + trn1 v29.4s, v17.4s, v18.4s + trn2 v30.4s, v17.4s, v18.4s + trn1 v17.2d, v29.2d, v30.2d + trn2 v18.2d, v29.2d, v30.2d + trn1 v29.4s, v19.4s, v20.4s + trn2 v30.4s, v19.4s, v20.4s + trn1 v19.2d, v29.2d, v30.2d + trn2 v20.2d, v29.2d, v30.2d + stp q5, q6, [x0, #256] + stp q7, q8, [x0, #288] + stp q9, q10, [x0, #320] + stp q11, q12, [x0, #352] + stp q13, q14, [x0, #384] + stp q15, q16, [x0, #416] + stp q17, q18, [x0, #448] + stp q19, q20, [x0, #480] + ldp q5, q6, [x0, #512] + ldp q7, q8, [x0, #544] + ldp q9, q10, [x0, #576] + ldp q11, q12, [x0, #608] + ldp q13, q14, [x0, #640] + ldp q15, q16, [x0, #672] + ldp q17, q18, [x0, #704] + ldp q19, q20, [x0, #736] + ldr q0, [x1, #96] + ldr q1, [x2, #96] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #192] + ldp q1, q3, [x2, #192] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #512] + ldp q1, q3, [x2, #512] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #544] + ldp q1, q3, [x2, #544] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #576] + ldp q1, q3, [x2, #576] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #608] + ldp q1, q3, [x2, #608] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldr q0, [x1, #1024] + ldr q2, [x1, #1040] + ldr q1, [x2, #1024] + ldr q3, [x2, #1040] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldr q0, [x1, #1056] + ldr q2, [x1, #1072] + ldr q1, [x2, #1056] + ldr q3, [x2, #1072] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldr q0, [x1, #1088] + ldr q2, [x1, #1104] + ldr q1, [x2, #1088] + ldr q3, [x2, #1104] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldr q0, [x1, #1120] + ldr q2, [x1, #1136] + ldr q1, [x2, #1120] + ldr q3, [x2, #1136] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + trn1 v29.4s, v5.4s, v6.4s + trn2 v30.4s, v5.4s, v6.4s + trn1 v5.2d, v29.2d, v30.2d + trn2 v6.2d, v29.2d, v30.2d + trn1 v29.4s, v7.4s, v8.4s + trn2 v30.4s, v7.4s, v8.4s + trn1 v7.2d, v29.2d, v30.2d + trn2 v8.2d, v29.2d, v30.2d + trn1 v29.4s, v9.4s, v10.4s + trn2 v30.4s, v9.4s, v10.4s + trn1 v9.2d, v29.2d, v30.2d + trn2 v10.2d, v29.2d, v30.2d + trn1 v29.4s, v11.4s, v12.4s + trn2 v30.4s, v11.4s, v12.4s + trn1 v11.2d, v29.2d, v30.2d + trn2 v12.2d, v29.2d, v30.2d + trn1 v29.4s, v13.4s, v14.4s + trn2 v30.4s, v13.4s, v14.4s + trn1 v13.2d, v29.2d, v30.2d + trn2 v14.2d, v29.2d, v30.2d + trn1 v29.4s, v15.4s, v16.4s + trn2 v30.4s, v15.4s, v16.4s + trn1 v15.2d, v29.2d, v30.2d + trn2 v16.2d, v29.2d, v30.2d + trn1 v29.4s, v17.4s, v18.4s + trn2 v30.4s, v17.4s, v18.4s + trn1 v17.2d, v29.2d, v30.2d + trn2 v18.2d, v29.2d, v30.2d + trn1 v29.4s, v19.4s, v20.4s + trn2 v30.4s, v19.4s, v20.4s + trn1 v19.2d, v29.2d, v30.2d + trn2 v20.2d, v29.2d, v30.2d + stp q5, q6, [x0, #512] + stp q7, q8, [x0, #544] + stp q9, q10, [x0, #576] + stp q11, q12, [x0, #608] + stp q13, q14, [x0, #640] + stp q15, q16, [x0, #672] + stp q17, q18, [x0, #704] + stp q19, q20, [x0, #736] + ldp q5, q6, [x0, #768] + ldp q7, q8, [x0, #800] + ldp q9, q10, [x0, #832] + ldp q11, q12, [x0, #864] + ldp q13, q14, [x0, #896] + ldp q15, q16, [x0, #928] + ldp q17, q18, [x0, #960] + ldp q19, q20, [x0, #992] + ldr q0, [x1, #112] + ldr q1, [x2, #112] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #224] + ldp q1, q3, [x2, #224] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #640] + ldp q1, q3, [x2, #640] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #672] + ldp q1, q3, [x2, #672] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #704] + ldp q1, q3, [x2, #704] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #736] + ldp q1, q3, [x2, #736] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldr q0, [x1, #1152] + ldr q2, [x1, #1168] + ldr q1, [x2, #1152] + ldr q3, [x2, #1168] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v21.4s, v21.4s, v29.4s + sub v22.4s, v22.4s, v30.4s + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldr q0, [x1, #1184] + ldr q2, [x1, #1200] + ldr q1, [x2, #1184] + ldr q3, [x2, #1200] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v23.4s, v23.4s, v29.4s + sub v24.4s, v24.4s, v30.4s + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldr q0, [x1, #1216] + ldr q2, [x1, #1232] + ldr q1, [x2, #1216] + ldr q3, [x2, #1232] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v25.4s, v25.4s, v29.4s + sub v26.4s, v26.4s, v30.4s + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x1, #1264] + ldr q1, [x2, #1248] + ldr q3, [x2, #1264] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmulh v29.4s, v29.4s, v4.s[0] + sqrdmulh v30.4s, v30.4s, v4.s[0] + sub v27.4s, v27.4s, v29.4s + sub v28.4s, v28.4s, v30.4s + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + trn1 v29.4s, v5.4s, v6.4s + trn2 v30.4s, v5.4s, v6.4s + trn1 v5.2d, v29.2d, v30.2d + trn2 v6.2d, v29.2d, v30.2d + trn1 v29.4s, v7.4s, v8.4s + trn2 v30.4s, v7.4s, v8.4s + trn1 v7.2d, v29.2d, v30.2d + trn2 v8.2d, v29.2d, v30.2d + trn1 v29.4s, v9.4s, v10.4s + trn2 v30.4s, v9.4s, v10.4s + trn1 v9.2d, v29.2d, v30.2d + trn2 v10.2d, v29.2d, v30.2d + trn1 v29.4s, v11.4s, v12.4s + trn2 v30.4s, v11.4s, v12.4s + trn1 v11.2d, v29.2d, v30.2d + trn2 v12.2d, v29.2d, v30.2d + trn1 v29.4s, v13.4s, v14.4s + trn2 v30.4s, v13.4s, v14.4s + trn1 v13.2d, v29.2d, v30.2d + trn2 v14.2d, v29.2d, v30.2d + trn1 v29.4s, v15.4s, v16.4s + trn2 v30.4s, v15.4s, v16.4s + trn1 v15.2d, v29.2d, v30.2d + trn2 v16.2d, v29.2d, v30.2d + trn1 v29.4s, v17.4s, v18.4s + trn2 v30.4s, v17.4s, v18.4s + trn1 v17.2d, v29.2d, v30.2d + trn2 v18.2d, v29.2d, v30.2d + trn1 v29.4s, v19.4s, v20.4s + trn2 v30.4s, v19.4s, v20.4s + trn1 v19.2d, v29.2d, v30.2d + trn2 v20.2d, v29.2d, v30.2d + stp q5, q6, [x0, #768] + stp q7, q8, [x0, #800] + stp q9, q10, [x0, #832] + stp q11, q12, [x0, #864] + stp q13, q14, [x0, #896] + stp q15, q16, [x0, #928] + stp q17, q18, [x0, #960] + stp q19, q20, [x0, #992] + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_ntt_full_neon,.-mldsa_ntt_full_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mldsa_invntt_neon +WC_ASM_ATT_HIDDEN(mldsa_invntt_neon) +.type mldsa_invntt_neon,@function +.align 2 +mldsa_invntt_neon: +#else +.section __TEXT,__text +.globl _mldsa_invntt_neon +WC_ASM_ATT_HIDDEN(_mldsa_invntt_neon) +.p2align 2 +_mldsa_invntt_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x1, L_mldsa_aarch64_zetas_inv + add x1, x1, :lo12:L_mldsa_aarch64_zetas_inv +#else + adrp x1, L_mldsa_aarch64_zetas_inv@PAGE + add x1, x1, L_mldsa_aarch64_zetas_inv@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x2, L_mldsa_aarch64_zetas_inv_qinv + add x2, x2, :lo12:L_mldsa_aarch64_zetas_inv_qinv +#else + adrp x2, L_mldsa_aarch64_zetas_inv_qinv@PAGE + add x2, x2, L_mldsa_aarch64_zetas_inv_qinv@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_aarch64_consts + add x3, x3, :lo12:L_mldsa_aarch64_consts +#else + adrp x3, L_mldsa_aarch64_consts@PAGE + add x3, x3, L_mldsa_aarch64_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q4, [x3] + ldp q5, q6, [x0] + ldp q7, q8, [x0, #32] + ldp q9, q10, [x0, #64] + ldp q11, q12, [x0, #96] + ldp q13, q14, [x0, #128] + ldp q15, q16, [x0, #160] + ldp q17, q18, [x0, #192] + ldp q19, q20, [x0, #224] + ldp q0, q1, [x1] + ldp q2, q3, [x2] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #32] + ldp q2, q3, [x2, #32] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #64] + ldp q2, q3, [x2, #64] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #96] + ldp q2, q3, [x2, #96] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #512] + ldp q2, q3, [x2, #512] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #544] + ldp q2, q3, [x2, #544] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #576] + ldp q2, q3, [x2, #576] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #608] + ldp q2, q3, [x2, #608] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1024] + ldr q1, [x1, #1040] + ldr q2, [x2, #1024] + ldr q3, [x2, #1040] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1152] + ldr q2, [x2, #1152] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0] + stp q7, q8, [x0, #32] + stp q9, q10, [x0, #64] + stp q11, q12, [x0, #96] + stp q13, q14, [x0, #128] + stp q15, q16, [x0, #160] + stp q17, q18, [x0, #192] + stp q19, q20, [x0, #224] + ldp q5, q6, [x0, #256] + ldp q7, q8, [x0, #288] + ldp q9, q10, [x0, #320] + ldp q11, q12, [x0, #352] + ldp q13, q14, [x0, #384] + ldp q15, q16, [x0, #416] + ldp q17, q18, [x0, #448] + ldp q19, q20, [x0, #480] + ldp q0, q1, [x1, #128] + ldp q2, q3, [x2, #128] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #160] + ldp q2, q3, [x2, #160] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #192] + ldp q2, q3, [x2, #192] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #224] + ldp q2, q3, [x2, #224] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #640] + ldp q2, q3, [x2, #640] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #672] + ldp q2, q3, [x2, #672] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #704] + ldp q2, q3, [x2, #704] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #736] + ldp q2, q3, [x2, #736] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1056] + ldr q1, [x1, #1072] + ldr q2, [x2, #1056] + ldr q3, [x2, #1072] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1168] + ldr q2, [x2, #1168] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0, #256] + stp q7, q8, [x0, #288] + stp q9, q10, [x0, #320] + stp q11, q12, [x0, #352] + stp q13, q14, [x0, #384] + stp q15, q16, [x0, #416] + stp q17, q18, [x0, #448] + stp q19, q20, [x0, #480] + ldp q5, q6, [x0, #512] + ldp q7, q8, [x0, #544] + ldp q9, q10, [x0, #576] + ldp q11, q12, [x0, #608] + ldp q13, q14, [x0, #640] + ldp q15, q16, [x0, #672] + ldp q17, q18, [x0, #704] + ldp q19, q20, [x0, #736] + ldp q0, q1, [x1, #256] + ldp q2, q3, [x2, #256] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #288] + ldp q2, q3, [x2, #288] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #320] + ldp q2, q3, [x2, #320] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #352] + ldp q2, q3, [x2, #352] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #768] + ldp q2, q3, [x2, #768] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #800] + ldp q2, q3, [x2, #800] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #832] + ldp q2, q3, [x2, #832] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #864] + ldp q2, q3, [x2, #864] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1088] + ldr q1, [x1, #1104] + ldr q2, [x2, #1088] + ldr q3, [x2, #1104] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1184] + ldr q2, [x2, #1184] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0, #512] + stp q7, q8, [x0, #544] + stp q9, q10, [x0, #576] + stp q11, q12, [x0, #608] + stp q13, q14, [x0, #640] + stp q15, q16, [x0, #672] + stp q17, q18, [x0, #704] + stp q19, q20, [x0, #736] + ldp q5, q6, [x0, #768] + ldp q7, q8, [x0, #800] + ldp q9, q10, [x0, #832] + ldp q11, q12, [x0, #864] + ldp q13, q14, [x0, #896] + ldp q15, q16, [x0, #928] + ldp q17, q18, [x0, #960] + ldp q19, q20, [x0, #992] + ldp q0, q1, [x1, #384] + ldp q2, q3, [x2, #384] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #416] + ldp q2, q3, [x2, #416] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #448] + ldp q2, q3, [x2, #448] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #480] + ldp q2, q3, [x2, #480] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #896] + ldp q2, q3, [x2, #896] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #928] + ldp q2, q3, [x2, #928] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #960] + ldp q2, q3, [x2, #960] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #992] + ldp q2, q3, [x2, #992] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1120] + ldr q1, [x1, #1136] + ldr q2, [x2, #1120] + ldr q3, [x2, #1136] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1200] + ldr q2, [x2, #1200] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0, #768] + stp q7, q8, [x0, #800] + stp q9, q10, [x0, #832] + stp q11, q12, [x0, #864] + stp q13, q14, [x0, #896] + stp q15, q16, [x0, #928] + stp q17, q18, [x0, #960] + stp q19, q20, [x0, #992] + ldr q5, [x0] + ldr q6, [x0, #64] + ldr q7, [x0, #128] + ldr q8, [x0, #192] + ldr q9, [x0, #256] + ldr q10, [x0, #320] + ldr q11, [x0, #384] + ldr q12, [x0, #448] + ldr q13, [x0, #512] + ldr q14, [x0, #576] + ldr q15, [x0, #640] + ldr q16, [x0, #704] + ldr q17, [x0, #768] + ldr q18, [x0, #832] + ldr q19, [x0, #896] + ldr q20, [x0, #960] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v23.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v13.4s, v13.4s, v21.4s + sub v14.4s, v14.4s, v23.4s + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v5.4s, v5.4s, v21.4s + sub v6.4s, v6.4s, v22.4s + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v22.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v22.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v22.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0] + str q6, [x0, #64] + str q7, [x0, #128] + str q8, [x0, #192] + str q9, [x0, #256] + str q10, [x0, #320] + str q11, [x0, #384] + str q12, [x0, #448] + str q13, [x0, #512] + str q14, [x0, #576] + str q15, [x0, #640] + str q16, [x0, #704] + str q17, [x0, #768] + str q18, [x0, #832] + str q19, [x0, #896] + str q20, [x0, #960] + ldr q5, [x0, #16] + ldr q6, [x0, #80] + ldr q7, [x0, #144] + ldr q8, [x0, #208] + ldr q9, [x0, #272] + ldr q10, [x0, #336] + ldr q11, [x0, #400] + ldr q12, [x0, #464] + ldr q13, [x0, #528] + ldr q14, [x0, #592] + ldr q15, [x0, #656] + ldr q16, [x0, #720] + ldr q17, [x0, #784] + ldr q18, [x0, #848] + ldr q19, [x0, #912] + ldr q20, [x0, #976] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v23.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v13.4s, v13.4s, v21.4s + sub v14.4s, v14.4s, v23.4s + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v5.4s, v5.4s, v21.4s + sub v6.4s, v6.4s, v22.4s + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v22.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v22.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v22.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0, #16] + str q6, [x0, #80] + str q7, [x0, #144] + str q8, [x0, #208] + str q9, [x0, #272] + str q10, [x0, #336] + str q11, [x0, #400] + str q12, [x0, #464] + str q13, [x0, #528] + str q14, [x0, #592] + str q15, [x0, #656] + str q16, [x0, #720] + str q17, [x0, #784] + str q18, [x0, #848] + str q19, [x0, #912] + str q20, [x0, #976] + ldr q5, [x0, #32] + ldr q6, [x0, #96] + ldr q7, [x0, #160] + ldr q8, [x0, #224] + ldr q9, [x0, #288] + ldr q10, [x0, #352] + ldr q11, [x0, #416] + ldr q12, [x0, #480] + ldr q13, [x0, #544] + ldr q14, [x0, #608] + ldr q15, [x0, #672] + ldr q16, [x0, #736] + ldr q17, [x0, #800] + ldr q18, [x0, #864] + ldr q19, [x0, #928] + ldr q20, [x0, #992] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v23.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v13.4s, v13.4s, v21.4s + sub v14.4s, v14.4s, v23.4s + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v5.4s, v5.4s, v21.4s + sub v6.4s, v6.4s, v22.4s + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v22.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v22.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v22.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0, #32] + str q6, [x0, #96] + str q7, [x0, #160] + str q8, [x0, #224] + str q9, [x0, #288] + str q10, [x0, #352] + str q11, [x0, #416] + str q12, [x0, #480] + str q13, [x0, #544] + str q14, [x0, #608] + str q15, [x0, #672] + str q16, [x0, #736] + str q17, [x0, #800] + str q18, [x0, #864] + str q19, [x0, #928] + str q20, [x0, #992] + ldr q5, [x0, #48] + ldr q6, [x0, #112] + ldr q7, [x0, #176] + ldr q8, [x0, #240] + ldr q9, [x0, #304] + ldr q10, [x0, #368] + ldr q11, [x0, #432] + ldr q12, [x0, #496] + ldr q13, [x0, #560] + ldr q14, [x0, #624] + ldr q15, [x0, #688] + ldr q16, [x0, #752] + ldr q17, [x0, #816] + ldr q18, [x0, #880] + ldr q19, [x0, #944] + ldr q20, [x0, #1008] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v23.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v13.4s, v13.4s, v21.4s + sub v14.4s, v14.4s, v23.4s + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v5.4s, v5.4s, v21.4s + sub v6.4s, v6.4s, v22.4s + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v22.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v22.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v22.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0, #48] + str q6, [x0, #112] + str q7, [x0, #176] + str q8, [x0, #240] + str q9, [x0, #304] + str q10, [x0, #368] + str q11, [x0, #432] + str q12, [x0, #496] + str q13, [x0, #560] + str q14, [x0, #624] + str q15, [x0, #688] + str q16, [x0, #752] + str q17, [x0, #816] + str q18, [x0, #880] + str q19, [x0, #944] + str q20, [x0, #1008] + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_invntt_neon,.-mldsa_invntt_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mldsa_invntt_full_neon +WC_ASM_ATT_HIDDEN(mldsa_invntt_full_neon) +.type mldsa_invntt_full_neon,@function +.align 2 +mldsa_invntt_full_neon: +#else +.section __TEXT,__text +.globl _mldsa_invntt_full_neon +WC_ASM_ATT_HIDDEN(_mldsa_invntt_full_neon) +.p2align 2 +_mldsa_invntt_full_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x1, L_mldsa_aarch64_zetas_inv + add x1, x1, :lo12:L_mldsa_aarch64_zetas_inv +#else + adrp x1, L_mldsa_aarch64_zetas_inv@PAGE + add x1, x1, L_mldsa_aarch64_zetas_inv@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x2, L_mldsa_aarch64_zetas_inv_qinv + add x2, x2, :lo12:L_mldsa_aarch64_zetas_inv_qinv +#else + adrp x2, L_mldsa_aarch64_zetas_inv_qinv@PAGE + add x2, x2, L_mldsa_aarch64_zetas_inv_qinv@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_aarch64_consts + add x3, x3, :lo12:L_mldsa_aarch64_consts +#else + adrp x3, L_mldsa_aarch64_consts@PAGE + add x3, x3, L_mldsa_aarch64_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q4, [x3] + ldp q5, q6, [x0] + ldp q7, q8, [x0, #32] + ldp q9, q10, [x0, #64] + ldp q11, q12, [x0, #96] + ldp q13, q14, [x0, #128] + ldp q15, q16, [x0, #160] + ldp q17, q18, [x0, #192] + ldp q19, q20, [x0, #224] + trn1 v21.2d, v5.2d, v6.2d + trn2 v22.2d, v5.2d, v6.2d + trn1 v5.4s, v21.4s, v22.4s + trn2 v6.4s, v21.4s, v22.4s + trn1 v21.2d, v7.2d, v8.2d + trn2 v22.2d, v7.2d, v8.2d + trn1 v7.4s, v21.4s, v22.4s + trn2 v8.4s, v21.4s, v22.4s + trn1 v21.2d, v9.2d, v10.2d + trn2 v22.2d, v9.2d, v10.2d + trn1 v9.4s, v21.4s, v22.4s + trn2 v10.4s, v21.4s, v22.4s + trn1 v21.2d, v11.2d, v12.2d + trn2 v22.2d, v11.2d, v12.2d + trn1 v11.4s, v21.4s, v22.4s + trn2 v12.4s, v21.4s, v22.4s + trn1 v21.2d, v13.2d, v14.2d + trn2 v22.2d, v13.2d, v14.2d + trn1 v13.4s, v21.4s, v22.4s + trn2 v14.4s, v21.4s, v22.4s + trn1 v21.2d, v15.2d, v16.2d + trn2 v22.2d, v15.2d, v16.2d + trn1 v15.4s, v21.4s, v22.4s + trn2 v16.4s, v21.4s, v22.4s + trn1 v21.2d, v17.2d, v18.2d + trn2 v22.2d, v17.2d, v18.2d + trn1 v17.4s, v21.4s, v22.4s + trn2 v18.4s, v21.4s, v22.4s + trn1 v21.2d, v19.2d, v20.2d + trn2 v22.2d, v19.2d, v20.2d + trn1 v19.4s, v21.4s, v22.4s + trn2 v20.4s, v21.4s, v22.4s + ldp q0, q1, [x1] + ldp q2, q3, [x2] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #32] + ldp q2, q3, [x2, #32] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #64] + ldp q2, q3, [x2, #64] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #96] + ldp q2, q3, [x2, #96] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #512] + ldp q2, q3, [x2, #512] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #544] + ldp q2, q3, [x2, #544] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #576] + ldp q2, q3, [x2, #576] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #608] + ldp q2, q3, [x2, #608] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1024] + ldr q1, [x1, #1040] + ldr q2, [x2, #1024] + ldr q3, [x2, #1040] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1152] + ldr q2, [x2, #1152] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0] + stp q7, q8, [x0, #32] + stp q9, q10, [x0, #64] + stp q11, q12, [x0, #96] + stp q13, q14, [x0, #128] + stp q15, q16, [x0, #160] + stp q17, q18, [x0, #192] + stp q19, q20, [x0, #224] + ldp q5, q6, [x0, #256] + ldp q7, q8, [x0, #288] + ldp q9, q10, [x0, #320] + ldp q11, q12, [x0, #352] + ldp q13, q14, [x0, #384] + ldp q15, q16, [x0, #416] + ldp q17, q18, [x0, #448] + ldp q19, q20, [x0, #480] + trn1 v21.2d, v5.2d, v6.2d + trn2 v22.2d, v5.2d, v6.2d + trn1 v5.4s, v21.4s, v22.4s + trn2 v6.4s, v21.4s, v22.4s + trn1 v21.2d, v7.2d, v8.2d + trn2 v22.2d, v7.2d, v8.2d + trn1 v7.4s, v21.4s, v22.4s + trn2 v8.4s, v21.4s, v22.4s + trn1 v21.2d, v9.2d, v10.2d + trn2 v22.2d, v9.2d, v10.2d + trn1 v9.4s, v21.4s, v22.4s + trn2 v10.4s, v21.4s, v22.4s + trn1 v21.2d, v11.2d, v12.2d + trn2 v22.2d, v11.2d, v12.2d + trn1 v11.4s, v21.4s, v22.4s + trn2 v12.4s, v21.4s, v22.4s + trn1 v21.2d, v13.2d, v14.2d + trn2 v22.2d, v13.2d, v14.2d + trn1 v13.4s, v21.4s, v22.4s + trn2 v14.4s, v21.4s, v22.4s + trn1 v21.2d, v15.2d, v16.2d + trn2 v22.2d, v15.2d, v16.2d + trn1 v15.4s, v21.4s, v22.4s + trn2 v16.4s, v21.4s, v22.4s + trn1 v21.2d, v17.2d, v18.2d + trn2 v22.2d, v17.2d, v18.2d + trn1 v17.4s, v21.4s, v22.4s + trn2 v18.4s, v21.4s, v22.4s + trn1 v21.2d, v19.2d, v20.2d + trn2 v22.2d, v19.2d, v20.2d + trn1 v19.4s, v21.4s, v22.4s + trn2 v20.4s, v21.4s, v22.4s + ldp q0, q1, [x1, #128] + ldp q2, q3, [x2, #128] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #160] + ldp q2, q3, [x2, #160] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #192] + ldp q2, q3, [x2, #192] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #224] + ldp q2, q3, [x2, #224] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #640] + ldp q2, q3, [x2, #640] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #672] + ldp q2, q3, [x2, #672] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #704] + ldp q2, q3, [x2, #704] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #736] + ldp q2, q3, [x2, #736] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1056] + ldr q1, [x1, #1072] + ldr q2, [x2, #1056] + ldr q3, [x2, #1072] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1168] + ldr q2, [x2, #1168] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0, #256] + stp q7, q8, [x0, #288] + stp q9, q10, [x0, #320] + stp q11, q12, [x0, #352] + stp q13, q14, [x0, #384] + stp q15, q16, [x0, #416] + stp q17, q18, [x0, #448] + stp q19, q20, [x0, #480] + ldp q5, q6, [x0, #512] + ldp q7, q8, [x0, #544] + ldp q9, q10, [x0, #576] + ldp q11, q12, [x0, #608] + ldp q13, q14, [x0, #640] + ldp q15, q16, [x0, #672] + ldp q17, q18, [x0, #704] + ldp q19, q20, [x0, #736] + trn1 v21.2d, v5.2d, v6.2d + trn2 v22.2d, v5.2d, v6.2d + trn1 v5.4s, v21.4s, v22.4s + trn2 v6.4s, v21.4s, v22.4s + trn1 v21.2d, v7.2d, v8.2d + trn2 v22.2d, v7.2d, v8.2d + trn1 v7.4s, v21.4s, v22.4s + trn2 v8.4s, v21.4s, v22.4s + trn1 v21.2d, v9.2d, v10.2d + trn2 v22.2d, v9.2d, v10.2d + trn1 v9.4s, v21.4s, v22.4s + trn2 v10.4s, v21.4s, v22.4s + trn1 v21.2d, v11.2d, v12.2d + trn2 v22.2d, v11.2d, v12.2d + trn1 v11.4s, v21.4s, v22.4s + trn2 v12.4s, v21.4s, v22.4s + trn1 v21.2d, v13.2d, v14.2d + trn2 v22.2d, v13.2d, v14.2d + trn1 v13.4s, v21.4s, v22.4s + trn2 v14.4s, v21.4s, v22.4s + trn1 v21.2d, v15.2d, v16.2d + trn2 v22.2d, v15.2d, v16.2d + trn1 v15.4s, v21.4s, v22.4s + trn2 v16.4s, v21.4s, v22.4s + trn1 v21.2d, v17.2d, v18.2d + trn2 v22.2d, v17.2d, v18.2d + trn1 v17.4s, v21.4s, v22.4s + trn2 v18.4s, v21.4s, v22.4s + trn1 v21.2d, v19.2d, v20.2d + trn2 v22.2d, v19.2d, v20.2d + trn1 v19.4s, v21.4s, v22.4s + trn2 v20.4s, v21.4s, v22.4s + ldp q0, q1, [x1, #256] + ldp q2, q3, [x2, #256] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #288] + ldp q2, q3, [x2, #288] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #320] + ldp q2, q3, [x2, #320] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #352] + ldp q2, q3, [x2, #352] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #768] + ldp q2, q3, [x2, #768] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #800] + ldp q2, q3, [x2, #800] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #832] + ldp q2, q3, [x2, #832] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #864] + ldp q2, q3, [x2, #864] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1088] + ldr q1, [x1, #1104] + ldr q2, [x2, #1088] + ldr q3, [x2, #1104] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1184] + ldr q2, [x2, #1184] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0, #512] + stp q7, q8, [x0, #544] + stp q9, q10, [x0, #576] + stp q11, q12, [x0, #608] + stp q13, q14, [x0, #640] + stp q15, q16, [x0, #672] + stp q17, q18, [x0, #704] + stp q19, q20, [x0, #736] + ldp q5, q6, [x0, #768] + ldp q7, q8, [x0, #800] + ldp q9, q10, [x0, #832] + ldp q11, q12, [x0, #864] + ldp q13, q14, [x0, #896] + ldp q15, q16, [x0, #928] + ldp q17, q18, [x0, #960] + ldp q19, q20, [x0, #992] + trn1 v21.2d, v5.2d, v6.2d + trn2 v22.2d, v5.2d, v6.2d + trn1 v5.4s, v21.4s, v22.4s + trn2 v6.4s, v21.4s, v22.4s + trn1 v21.2d, v7.2d, v8.2d + trn2 v22.2d, v7.2d, v8.2d + trn1 v7.4s, v21.4s, v22.4s + trn2 v8.4s, v21.4s, v22.4s + trn1 v21.2d, v9.2d, v10.2d + trn2 v22.2d, v9.2d, v10.2d + trn1 v9.4s, v21.4s, v22.4s + trn2 v10.4s, v21.4s, v22.4s + trn1 v21.2d, v11.2d, v12.2d + trn2 v22.2d, v11.2d, v12.2d + trn1 v11.4s, v21.4s, v22.4s + trn2 v12.4s, v21.4s, v22.4s + trn1 v21.2d, v13.2d, v14.2d + trn2 v22.2d, v13.2d, v14.2d + trn1 v13.4s, v21.4s, v22.4s + trn2 v14.4s, v21.4s, v22.4s + trn1 v21.2d, v15.2d, v16.2d + trn2 v22.2d, v15.2d, v16.2d + trn1 v15.4s, v21.4s, v22.4s + trn2 v16.4s, v21.4s, v22.4s + trn1 v21.2d, v17.2d, v18.2d + trn2 v22.2d, v17.2d, v18.2d + trn1 v17.4s, v21.4s, v22.4s + trn2 v18.4s, v21.4s, v22.4s + trn1 v21.2d, v19.2d, v20.2d + trn2 v22.2d, v19.2d, v20.2d + trn1 v19.4s, v21.4s, v22.4s + trn2 v20.4s, v21.4s, v22.4s + ldp q0, q1, [x1, #384] + ldp q2, q3, [x2, #384] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #416] + ldp q2, q3, [x2, #416] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #448] + ldp q2, q3, [x2, #448] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #480] + ldp q2, q3, [x2, #480] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #896] + ldp q2, q3, [x2, #896] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #928] + ldp q2, q3, [x2, #928] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #960] + ldp q2, q3, [x2, #960] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #992] + ldp q2, q3, [x2, #992] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1120] + ldr q1, [x1, #1136] + ldr q2, [x2, #1120] + ldr q3, [x2, #1136] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1200] + ldr q2, [x2, #1200] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0, #768] + stp q7, q8, [x0, #800] + stp q9, q10, [x0, #832] + stp q11, q12, [x0, #864] + stp q13, q14, [x0, #896] + stp q15, q16, [x0, #928] + stp q17, q18, [x0, #960] + stp q19, q20, [x0, #992] + ldr q5, [x0] + ldr q6, [x0, #64] + ldr q7, [x0, #128] + ldr q8, [x0, #192] + ldr q9, [x0, #256] + ldr q10, [x0, #320] + ldr q11, [x0, #384] + ldr q12, [x0, #448] + ldr q13, [x0, #512] + ldr q14, [x0, #576] + ldr q15, [x0, #640] + ldr q16, [x0, #704] + ldr q17, [x0, #768] + ldr q18, [x0, #832] + ldr q19, [x0, #896] + ldr q20, [x0, #960] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v23.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v13.4s, v13.4s, v21.4s + sub v14.4s, v14.4s, v23.4s + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v5.4s, v5.4s, v21.4s + sub v6.4s, v6.4s, v22.4s + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v22.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v22.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v22.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0] + str q6, [x0, #64] + str q7, [x0, #128] + str q8, [x0, #192] + str q9, [x0, #256] + str q10, [x0, #320] + str q11, [x0, #384] + str q12, [x0, #448] + str q13, [x0, #512] + str q14, [x0, #576] + str q15, [x0, #640] + str q16, [x0, #704] + str q17, [x0, #768] + str q18, [x0, #832] + str q19, [x0, #896] + str q20, [x0, #960] + ldr q5, [x0, #16] + ldr q6, [x0, #80] + ldr q7, [x0, #144] + ldr q8, [x0, #208] + ldr q9, [x0, #272] + ldr q10, [x0, #336] + ldr q11, [x0, #400] + ldr q12, [x0, #464] + ldr q13, [x0, #528] + ldr q14, [x0, #592] + ldr q15, [x0, #656] + ldr q16, [x0, #720] + ldr q17, [x0, #784] + ldr q18, [x0, #848] + ldr q19, [x0, #912] + ldr q20, [x0, #976] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v23.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v13.4s, v13.4s, v21.4s + sub v14.4s, v14.4s, v23.4s + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v5.4s, v5.4s, v21.4s + sub v6.4s, v6.4s, v22.4s + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v22.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v22.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v22.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0, #16] + str q6, [x0, #80] + str q7, [x0, #144] + str q8, [x0, #208] + str q9, [x0, #272] + str q10, [x0, #336] + str q11, [x0, #400] + str q12, [x0, #464] + str q13, [x0, #528] + str q14, [x0, #592] + str q15, [x0, #656] + str q16, [x0, #720] + str q17, [x0, #784] + str q18, [x0, #848] + str q19, [x0, #912] + str q20, [x0, #976] + ldr q5, [x0, #32] + ldr q6, [x0, #96] + ldr q7, [x0, #160] + ldr q8, [x0, #224] + ldr q9, [x0, #288] + ldr q10, [x0, #352] + ldr q11, [x0, #416] + ldr q12, [x0, #480] + ldr q13, [x0, #544] + ldr q14, [x0, #608] + ldr q15, [x0, #672] + ldr q16, [x0, #736] + ldr q17, [x0, #800] + ldr q18, [x0, #864] + ldr q19, [x0, #928] + ldr q20, [x0, #992] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v23.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v13.4s, v13.4s, v21.4s + sub v14.4s, v14.4s, v23.4s + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v5.4s, v5.4s, v21.4s + sub v6.4s, v6.4s, v22.4s + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v22.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v22.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v22.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0, #32] + str q6, [x0, #96] + str q7, [x0, #160] + str q8, [x0, #224] + str q9, [x0, #288] + str q10, [x0, #352] + str q11, [x0, #416] + str q12, [x0, #480] + str q13, [x0, #544] + str q14, [x0, #608] + str q15, [x0, #672] + str q16, [x0, #736] + str q17, [x0, #800] + str q18, [x0, #864] + str q19, [x0, #928] + str q20, [x0, #992] + ldr q5, [x0, #48] + ldr q6, [x0, #112] + ldr q7, [x0, #176] + ldr q8, [x0, #240] + ldr q9, [x0, #304] + ldr q10, [x0, #368] + ldr q11, [x0, #432] + ldr q12, [x0, #496] + ldr q13, [x0, #560] + ldr q14, [x0, #624] + ldr q15, [x0, #688] + ldr q16, [x0, #752] + ldr q17, [x0, #816] + ldr q18, [x0, #880] + ldr q19, [x0, #944] + ldr q20, [x0, #1008] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v6.4s, v6.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v10.4s, v10.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v14.4s, v14.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v18.4s, v18.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v23.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v23.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v23.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v13.4s, v13.4s, v21.4s + sub v14.4s, v14.4s, v23.4s + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v15.4s, v15.4s, v21.4s + sub v16.4s, v16.4s, v23.4s + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v17.4s, v17.4s, v21.4s + sub v18.4s, v18.4s, v23.4s + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v23.4s, v23.4s, v4.s[0] + sub v19.4s, v19.4s, v21.4s + sub v20.4s, v20.4s, v23.4s + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v5.4s, v5.4s, v21.4s + sub v6.4s, v6.4s, v22.4s + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v7.4s, v7.4s, v21.4s + sub v8.4s, v8.4s, v22.4s + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v9.4s, v9.4s, v21.4s + sub v10.4s, v10.4s, v22.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmulh v21.4s, v21.4s, v4.s[0] + sqrdmulh v22.4s, v22.4s, v4.s[0] + sub v11.4s, v11.4s, v21.4s + sub v12.4s, v12.4s, v22.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0, #48] + str q6, [x0, #112] + str q7, [x0, #176] + str q8, [x0, #240] + str q9, [x0, #304] + str q10, [x0, #368] + str q11, [x0, #432] + str q12, [x0, #496] + str q13, [x0, #560] + str q14, [x0, #624] + str q15, [x0, #688] + str q16, [x0, #752] + str q17, [x0, #816] + str q18, [x0, #880] + str q19, [x0, #944] + str q20, [x0, #1008] + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_invntt_full_neon,.-mldsa_invntt_full_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mldsa_mul_neon +WC_ASM_ATT_HIDDEN(mldsa_mul_neon) +.type mldsa_mul_neon,@function +.align 2 +mldsa_mul_neon: +#else +.section __TEXT,__text +.globl _mldsa_mul_neon +WC_ASM_ATT_HIDDEN(_mldsa_mul_neon) +.p2align 2 +_mldsa_mul_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x3, L_mldsa_aarch64_consts + add x3, x3, :lo12:L_mldsa_aarch64_consts +#else + adrp x3, L_mldsa_aarch64_consts@PAGE + add x3, x3, L_mldsa_aarch64_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q0, [x3] + ldr q1, [x1] + ldr q5, [x2] + ldr q2, [x1, #16] + ldr q6, [x2, #16] + ldr q3, [x1, #32] + ldr q7, [x2, #32] + ldr q4, [x1, #48] + ldr q8, [x2, #48] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0] + str q10, [x0, #16] + str q11, [x0, #32] + str q12, [x0, #48] + ldr q1, [x1, #64] + ldr q5, [x2, #64] + ldr q2, [x1, #80] + ldr q6, [x2, #80] + ldr q3, [x1, #96] + ldr q7, [x2, #96] + ldr q4, [x1, #112] + ldr q8, [x2, #112] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #64] + str q10, [x0, #80] + str q11, [x0, #96] + str q12, [x0, #112] + ldr q1, [x1, #128] + ldr q5, [x2, #128] + ldr q2, [x1, #144] + ldr q6, [x2, #144] + ldr q3, [x1, #160] + ldr q7, [x2, #160] + ldr q4, [x1, #176] + ldr q8, [x2, #176] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #128] + str q10, [x0, #144] + str q11, [x0, #160] + str q12, [x0, #176] + ldr q1, [x1, #192] + ldr q5, [x2, #192] + ldr q2, [x1, #208] + ldr q6, [x2, #208] + ldr q3, [x1, #224] + ldr q7, [x2, #224] + ldr q4, [x1, #240] + ldr q8, [x2, #240] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #192] + str q10, [x0, #208] + str q11, [x0, #224] + str q12, [x0, #240] + ldr q1, [x1, #256] + ldr q5, [x2, #256] + ldr q2, [x1, #272] + ldr q6, [x2, #272] + ldr q3, [x1, #288] + ldr q7, [x2, #288] + ldr q4, [x1, #304] + ldr q8, [x2, #304] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #256] + str q10, [x0, #272] + str q11, [x0, #288] + str q12, [x0, #304] + ldr q1, [x1, #320] + ldr q5, [x2, #320] + ldr q2, [x1, #336] + ldr q6, [x2, #336] + ldr q3, [x1, #352] + ldr q7, [x2, #352] + ldr q4, [x1, #368] + ldr q8, [x2, #368] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #320] + str q10, [x0, #336] + str q11, [x0, #352] + str q12, [x0, #368] + ldr q1, [x1, #384] + ldr q5, [x2, #384] + ldr q2, [x1, #400] + ldr q6, [x2, #400] + ldr q3, [x1, #416] + ldr q7, [x2, #416] + ldr q4, [x1, #432] + ldr q8, [x2, #432] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #384] + str q10, [x0, #400] + str q11, [x0, #416] + str q12, [x0, #432] + ldr q1, [x1, #448] + ldr q5, [x2, #448] + ldr q2, [x1, #464] + ldr q6, [x2, #464] + ldr q3, [x1, #480] + ldr q7, [x2, #480] + ldr q4, [x1, #496] + ldr q8, [x2, #496] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #448] + str q10, [x0, #464] + str q11, [x0, #480] + str q12, [x0, #496] + ldr q1, [x1, #512] + ldr q5, [x2, #512] + ldr q2, [x1, #528] + ldr q6, [x2, #528] + ldr q3, [x1, #544] + ldr q7, [x2, #544] + ldr q4, [x1, #560] + ldr q8, [x2, #560] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #512] + str q10, [x0, #528] + str q11, [x0, #544] + str q12, [x0, #560] + ldr q1, [x1, #576] + ldr q5, [x2, #576] + ldr q2, [x1, #592] + ldr q6, [x2, #592] + ldr q3, [x1, #608] + ldr q7, [x2, #608] + ldr q4, [x1, #624] + ldr q8, [x2, #624] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #576] + str q10, [x0, #592] + str q11, [x0, #608] + str q12, [x0, #624] + ldr q1, [x1, #640] + ldr q5, [x2, #640] + ldr q2, [x1, #656] + ldr q6, [x2, #656] + ldr q3, [x1, #672] + ldr q7, [x2, #672] + ldr q4, [x1, #688] + ldr q8, [x2, #688] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #640] + str q10, [x0, #656] + str q11, [x0, #672] + str q12, [x0, #688] + ldr q1, [x1, #704] + ldr q5, [x2, #704] + ldr q2, [x1, #720] + ldr q6, [x2, #720] + ldr q3, [x1, #736] + ldr q7, [x2, #736] + ldr q4, [x1, #752] + ldr q8, [x2, #752] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #704] + str q10, [x0, #720] + str q11, [x0, #736] + str q12, [x0, #752] + ldr q1, [x1, #768] + ldr q5, [x2, #768] + ldr q2, [x1, #784] + ldr q6, [x2, #784] + ldr q3, [x1, #800] + ldr q7, [x2, #800] + ldr q4, [x1, #816] + ldr q8, [x2, #816] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #768] + str q10, [x0, #784] + str q11, [x0, #800] + str q12, [x0, #816] + ldr q1, [x1, #832] + ldr q5, [x2, #832] + ldr q2, [x1, #848] + ldr q6, [x2, #848] + ldr q3, [x1, #864] + ldr q7, [x2, #864] + ldr q4, [x1, #880] + ldr q8, [x2, #880] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #832] + str q10, [x0, #848] + str q11, [x0, #864] + str q12, [x0, #880] + ldr q1, [x1, #896] + ldr q5, [x2, #896] + ldr q2, [x1, #912] + ldr q6, [x2, #912] + ldr q3, [x1, #928] + ldr q7, [x2, #928] + ldr q4, [x1, #944] + ldr q8, [x2, #944] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #896] + str q10, [x0, #912] + str q11, [x0, #928] + str q12, [x0, #944] + ldr q1, [x1, #960] + ldr q5, [x2, #960] + ldr q2, [x1, #976] + ldr q6, [x2, #976] + ldr q3, [x1, #992] + ldr q7, [x2, #992] + ldr q4, [x1, #1008] + ldr q8, [x2, #1008] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v9.4s, v9.4s, v17.4s + sub v10.4s, v10.4s, v18.4s + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmulh v17.4s, v17.4s, v0.s[0] + sqrdmulh v18.4s, v18.4s, v0.s[0] + sub v11.4s, v11.4s, v17.4s + sub v12.4s, v12.4s, v18.4s + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #960] + str q10, [x0, #976] + str q11, [x0, #992] + str q12, [x0, #1008] + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_mul_neon,.-mldsa_mul_neon +#endif /* __APPLE__ */ +#ifndef WOLFSSL_AARCH64_NO_SQRDMLSH +#ifndef __APPLE__ +.text +.globl mldsa_ntt_sqrdmlsh_neon +WC_ASM_ATT_HIDDEN(mldsa_ntt_sqrdmlsh_neon) +.type mldsa_ntt_sqrdmlsh_neon,@function +.align 2 +mldsa_ntt_sqrdmlsh_neon: +#else +.section __TEXT,__text +.globl _mldsa_ntt_sqrdmlsh_neon +WC_ASM_ATT_HIDDEN(_mldsa_ntt_sqrdmlsh_neon) +.p2align 2 +_mldsa_ntt_sqrdmlsh_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x1, L_mldsa_aarch64_zetas + add x1, x1, :lo12:L_mldsa_aarch64_zetas +#else + adrp x1, L_mldsa_aarch64_zetas@PAGE + add x1, x1, L_mldsa_aarch64_zetas@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x2, L_mldsa_aarch64_zetas_qinv + add x2, x2, :lo12:L_mldsa_aarch64_zetas_qinv +#else + adrp x2, L_mldsa_aarch64_zetas_qinv@PAGE + add x2, x2, L_mldsa_aarch64_zetas_qinv@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_aarch64_consts + add x3, x3, :lo12:L_mldsa_aarch64_consts +#else + adrp x3, L_mldsa_aarch64_consts@PAGE + add x3, x3, L_mldsa_aarch64_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q4, [x3] + ldr q5, [x0] + ldr q6, [x0, #64] + ldr q7, [x0, #128] + ldr q8, [x0, #192] + ldr q9, [x0, #256] + ldr q10, [x0, #320] + ldr q11, [x0, #384] + ldr q12, [x0, #448] + ldr q13, [x0, #512] + ldr q14, [x0, #576] + ldr q15, [x0, #640] + ldr q16, [x0, #704] + ldr q17, [x0, #768] + ldr q18, [x0, #832] + ldr q19, [x0, #896] + ldr q20, [x0, #960] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0] + str q6, [x0, #64] + str q7, [x0, #128] + str q8, [x0, #192] + str q9, [x0, #256] + str q10, [x0, #320] + str q11, [x0, #384] + str q12, [x0, #448] + str q13, [x0, #512] + str q14, [x0, #576] + str q15, [x0, #640] + str q16, [x0, #704] + str q17, [x0, #768] + str q18, [x0, #832] + str q19, [x0, #896] + str q20, [x0, #960] + ldr q5, [x0, #16] + ldr q6, [x0, #80] + ldr q7, [x0, #144] + ldr q8, [x0, #208] + ldr q9, [x0, #272] + ldr q10, [x0, #336] + ldr q11, [x0, #400] + ldr q12, [x0, #464] + ldr q13, [x0, #528] + ldr q14, [x0, #592] + ldr q15, [x0, #656] + ldr q16, [x0, #720] + ldr q17, [x0, #784] + ldr q18, [x0, #848] + ldr q19, [x0, #912] + ldr q20, [x0, #976] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0, #16] + str q6, [x0, #80] + str q7, [x0, #144] + str q8, [x0, #208] + str q9, [x0, #272] + str q10, [x0, #336] + str q11, [x0, #400] + str q12, [x0, #464] + str q13, [x0, #528] + str q14, [x0, #592] + str q15, [x0, #656] + str q16, [x0, #720] + str q17, [x0, #784] + str q18, [x0, #848] + str q19, [x0, #912] + str q20, [x0, #976] + ldr q5, [x0, #32] + ldr q6, [x0, #96] + ldr q7, [x0, #160] + ldr q8, [x0, #224] + ldr q9, [x0, #288] + ldr q10, [x0, #352] + ldr q11, [x0, #416] + ldr q12, [x0, #480] + ldr q13, [x0, #544] + ldr q14, [x0, #608] + ldr q15, [x0, #672] + ldr q16, [x0, #736] + ldr q17, [x0, #800] + ldr q18, [x0, #864] + ldr q19, [x0, #928] + ldr q20, [x0, #992] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0, #32] + str q6, [x0, #96] + str q7, [x0, #160] + str q8, [x0, #224] + str q9, [x0, #288] + str q10, [x0, #352] + str q11, [x0, #416] + str q12, [x0, #480] + str q13, [x0, #544] + str q14, [x0, #608] + str q15, [x0, #672] + str q16, [x0, #736] + str q17, [x0, #800] + str q18, [x0, #864] + str q19, [x0, #928] + str q20, [x0, #992] + ldr q5, [x0, #48] + ldr q6, [x0, #112] + ldr q7, [x0, #176] + ldr q8, [x0, #240] + ldr q9, [x0, #304] + ldr q10, [x0, #368] + ldr q11, [x0, #432] + ldr q12, [x0, #496] + ldr q13, [x0, #560] + ldr q14, [x0, #624] + ldr q15, [x0, #688] + ldr q16, [x0, #752] + ldr q17, [x0, #816] + ldr q18, [x0, #880] + ldr q19, [x0, #944] + ldr q20, [x0, #1008] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0, #48] + str q6, [x0, #112] + str q7, [x0, #176] + str q8, [x0, #240] + str q9, [x0, #304] + str q10, [x0, #368] + str q11, [x0, #432] + str q12, [x0, #496] + str q13, [x0, #560] + str q14, [x0, #624] + str q15, [x0, #688] + str q16, [x0, #752] + str q17, [x0, #816] + str q18, [x0, #880] + str q19, [x0, #944] + str q20, [x0, #1008] + ldp q5, q6, [x0] + ldp q7, q8, [x0, #32] + ldp q9, q10, [x0, #64] + ldp q11, q12, [x0, #96] + ldp q13, q14, [x0, #128] + ldp q15, q16, [x0, #160] + ldp q17, q18, [x0, #192] + ldp q19, q20, [x0, #224] + ldr q0, [x1, #64] + ldr q1, [x2, #64] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #128] + ldp q1, q3, [x2, #128] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #256] + ldp q1, q3, [x2, #256] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #288] + ldp q1, q3, [x2, #288] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #320] + ldp q1, q3, [x2, #320] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #352] + ldp q1, q3, [x2, #352] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #768] + ldp q1, q3, [x2, #768] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #800] + ldp q1, q3, [x2, #800] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #832] + ldp q1, q3, [x2, #832] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #864] + ldp q1, q3, [x2, #864] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + stp q5, q6, [x0] + stp q7, q8, [x0, #32] + stp q9, q10, [x0, #64] + stp q11, q12, [x0, #96] + stp q13, q14, [x0, #128] + stp q15, q16, [x0, #160] + stp q17, q18, [x0, #192] + stp q19, q20, [x0, #224] + ldp q5, q6, [x0, #256] + ldp q7, q8, [x0, #288] + ldp q9, q10, [x0, #320] + ldp q11, q12, [x0, #352] + ldp q13, q14, [x0, #384] + ldp q15, q16, [x0, #416] + ldp q17, q18, [x0, #448] + ldp q19, q20, [x0, #480] + ldr q0, [x1, #80] + ldr q1, [x2, #80] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #160] + ldp q1, q3, [x2, #160] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #384] + ldp q1, q3, [x2, #384] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #416] + ldp q1, q3, [x2, #416] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #448] + ldp q1, q3, [x2, #448] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #480] + ldp q1, q3, [x2, #480] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #896] + ldp q1, q3, [x2, #896] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #928] + ldp q1, q3, [x2, #928] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #960] + ldp q1, q3, [x2, #960] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #992] + ldp q1, q3, [x2, #992] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + stp q5, q6, [x0, #256] + stp q7, q8, [x0, #288] + stp q9, q10, [x0, #320] + stp q11, q12, [x0, #352] + stp q13, q14, [x0, #384] + stp q15, q16, [x0, #416] + stp q17, q18, [x0, #448] + stp q19, q20, [x0, #480] + ldp q5, q6, [x0, #512] + ldp q7, q8, [x0, #544] + ldp q9, q10, [x0, #576] + ldp q11, q12, [x0, #608] + ldp q13, q14, [x0, #640] + ldp q15, q16, [x0, #672] + ldp q17, q18, [x0, #704] + ldp q19, q20, [x0, #736] + ldr q0, [x1, #96] + ldr q1, [x2, #96] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #192] + ldp q1, q3, [x2, #192] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #512] + ldp q1, q3, [x2, #512] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #544] + ldp q1, q3, [x2, #544] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #576] + ldp q1, q3, [x2, #576] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #608] + ldp q1, q3, [x2, #608] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldr q0, [x1, #1024] + ldr q2, [x1, #1040] + ldr q1, [x2, #1024] + ldr q3, [x2, #1040] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldr q0, [x1, #1056] + ldr q2, [x1, #1072] + ldr q1, [x2, #1056] + ldr q3, [x2, #1072] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldr q0, [x1, #1088] + ldr q2, [x1, #1104] + ldr q1, [x2, #1088] + ldr q3, [x2, #1104] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldr q0, [x1, #1120] + ldr q2, [x1, #1136] + ldr q1, [x2, #1120] + ldr q3, [x2, #1136] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + stp q5, q6, [x0, #512] + stp q7, q8, [x0, #544] + stp q9, q10, [x0, #576] + stp q11, q12, [x0, #608] + stp q13, q14, [x0, #640] + stp q15, q16, [x0, #672] + stp q17, q18, [x0, #704] + stp q19, q20, [x0, #736] + ldp q5, q6, [x0, #768] + ldp q7, q8, [x0, #800] + ldp q9, q10, [x0, #832] + ldp q11, q12, [x0, #864] + ldp q13, q14, [x0, #896] + ldp q15, q16, [x0, #928] + ldp q17, q18, [x0, #960] + ldp q19, q20, [x0, #992] + ldr q0, [x1, #112] + ldr q1, [x2, #112] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #224] + ldp q1, q3, [x2, #224] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #640] + ldp q1, q3, [x2, #640] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #672] + ldp q1, q3, [x2, #672] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #704] + ldp q1, q3, [x2, #704] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #736] + ldp q1, q3, [x2, #736] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldr q0, [x1, #1152] + ldr q2, [x1, #1168] + ldr q1, [x2, #1152] + ldr q3, [x2, #1168] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldr q0, [x1, #1184] + ldr q2, [x1, #1200] + ldr q1, [x2, #1184] + ldr q3, [x2, #1200] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldr q0, [x1, #1216] + ldr q2, [x1, #1232] + ldr q1, [x2, #1216] + ldr q3, [x2, #1232] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x1, #1264] + ldr q1, [x2, #1248] + ldr q3, [x2, #1264] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + stp q5, q6, [x0, #768] + stp q7, q8, [x0, #800] + stp q9, q10, [x0, #832] + stp q11, q12, [x0, #864] + stp q13, q14, [x0, #896] + stp q15, q16, [x0, #928] + stp q17, q18, [x0, #960] + stp q19, q20, [x0, #992] + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_ntt_sqrdmlsh_neon,.-mldsa_ntt_sqrdmlsh_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mldsa_ntt_full_sqrdmlsh_neon +WC_ASM_ATT_HIDDEN(mldsa_ntt_full_sqrdmlsh_neon) +.type mldsa_ntt_full_sqrdmlsh_neon,@function +.align 2 +mldsa_ntt_full_sqrdmlsh_neon: +#else +.section __TEXT,__text +.globl _mldsa_ntt_full_sqrdmlsh_neon +WC_ASM_ATT_HIDDEN(_mldsa_ntt_full_sqrdmlsh_neon) +.p2align 2 +_mldsa_ntt_full_sqrdmlsh_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x1, L_mldsa_aarch64_zetas + add x1, x1, :lo12:L_mldsa_aarch64_zetas +#else + adrp x1, L_mldsa_aarch64_zetas@PAGE + add x1, x1, L_mldsa_aarch64_zetas@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x2, L_mldsa_aarch64_zetas_qinv + add x2, x2, :lo12:L_mldsa_aarch64_zetas_qinv +#else + adrp x2, L_mldsa_aarch64_zetas_qinv@PAGE + add x2, x2, L_mldsa_aarch64_zetas_qinv@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_aarch64_consts + add x3, x3, :lo12:L_mldsa_aarch64_consts +#else + adrp x3, L_mldsa_aarch64_consts@PAGE + add x3, x3, L_mldsa_aarch64_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q4, [x3] + ldr q5, [x0] + ldr q6, [x0, #64] + ldr q7, [x0, #128] + ldr q8, [x0, #192] + ldr q9, [x0, #256] + ldr q10, [x0, #320] + ldr q11, [x0, #384] + ldr q12, [x0, #448] + ldr q13, [x0, #512] + ldr q14, [x0, #576] + ldr q15, [x0, #640] + ldr q16, [x0, #704] + ldr q17, [x0, #768] + ldr q18, [x0, #832] + ldr q19, [x0, #896] + ldr q20, [x0, #960] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0] + str q6, [x0, #64] + str q7, [x0, #128] + str q8, [x0, #192] + str q9, [x0, #256] + str q10, [x0, #320] + str q11, [x0, #384] + str q12, [x0, #448] + str q13, [x0, #512] + str q14, [x0, #576] + str q15, [x0, #640] + str q16, [x0, #704] + str q17, [x0, #768] + str q18, [x0, #832] + str q19, [x0, #896] + str q20, [x0, #960] + ldr q5, [x0, #16] + ldr q6, [x0, #80] + ldr q7, [x0, #144] + ldr q8, [x0, #208] + ldr q9, [x0, #272] + ldr q10, [x0, #336] + ldr q11, [x0, #400] + ldr q12, [x0, #464] + ldr q13, [x0, #528] + ldr q14, [x0, #592] + ldr q15, [x0, #656] + ldr q16, [x0, #720] + ldr q17, [x0, #784] + ldr q18, [x0, #848] + ldr q19, [x0, #912] + ldr q20, [x0, #976] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0, #16] + str q6, [x0, #80] + str q7, [x0, #144] + str q8, [x0, #208] + str q9, [x0, #272] + str q10, [x0, #336] + str q11, [x0, #400] + str q12, [x0, #464] + str q13, [x0, #528] + str q14, [x0, #592] + str q15, [x0, #656] + str q16, [x0, #720] + str q17, [x0, #784] + str q18, [x0, #848] + str q19, [x0, #912] + str q20, [x0, #976] + ldr q5, [x0, #32] + ldr q6, [x0, #96] + ldr q7, [x0, #160] + ldr q8, [x0, #224] + ldr q9, [x0, #288] + ldr q10, [x0, #352] + ldr q11, [x0, #416] + ldr q12, [x0, #480] + ldr q13, [x0, #544] + ldr q14, [x0, #608] + ldr q15, [x0, #672] + ldr q16, [x0, #736] + ldr q17, [x0, #800] + ldr q18, [x0, #864] + ldr q19, [x0, #928] + ldr q20, [x0, #992] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0, #32] + str q6, [x0, #96] + str q7, [x0, #160] + str q8, [x0, #224] + str q9, [x0, #288] + str q10, [x0, #352] + str q11, [x0, #416] + str q12, [x0, #480] + str q13, [x0, #544] + str q14, [x0, #608] + str q15, [x0, #672] + str q16, [x0, #736] + str q17, [x0, #800] + str q18, [x0, #864] + str q19, [x0, #928] + str q20, [x0, #992] + ldr q5, [x0, #48] + ldr q6, [x0, #112] + ldr q7, [x0, #176] + ldr q8, [x0, #240] + ldr q9, [x0, #304] + ldr q10, [x0, #368] + ldr q11, [x0, #432] + ldr q12, [x0, #496] + ldr q13, [x0, #560] + ldr q14, [x0, #624] + ldr q15, [x0, #688] + ldr q16, [x0, #752] + ldr q17, [x0, #816] + ldr q18, [x0, #880] + ldr q19, [x0, #944] + ldr q20, [x0, #1008] + ldr q0, [x1] + ldr q1, [x2] + mul v29.4s, v13.4s, v1.s[1] + mul v30.4s, v14.4s, v1.s[1] + sqrdmulh v21.4s, v13.4s, v0.s[1] + sqrdmulh v22.4s, v14.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v15.4s, v1.s[1] + mul v30.4s, v16.4s, v1.s[1] + sqrdmulh v23.4s, v15.4s, v0.s[1] + sqrdmulh v24.4s, v16.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[1] + mul v30.4s, v18.4s, v1.s[1] + sqrdmulh v25.4s, v17.4s, v0.s[1] + sqrdmulh v26.4s, v18.4s, v0.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[1] + mul v30.4s, v20.4s, v1.s[1] + sqrdmulh v27.4s, v19.4s, v0.s[1] + sqrdmulh v28.4s, v20.4s, v0.s[1] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v13.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v14.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v15.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v16.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v9.4s, v25.4s + add v9.4s, v9.4s, v25.4s + sub v18.4s, v10.4s, v26.4s + add v10.4s, v10.4s, v26.4s + sub v19.4s, v11.4s, v27.4s + add v11.4s, v11.4s, v27.4s + sub v20.4s, v12.4s, v28.4s + add v12.4s, v12.4s, v28.4s + mul v29.4s, v9.4s, v1.s[2] + mul v30.4s, v10.4s, v1.s[2] + sqrdmulh v21.4s, v9.4s, v0.s[2] + sqrdmulh v22.4s, v10.4s, v0.s[2] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[2] + sqrdmulh v23.4s, v11.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[2] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v17.4s, v1.s[3] + mul v30.4s, v18.4s, v1.s[3] + sqrdmulh v25.4s, v17.4s, v0.s[3] + sqrdmulh v26.4s, v18.4s, v0.s[3] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v9.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v10.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v7.4s, v23.4s + add v7.4s, v7.4s, v23.4s + sub v12.4s, v8.4s, v24.4s + add v8.4s, v8.4s, v24.4s + sub v17.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v18.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v15.4s, v27.4s + add v15.4s, v15.4s, v27.4s + sub v20.4s, v16.4s, v28.4s + add v16.4s, v16.4s, v28.4s + ldr q0, [x1, #16] + ldr q1, [x2, #16] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #32] + ldp q1, q3, [x2, #32] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + str q5, [x0, #48] + str q6, [x0, #112] + str q7, [x0, #176] + str q8, [x0, #240] + str q9, [x0, #304] + str q10, [x0, #368] + str q11, [x0, #432] + str q12, [x0, #496] + str q13, [x0, #560] + str q14, [x0, #624] + str q15, [x0, #688] + str q16, [x0, #752] + str q17, [x0, #816] + str q18, [x0, #880] + str q19, [x0, #944] + str q20, [x0, #1008] + ldp q5, q6, [x0] + ldp q7, q8, [x0, #32] + ldp q9, q10, [x0, #64] + ldp q11, q12, [x0, #96] + ldp q13, q14, [x0, #128] + ldp q15, q16, [x0, #160] + ldp q17, q18, [x0, #192] + ldp q19, q20, [x0, #224] + ldr q0, [x1, #64] + ldr q1, [x2, #64] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #128] + ldp q1, q3, [x2, #128] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #256] + ldp q1, q3, [x2, #256] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #288] + ldp q1, q3, [x2, #288] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #320] + ldp q1, q3, [x2, #320] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #352] + ldp q1, q3, [x2, #352] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #768] + ldp q1, q3, [x2, #768] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #800] + ldp q1, q3, [x2, #800] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #832] + ldp q1, q3, [x2, #832] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #864] + ldp q1, q3, [x2, #864] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + trn1 v29.4s, v5.4s, v6.4s + trn2 v30.4s, v5.4s, v6.4s + trn1 v5.2d, v29.2d, v30.2d + trn2 v6.2d, v29.2d, v30.2d + trn1 v29.4s, v7.4s, v8.4s + trn2 v30.4s, v7.4s, v8.4s + trn1 v7.2d, v29.2d, v30.2d + trn2 v8.2d, v29.2d, v30.2d + trn1 v29.4s, v9.4s, v10.4s + trn2 v30.4s, v9.4s, v10.4s + trn1 v9.2d, v29.2d, v30.2d + trn2 v10.2d, v29.2d, v30.2d + trn1 v29.4s, v11.4s, v12.4s + trn2 v30.4s, v11.4s, v12.4s + trn1 v11.2d, v29.2d, v30.2d + trn2 v12.2d, v29.2d, v30.2d + trn1 v29.4s, v13.4s, v14.4s + trn2 v30.4s, v13.4s, v14.4s + trn1 v13.2d, v29.2d, v30.2d + trn2 v14.2d, v29.2d, v30.2d + trn1 v29.4s, v15.4s, v16.4s + trn2 v30.4s, v15.4s, v16.4s + trn1 v15.2d, v29.2d, v30.2d + trn2 v16.2d, v29.2d, v30.2d + trn1 v29.4s, v17.4s, v18.4s + trn2 v30.4s, v17.4s, v18.4s + trn1 v17.2d, v29.2d, v30.2d + trn2 v18.2d, v29.2d, v30.2d + trn1 v29.4s, v19.4s, v20.4s + trn2 v30.4s, v19.4s, v20.4s + trn1 v19.2d, v29.2d, v30.2d + trn2 v20.2d, v29.2d, v30.2d + stp q5, q6, [x0] + stp q7, q8, [x0, #32] + stp q9, q10, [x0, #64] + stp q11, q12, [x0, #96] + stp q13, q14, [x0, #128] + stp q15, q16, [x0, #160] + stp q17, q18, [x0, #192] + stp q19, q20, [x0, #224] + ldp q5, q6, [x0, #256] + ldp q7, q8, [x0, #288] + ldp q9, q10, [x0, #320] + ldp q11, q12, [x0, #352] + ldp q13, q14, [x0, #384] + ldp q15, q16, [x0, #416] + ldp q17, q18, [x0, #448] + ldp q19, q20, [x0, #480] + ldr q0, [x1, #80] + ldr q1, [x2, #80] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #160] + ldp q1, q3, [x2, #160] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #384] + ldp q1, q3, [x2, #384] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #416] + ldp q1, q3, [x2, #416] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #448] + ldp q1, q3, [x2, #448] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #480] + ldp q1, q3, [x2, #480] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #896] + ldp q1, q3, [x2, #896] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #928] + ldp q1, q3, [x2, #928] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #960] + ldp q1, q3, [x2, #960] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #992] + ldp q1, q3, [x2, #992] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + trn1 v29.4s, v5.4s, v6.4s + trn2 v30.4s, v5.4s, v6.4s + trn1 v5.2d, v29.2d, v30.2d + trn2 v6.2d, v29.2d, v30.2d + trn1 v29.4s, v7.4s, v8.4s + trn2 v30.4s, v7.4s, v8.4s + trn1 v7.2d, v29.2d, v30.2d + trn2 v8.2d, v29.2d, v30.2d + trn1 v29.4s, v9.4s, v10.4s + trn2 v30.4s, v9.4s, v10.4s + trn1 v9.2d, v29.2d, v30.2d + trn2 v10.2d, v29.2d, v30.2d + trn1 v29.4s, v11.4s, v12.4s + trn2 v30.4s, v11.4s, v12.4s + trn1 v11.2d, v29.2d, v30.2d + trn2 v12.2d, v29.2d, v30.2d + trn1 v29.4s, v13.4s, v14.4s + trn2 v30.4s, v13.4s, v14.4s + trn1 v13.2d, v29.2d, v30.2d + trn2 v14.2d, v29.2d, v30.2d + trn1 v29.4s, v15.4s, v16.4s + trn2 v30.4s, v15.4s, v16.4s + trn1 v15.2d, v29.2d, v30.2d + trn2 v16.2d, v29.2d, v30.2d + trn1 v29.4s, v17.4s, v18.4s + trn2 v30.4s, v17.4s, v18.4s + trn1 v17.2d, v29.2d, v30.2d + trn2 v18.2d, v29.2d, v30.2d + trn1 v29.4s, v19.4s, v20.4s + trn2 v30.4s, v19.4s, v20.4s + trn1 v19.2d, v29.2d, v30.2d + trn2 v20.2d, v29.2d, v30.2d + stp q5, q6, [x0, #256] + stp q7, q8, [x0, #288] + stp q9, q10, [x0, #320] + stp q11, q12, [x0, #352] + stp q13, q14, [x0, #384] + stp q15, q16, [x0, #416] + stp q17, q18, [x0, #448] + stp q19, q20, [x0, #480] + ldp q5, q6, [x0, #512] + ldp q7, q8, [x0, #544] + ldp q9, q10, [x0, #576] + ldp q11, q12, [x0, #608] + ldp q13, q14, [x0, #640] + ldp q15, q16, [x0, #672] + ldp q17, q18, [x0, #704] + ldp q19, q20, [x0, #736] + ldr q0, [x1, #96] + ldr q1, [x2, #96] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #192] + ldp q1, q3, [x2, #192] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #512] + ldp q1, q3, [x2, #512] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #544] + ldp q1, q3, [x2, #544] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #576] + ldp q1, q3, [x2, #576] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #608] + ldp q1, q3, [x2, #608] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldr q0, [x1, #1024] + ldr q2, [x1, #1040] + ldr q1, [x2, #1024] + ldr q3, [x2, #1040] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldr q0, [x1, #1056] + ldr q2, [x1, #1072] + ldr q1, [x2, #1056] + ldr q3, [x2, #1072] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldr q0, [x1, #1088] + ldr q2, [x1, #1104] + ldr q1, [x2, #1088] + ldr q3, [x2, #1104] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldr q0, [x1, #1120] + ldr q2, [x1, #1136] + ldr q1, [x2, #1120] + ldr q3, [x2, #1136] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + trn1 v29.4s, v5.4s, v6.4s + trn2 v30.4s, v5.4s, v6.4s + trn1 v5.2d, v29.2d, v30.2d + trn2 v6.2d, v29.2d, v30.2d + trn1 v29.4s, v7.4s, v8.4s + trn2 v30.4s, v7.4s, v8.4s + trn1 v7.2d, v29.2d, v30.2d + trn2 v8.2d, v29.2d, v30.2d + trn1 v29.4s, v9.4s, v10.4s + trn2 v30.4s, v9.4s, v10.4s + trn1 v9.2d, v29.2d, v30.2d + trn2 v10.2d, v29.2d, v30.2d + trn1 v29.4s, v11.4s, v12.4s + trn2 v30.4s, v11.4s, v12.4s + trn1 v11.2d, v29.2d, v30.2d + trn2 v12.2d, v29.2d, v30.2d + trn1 v29.4s, v13.4s, v14.4s + trn2 v30.4s, v13.4s, v14.4s + trn1 v13.2d, v29.2d, v30.2d + trn2 v14.2d, v29.2d, v30.2d + trn1 v29.4s, v15.4s, v16.4s + trn2 v30.4s, v15.4s, v16.4s + trn1 v15.2d, v29.2d, v30.2d + trn2 v16.2d, v29.2d, v30.2d + trn1 v29.4s, v17.4s, v18.4s + trn2 v30.4s, v17.4s, v18.4s + trn1 v17.2d, v29.2d, v30.2d + trn2 v18.2d, v29.2d, v30.2d + trn1 v29.4s, v19.4s, v20.4s + trn2 v30.4s, v19.4s, v20.4s + trn1 v19.2d, v29.2d, v30.2d + trn2 v20.2d, v29.2d, v30.2d + stp q5, q6, [x0, #512] + stp q7, q8, [x0, #544] + stp q9, q10, [x0, #576] + stp q11, q12, [x0, #608] + stp q13, q14, [x0, #640] + stp q15, q16, [x0, #672] + stp q17, q18, [x0, #704] + stp q19, q20, [x0, #736] + ldp q5, q6, [x0, #768] + ldp q7, q8, [x0, #800] + ldp q9, q10, [x0, #832] + ldp q11, q12, [x0, #864] + ldp q13, q14, [x0, #896] + ldp q15, q16, [x0, #928] + ldp q17, q18, [x0, #960] + ldp q19, q20, [x0, #992] + ldr q0, [x1, #112] + ldr q1, [x2, #112] + mul v29.4s, v7.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[0] + sqrdmulh v21.4s, v7.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[0] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v11.4s, v1.s[1] + mul v30.4s, v12.4s, v1.s[1] + sqrdmulh v23.4s, v11.4s, v0.s[1] + sqrdmulh v24.4s, v12.4s, v0.s[1] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v15.4s, v1.s[2] + mul v30.4s, v16.4s, v1.s[2] + sqrdmulh v25.4s, v15.4s, v0.s[2] + sqrdmulh v26.4s, v16.4s, v0.s[2] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v19.4s, v1.s[3] + mul v30.4s, v20.4s, v1.s[3] + sqrdmulh v27.4s, v19.4s, v0.s[3] + sqrdmulh v28.4s, v20.4s, v0.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v7.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v6.4s, v22.4s + add v6.4s, v6.4s, v22.4s + sub v11.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v10.4s, v24.4s + add v10.4s, v10.4s, v24.4s + sub v15.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v14.4s, v26.4s + add v14.4s, v14.4s, v26.4s + sub v19.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v18.4s, v28.4s + add v18.4s, v18.4s, v28.4s + ldp q0, q2, [x1, #224] + ldp q1, q3, [x2, #224] + mul v29.4s, v6.4s, v1.s[0] + mul v30.4s, v8.4s, v1.s[1] + sqrdmulh v21.4s, v6.4s, v0.s[0] + sqrdmulh v22.4s, v8.4s, v0.s[1] + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + mul v29.4s, v10.4s, v1.s[2] + mul v30.4s, v12.4s, v1.s[3] + sqrdmulh v23.4s, v10.4s, v0.s[2] + sqrdmulh v24.4s, v12.4s, v0.s[3] + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + mul v29.4s, v14.4s, v3.s[0] + mul v30.4s, v16.4s, v3.s[1] + sqrdmulh v25.4s, v14.4s, v2.s[0] + sqrdmulh v26.4s, v16.4s, v2.s[1] + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + mul v29.4s, v18.4s, v3.s[2] + mul v30.4s, v20.4s, v3.s[3] + sqrdmulh v27.4s, v18.4s, v2.s[2] + sqrdmulh v28.4s, v20.4s, v2.s[3] + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldp q0, q2, [x1, #640] + ldp q1, q3, [x2, #640] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v29.2d, v6.2d + trn2 v8.2d, v30.2d, v8.2d + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldp q0, q2, [x1, #672] + ldp q1, q3, [x2, #672] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v29.2d, v10.2d + trn2 v12.2d, v30.2d, v12.2d + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldp q0, q2, [x1, #704] + ldp q1, q3, [x2, #704] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v29.2d, v14.2d + trn2 v16.2d, v30.2d, v16.2d + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldp q0, q2, [x1, #736] + ldp q1, q3, [x2, #736] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v29.2d, v18.2d + trn2 v20.2d, v30.2d, v20.2d + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + ldr q0, [x1, #1152] + ldr q2, [x1, #1168] + ldr q1, [x2, #1152] + ldr q3, [x2, #1168] + mov v29.16b, v5.16b + mov v30.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v29.4s, v6.4s + trn2 v8.4s, v30.4s, v8.4s + mul v29.4s, v6.4s, v1.4s + mul v30.4s, v8.4s, v3.4s + sqrdmulh v21.4s, v6.4s, v0.4s + sqrdmulh v22.4s, v8.4s, v2.4s + sqrdmlsh v21.4s, v29.4s, v4.s[0] + sqrdmlsh v22.4s, v30.4s, v4.s[0] + sshr v21.4s, v21.4s, #1 + sshr v22.4s, v22.4s, #1 + ldr q0, [x1, #1184] + ldr q2, [x1, #1200] + ldr q1, [x2, #1184] + ldr q3, [x2, #1200] + mov v29.16b, v9.16b + mov v30.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v29.4s, v10.4s + trn2 v12.4s, v30.4s, v12.4s + mul v29.4s, v10.4s, v1.4s + mul v30.4s, v12.4s, v3.4s + sqrdmulh v23.4s, v10.4s, v0.4s + sqrdmulh v24.4s, v12.4s, v2.4s + sqrdmlsh v23.4s, v29.4s, v4.s[0] + sqrdmlsh v24.4s, v30.4s, v4.s[0] + sshr v23.4s, v23.4s, #1 + sshr v24.4s, v24.4s, #1 + ldr q0, [x1, #1216] + ldr q2, [x1, #1232] + ldr q1, [x2, #1216] + ldr q3, [x2, #1232] + mov v29.16b, v13.16b + mov v30.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v29.4s, v14.4s + trn2 v16.4s, v30.4s, v16.4s + mul v29.4s, v14.4s, v1.4s + mul v30.4s, v16.4s, v3.4s + sqrdmulh v25.4s, v14.4s, v0.4s + sqrdmulh v26.4s, v16.4s, v2.4s + sqrdmlsh v25.4s, v29.4s, v4.s[0] + sqrdmlsh v26.4s, v30.4s, v4.s[0] + sshr v25.4s, v25.4s, #1 + sshr v26.4s, v26.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x1, #1264] + ldr q1, [x2, #1248] + ldr q3, [x2, #1264] + mov v29.16b, v17.16b + mov v30.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v29.4s, v18.4s + trn2 v20.4s, v30.4s, v20.4s + mul v29.4s, v18.4s, v1.4s + mul v30.4s, v20.4s, v3.4s + sqrdmulh v27.4s, v18.4s, v0.4s + sqrdmulh v28.4s, v20.4s, v2.4s + sqrdmlsh v27.4s, v29.4s, v4.s[0] + sqrdmlsh v28.4s, v30.4s, v4.s[0] + sshr v27.4s, v27.4s, #1 + sshr v28.4s, v28.4s, #1 + sub v6.4s, v5.4s, v21.4s + add v5.4s, v5.4s, v21.4s + sub v8.4s, v7.4s, v22.4s + add v7.4s, v7.4s, v22.4s + sub v10.4s, v9.4s, v23.4s + add v9.4s, v9.4s, v23.4s + sub v12.4s, v11.4s, v24.4s + add v11.4s, v11.4s, v24.4s + sub v14.4s, v13.4s, v25.4s + add v13.4s, v13.4s, v25.4s + sub v16.4s, v15.4s, v26.4s + add v15.4s, v15.4s, v26.4s + sub v18.4s, v17.4s, v27.4s + add v17.4s, v17.4s, v27.4s + sub v20.4s, v19.4s, v28.4s + add v19.4s, v19.4s, v28.4s + trn1 v29.4s, v5.4s, v6.4s + trn2 v30.4s, v5.4s, v6.4s + trn1 v5.2d, v29.2d, v30.2d + trn2 v6.2d, v29.2d, v30.2d + trn1 v29.4s, v7.4s, v8.4s + trn2 v30.4s, v7.4s, v8.4s + trn1 v7.2d, v29.2d, v30.2d + trn2 v8.2d, v29.2d, v30.2d + trn1 v29.4s, v9.4s, v10.4s + trn2 v30.4s, v9.4s, v10.4s + trn1 v9.2d, v29.2d, v30.2d + trn2 v10.2d, v29.2d, v30.2d + trn1 v29.4s, v11.4s, v12.4s + trn2 v30.4s, v11.4s, v12.4s + trn1 v11.2d, v29.2d, v30.2d + trn2 v12.2d, v29.2d, v30.2d + trn1 v29.4s, v13.4s, v14.4s + trn2 v30.4s, v13.4s, v14.4s + trn1 v13.2d, v29.2d, v30.2d + trn2 v14.2d, v29.2d, v30.2d + trn1 v29.4s, v15.4s, v16.4s + trn2 v30.4s, v15.4s, v16.4s + trn1 v15.2d, v29.2d, v30.2d + trn2 v16.2d, v29.2d, v30.2d + trn1 v29.4s, v17.4s, v18.4s + trn2 v30.4s, v17.4s, v18.4s + trn1 v17.2d, v29.2d, v30.2d + trn2 v18.2d, v29.2d, v30.2d + trn1 v29.4s, v19.4s, v20.4s + trn2 v30.4s, v19.4s, v20.4s + trn1 v19.2d, v29.2d, v30.2d + trn2 v20.2d, v29.2d, v30.2d + stp q5, q6, [x0, #768] + stp q7, q8, [x0, #800] + stp q9, q10, [x0, #832] + stp q11, q12, [x0, #864] + stp q13, q14, [x0, #896] + stp q15, q16, [x0, #928] + stp q17, q18, [x0, #960] + stp q19, q20, [x0, #992] + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_ntt_full_sqrdmlsh_neon,.-mldsa_ntt_full_sqrdmlsh_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mldsa_invntt_sqrdmlsh_neon +WC_ASM_ATT_HIDDEN(mldsa_invntt_sqrdmlsh_neon) +.type mldsa_invntt_sqrdmlsh_neon,@function +.align 2 +mldsa_invntt_sqrdmlsh_neon: +#else +.section __TEXT,__text +.globl _mldsa_invntt_sqrdmlsh_neon +WC_ASM_ATT_HIDDEN(_mldsa_invntt_sqrdmlsh_neon) +.p2align 2 +_mldsa_invntt_sqrdmlsh_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x1, L_mldsa_aarch64_zetas_inv + add x1, x1, :lo12:L_mldsa_aarch64_zetas_inv +#else + adrp x1, L_mldsa_aarch64_zetas_inv@PAGE + add x1, x1, L_mldsa_aarch64_zetas_inv@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x2, L_mldsa_aarch64_zetas_inv_qinv + add x2, x2, :lo12:L_mldsa_aarch64_zetas_inv_qinv +#else + adrp x2, L_mldsa_aarch64_zetas_inv_qinv@PAGE + add x2, x2, L_mldsa_aarch64_zetas_inv_qinv@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_aarch64_consts + add x3, x3, :lo12:L_mldsa_aarch64_consts +#else + adrp x3, L_mldsa_aarch64_consts@PAGE + add x3, x3, L_mldsa_aarch64_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q4, [x3] + ldp q5, q6, [x0] + ldp q7, q8, [x0, #32] + ldp q9, q10, [x0, #64] + ldp q11, q12, [x0, #96] + ldp q13, q14, [x0, #128] + ldp q15, q16, [x0, #160] + ldp q17, q18, [x0, #192] + ldp q19, q20, [x0, #224] + ldp q0, q1, [x1] + ldp q2, q3, [x2] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #32] + ldp q2, q3, [x2, #32] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #64] + ldp q2, q3, [x2, #64] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #96] + ldp q2, q3, [x2, #96] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #512] + ldp q2, q3, [x2, #512] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #544] + ldp q2, q3, [x2, #544] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #576] + ldp q2, q3, [x2, #576] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #608] + ldp q2, q3, [x2, #608] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1024] + ldr q1, [x1, #1040] + ldr q2, [x2, #1024] + ldr q3, [x2, #1040] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1152] + ldr q2, [x2, #1152] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0] + stp q7, q8, [x0, #32] + stp q9, q10, [x0, #64] + stp q11, q12, [x0, #96] + stp q13, q14, [x0, #128] + stp q15, q16, [x0, #160] + stp q17, q18, [x0, #192] + stp q19, q20, [x0, #224] + ldp q5, q6, [x0, #256] + ldp q7, q8, [x0, #288] + ldp q9, q10, [x0, #320] + ldp q11, q12, [x0, #352] + ldp q13, q14, [x0, #384] + ldp q15, q16, [x0, #416] + ldp q17, q18, [x0, #448] + ldp q19, q20, [x0, #480] + ldp q0, q1, [x1, #128] + ldp q2, q3, [x2, #128] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #160] + ldp q2, q3, [x2, #160] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #192] + ldp q2, q3, [x2, #192] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #224] + ldp q2, q3, [x2, #224] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #640] + ldp q2, q3, [x2, #640] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #672] + ldp q2, q3, [x2, #672] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #704] + ldp q2, q3, [x2, #704] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #736] + ldp q2, q3, [x2, #736] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1056] + ldr q1, [x1, #1072] + ldr q2, [x2, #1056] + ldr q3, [x2, #1072] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1168] + ldr q2, [x2, #1168] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0, #256] + stp q7, q8, [x0, #288] + stp q9, q10, [x0, #320] + stp q11, q12, [x0, #352] + stp q13, q14, [x0, #384] + stp q15, q16, [x0, #416] + stp q17, q18, [x0, #448] + stp q19, q20, [x0, #480] + ldp q5, q6, [x0, #512] + ldp q7, q8, [x0, #544] + ldp q9, q10, [x0, #576] + ldp q11, q12, [x0, #608] + ldp q13, q14, [x0, #640] + ldp q15, q16, [x0, #672] + ldp q17, q18, [x0, #704] + ldp q19, q20, [x0, #736] + ldp q0, q1, [x1, #256] + ldp q2, q3, [x2, #256] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #288] + ldp q2, q3, [x2, #288] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #320] + ldp q2, q3, [x2, #320] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #352] + ldp q2, q3, [x2, #352] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #768] + ldp q2, q3, [x2, #768] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #800] + ldp q2, q3, [x2, #800] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #832] + ldp q2, q3, [x2, #832] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #864] + ldp q2, q3, [x2, #864] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1088] + ldr q1, [x1, #1104] + ldr q2, [x2, #1088] + ldr q3, [x2, #1104] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1184] + ldr q2, [x2, #1184] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0, #512] + stp q7, q8, [x0, #544] + stp q9, q10, [x0, #576] + stp q11, q12, [x0, #608] + stp q13, q14, [x0, #640] + stp q15, q16, [x0, #672] + stp q17, q18, [x0, #704] + stp q19, q20, [x0, #736] + ldp q5, q6, [x0, #768] + ldp q7, q8, [x0, #800] + ldp q9, q10, [x0, #832] + ldp q11, q12, [x0, #864] + ldp q13, q14, [x0, #896] + ldp q15, q16, [x0, #928] + ldp q17, q18, [x0, #960] + ldp q19, q20, [x0, #992] + ldp q0, q1, [x1, #384] + ldp q2, q3, [x2, #384] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #416] + ldp q2, q3, [x2, #416] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #448] + ldp q2, q3, [x2, #448] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #480] + ldp q2, q3, [x2, #480] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #896] + ldp q2, q3, [x2, #896] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #928] + ldp q2, q3, [x2, #928] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #960] + ldp q2, q3, [x2, #960] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #992] + ldp q2, q3, [x2, #992] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1120] + ldr q1, [x1, #1136] + ldr q2, [x2, #1120] + ldr q3, [x2, #1136] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1200] + ldr q2, [x2, #1200] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0, #768] + stp q7, q8, [x0, #800] + stp q9, q10, [x0, #832] + stp q11, q12, [x0, #864] + stp q13, q14, [x0, #896] + stp q15, q16, [x0, #928] + stp q17, q18, [x0, #960] + stp q19, q20, [x0, #992] + ldr q5, [x0] + ldr q6, [x0, #64] + ldr q7, [x0, #128] + ldr q8, [x0, #192] + ldr q9, [x0, #256] + ldr q10, [x0, #320] + ldr q11, [x0, #384] + ldr q12, [x0, #448] + ldr q13, [x0, #512] + ldr q14, [x0, #576] + ldr q15, [x0, #640] + ldr q16, [x0, #704] + ldr q17, [x0, #768] + ldr q18, [x0, #832] + ldr q19, [x0, #896] + ldr q20, [x0, #960] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v23.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmlsh v13.4s, v21.4s, v4.s[0] + sqrdmlsh v14.4s, v23.4s, v4.s[0] + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmlsh v5.4s, v21.4s, v4.s[0] + sqrdmlsh v6.4s, v22.4s, v4.s[0] + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v22.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v22.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v22.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0] + str q6, [x0, #64] + str q7, [x0, #128] + str q8, [x0, #192] + str q9, [x0, #256] + str q10, [x0, #320] + str q11, [x0, #384] + str q12, [x0, #448] + str q13, [x0, #512] + str q14, [x0, #576] + str q15, [x0, #640] + str q16, [x0, #704] + str q17, [x0, #768] + str q18, [x0, #832] + str q19, [x0, #896] + str q20, [x0, #960] + ldr q5, [x0, #16] + ldr q6, [x0, #80] + ldr q7, [x0, #144] + ldr q8, [x0, #208] + ldr q9, [x0, #272] + ldr q10, [x0, #336] + ldr q11, [x0, #400] + ldr q12, [x0, #464] + ldr q13, [x0, #528] + ldr q14, [x0, #592] + ldr q15, [x0, #656] + ldr q16, [x0, #720] + ldr q17, [x0, #784] + ldr q18, [x0, #848] + ldr q19, [x0, #912] + ldr q20, [x0, #976] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v23.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmlsh v13.4s, v21.4s, v4.s[0] + sqrdmlsh v14.4s, v23.4s, v4.s[0] + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmlsh v5.4s, v21.4s, v4.s[0] + sqrdmlsh v6.4s, v22.4s, v4.s[0] + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v22.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v22.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v22.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0, #16] + str q6, [x0, #80] + str q7, [x0, #144] + str q8, [x0, #208] + str q9, [x0, #272] + str q10, [x0, #336] + str q11, [x0, #400] + str q12, [x0, #464] + str q13, [x0, #528] + str q14, [x0, #592] + str q15, [x0, #656] + str q16, [x0, #720] + str q17, [x0, #784] + str q18, [x0, #848] + str q19, [x0, #912] + str q20, [x0, #976] + ldr q5, [x0, #32] + ldr q6, [x0, #96] + ldr q7, [x0, #160] + ldr q8, [x0, #224] + ldr q9, [x0, #288] + ldr q10, [x0, #352] + ldr q11, [x0, #416] + ldr q12, [x0, #480] + ldr q13, [x0, #544] + ldr q14, [x0, #608] + ldr q15, [x0, #672] + ldr q16, [x0, #736] + ldr q17, [x0, #800] + ldr q18, [x0, #864] + ldr q19, [x0, #928] + ldr q20, [x0, #992] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v23.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmlsh v13.4s, v21.4s, v4.s[0] + sqrdmlsh v14.4s, v23.4s, v4.s[0] + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmlsh v5.4s, v21.4s, v4.s[0] + sqrdmlsh v6.4s, v22.4s, v4.s[0] + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v22.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v22.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v22.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0, #32] + str q6, [x0, #96] + str q7, [x0, #160] + str q8, [x0, #224] + str q9, [x0, #288] + str q10, [x0, #352] + str q11, [x0, #416] + str q12, [x0, #480] + str q13, [x0, #544] + str q14, [x0, #608] + str q15, [x0, #672] + str q16, [x0, #736] + str q17, [x0, #800] + str q18, [x0, #864] + str q19, [x0, #928] + str q20, [x0, #992] + ldr q5, [x0, #48] + ldr q6, [x0, #112] + ldr q7, [x0, #176] + ldr q8, [x0, #240] + ldr q9, [x0, #304] + ldr q10, [x0, #368] + ldr q11, [x0, #432] + ldr q12, [x0, #496] + ldr q13, [x0, #560] + ldr q14, [x0, #624] + ldr q15, [x0, #688] + ldr q16, [x0, #752] + ldr q17, [x0, #816] + ldr q18, [x0, #880] + ldr q19, [x0, #944] + ldr q20, [x0, #1008] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v23.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmlsh v13.4s, v21.4s, v4.s[0] + sqrdmlsh v14.4s, v23.4s, v4.s[0] + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmlsh v5.4s, v21.4s, v4.s[0] + sqrdmlsh v6.4s, v22.4s, v4.s[0] + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v22.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v22.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v22.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0, #48] + str q6, [x0, #112] + str q7, [x0, #176] + str q8, [x0, #240] + str q9, [x0, #304] + str q10, [x0, #368] + str q11, [x0, #432] + str q12, [x0, #496] + str q13, [x0, #560] + str q14, [x0, #624] + str q15, [x0, #688] + str q16, [x0, #752] + str q17, [x0, #816] + str q18, [x0, #880] + str q19, [x0, #944] + str q20, [x0, #1008] + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_invntt_sqrdmlsh_neon,.-mldsa_invntt_sqrdmlsh_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mldsa_invntt_full_sqrdmlsh_neon +WC_ASM_ATT_HIDDEN(mldsa_invntt_full_sqrdmlsh_neon) +.type mldsa_invntt_full_sqrdmlsh_neon,@function +.align 2 +mldsa_invntt_full_sqrdmlsh_neon: +#else +.section __TEXT,__text +.globl _mldsa_invntt_full_sqrdmlsh_neon +WC_ASM_ATT_HIDDEN(_mldsa_invntt_full_sqrdmlsh_neon) +.p2align 2 +_mldsa_invntt_full_sqrdmlsh_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x1, L_mldsa_aarch64_zetas_inv + add x1, x1, :lo12:L_mldsa_aarch64_zetas_inv +#else + adrp x1, L_mldsa_aarch64_zetas_inv@PAGE + add x1, x1, L_mldsa_aarch64_zetas_inv@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x2, L_mldsa_aarch64_zetas_inv_qinv + add x2, x2, :lo12:L_mldsa_aarch64_zetas_inv_qinv +#else + adrp x2, L_mldsa_aarch64_zetas_inv_qinv@PAGE + add x2, x2, L_mldsa_aarch64_zetas_inv_qinv@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_aarch64_consts + add x3, x3, :lo12:L_mldsa_aarch64_consts +#else + adrp x3, L_mldsa_aarch64_consts@PAGE + add x3, x3, L_mldsa_aarch64_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q4, [x3] + ldp q5, q6, [x0] + ldp q7, q8, [x0, #32] + ldp q9, q10, [x0, #64] + ldp q11, q12, [x0, #96] + ldp q13, q14, [x0, #128] + ldp q15, q16, [x0, #160] + ldp q17, q18, [x0, #192] + ldp q19, q20, [x0, #224] + trn1 v21.2d, v5.2d, v6.2d + trn2 v22.2d, v5.2d, v6.2d + trn1 v5.4s, v21.4s, v22.4s + trn2 v6.4s, v21.4s, v22.4s + trn1 v21.2d, v7.2d, v8.2d + trn2 v22.2d, v7.2d, v8.2d + trn1 v7.4s, v21.4s, v22.4s + trn2 v8.4s, v21.4s, v22.4s + trn1 v21.2d, v9.2d, v10.2d + trn2 v22.2d, v9.2d, v10.2d + trn1 v9.4s, v21.4s, v22.4s + trn2 v10.4s, v21.4s, v22.4s + trn1 v21.2d, v11.2d, v12.2d + trn2 v22.2d, v11.2d, v12.2d + trn1 v11.4s, v21.4s, v22.4s + trn2 v12.4s, v21.4s, v22.4s + trn1 v21.2d, v13.2d, v14.2d + trn2 v22.2d, v13.2d, v14.2d + trn1 v13.4s, v21.4s, v22.4s + trn2 v14.4s, v21.4s, v22.4s + trn1 v21.2d, v15.2d, v16.2d + trn2 v22.2d, v15.2d, v16.2d + trn1 v15.4s, v21.4s, v22.4s + trn2 v16.4s, v21.4s, v22.4s + trn1 v21.2d, v17.2d, v18.2d + trn2 v22.2d, v17.2d, v18.2d + trn1 v17.4s, v21.4s, v22.4s + trn2 v18.4s, v21.4s, v22.4s + trn1 v21.2d, v19.2d, v20.2d + trn2 v22.2d, v19.2d, v20.2d + trn1 v19.4s, v21.4s, v22.4s + trn2 v20.4s, v21.4s, v22.4s + ldp q0, q1, [x1] + ldp q2, q3, [x2] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #32] + ldp q2, q3, [x2, #32] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #64] + ldp q2, q3, [x2, #64] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #96] + ldp q2, q3, [x2, #96] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #512] + ldp q2, q3, [x2, #512] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #544] + ldp q2, q3, [x2, #544] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #576] + ldp q2, q3, [x2, #576] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #608] + ldp q2, q3, [x2, #608] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1024] + ldr q1, [x1, #1040] + ldr q2, [x2, #1024] + ldr q3, [x2, #1040] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1152] + ldr q2, [x2, #1152] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0] + stp q7, q8, [x0, #32] + stp q9, q10, [x0, #64] + stp q11, q12, [x0, #96] + stp q13, q14, [x0, #128] + stp q15, q16, [x0, #160] + stp q17, q18, [x0, #192] + stp q19, q20, [x0, #224] + ldp q5, q6, [x0, #256] + ldp q7, q8, [x0, #288] + ldp q9, q10, [x0, #320] + ldp q11, q12, [x0, #352] + ldp q13, q14, [x0, #384] + ldp q15, q16, [x0, #416] + ldp q17, q18, [x0, #448] + ldp q19, q20, [x0, #480] + trn1 v21.2d, v5.2d, v6.2d + trn2 v22.2d, v5.2d, v6.2d + trn1 v5.4s, v21.4s, v22.4s + trn2 v6.4s, v21.4s, v22.4s + trn1 v21.2d, v7.2d, v8.2d + trn2 v22.2d, v7.2d, v8.2d + trn1 v7.4s, v21.4s, v22.4s + trn2 v8.4s, v21.4s, v22.4s + trn1 v21.2d, v9.2d, v10.2d + trn2 v22.2d, v9.2d, v10.2d + trn1 v9.4s, v21.4s, v22.4s + trn2 v10.4s, v21.4s, v22.4s + trn1 v21.2d, v11.2d, v12.2d + trn2 v22.2d, v11.2d, v12.2d + trn1 v11.4s, v21.4s, v22.4s + trn2 v12.4s, v21.4s, v22.4s + trn1 v21.2d, v13.2d, v14.2d + trn2 v22.2d, v13.2d, v14.2d + trn1 v13.4s, v21.4s, v22.4s + trn2 v14.4s, v21.4s, v22.4s + trn1 v21.2d, v15.2d, v16.2d + trn2 v22.2d, v15.2d, v16.2d + trn1 v15.4s, v21.4s, v22.4s + trn2 v16.4s, v21.4s, v22.4s + trn1 v21.2d, v17.2d, v18.2d + trn2 v22.2d, v17.2d, v18.2d + trn1 v17.4s, v21.4s, v22.4s + trn2 v18.4s, v21.4s, v22.4s + trn1 v21.2d, v19.2d, v20.2d + trn2 v22.2d, v19.2d, v20.2d + trn1 v19.4s, v21.4s, v22.4s + trn2 v20.4s, v21.4s, v22.4s + ldp q0, q1, [x1, #128] + ldp q2, q3, [x2, #128] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #160] + ldp q2, q3, [x2, #160] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #192] + ldp q2, q3, [x2, #192] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #224] + ldp q2, q3, [x2, #224] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #640] + ldp q2, q3, [x2, #640] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #672] + ldp q2, q3, [x2, #672] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #704] + ldp q2, q3, [x2, #704] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #736] + ldp q2, q3, [x2, #736] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1056] + ldr q1, [x1, #1072] + ldr q2, [x2, #1056] + ldr q3, [x2, #1072] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1168] + ldr q2, [x2, #1168] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0, #256] + stp q7, q8, [x0, #288] + stp q9, q10, [x0, #320] + stp q11, q12, [x0, #352] + stp q13, q14, [x0, #384] + stp q15, q16, [x0, #416] + stp q17, q18, [x0, #448] + stp q19, q20, [x0, #480] + ldp q5, q6, [x0, #512] + ldp q7, q8, [x0, #544] + ldp q9, q10, [x0, #576] + ldp q11, q12, [x0, #608] + ldp q13, q14, [x0, #640] + ldp q15, q16, [x0, #672] + ldp q17, q18, [x0, #704] + ldp q19, q20, [x0, #736] + trn1 v21.2d, v5.2d, v6.2d + trn2 v22.2d, v5.2d, v6.2d + trn1 v5.4s, v21.4s, v22.4s + trn2 v6.4s, v21.4s, v22.4s + trn1 v21.2d, v7.2d, v8.2d + trn2 v22.2d, v7.2d, v8.2d + trn1 v7.4s, v21.4s, v22.4s + trn2 v8.4s, v21.4s, v22.4s + trn1 v21.2d, v9.2d, v10.2d + trn2 v22.2d, v9.2d, v10.2d + trn1 v9.4s, v21.4s, v22.4s + trn2 v10.4s, v21.4s, v22.4s + trn1 v21.2d, v11.2d, v12.2d + trn2 v22.2d, v11.2d, v12.2d + trn1 v11.4s, v21.4s, v22.4s + trn2 v12.4s, v21.4s, v22.4s + trn1 v21.2d, v13.2d, v14.2d + trn2 v22.2d, v13.2d, v14.2d + trn1 v13.4s, v21.4s, v22.4s + trn2 v14.4s, v21.4s, v22.4s + trn1 v21.2d, v15.2d, v16.2d + trn2 v22.2d, v15.2d, v16.2d + trn1 v15.4s, v21.4s, v22.4s + trn2 v16.4s, v21.4s, v22.4s + trn1 v21.2d, v17.2d, v18.2d + trn2 v22.2d, v17.2d, v18.2d + trn1 v17.4s, v21.4s, v22.4s + trn2 v18.4s, v21.4s, v22.4s + trn1 v21.2d, v19.2d, v20.2d + trn2 v22.2d, v19.2d, v20.2d + trn1 v19.4s, v21.4s, v22.4s + trn2 v20.4s, v21.4s, v22.4s + ldp q0, q1, [x1, #256] + ldp q2, q3, [x2, #256] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #288] + ldp q2, q3, [x2, #288] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #320] + ldp q2, q3, [x2, #320] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #352] + ldp q2, q3, [x2, #352] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #768] + ldp q2, q3, [x2, #768] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #800] + ldp q2, q3, [x2, #800] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #832] + ldp q2, q3, [x2, #832] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #864] + ldp q2, q3, [x2, #864] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1088] + ldr q1, [x1, #1104] + ldr q2, [x2, #1088] + ldr q3, [x2, #1104] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1184] + ldr q2, [x2, #1184] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0, #512] + stp q7, q8, [x0, #544] + stp q9, q10, [x0, #576] + stp q11, q12, [x0, #608] + stp q13, q14, [x0, #640] + stp q15, q16, [x0, #672] + stp q17, q18, [x0, #704] + stp q19, q20, [x0, #736] + ldp q5, q6, [x0, #768] + ldp q7, q8, [x0, #800] + ldp q9, q10, [x0, #832] + ldp q11, q12, [x0, #864] + ldp q13, q14, [x0, #896] + ldp q15, q16, [x0, #928] + ldp q17, q18, [x0, #960] + ldp q19, q20, [x0, #992] + trn1 v21.2d, v5.2d, v6.2d + trn2 v22.2d, v5.2d, v6.2d + trn1 v5.4s, v21.4s, v22.4s + trn2 v6.4s, v21.4s, v22.4s + trn1 v21.2d, v7.2d, v8.2d + trn2 v22.2d, v7.2d, v8.2d + trn1 v7.4s, v21.4s, v22.4s + trn2 v8.4s, v21.4s, v22.4s + trn1 v21.2d, v9.2d, v10.2d + trn2 v22.2d, v9.2d, v10.2d + trn1 v9.4s, v21.4s, v22.4s + trn2 v10.4s, v21.4s, v22.4s + trn1 v21.2d, v11.2d, v12.2d + trn2 v22.2d, v11.2d, v12.2d + trn1 v11.4s, v21.4s, v22.4s + trn2 v12.4s, v21.4s, v22.4s + trn1 v21.2d, v13.2d, v14.2d + trn2 v22.2d, v13.2d, v14.2d + trn1 v13.4s, v21.4s, v22.4s + trn2 v14.4s, v21.4s, v22.4s + trn1 v21.2d, v15.2d, v16.2d + trn2 v22.2d, v15.2d, v16.2d + trn1 v15.4s, v21.4s, v22.4s + trn2 v16.4s, v21.4s, v22.4s + trn1 v21.2d, v17.2d, v18.2d + trn2 v22.2d, v17.2d, v18.2d + trn1 v17.4s, v21.4s, v22.4s + trn2 v18.4s, v21.4s, v22.4s + trn1 v21.2d, v19.2d, v20.2d + trn2 v22.2d, v19.2d, v20.2d + trn1 v19.4s, v21.4s, v22.4s + trn2 v20.4s, v21.4s, v22.4s + ldp q0, q1, [x1, #384] + ldp q2, q3, [x2, #384] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #416] + ldp q2, q3, [x2, #416] + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #448] + ldp q2, q3, [x2, #448] + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #480] + ldp q2, q3, [x2, #480] + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldp q0, q1, [x1, #896] + ldp q2, q3, [x2, #896] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.4s, v5.4s, v6.4s + trn1 v7.4s, v7.4s, v8.4s + trn2 v6.4s, v21.4s, v6.4s + trn2 v8.4s, v22.4s, v8.4s + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v6.4s, v22.4s, v0.4s + sqrdmulh v8.4s, v24.4s, v1.4s + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + ldp q0, q1, [x1, #928] + ldp q2, q3, [x2, #928] + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.4s, v9.4s, v10.4s + trn1 v11.4s, v11.4s, v12.4s + trn2 v10.4s, v21.4s, v10.4s + trn2 v12.4s, v22.4s, v12.4s + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v10.4s, v22.4s, v0.4s + sqrdmulh v12.4s, v24.4s, v1.4s + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + ldp q0, q1, [x1, #960] + ldp q2, q3, [x2, #960] + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.4s, v13.4s, v14.4s + trn1 v15.4s, v15.4s, v16.4s + trn2 v14.4s, v21.4s, v14.4s + trn2 v16.4s, v22.4s, v16.4s + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v14.4s, v22.4s, v0.4s + sqrdmulh v16.4s, v24.4s, v1.4s + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + ldp q0, q1, [x1, #992] + ldp q2, q3, [x2, #992] + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.4s, v17.4s, v18.4s + trn1 v19.4s, v19.4s, v20.4s + trn2 v18.4s, v21.4s, v18.4s + trn2 v20.4s, v22.4s, v20.4s + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v2.4s + mul v23.4s, v24.4s, v3.4s + sqrdmulh v18.4s, v22.4s, v0.4s + sqrdmulh v20.4s, v24.4s, v1.4s + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1120] + ldr q1, [x1, #1136] + ldr q2, [x2, #1120] + ldr q3, [x2, #1136] + mov v21.16b, v5.16b + mov v22.16b, v7.16b + trn1 v5.2d, v5.2d, v6.2d + trn1 v7.2d, v7.2d, v8.2d + trn2 v6.2d, v21.2d, v6.2d + trn2 v8.2d, v22.2d, v8.2d + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + mov v21.16b, v9.16b + mov v22.16b, v11.16b + trn1 v9.2d, v9.2d, v10.2d + trn1 v11.2d, v11.2d, v12.2d + trn2 v10.2d, v21.2d, v10.2d + trn2 v12.2d, v22.2d, v12.2d + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + mov v21.16b, v13.16b + mov v22.16b, v15.16b + trn1 v13.2d, v13.2d, v14.2d + trn1 v15.2d, v15.2d, v16.2d + trn2 v14.2d, v21.2d, v14.2d + trn2 v16.2d, v22.2d, v16.2d + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + mov v21.16b, v17.16b + mov v22.16b, v19.16b + trn1 v17.2d, v17.2d, v18.2d + trn1 v19.2d, v19.2d, v20.2d + trn2 v18.2d, v21.2d, v18.2d + trn2 v20.2d, v22.2d, v20.2d + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1200] + ldr q2, [x2, #1200] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + stp q5, q6, [x0, #768] + stp q7, q8, [x0, #800] + stp q9, q10, [x0, #832] + stp q11, q12, [x0, #864] + stp q13, q14, [x0, #896] + stp q15, q16, [x0, #928] + stp q17, q18, [x0, #960] + stp q19, q20, [x0, #992] + ldr q5, [x0] + ldr q6, [x0, #64] + ldr q7, [x0, #128] + ldr q8, [x0, #192] + ldr q9, [x0, #256] + ldr q10, [x0, #320] + ldr q11, [x0, #384] + ldr q12, [x0, #448] + ldr q13, [x0, #512] + ldr q14, [x0, #576] + ldr q15, [x0, #640] + ldr q16, [x0, #704] + ldr q17, [x0, #768] + ldr q18, [x0, #832] + ldr q19, [x0, #896] + ldr q20, [x0, #960] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v23.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmlsh v13.4s, v21.4s, v4.s[0] + sqrdmlsh v14.4s, v23.4s, v4.s[0] + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmlsh v5.4s, v21.4s, v4.s[0] + sqrdmlsh v6.4s, v22.4s, v4.s[0] + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v22.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v22.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v22.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0] + str q6, [x0, #64] + str q7, [x0, #128] + str q8, [x0, #192] + str q9, [x0, #256] + str q10, [x0, #320] + str q11, [x0, #384] + str q12, [x0, #448] + str q13, [x0, #512] + str q14, [x0, #576] + str q15, [x0, #640] + str q16, [x0, #704] + str q17, [x0, #768] + str q18, [x0, #832] + str q19, [x0, #896] + str q20, [x0, #960] + ldr q5, [x0, #16] + ldr q6, [x0, #80] + ldr q7, [x0, #144] + ldr q8, [x0, #208] + ldr q9, [x0, #272] + ldr q10, [x0, #336] + ldr q11, [x0, #400] + ldr q12, [x0, #464] + ldr q13, [x0, #528] + ldr q14, [x0, #592] + ldr q15, [x0, #656] + ldr q16, [x0, #720] + ldr q17, [x0, #784] + ldr q18, [x0, #848] + ldr q19, [x0, #912] + ldr q20, [x0, #976] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v23.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmlsh v13.4s, v21.4s, v4.s[0] + sqrdmlsh v14.4s, v23.4s, v4.s[0] + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmlsh v5.4s, v21.4s, v4.s[0] + sqrdmlsh v6.4s, v22.4s, v4.s[0] + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v22.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v22.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v22.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0, #16] + str q6, [x0, #80] + str q7, [x0, #144] + str q8, [x0, #208] + str q9, [x0, #272] + str q10, [x0, #336] + str q11, [x0, #400] + str q12, [x0, #464] + str q13, [x0, #528] + str q14, [x0, #592] + str q15, [x0, #656] + str q16, [x0, #720] + str q17, [x0, #784] + str q18, [x0, #848] + str q19, [x0, #912] + str q20, [x0, #976] + ldr q5, [x0, #32] + ldr q6, [x0, #96] + ldr q7, [x0, #160] + ldr q8, [x0, #224] + ldr q9, [x0, #288] + ldr q10, [x0, #352] + ldr q11, [x0, #416] + ldr q12, [x0, #480] + ldr q13, [x0, #544] + ldr q14, [x0, #608] + ldr q15, [x0, #672] + ldr q16, [x0, #736] + ldr q17, [x0, #800] + ldr q18, [x0, #864] + ldr q19, [x0, #928] + ldr q20, [x0, #992] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v23.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmlsh v13.4s, v21.4s, v4.s[0] + sqrdmlsh v14.4s, v23.4s, v4.s[0] + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmlsh v5.4s, v21.4s, v4.s[0] + sqrdmlsh v6.4s, v22.4s, v4.s[0] + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v22.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v22.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v22.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0, #32] + str q6, [x0, #96] + str q7, [x0, #160] + str q8, [x0, #224] + str q9, [x0, #288] + str q10, [x0, #352] + str q11, [x0, #416] + str q12, [x0, #480] + str q13, [x0, #544] + str q14, [x0, #608] + str q15, [x0, #672] + str q16, [x0, #736] + str q17, [x0, #800] + str q18, [x0, #864] + str q19, [x0, #928] + str q20, [x0, #992] + ldr q5, [x0, #48] + ldr q6, [x0, #112] + ldr q7, [x0, #176] + ldr q8, [x0, #240] + ldr q9, [x0, #304] + ldr q10, [x0, #368] + ldr q11, [x0, #432] + ldr q12, [x0, #496] + ldr q13, [x0, #560] + ldr q14, [x0, #624] + ldr q15, [x0, #688] + ldr q16, [x0, #752] + ldr q17, [x0, #816] + ldr q18, [x0, #880] + ldr q19, [x0, #944] + ldr q20, [x0, #1008] + ldr q0, [x1, #1216] + ldr q1, [x1, #1232] + ldr q2, [x2, #1216] + ldr q3, [x2, #1232] + sub v22.4s, v5.4s, v6.4s + sub v24.4s, v7.4s, v8.4s + add v5.4s, v5.4s, v6.4s + add v7.4s, v7.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v6.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[1] + sqrdmlsh v6.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v6.4s, v6.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v10.4s + sub v24.4s, v11.4s, v12.4s + add v9.4s, v9.4s, v10.4s + add v11.4s, v11.4s, v12.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v10.4s, v22.4s, v0.s[2] + sqrdmulh v12.4s, v24.4s, v0.s[3] + sqrdmlsh v10.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v10.4s, v10.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v14.4s + sub v24.4s, v15.4s, v16.4s + add v13.4s, v13.4s, v14.4s + add v15.4s, v15.4s, v16.4s + mul v21.4s, v22.4s, v3.s[0] + mul v23.4s, v24.4s, v3.s[1] + sqrdmulh v14.4s, v22.4s, v1.s[0] + sqrdmulh v16.4s, v24.4s, v1.s[1] + sqrdmlsh v14.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v14.4s, v14.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v18.4s + sub v24.4s, v19.4s, v20.4s + add v17.4s, v17.4s, v18.4s + add v19.4s, v19.4s, v20.4s + mul v21.4s, v22.4s, v3.s[2] + mul v23.4s, v24.4s, v3.s[3] + sqrdmulh v18.4s, v22.4s, v1.s[2] + sqrdmulh v20.4s, v24.4s, v1.s[3] + sqrdmlsh v18.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v18.4s, v18.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1248] + ldr q2, [x2, #1248] + sub v22.4s, v5.4s, v7.4s + sub v24.4s, v6.4s, v8.4s + add v5.4s, v5.4s, v7.4s + add v6.4s, v6.4s, v8.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v7.4s, v22.4s, v0.s[0] + sqrdmulh v8.4s, v24.4s, v0.s[0] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v23.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + sub v22.4s, v9.4s, v11.4s + sub v24.4s, v10.4s, v12.4s + add v9.4s, v9.4s, v11.4s + add v10.4s, v10.4s, v12.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v11.4s, v22.4s, v0.s[1] + sqrdmulh v12.4s, v24.4s, v0.s[1] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v15.4s + sub v24.4s, v14.4s, v16.4s + add v13.4s, v13.4s, v15.4s + add v14.4s, v14.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v17.4s, v19.4s + sub v24.4s, v18.4s, v20.4s + add v17.4s, v17.4s, v19.4s + add v18.4s, v18.4s, v20.4s + mul v21.4s, v22.4s, v2.s[3] + mul v23.4s, v24.4s, v2.s[3] + sqrdmulh v19.4s, v22.4s, v0.s[3] + sqrdmulh v20.4s, v24.4s, v0.s[3] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + ldr q0, [x1, #1264] + ldr q2, [x2, #1264] + sub v22.4s, v5.4s, v9.4s + sub v24.4s, v6.4s, v10.4s + add v5.4s, v5.4s, v9.4s + add v6.4s, v6.4s, v10.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v9.4s, v22.4s, v0.s[0] + sqrdmulh v10.4s, v24.4s, v0.s[0] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v23.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + sub v22.4s, v7.4s, v11.4s + sub v24.4s, v8.4s, v12.4s + add v7.4s, v7.4s, v11.4s + add v8.4s, v8.4s, v12.4s + mul v21.4s, v22.4s, v2.s[0] + mul v23.4s, v24.4s, v2.s[0] + sqrdmulh v11.4s, v22.4s, v0.s[0] + sqrdmulh v12.4s, v24.4s, v0.s[0] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v23.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + sub v22.4s, v13.4s, v17.4s + sub v24.4s, v14.4s, v18.4s + add v13.4s, v13.4s, v17.4s + add v14.4s, v14.4s, v18.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v17.4s, v22.4s, v0.s[1] + sqrdmulh v18.4s, v24.4s, v0.s[1] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v15.4s, v19.4s + sub v24.4s, v16.4s, v20.4s + add v15.4s, v15.4s, v19.4s + add v16.4s, v16.4s, v20.4s + mul v21.4s, v22.4s, v2.s[1] + mul v23.4s, v24.4s, v2.s[1] + sqrdmulh v19.4s, v22.4s, v0.s[1] + sqrdmulh v20.4s, v24.4s, v0.s[1] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + sub v22.4s, v5.4s, v13.4s + sub v24.4s, v6.4s, v14.4s + add v5.4s, v5.4s, v13.4s + add v6.4s, v6.4s, v14.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v13.4s, v22.4s, v0.s[2] + sqrdmulh v14.4s, v24.4s, v0.s[2] + sqrdmlsh v13.4s, v21.4s, v4.s[0] + sqrdmlsh v14.4s, v23.4s, v4.s[0] + sshr v13.4s, v13.4s, #1 + sshr v14.4s, v14.4s, #1 + sub v22.4s, v7.4s, v15.4s + sub v24.4s, v8.4s, v16.4s + add v7.4s, v7.4s, v15.4s + add v8.4s, v8.4s, v16.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v15.4s, v22.4s, v0.s[2] + sqrdmulh v16.4s, v24.4s, v0.s[2] + sqrdmlsh v15.4s, v21.4s, v4.s[0] + sqrdmlsh v16.4s, v23.4s, v4.s[0] + sshr v15.4s, v15.4s, #1 + sshr v16.4s, v16.4s, #1 + sub v22.4s, v9.4s, v17.4s + sub v24.4s, v10.4s, v18.4s + add v9.4s, v9.4s, v17.4s + add v10.4s, v10.4s, v18.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v17.4s, v22.4s, v0.s[2] + sqrdmulh v18.4s, v24.4s, v0.s[2] + sqrdmlsh v17.4s, v21.4s, v4.s[0] + sqrdmlsh v18.4s, v23.4s, v4.s[0] + sshr v17.4s, v17.4s, #1 + sshr v18.4s, v18.4s, #1 + sub v22.4s, v11.4s, v19.4s + sub v24.4s, v12.4s, v20.4s + add v11.4s, v11.4s, v19.4s + add v12.4s, v12.4s, v20.4s + mul v21.4s, v22.4s, v2.s[2] + mul v23.4s, v24.4s, v2.s[2] + sqrdmulh v19.4s, v22.4s, v0.s[2] + sqrdmulh v20.4s, v24.4s, v0.s[2] + sqrdmlsh v19.4s, v21.4s, v4.s[0] + sqrdmlsh v20.4s, v23.4s, v4.s[0] + sshr v19.4s, v19.4s, #1 + sshr v20.4s, v20.4s, #1 + mul v21.4s, v5.4s, v2.s[3] + mul v22.4s, v6.4s, v2.s[3] + sqrdmulh v5.4s, v5.4s, v0.s[3] + sqrdmulh v6.4s, v6.4s, v0.s[3] + sqrdmlsh v5.4s, v21.4s, v4.s[0] + sqrdmlsh v6.4s, v22.4s, v4.s[0] + sshr v5.4s, v5.4s, #1 + sshr v6.4s, v6.4s, #1 + mul v21.4s, v7.4s, v2.s[3] + mul v22.4s, v8.4s, v2.s[3] + sqrdmulh v7.4s, v7.4s, v0.s[3] + sqrdmulh v8.4s, v8.4s, v0.s[3] + sqrdmlsh v7.4s, v21.4s, v4.s[0] + sqrdmlsh v8.4s, v22.4s, v4.s[0] + sshr v7.4s, v7.4s, #1 + sshr v8.4s, v8.4s, #1 + mul v21.4s, v9.4s, v2.s[3] + mul v22.4s, v10.4s, v2.s[3] + sqrdmulh v9.4s, v9.4s, v0.s[3] + sqrdmulh v10.4s, v10.4s, v0.s[3] + sqrdmlsh v9.4s, v21.4s, v4.s[0] + sqrdmlsh v10.4s, v22.4s, v4.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v21.4s, v11.4s, v2.s[3] + mul v22.4s, v12.4s, v2.s[3] + sqrdmulh v11.4s, v11.4s, v0.s[3] + sqrdmulh v12.4s, v12.4s, v0.s[3] + sqrdmlsh v11.4s, v21.4s, v4.s[0] + sqrdmlsh v12.4s, v22.4s, v4.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q5, [x0, #48] + str q6, [x0, #112] + str q7, [x0, #176] + str q8, [x0, #240] + str q9, [x0, #304] + str q10, [x0, #368] + str q11, [x0, #432] + str q12, [x0, #496] + str q13, [x0, #560] + str q14, [x0, #624] + str q15, [x0, #688] + str q16, [x0, #752] + str q17, [x0, #816] + str q18, [x0, #880] + str q19, [x0, #944] + str q20, [x0, #1008] + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_invntt_full_sqrdmlsh_neon,.-mldsa_invntt_full_sqrdmlsh_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mldsa_mul_sqrdmlsh_neon +WC_ASM_ATT_HIDDEN(mldsa_mul_sqrdmlsh_neon) +.type mldsa_mul_sqrdmlsh_neon,@function +.align 2 +mldsa_mul_sqrdmlsh_neon: +#else +.section __TEXT,__text +.globl _mldsa_mul_sqrdmlsh_neon +WC_ASM_ATT_HIDDEN(_mldsa_mul_sqrdmlsh_neon) +.p2align 2 +_mldsa_mul_sqrdmlsh_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x3, L_mldsa_aarch64_consts + add x3, x3, :lo12:L_mldsa_aarch64_consts +#else + adrp x3, L_mldsa_aarch64_consts@PAGE + add x3, x3, L_mldsa_aarch64_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q0, [x3] + ldr q1, [x1] + ldr q5, [x2] + ldr q2, [x1, #16] + ldr q6, [x2, #16] + ldr q3, [x1, #32] + ldr q7, [x2, #32] + ldr q4, [x1, #48] + ldr q8, [x2, #48] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0] + str q10, [x0, #16] + str q11, [x0, #32] + str q12, [x0, #48] + ldr q1, [x1, #64] + ldr q5, [x2, #64] + ldr q2, [x1, #80] + ldr q6, [x2, #80] + ldr q3, [x1, #96] + ldr q7, [x2, #96] + ldr q4, [x1, #112] + ldr q8, [x2, #112] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #64] + str q10, [x0, #80] + str q11, [x0, #96] + str q12, [x0, #112] + ldr q1, [x1, #128] + ldr q5, [x2, #128] + ldr q2, [x1, #144] + ldr q6, [x2, #144] + ldr q3, [x1, #160] + ldr q7, [x2, #160] + ldr q4, [x1, #176] + ldr q8, [x2, #176] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #128] + str q10, [x0, #144] + str q11, [x0, #160] + str q12, [x0, #176] + ldr q1, [x1, #192] + ldr q5, [x2, #192] + ldr q2, [x1, #208] + ldr q6, [x2, #208] + ldr q3, [x1, #224] + ldr q7, [x2, #224] + ldr q4, [x1, #240] + ldr q8, [x2, #240] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #192] + str q10, [x0, #208] + str q11, [x0, #224] + str q12, [x0, #240] + ldr q1, [x1, #256] + ldr q5, [x2, #256] + ldr q2, [x1, #272] + ldr q6, [x2, #272] + ldr q3, [x1, #288] + ldr q7, [x2, #288] + ldr q4, [x1, #304] + ldr q8, [x2, #304] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #256] + str q10, [x0, #272] + str q11, [x0, #288] + str q12, [x0, #304] + ldr q1, [x1, #320] + ldr q5, [x2, #320] + ldr q2, [x1, #336] + ldr q6, [x2, #336] + ldr q3, [x1, #352] + ldr q7, [x2, #352] + ldr q4, [x1, #368] + ldr q8, [x2, #368] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #320] + str q10, [x0, #336] + str q11, [x0, #352] + str q12, [x0, #368] + ldr q1, [x1, #384] + ldr q5, [x2, #384] + ldr q2, [x1, #400] + ldr q6, [x2, #400] + ldr q3, [x1, #416] + ldr q7, [x2, #416] + ldr q4, [x1, #432] + ldr q8, [x2, #432] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #384] + str q10, [x0, #400] + str q11, [x0, #416] + str q12, [x0, #432] + ldr q1, [x1, #448] + ldr q5, [x2, #448] + ldr q2, [x1, #464] + ldr q6, [x2, #464] + ldr q3, [x1, #480] + ldr q7, [x2, #480] + ldr q4, [x1, #496] + ldr q8, [x2, #496] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #448] + str q10, [x0, #464] + str q11, [x0, #480] + str q12, [x0, #496] + ldr q1, [x1, #512] + ldr q5, [x2, #512] + ldr q2, [x1, #528] + ldr q6, [x2, #528] + ldr q3, [x1, #544] + ldr q7, [x2, #544] + ldr q4, [x1, #560] + ldr q8, [x2, #560] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #512] + str q10, [x0, #528] + str q11, [x0, #544] + str q12, [x0, #560] + ldr q1, [x1, #576] + ldr q5, [x2, #576] + ldr q2, [x1, #592] + ldr q6, [x2, #592] + ldr q3, [x1, #608] + ldr q7, [x2, #608] + ldr q4, [x1, #624] + ldr q8, [x2, #624] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #576] + str q10, [x0, #592] + str q11, [x0, #608] + str q12, [x0, #624] + ldr q1, [x1, #640] + ldr q5, [x2, #640] + ldr q2, [x1, #656] + ldr q6, [x2, #656] + ldr q3, [x1, #672] + ldr q7, [x2, #672] + ldr q4, [x1, #688] + ldr q8, [x2, #688] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #640] + str q10, [x0, #656] + str q11, [x0, #672] + str q12, [x0, #688] + ldr q1, [x1, #704] + ldr q5, [x2, #704] + ldr q2, [x1, #720] + ldr q6, [x2, #720] + ldr q3, [x1, #736] + ldr q7, [x2, #736] + ldr q4, [x1, #752] + ldr q8, [x2, #752] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #704] + str q10, [x0, #720] + str q11, [x0, #736] + str q12, [x0, #752] + ldr q1, [x1, #768] + ldr q5, [x2, #768] + ldr q2, [x1, #784] + ldr q6, [x2, #784] + ldr q3, [x1, #800] + ldr q7, [x2, #800] + ldr q4, [x1, #816] + ldr q8, [x2, #816] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #768] + str q10, [x0, #784] + str q11, [x0, #800] + str q12, [x0, #816] + ldr q1, [x1, #832] + ldr q5, [x2, #832] + ldr q2, [x1, #848] + ldr q6, [x2, #848] + ldr q3, [x1, #864] + ldr q7, [x2, #864] + ldr q4, [x1, #880] + ldr q8, [x2, #880] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #832] + str q10, [x0, #848] + str q11, [x0, #864] + str q12, [x0, #880] + ldr q1, [x1, #896] + ldr q5, [x2, #896] + ldr q2, [x1, #912] + ldr q6, [x2, #912] + ldr q3, [x1, #928] + ldr q7, [x2, #928] + ldr q4, [x1, #944] + ldr q8, [x2, #944] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #896] + str q10, [x0, #912] + str q11, [x0, #928] + str q12, [x0, #944] + ldr q1, [x1, #960] + ldr q5, [x2, #960] + ldr q2, [x1, #976] + ldr q6, [x2, #976] + ldr q3, [x1, #992] + ldr q7, [x2, #992] + ldr q4, [x1, #1008] + ldr q8, [x2, #1008] + mul v13.4s, v5.4s, v0.s[1] + mul v14.4s, v6.4s, v0.s[1] + mul v17.4s, v1.4s, v13.4s + mul v18.4s, v2.4s, v14.4s + sqrdmulh v9.4s, v1.4s, v5.4s + sqrdmulh v10.4s, v2.4s, v6.4s + sqrdmlsh v9.4s, v17.4s, v0.s[0] + sqrdmlsh v10.4s, v18.4s, v0.s[0] + sshr v9.4s, v9.4s, #1 + sshr v10.4s, v10.4s, #1 + mul v15.4s, v7.4s, v0.s[1] + mul v16.4s, v8.4s, v0.s[1] + mul v17.4s, v3.4s, v15.4s + mul v18.4s, v4.4s, v16.4s + sqrdmulh v11.4s, v3.4s, v7.4s + sqrdmulh v12.4s, v4.4s, v8.4s + sqrdmlsh v11.4s, v17.4s, v0.s[0] + sqrdmlsh v12.4s, v18.4s, v0.s[0] + sshr v11.4s, v11.4s, #1 + sshr v12.4s, v12.4s, #1 + str q9, [x0, #960] + str q10, [x0, #976] + str q11, [x0, #992] + str q12, [x0, #1008] + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_mul_sqrdmlsh_neon,.-mldsa_mul_sqrdmlsh_neon +#endif /* __APPLE__ */ +#endif /* WOLFSSL_AARCH64_NO_SQRDMLSH */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decompose_q88_consts, %object + .size L_mldsa_decompose_q88_consts, 96 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decompose_q88_consts: + .long 0x0000002c,0x0000002c,0x0000002c,0x0000002c + .long 0x003fefd4,0x003fefd4,0x003fefd4,0x003fefd4 + .long 0x0002e800,0x0002e800,0x0002e800,0x0002e800 + .long 0x00017400,0x00017400,0x00017400,0x00017400 + .long 0x0000002b,0x0000002b,0x0000002b,0x0000002b + .long 0x007fe001,0x007fe001,0x007fe001,0x007fe001 +#ifndef __APPLE__ +.text +.globl mldsa_decompose_q88_neon +WC_ASM_ATT_HIDDEN(mldsa_decompose_q88_neon) +.type mldsa_decompose_q88_neon,@function +.align 2 +mldsa_decompose_q88_neon: +#else +.section __TEXT,__text +.globl _mldsa_decompose_q88_neon +WC_ASM_ATT_HIDDEN(_mldsa_decompose_q88_neon) +.p2align 2 +_mldsa_decompose_q88_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x3, L_mldsa_decompose_q88_consts + add x3, x3, :lo12:L_mldsa_decompose_q88_consts +#else + adrp x3, L_mldsa_decompose_q88_consts@PAGE + add x3, x3, L_mldsa_decompose_q88_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q0, [x3] + ldr q1, [x3, #16] + ldr q2, [x3, #32] + ldr q3, [x3, #48] + ldr q4, [x3, #64] + mov x4, #16 +L_mldsa_decompose_q88_loop: + ldr q5, [x0] + ldr q6, [x0, #16] + ldr q7, [x0, #32] + ldr q8, [x0, #48] + mov v13.16b, v1.16b + mla v13.4s, v5.4s, v0.4s + sshr v13.4s, v13.4s, #23 + mov v9.16b, v5.16b + mls v9.4s, v13.4s, v2.4s + sub v17.4s, v3.4s, v9.4s + usra v13.4s, v17.4s, #31 + mov v9.16b, v5.16b + mls v9.4s, v13.4s, v2.4s + sub v17.4s, v4.4s, v13.4s + ushr v17.4s, v17.4s, #31 + sub v9.4s, v9.4s, v17.4s + sub v17.4s, v13.4s, v0.4s + ushr v17.4s, v17.4s, #31 + neg v17.4s, v17.4s + and v13.16b, v13.16b, v17.16b + mov v14.16b, v1.16b + mla v14.4s, v6.4s, v0.4s + sshr v14.4s, v14.4s, #23 + mov v10.16b, v6.16b + mls v10.4s, v14.4s, v2.4s + sub v18.4s, v3.4s, v10.4s + usra v14.4s, v18.4s, #31 + mov v10.16b, v6.16b + mls v10.4s, v14.4s, v2.4s + sub v18.4s, v4.4s, v14.4s + ushr v18.4s, v18.4s, #31 + sub v10.4s, v10.4s, v18.4s + sub v18.4s, v14.4s, v0.4s + ushr v18.4s, v18.4s, #31 + neg v18.4s, v18.4s + and v14.16b, v14.16b, v18.16b + mov v15.16b, v1.16b + mla v15.4s, v7.4s, v0.4s + sshr v15.4s, v15.4s, #23 + mov v11.16b, v7.16b + mls v11.4s, v15.4s, v2.4s + sub v19.4s, v3.4s, v11.4s + usra v15.4s, v19.4s, #31 + mov v11.16b, v7.16b + mls v11.4s, v15.4s, v2.4s + sub v19.4s, v4.4s, v15.4s + ushr v19.4s, v19.4s, #31 + sub v11.4s, v11.4s, v19.4s + sub v19.4s, v15.4s, v0.4s + ushr v19.4s, v19.4s, #31 + neg v19.4s, v19.4s + and v15.16b, v15.16b, v19.16b + mov v16.16b, v1.16b + mla v16.4s, v8.4s, v0.4s + sshr v16.4s, v16.4s, #23 + mov v12.16b, v8.16b + mls v12.4s, v16.4s, v2.4s + sub v20.4s, v3.4s, v12.4s + usra v16.4s, v20.4s, #31 + mov v12.16b, v8.16b + mls v12.4s, v16.4s, v2.4s + sub v20.4s, v4.4s, v16.4s + ushr v20.4s, v20.4s, #31 + sub v12.4s, v12.4s, v20.4s + sub v20.4s, v16.4s, v0.4s + ushr v20.4s, v20.4s, #31 + neg v20.4s, v20.4s + and v16.16b, v16.16b, v20.16b + str q9, [x1] + str q13, [x2] + str q10, [x1, #16] + str q14, [x2, #16] + str q11, [x1, #32] + str q15, [x2, #32] + str q12, [x1, #48] + str q16, [x2, #48] + add x0, x0, #0x40 + add x1, x1, #0x40 + add x2, x2, #0x40 + subs x4, x4, #1 + bne L_mldsa_decompose_q88_loop + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_decompose_q88_neon,.-mldsa_decompose_q88_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decompose_q32_consts, %object + .size L_mldsa_decompose_q32_consts, 80 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decompose_q32_consts: + .long 0x0003feff,0x0003feff,0x0003feff,0x0003feff + .long 0x0003ff00,0x0003ff00,0x0003ff00,0x0003ff00 + .long 0x0000000f,0x0000000f,0x0000000f,0x0000000f + .long 0x0007fe00,0x0007fe00,0x0007fe00,0x0007fe00 + .long 0x007fe001,0x007fe001,0x007fe001,0x007fe001 +#ifndef __APPLE__ +.text +.globl mldsa_decompose_q32_neon +WC_ASM_ATT_HIDDEN(mldsa_decompose_q32_neon) +.type mldsa_decompose_q32_neon,@function +.align 2 +mldsa_decompose_q32_neon: +#else +.section __TEXT,__text +.globl _mldsa_decompose_q32_neon +WC_ASM_ATT_HIDDEN(_mldsa_decompose_q32_neon) +.p2align 2 +_mldsa_decompose_q32_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x3, L_mldsa_decompose_q32_consts + add x3, x3, :lo12:L_mldsa_decompose_q32_consts +#else + adrp x3, L_mldsa_decompose_q32_consts@PAGE + add x3, x3, L_mldsa_decompose_q32_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q0, [x3] + ldr q1, [x3, #16] + ldr q2, [x3, #32] + ldr q3, [x3, #48] + mov x4, #16 +L_mldsa_decompose_q32_loop: + ldr q4, [x0] + ldr q5, [x0, #16] + ldr q6, [x0, #32] + ldr q7, [x0, #48] + add v12.4s, v4.4s, v0.4s + sshr v12.4s, v12.4s, #19 + mov v8.16b, v4.16b + mls v8.4s, v12.4s, v3.4s + sub v16.4s, v1.4s, v8.4s + usra v12.4s, v16.4s, #31 + mov v8.16b, v4.16b + mls v8.4s, v12.4s, v3.4s + sshr v16.4s, v12.4s, #4 + sub v8.4s, v8.4s, v16.4s + and v12.16b, v12.16b, v2.16b + add v13.4s, v5.4s, v0.4s + sshr v13.4s, v13.4s, #19 + mov v9.16b, v5.16b + mls v9.4s, v13.4s, v3.4s + sub v17.4s, v1.4s, v9.4s + usra v13.4s, v17.4s, #31 + mov v9.16b, v5.16b + mls v9.4s, v13.4s, v3.4s + sshr v17.4s, v13.4s, #4 + sub v9.4s, v9.4s, v17.4s + and v13.16b, v13.16b, v2.16b + add v14.4s, v6.4s, v0.4s + sshr v14.4s, v14.4s, #19 + mov v10.16b, v6.16b + mls v10.4s, v14.4s, v3.4s + sub v18.4s, v1.4s, v10.4s + usra v14.4s, v18.4s, #31 + mov v10.16b, v6.16b + mls v10.4s, v14.4s, v3.4s + sshr v18.4s, v14.4s, #4 + sub v10.4s, v10.4s, v18.4s + and v14.16b, v14.16b, v2.16b + add v15.4s, v7.4s, v0.4s + sshr v15.4s, v15.4s, #19 + mov v11.16b, v7.16b + mls v11.4s, v15.4s, v3.4s + sub v19.4s, v1.4s, v11.4s + usra v15.4s, v19.4s, #31 + mov v11.16b, v7.16b + mls v11.4s, v15.4s, v3.4s + sshr v19.4s, v15.4s, #4 + sub v11.4s, v11.4s, v19.4s + and v15.16b, v15.16b, v2.16b + str q8, [x1] + str q12, [x2] + str q9, [x1, #16] + str q13, [x2, #16] + str q10, [x1, #32] + str q14, [x2, #32] + str q11, [x1, #48] + str q15, [x2, #48] + add x0, x0, #0x40 + add x1, x1, #0x40 + add x2, x2, #0x40 + subs x4, x4, #1 + bne L_mldsa_decompose_q32_loop + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_decompose_q32_neon,.-mldsa_decompose_q32_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_use_hint_q88_consts, %object + .size L_mldsa_use_hint_q88_consts, 96 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_use_hint_q88_consts: + .long 0x0000002c,0x0000002c,0x0000002c,0x0000002c + .long 0x003fefd4,0x003fefd4,0x003fefd4,0x003fefd4 + .long 0x0002e800,0x0002e800,0x0002e800,0x0002e800 + .long 0x00017400,0x00017400,0x00017400,0x00017400 + .long 0x0000002b,0x0000002b,0x0000002b,0x0000002b + .long 0x007fe001,0x007fe001,0x007fe001,0x007fe001 +#ifndef __APPLE__ +.text +.globl mldsa_use_hint_q88_neon +WC_ASM_ATT_HIDDEN(mldsa_use_hint_q88_neon) +.type mldsa_use_hint_q88_neon,@function +.align 2 +mldsa_use_hint_q88_neon: +#else +.section __TEXT,__text +.globl _mldsa_use_hint_q88_neon +WC_ASM_ATT_HIDDEN(_mldsa_use_hint_q88_neon) +.p2align 2 +_mldsa_use_hint_q88_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x1, L_mldsa_use_hint_q88_consts + add x1, x1, :lo12:L_mldsa_use_hint_q88_consts +#else + adrp x1, L_mldsa_use_hint_q88_consts@PAGE + add x1, x1, L_mldsa_use_hint_q88_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q0, [x1] + ldr q1, [x1, #16] + ldr q2, [x1, #32] + ldr q3, [x1, #48] + ldr q4, [x1, #64] + ldr q5, [x1, #80] + mov x2, #16 +L_mldsa_use_hint_q88_loop: + ldr q6, [x0] + ldr q7, [x0, #16] + ldr q8, [x0, #32] + ldr q9, [x0, #48] + sshr v18.4s, v6.4s, #31 + and v18.16b, v18.16b, v5.16b + add v6.4s, v6.4s, v18.4s + sshr v19.4s, v7.4s, #31 + and v19.16b, v19.16b, v5.16b + add v7.4s, v7.4s, v19.4s + sshr v20.4s, v8.4s, #31 + and v20.16b, v20.16b, v5.16b + add v8.4s, v8.4s, v20.4s + sshr v21.4s, v9.4s, #31 + and v21.16b, v21.16b, v5.16b + add v9.4s, v9.4s, v21.4s + mov v14.16b, v1.16b + mla v14.4s, v6.4s, v0.4s + sshr v14.4s, v14.4s, #23 + mov v10.16b, v6.16b + mls v10.4s, v14.4s, v2.4s + sub v18.4s, v3.4s, v10.4s + usra v14.4s, v18.4s, #31 + mov v10.16b, v6.16b + mls v10.4s, v14.4s, v2.4s + sub v18.4s, v4.4s, v14.4s + ushr v18.4s, v18.4s, #31 + sub v10.4s, v10.4s, v18.4s + sub v18.4s, v14.4s, v0.4s + ushr v18.4s, v18.4s, #31 + neg v18.4s, v18.4s + and v14.16b, v14.16b, v18.16b + mov v15.16b, v1.16b + mla v15.4s, v7.4s, v0.4s + sshr v15.4s, v15.4s, #23 + mov v11.16b, v7.16b + mls v11.4s, v15.4s, v2.4s + sub v19.4s, v3.4s, v11.4s + usra v15.4s, v19.4s, #31 + mov v11.16b, v7.16b + mls v11.4s, v15.4s, v2.4s + sub v19.4s, v4.4s, v15.4s + ushr v19.4s, v19.4s, #31 + sub v11.4s, v11.4s, v19.4s + sub v19.4s, v15.4s, v0.4s + ushr v19.4s, v19.4s, #31 + neg v19.4s, v19.4s + and v15.16b, v15.16b, v19.16b + mov v16.16b, v1.16b + mla v16.4s, v8.4s, v0.4s + sshr v16.4s, v16.4s, #23 + mov v12.16b, v8.16b + mls v12.4s, v16.4s, v2.4s + sub v20.4s, v3.4s, v12.4s + usra v16.4s, v20.4s, #31 + mov v12.16b, v8.16b + mls v12.4s, v16.4s, v2.4s + sub v20.4s, v4.4s, v16.4s + ushr v20.4s, v20.4s, #31 + sub v12.4s, v12.4s, v20.4s + sub v20.4s, v16.4s, v0.4s + ushr v20.4s, v20.4s, #31 + neg v20.4s, v20.4s + and v16.16b, v16.16b, v20.16b + mov v17.16b, v1.16b + mla v17.4s, v9.4s, v0.4s + sshr v17.4s, v17.4s, #23 + mov v13.16b, v9.16b + mls v13.4s, v17.4s, v2.4s + sub v21.4s, v3.4s, v13.4s + usra v17.4s, v21.4s, #31 + mov v13.16b, v9.16b + mls v13.4s, v17.4s, v2.4s + sub v21.4s, v4.4s, v17.4s + ushr v21.4s, v21.4s, #31 + sub v13.4s, v13.4s, v21.4s + sub v21.4s, v17.4s, v0.4s + ushr v21.4s, v21.4s, #31 + neg v21.4s, v21.4s + and v17.16b, v17.16b, v21.16b + str q14, [x0] + str q15, [x0, #16] + str q16, [x0, #32] + str q17, [x0, #48] + add x0, x0, #0x40 + subs x2, x2, #1 + bne L_mldsa_use_hint_q88_loop + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_use_hint_q88_neon,.-mldsa_use_hint_q88_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_use_hint_q32_consts, %object + .size L_mldsa_use_hint_q32_consts, 80 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_use_hint_q32_consts: + .long 0x0003feff,0x0003feff,0x0003feff,0x0003feff + .long 0x0003ff00,0x0003ff00,0x0003ff00,0x0003ff00 + .long 0x0000000f,0x0000000f,0x0000000f,0x0000000f + .long 0x0007fe00,0x0007fe00,0x0007fe00,0x0007fe00 + .long 0x007fe001,0x007fe001,0x007fe001,0x007fe001 +#ifndef __APPLE__ +.text +.globl mldsa_use_hint_q32_neon +WC_ASM_ATT_HIDDEN(mldsa_use_hint_q32_neon) +.type mldsa_use_hint_q32_neon,@function +.align 2 +mldsa_use_hint_q32_neon: +#else +.section __TEXT,__text +.globl _mldsa_use_hint_q32_neon +WC_ASM_ATT_HIDDEN(_mldsa_use_hint_q32_neon) +.p2align 2 +_mldsa_use_hint_q32_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x1, L_mldsa_use_hint_q32_consts + add x1, x1, :lo12:L_mldsa_use_hint_q32_consts +#else + adrp x1, L_mldsa_use_hint_q32_consts@PAGE + add x1, x1, L_mldsa_use_hint_q32_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q0, [x1] + ldr q1, [x1, #16] + ldr q2, [x1, #32] + ldr q3, [x1, #48] + ldr q4, [x1, #64] + mov x2, #16 +L_mldsa_use_hint_q32_loop: + ldr q5, [x0] + ldr q6, [x0, #16] + ldr q7, [x0, #32] + ldr q8, [x0, #48] + sshr v17.4s, v5.4s, #31 + and v17.16b, v17.16b, v4.16b + add v5.4s, v5.4s, v17.4s + sshr v18.4s, v6.4s, #31 + and v18.16b, v18.16b, v4.16b + add v6.4s, v6.4s, v18.4s + sshr v19.4s, v7.4s, #31 + and v19.16b, v19.16b, v4.16b + add v7.4s, v7.4s, v19.4s + sshr v20.4s, v8.4s, #31 + and v20.16b, v20.16b, v4.16b + add v8.4s, v8.4s, v20.4s + add v13.4s, v5.4s, v0.4s + sshr v13.4s, v13.4s, #19 + mov v9.16b, v5.16b + mls v9.4s, v13.4s, v3.4s + sub v17.4s, v1.4s, v9.4s + usra v13.4s, v17.4s, #31 + mov v9.16b, v5.16b + mls v9.4s, v13.4s, v3.4s + sshr v17.4s, v13.4s, #4 + sub v9.4s, v9.4s, v17.4s + and v13.16b, v13.16b, v2.16b + add v14.4s, v6.4s, v0.4s + sshr v14.4s, v14.4s, #19 + mov v10.16b, v6.16b + mls v10.4s, v14.4s, v3.4s + sub v18.4s, v1.4s, v10.4s + usra v14.4s, v18.4s, #31 + mov v10.16b, v6.16b + mls v10.4s, v14.4s, v3.4s + sshr v18.4s, v14.4s, #4 + sub v10.4s, v10.4s, v18.4s + and v14.16b, v14.16b, v2.16b + add v15.4s, v7.4s, v0.4s + sshr v15.4s, v15.4s, #19 + mov v11.16b, v7.16b + mls v11.4s, v15.4s, v3.4s + sub v19.4s, v1.4s, v11.4s + usra v15.4s, v19.4s, #31 + mov v11.16b, v7.16b + mls v11.4s, v15.4s, v3.4s + sshr v19.4s, v15.4s, #4 + sub v11.4s, v11.4s, v19.4s + and v15.16b, v15.16b, v2.16b + add v16.4s, v8.4s, v0.4s + sshr v16.4s, v16.4s, #19 + mov v12.16b, v8.16b + mls v12.4s, v16.4s, v3.4s + sub v20.4s, v1.4s, v12.4s + usra v16.4s, v20.4s, #31 + mov v12.16b, v8.16b + mls v12.4s, v16.4s, v3.4s + sshr v20.4s, v16.4s, #4 + sub v12.4s, v12.4s, v20.4s + and v16.16b, v16.16b, v2.16b + str q13, [x0] + str q14, [x0, #16] + str q15, [x0, #32] + str q16, [x0, #48] + add x0, x0, #0x40 + subs x2, x2, #1 + bne L_mldsa_use_hint_q32_loop + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_use_hint_q32_neon,.-mldsa_use_hint_q32_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_make_hint_bits, %object + .size L_mldsa_make_hint_bits, 64 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_make_hint_bits: + .long 0x00000001,0x00000002,0x00000004,0x00000008 + .long 0x00000010,0x00000020,0x00000040,0x00000080 + .long 0x00000100,0x00000200,0x00000400,0x00000800 + .long 0x00001000,0x00002000,0x00004000,0x00008000 +#ifndef __APPLE__ +.text +.globl mldsa_make_hint_neon +WC_ASM_ATT_HIDDEN(mldsa_make_hint_neon) +.type mldsa_make_hint_neon,@function +.align 2 +mldsa_make_hint_neon: +#else +.section __TEXT,__text +.globl _mldsa_make_hint_neon +WC_ASM_ATT_HIDDEN(_mldsa_make_hint_neon) +.p2align 2 +_mldsa_make_hint_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x6, L_mldsa_make_hint_bits + add x6, x6, :lo12:L_mldsa_make_hint_bits +#else + adrp x6, L_mldsa_make_hint_bits@PAGE + add x6, x6, L_mldsa_make_hint_bits@PAGEOFF +#endif /* __APPLE__ */ + dup v0.4s, w5 + neg v1.4s, v0.4s + ldr q2, [x6] + ldr q3, [x6, #16] + ldr q4, [x6, #32] + ldr q5, [x6, #48] + eor x7, x7, x7 +L_mldsa_make_hint_loop: + ldr q6, [x0] + ldr q10, [x1] + ldr q7, [x0, #16] + ldr q11, [x1, #16] + ldr q8, [x0, #32] + ldr q12, [x1, #32] + ldr q9, [x0, #48] + ldr q13, [x1, #48] + cmgt v14.4s, v6.4s, v0.4s + cmgt v18.4s, v1.4s, v6.4s + cmeq v22.4s, v6.4s, v1.4s + cmeq v10.4s, v10.4s, #0 + bic v22.16b, v22.16b, v10.16b + orr v14.16b, v14.16b, v18.16b + orr v14.16b, v14.16b, v22.16b + and v14.16b, v14.16b, v2.16b + cmgt v15.4s, v7.4s, v0.4s + cmgt v19.4s, v1.4s, v7.4s + cmeq v23.4s, v7.4s, v1.4s + cmeq v11.4s, v11.4s, #0 + bic v23.16b, v23.16b, v11.16b + orr v15.16b, v15.16b, v19.16b + orr v15.16b, v15.16b, v23.16b + and v15.16b, v15.16b, v3.16b + cmgt v16.4s, v8.4s, v0.4s + cmgt v20.4s, v1.4s, v8.4s + cmeq v24.4s, v8.4s, v1.4s + cmeq v12.4s, v12.4s, #0 + bic v24.16b, v24.16b, v12.16b + orr v16.16b, v16.16b, v20.16b + orr v16.16b, v16.16b, v24.16b + and v16.16b, v16.16b, v4.16b + cmgt v17.4s, v9.4s, v0.4s + cmgt v21.4s, v1.4s, v9.4s + cmeq v25.4s, v9.4s, v1.4s + cmeq v13.4s, v13.4s, #0 + bic v25.16b, v25.16b, v13.16b + orr v17.16b, v17.16b, v21.16b + orr v17.16b, v17.16b, v25.16b + and v17.16b, v17.16b, v5.16b + orr v14.16b, v14.16b, v15.16b + orr v16.16b, v16.16b, v17.16b + orr v14.16b, v14.16b, v16.16b + addv s14, v14.4s + mov w8, v14.s[0] + cbz w8, L_mldsa_make_hint_next +L_mldsa_make_hint_bit: + rbit w9, w8 + clz w9, w9 + add w9, w9, w7 + strb w9, [x2] + add x2, x2, #1 + add w3, w3, #1 + subs wzr, w3, w4 + bgt L_mldsa_make_hint_fail + sub w10, w8, #1 + and w8, w8, w10 + cbnz w8, L_mldsa_make_hint_bit +L_mldsa_make_hint_next: + add x0, x0, #0x40 + add x1, x1, #0x40 + add x7, x7, #16 + subs wzr, w7, #0x100 + blt L_mldsa_make_hint_loop + mov x0, x3 + b L_mldsa_make_hint_done +L_mldsa_make_hint_fail: + movn x0, #0 +L_mldsa_make_hint_done: + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_make_hint_neon,.-mldsa_make_hint_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mldsa_mul_vec_4_neon +WC_ASM_ATT_HIDDEN(mldsa_mul_vec_4_neon) +.type mldsa_mul_vec_4_neon,@function +.align 2 +mldsa_mul_vec_4_neon: +#else +.section __TEXT,__text +.globl _mldsa_mul_vec_4_neon +WC_ASM_ATT_HIDDEN(_mldsa_mul_vec_4_neon) +.p2align 2 +_mldsa_mul_vec_4_neon: +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_aarch64_consts + add x3, x3, :lo12:L_mldsa_aarch64_consts +#else + adrp x3, L_mldsa_aarch64_consts@PAGE + add x3, x3, L_mldsa_aarch64_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q0, [x3] + ldr q1, [x1] + ldr q2, [x2] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1024] + ldr q2, [x2, #1024] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2048] + ldr q2, [x2, #2048] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3072] + ldr q2, [x2, #3072] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0] + ldr q1, [x1, #16] + ldr q2, [x2, #16] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1040] + ldr q2, [x2, #1040] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2064] + ldr q2, [x2, #2064] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3088] + ldr q2, [x2, #3088] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #16] + ldr q1, [x1, #32] + ldr q2, [x2, #32] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1056] + ldr q2, [x2, #1056] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2080] + ldr q2, [x2, #2080] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3104] + ldr q2, [x2, #3104] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #32] + ldr q1, [x1, #48] + ldr q2, [x2, #48] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1072] + ldr q2, [x2, #1072] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2096] + ldr q2, [x2, #2096] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3120] + ldr q2, [x2, #3120] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #48] + ldr q1, [x1, #64] + ldr q2, [x2, #64] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1088] + ldr q2, [x2, #1088] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2112] + ldr q2, [x2, #2112] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3136] + ldr q2, [x2, #3136] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #64] + ldr q1, [x1, #80] + ldr q2, [x2, #80] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1104] + ldr q2, [x2, #1104] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2128] + ldr q2, [x2, #2128] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3152] + ldr q2, [x2, #3152] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #80] + ldr q1, [x1, #96] + ldr q2, [x2, #96] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1120] + ldr q2, [x2, #1120] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2144] + ldr q2, [x2, #2144] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3168] + ldr q2, [x2, #3168] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #96] + ldr q1, [x1, #112] + ldr q2, [x2, #112] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1136] + ldr q2, [x2, #1136] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2160] + ldr q2, [x2, #2160] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3184] + ldr q2, [x2, #3184] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #112] + ldr q1, [x1, #128] + ldr q2, [x2, #128] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1152] + ldr q2, [x2, #1152] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2176] + ldr q2, [x2, #2176] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3200] + ldr q2, [x2, #3200] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #128] + ldr q1, [x1, #144] + ldr q2, [x2, #144] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1168] + ldr q2, [x2, #1168] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2192] + ldr q2, [x2, #2192] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3216] + ldr q2, [x2, #3216] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #144] + ldr q1, [x1, #160] + ldr q2, [x2, #160] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1184] + ldr q2, [x2, #1184] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2208] + ldr q2, [x2, #2208] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3232] + ldr q2, [x2, #3232] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #160] + ldr q1, [x1, #176] + ldr q2, [x2, #176] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1200] + ldr q2, [x2, #1200] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2224] + ldr q2, [x2, #2224] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3248] + ldr q2, [x2, #3248] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #176] + ldr q1, [x1, #192] + ldr q2, [x2, #192] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1216] + ldr q2, [x2, #1216] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2240] + ldr q2, [x2, #2240] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3264] + ldr q2, [x2, #3264] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #192] + ldr q1, [x1, #208] + ldr q2, [x2, #208] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1232] + ldr q2, [x2, #1232] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2256] + ldr q2, [x2, #2256] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3280] + ldr q2, [x2, #3280] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #208] + ldr q1, [x1, #224] + ldr q2, [x2, #224] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1248] + ldr q2, [x2, #1248] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2272] + ldr q2, [x2, #2272] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3296] + ldr q2, [x2, #3296] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #224] + ldr q1, [x1, #240] + ldr q2, [x2, #240] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1264] + ldr q2, [x2, #1264] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2288] + ldr q2, [x2, #2288] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3312] + ldr q2, [x2, #3312] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #240] + ldr q1, [x1, #256] + ldr q2, [x2, #256] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1280] + ldr q2, [x2, #1280] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2304] + ldr q2, [x2, #2304] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3328] + ldr q2, [x2, #3328] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #256] + ldr q1, [x1, #272] + ldr q2, [x2, #272] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1296] + ldr q2, [x2, #1296] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2320] + ldr q2, [x2, #2320] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3344] + ldr q2, [x2, #3344] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #272] + ldr q1, [x1, #288] + ldr q2, [x2, #288] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1312] + ldr q2, [x2, #1312] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2336] + ldr q2, [x2, #2336] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3360] + ldr q2, [x2, #3360] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #288] + ldr q1, [x1, #304] + ldr q2, [x2, #304] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1328] + ldr q2, [x2, #1328] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2352] + ldr q2, [x2, #2352] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3376] + ldr q2, [x2, #3376] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #304] + ldr q1, [x1, #320] + ldr q2, [x2, #320] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1344] + ldr q2, [x2, #1344] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2368] + ldr q2, [x2, #2368] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3392] + ldr q2, [x2, #3392] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #320] + ldr q1, [x1, #336] + ldr q2, [x2, #336] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1360] + ldr q2, [x2, #1360] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2384] + ldr q2, [x2, #2384] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3408] + ldr q2, [x2, #3408] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #336] + ldr q1, [x1, #352] + ldr q2, [x2, #352] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1376] + ldr q2, [x2, #1376] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2400] + ldr q2, [x2, #2400] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3424] + ldr q2, [x2, #3424] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #352] + ldr q1, [x1, #368] + ldr q2, [x2, #368] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1392] + ldr q2, [x2, #1392] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2416] + ldr q2, [x2, #2416] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3440] + ldr q2, [x2, #3440] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #368] + ldr q1, [x1, #384] + ldr q2, [x2, #384] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1408] + ldr q2, [x2, #1408] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2432] + ldr q2, [x2, #2432] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3456] + ldr q2, [x2, #3456] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #384] + ldr q1, [x1, #400] + ldr q2, [x2, #400] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1424] + ldr q2, [x2, #1424] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2448] + ldr q2, [x2, #2448] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3472] + ldr q2, [x2, #3472] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #400] + ldr q1, [x1, #416] + ldr q2, [x2, #416] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1440] + ldr q2, [x2, #1440] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2464] + ldr q2, [x2, #2464] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3488] + ldr q2, [x2, #3488] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #416] + ldr q1, [x1, #432] + ldr q2, [x2, #432] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1456] + ldr q2, [x2, #1456] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2480] + ldr q2, [x2, #2480] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3504] + ldr q2, [x2, #3504] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #432] + ldr q1, [x1, #448] + ldr q2, [x2, #448] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1472] + ldr q2, [x2, #1472] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2496] + ldr q2, [x2, #2496] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3520] + ldr q2, [x2, #3520] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #448] + ldr q1, [x1, #464] + ldr q2, [x2, #464] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1488] + ldr q2, [x2, #1488] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2512] + ldr q2, [x2, #2512] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3536] + ldr q2, [x2, #3536] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #464] + ldr q1, [x1, #480] + ldr q2, [x2, #480] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1504] + ldr q2, [x2, #1504] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2528] + ldr q2, [x2, #2528] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3552] + ldr q2, [x2, #3552] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #480] + ldr q1, [x1, #496] + ldr q2, [x2, #496] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1520] + ldr q2, [x2, #1520] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2544] + ldr q2, [x2, #2544] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3568] + ldr q2, [x2, #3568] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #496] + ldr q1, [x1, #512] + ldr q2, [x2, #512] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1536] + ldr q2, [x2, #1536] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2560] + ldr q2, [x2, #2560] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3584] + ldr q2, [x2, #3584] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #512] + ldr q1, [x1, #528] + ldr q2, [x2, #528] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1552] + ldr q2, [x2, #1552] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2576] + ldr q2, [x2, #2576] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3600] + ldr q2, [x2, #3600] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #528] + ldr q1, [x1, #544] + ldr q2, [x2, #544] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1568] + ldr q2, [x2, #1568] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2592] + ldr q2, [x2, #2592] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3616] + ldr q2, [x2, #3616] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #544] + ldr q1, [x1, #560] + ldr q2, [x2, #560] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1584] + ldr q2, [x2, #1584] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2608] + ldr q2, [x2, #2608] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3632] + ldr q2, [x2, #3632] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #560] + ldr q1, [x1, #576] + ldr q2, [x2, #576] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1600] + ldr q2, [x2, #1600] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2624] + ldr q2, [x2, #2624] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3648] + ldr q2, [x2, #3648] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #576] + ldr q1, [x1, #592] + ldr q2, [x2, #592] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1616] + ldr q2, [x2, #1616] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2640] + ldr q2, [x2, #2640] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3664] + ldr q2, [x2, #3664] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #592] + ldr q1, [x1, #608] + ldr q2, [x2, #608] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1632] + ldr q2, [x2, #1632] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2656] + ldr q2, [x2, #2656] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3680] + ldr q2, [x2, #3680] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #608] + ldr q1, [x1, #624] + ldr q2, [x2, #624] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1648] + ldr q2, [x2, #1648] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2672] + ldr q2, [x2, #2672] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3696] + ldr q2, [x2, #3696] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #624] + ldr q1, [x1, #640] + ldr q2, [x2, #640] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1664] + ldr q2, [x2, #1664] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2688] + ldr q2, [x2, #2688] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3712] + ldr q2, [x2, #3712] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #640] + ldr q1, [x1, #656] + ldr q2, [x2, #656] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1680] + ldr q2, [x2, #1680] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2704] + ldr q2, [x2, #2704] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3728] + ldr q2, [x2, #3728] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #656] + ldr q1, [x1, #672] + ldr q2, [x2, #672] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1696] + ldr q2, [x2, #1696] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2720] + ldr q2, [x2, #2720] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3744] + ldr q2, [x2, #3744] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #672] + ldr q1, [x1, #688] + ldr q2, [x2, #688] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1712] + ldr q2, [x2, #1712] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2736] + ldr q2, [x2, #2736] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3760] + ldr q2, [x2, #3760] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #688] + ldr q1, [x1, #704] + ldr q2, [x2, #704] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1728] + ldr q2, [x2, #1728] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2752] + ldr q2, [x2, #2752] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3776] + ldr q2, [x2, #3776] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #704] + ldr q1, [x1, #720] + ldr q2, [x2, #720] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1744] + ldr q2, [x2, #1744] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2768] + ldr q2, [x2, #2768] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3792] + ldr q2, [x2, #3792] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #720] + ldr q1, [x1, #736] + ldr q2, [x2, #736] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1760] + ldr q2, [x2, #1760] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2784] + ldr q2, [x2, #2784] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3808] + ldr q2, [x2, #3808] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #736] + ldr q1, [x1, #752] + ldr q2, [x2, #752] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1776] + ldr q2, [x2, #1776] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2800] + ldr q2, [x2, #2800] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3824] + ldr q2, [x2, #3824] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #752] + ldr q1, [x1, #768] + ldr q2, [x2, #768] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1792] + ldr q2, [x2, #1792] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2816] + ldr q2, [x2, #2816] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3840] + ldr q2, [x2, #3840] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #768] + ldr q1, [x1, #784] + ldr q2, [x2, #784] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1808] + ldr q2, [x2, #1808] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2832] + ldr q2, [x2, #2832] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3856] + ldr q2, [x2, #3856] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #784] + ldr q1, [x1, #800] + ldr q2, [x2, #800] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1824] + ldr q2, [x2, #1824] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2848] + ldr q2, [x2, #2848] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3872] + ldr q2, [x2, #3872] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #800] + ldr q1, [x1, #816] + ldr q2, [x2, #816] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1840] + ldr q2, [x2, #1840] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2864] + ldr q2, [x2, #2864] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3888] + ldr q2, [x2, #3888] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #816] + ldr q1, [x1, #832] + ldr q2, [x2, #832] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1856] + ldr q2, [x2, #1856] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2880] + ldr q2, [x2, #2880] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3904] + ldr q2, [x2, #3904] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #832] + ldr q1, [x1, #848] + ldr q2, [x2, #848] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1872] + ldr q2, [x2, #1872] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2896] + ldr q2, [x2, #2896] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3920] + ldr q2, [x2, #3920] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #848] + ldr q1, [x1, #864] + ldr q2, [x2, #864] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1888] + ldr q2, [x2, #1888] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2912] + ldr q2, [x2, #2912] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3936] + ldr q2, [x2, #3936] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #864] + ldr q1, [x1, #880] + ldr q2, [x2, #880] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1904] + ldr q2, [x2, #1904] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2928] + ldr q2, [x2, #2928] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3952] + ldr q2, [x2, #3952] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #880] + ldr q1, [x1, #896] + ldr q2, [x2, #896] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1920] + ldr q2, [x2, #1920] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2944] + ldr q2, [x2, #2944] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3968] + ldr q2, [x2, #3968] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #896] + ldr q1, [x1, #912] + ldr q2, [x2, #912] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1936] + ldr q2, [x2, #1936] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2960] + ldr q2, [x2, #2960] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3984] + ldr q2, [x2, #3984] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #912] + ldr q1, [x1, #928] + ldr q2, [x2, #928] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1952] + ldr q2, [x2, #1952] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2976] + ldr q2, [x2, #2976] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4000] + ldr q2, [x2, #4000] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #928] + ldr q1, [x1, #944] + ldr q2, [x2, #944] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1968] + ldr q2, [x2, #1968] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2992] + ldr q2, [x2, #2992] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4016] + ldr q2, [x2, #4016] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #944] + ldr q1, [x1, #960] + ldr q2, [x2, #960] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1984] + ldr q2, [x2, #1984] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3008] + ldr q2, [x2, #3008] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4032] + ldr q2, [x2, #4032] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #960] + ldr q1, [x1, #976] + ldr q2, [x2, #976] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2000] + ldr q2, [x2, #2000] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3024] + ldr q2, [x2, #3024] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4048] + ldr q2, [x2, #4048] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #976] + ldr q1, [x1, #992] + ldr q2, [x2, #992] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2016] + ldr q2, [x2, #2016] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3040] + ldr q2, [x2, #3040] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4064] + ldr q2, [x2, #4064] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #992] + ldr q1, [x1, #1008] + ldr q2, [x2, #1008] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2032] + ldr q2, [x2, #2032] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3056] + ldr q2, [x2, #3056] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4080] + ldr q2, [x2, #4080] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #1008] + ret +#ifndef __APPLE__ + .size mldsa_mul_vec_4_neon,.-mldsa_mul_vec_4_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mldsa_mul_vec_5_neon +WC_ASM_ATT_HIDDEN(mldsa_mul_vec_5_neon) +.type mldsa_mul_vec_5_neon,@function +.align 2 +mldsa_mul_vec_5_neon: +#else +.section __TEXT,__text +.globl _mldsa_mul_vec_5_neon +WC_ASM_ATT_HIDDEN(_mldsa_mul_vec_5_neon) +.p2align 2 +_mldsa_mul_vec_5_neon: +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_aarch64_consts + add x3, x3, :lo12:L_mldsa_aarch64_consts +#else + adrp x3, L_mldsa_aarch64_consts@PAGE + add x3, x3, L_mldsa_aarch64_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q0, [x3] + ldr q1, [x1] + ldr q2, [x2] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1024] + ldr q2, [x2, #1024] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2048] + ldr q2, [x2, #2048] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3072] + ldr q2, [x2, #3072] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4096] + ldr q2, [x2, #4096] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0] + ldr q1, [x1, #16] + ldr q2, [x2, #16] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1040] + ldr q2, [x2, #1040] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2064] + ldr q2, [x2, #2064] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3088] + ldr q2, [x2, #3088] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4112] + ldr q2, [x2, #4112] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #16] + ldr q1, [x1, #32] + ldr q2, [x2, #32] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1056] + ldr q2, [x2, #1056] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2080] + ldr q2, [x2, #2080] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3104] + ldr q2, [x2, #3104] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4128] + ldr q2, [x2, #4128] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #32] + ldr q1, [x1, #48] + ldr q2, [x2, #48] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1072] + ldr q2, [x2, #1072] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2096] + ldr q2, [x2, #2096] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3120] + ldr q2, [x2, #3120] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4144] + ldr q2, [x2, #4144] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #48] + ldr q1, [x1, #64] + ldr q2, [x2, #64] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1088] + ldr q2, [x2, #1088] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2112] + ldr q2, [x2, #2112] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3136] + ldr q2, [x2, #3136] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4160] + ldr q2, [x2, #4160] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #64] + ldr q1, [x1, #80] + ldr q2, [x2, #80] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1104] + ldr q2, [x2, #1104] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2128] + ldr q2, [x2, #2128] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3152] + ldr q2, [x2, #3152] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4176] + ldr q2, [x2, #4176] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #80] + ldr q1, [x1, #96] + ldr q2, [x2, #96] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1120] + ldr q2, [x2, #1120] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2144] + ldr q2, [x2, #2144] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3168] + ldr q2, [x2, #3168] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4192] + ldr q2, [x2, #4192] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #96] + ldr q1, [x1, #112] + ldr q2, [x2, #112] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1136] + ldr q2, [x2, #1136] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2160] + ldr q2, [x2, #2160] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3184] + ldr q2, [x2, #3184] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4208] + ldr q2, [x2, #4208] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #112] + ldr q1, [x1, #128] + ldr q2, [x2, #128] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1152] + ldr q2, [x2, #1152] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2176] + ldr q2, [x2, #2176] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3200] + ldr q2, [x2, #3200] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4224] + ldr q2, [x2, #4224] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #128] + ldr q1, [x1, #144] + ldr q2, [x2, #144] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1168] + ldr q2, [x2, #1168] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2192] + ldr q2, [x2, #2192] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3216] + ldr q2, [x2, #3216] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4240] + ldr q2, [x2, #4240] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #144] + ldr q1, [x1, #160] + ldr q2, [x2, #160] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1184] + ldr q2, [x2, #1184] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2208] + ldr q2, [x2, #2208] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3232] + ldr q2, [x2, #3232] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4256] + ldr q2, [x2, #4256] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #160] + ldr q1, [x1, #176] + ldr q2, [x2, #176] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1200] + ldr q2, [x2, #1200] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2224] + ldr q2, [x2, #2224] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3248] + ldr q2, [x2, #3248] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4272] + ldr q2, [x2, #4272] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #176] + ldr q1, [x1, #192] + ldr q2, [x2, #192] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1216] + ldr q2, [x2, #1216] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2240] + ldr q2, [x2, #2240] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3264] + ldr q2, [x2, #3264] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4288] + ldr q2, [x2, #4288] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #192] + ldr q1, [x1, #208] + ldr q2, [x2, #208] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1232] + ldr q2, [x2, #1232] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2256] + ldr q2, [x2, #2256] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3280] + ldr q2, [x2, #3280] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4304] + ldr q2, [x2, #4304] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #208] + ldr q1, [x1, #224] + ldr q2, [x2, #224] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1248] + ldr q2, [x2, #1248] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2272] + ldr q2, [x2, #2272] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3296] + ldr q2, [x2, #3296] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4320] + ldr q2, [x2, #4320] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #224] + ldr q1, [x1, #240] + ldr q2, [x2, #240] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1264] + ldr q2, [x2, #1264] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2288] + ldr q2, [x2, #2288] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3312] + ldr q2, [x2, #3312] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4336] + ldr q2, [x2, #4336] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #240] + ldr q1, [x1, #256] + ldr q2, [x2, #256] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1280] + ldr q2, [x2, #1280] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2304] + ldr q2, [x2, #2304] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3328] + ldr q2, [x2, #3328] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4352] + ldr q2, [x2, #4352] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #256] + ldr q1, [x1, #272] + ldr q2, [x2, #272] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1296] + ldr q2, [x2, #1296] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2320] + ldr q2, [x2, #2320] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3344] + ldr q2, [x2, #3344] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4368] + ldr q2, [x2, #4368] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #272] + ldr q1, [x1, #288] + ldr q2, [x2, #288] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1312] + ldr q2, [x2, #1312] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2336] + ldr q2, [x2, #2336] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3360] + ldr q2, [x2, #3360] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4384] + ldr q2, [x2, #4384] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #288] + ldr q1, [x1, #304] + ldr q2, [x2, #304] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1328] + ldr q2, [x2, #1328] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2352] + ldr q2, [x2, #2352] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3376] + ldr q2, [x2, #3376] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4400] + ldr q2, [x2, #4400] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #304] + ldr q1, [x1, #320] + ldr q2, [x2, #320] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1344] + ldr q2, [x2, #1344] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2368] + ldr q2, [x2, #2368] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3392] + ldr q2, [x2, #3392] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4416] + ldr q2, [x2, #4416] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #320] + ldr q1, [x1, #336] + ldr q2, [x2, #336] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1360] + ldr q2, [x2, #1360] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2384] + ldr q2, [x2, #2384] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3408] + ldr q2, [x2, #3408] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4432] + ldr q2, [x2, #4432] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #336] + ldr q1, [x1, #352] + ldr q2, [x2, #352] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1376] + ldr q2, [x2, #1376] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2400] + ldr q2, [x2, #2400] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3424] + ldr q2, [x2, #3424] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4448] + ldr q2, [x2, #4448] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #352] + ldr q1, [x1, #368] + ldr q2, [x2, #368] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1392] + ldr q2, [x2, #1392] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2416] + ldr q2, [x2, #2416] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3440] + ldr q2, [x2, #3440] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4464] + ldr q2, [x2, #4464] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #368] + ldr q1, [x1, #384] + ldr q2, [x2, #384] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1408] + ldr q2, [x2, #1408] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2432] + ldr q2, [x2, #2432] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3456] + ldr q2, [x2, #3456] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4480] + ldr q2, [x2, #4480] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #384] + ldr q1, [x1, #400] + ldr q2, [x2, #400] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1424] + ldr q2, [x2, #1424] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2448] + ldr q2, [x2, #2448] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3472] + ldr q2, [x2, #3472] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4496] + ldr q2, [x2, #4496] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #400] + ldr q1, [x1, #416] + ldr q2, [x2, #416] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1440] + ldr q2, [x2, #1440] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2464] + ldr q2, [x2, #2464] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3488] + ldr q2, [x2, #3488] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4512] + ldr q2, [x2, #4512] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #416] + ldr q1, [x1, #432] + ldr q2, [x2, #432] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1456] + ldr q2, [x2, #1456] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2480] + ldr q2, [x2, #2480] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3504] + ldr q2, [x2, #3504] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4528] + ldr q2, [x2, #4528] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #432] + ldr q1, [x1, #448] + ldr q2, [x2, #448] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1472] + ldr q2, [x2, #1472] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2496] + ldr q2, [x2, #2496] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3520] + ldr q2, [x2, #3520] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4544] + ldr q2, [x2, #4544] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #448] + ldr q1, [x1, #464] + ldr q2, [x2, #464] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1488] + ldr q2, [x2, #1488] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2512] + ldr q2, [x2, #2512] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3536] + ldr q2, [x2, #3536] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4560] + ldr q2, [x2, #4560] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #464] + ldr q1, [x1, #480] + ldr q2, [x2, #480] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1504] + ldr q2, [x2, #1504] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2528] + ldr q2, [x2, #2528] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3552] + ldr q2, [x2, #3552] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4576] + ldr q2, [x2, #4576] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #480] + ldr q1, [x1, #496] + ldr q2, [x2, #496] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1520] + ldr q2, [x2, #1520] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2544] + ldr q2, [x2, #2544] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3568] + ldr q2, [x2, #3568] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4592] + ldr q2, [x2, #4592] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #496] + ldr q1, [x1, #512] + ldr q2, [x2, #512] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1536] + ldr q2, [x2, #1536] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2560] + ldr q2, [x2, #2560] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3584] + ldr q2, [x2, #3584] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4608] + ldr q2, [x2, #4608] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #512] + ldr q1, [x1, #528] + ldr q2, [x2, #528] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1552] + ldr q2, [x2, #1552] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2576] + ldr q2, [x2, #2576] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3600] + ldr q2, [x2, #3600] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4624] + ldr q2, [x2, #4624] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #528] + ldr q1, [x1, #544] + ldr q2, [x2, #544] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1568] + ldr q2, [x2, #1568] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2592] + ldr q2, [x2, #2592] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3616] + ldr q2, [x2, #3616] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4640] + ldr q2, [x2, #4640] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #544] + ldr q1, [x1, #560] + ldr q2, [x2, #560] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1584] + ldr q2, [x2, #1584] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2608] + ldr q2, [x2, #2608] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3632] + ldr q2, [x2, #3632] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4656] + ldr q2, [x2, #4656] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #560] + ldr q1, [x1, #576] + ldr q2, [x2, #576] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1600] + ldr q2, [x2, #1600] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2624] + ldr q2, [x2, #2624] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3648] + ldr q2, [x2, #3648] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4672] + ldr q2, [x2, #4672] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #576] + ldr q1, [x1, #592] + ldr q2, [x2, #592] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1616] + ldr q2, [x2, #1616] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2640] + ldr q2, [x2, #2640] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3664] + ldr q2, [x2, #3664] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4688] + ldr q2, [x2, #4688] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #592] + ldr q1, [x1, #608] + ldr q2, [x2, #608] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1632] + ldr q2, [x2, #1632] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2656] + ldr q2, [x2, #2656] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3680] + ldr q2, [x2, #3680] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4704] + ldr q2, [x2, #4704] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #608] + ldr q1, [x1, #624] + ldr q2, [x2, #624] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1648] + ldr q2, [x2, #1648] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2672] + ldr q2, [x2, #2672] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3696] + ldr q2, [x2, #3696] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4720] + ldr q2, [x2, #4720] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #624] + ldr q1, [x1, #640] + ldr q2, [x2, #640] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1664] + ldr q2, [x2, #1664] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2688] + ldr q2, [x2, #2688] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3712] + ldr q2, [x2, #3712] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4736] + ldr q2, [x2, #4736] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #640] + ldr q1, [x1, #656] + ldr q2, [x2, #656] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1680] + ldr q2, [x2, #1680] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2704] + ldr q2, [x2, #2704] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3728] + ldr q2, [x2, #3728] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4752] + ldr q2, [x2, #4752] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #656] + ldr q1, [x1, #672] + ldr q2, [x2, #672] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1696] + ldr q2, [x2, #1696] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2720] + ldr q2, [x2, #2720] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3744] + ldr q2, [x2, #3744] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4768] + ldr q2, [x2, #4768] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #672] + ldr q1, [x1, #688] + ldr q2, [x2, #688] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1712] + ldr q2, [x2, #1712] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2736] + ldr q2, [x2, #2736] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3760] + ldr q2, [x2, #3760] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4784] + ldr q2, [x2, #4784] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #688] + ldr q1, [x1, #704] + ldr q2, [x2, #704] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1728] + ldr q2, [x2, #1728] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2752] + ldr q2, [x2, #2752] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3776] + ldr q2, [x2, #3776] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4800] + ldr q2, [x2, #4800] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #704] + ldr q1, [x1, #720] + ldr q2, [x2, #720] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1744] + ldr q2, [x2, #1744] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2768] + ldr q2, [x2, #2768] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3792] + ldr q2, [x2, #3792] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4816] + ldr q2, [x2, #4816] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #720] + ldr q1, [x1, #736] + ldr q2, [x2, #736] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1760] + ldr q2, [x2, #1760] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2784] + ldr q2, [x2, #2784] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3808] + ldr q2, [x2, #3808] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4832] + ldr q2, [x2, #4832] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #736] + ldr q1, [x1, #752] + ldr q2, [x2, #752] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1776] + ldr q2, [x2, #1776] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2800] + ldr q2, [x2, #2800] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3824] + ldr q2, [x2, #3824] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4848] + ldr q2, [x2, #4848] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #752] + ldr q1, [x1, #768] + ldr q2, [x2, #768] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1792] + ldr q2, [x2, #1792] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2816] + ldr q2, [x2, #2816] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3840] + ldr q2, [x2, #3840] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4864] + ldr q2, [x2, #4864] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #768] + ldr q1, [x1, #784] + ldr q2, [x2, #784] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1808] + ldr q2, [x2, #1808] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2832] + ldr q2, [x2, #2832] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3856] + ldr q2, [x2, #3856] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4880] + ldr q2, [x2, #4880] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #784] + ldr q1, [x1, #800] + ldr q2, [x2, #800] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1824] + ldr q2, [x2, #1824] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2848] + ldr q2, [x2, #2848] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3872] + ldr q2, [x2, #3872] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4896] + ldr q2, [x2, #4896] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #800] + ldr q1, [x1, #816] + ldr q2, [x2, #816] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1840] + ldr q2, [x2, #1840] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2864] + ldr q2, [x2, #2864] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3888] + ldr q2, [x2, #3888] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4912] + ldr q2, [x2, #4912] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #816] + ldr q1, [x1, #832] + ldr q2, [x2, #832] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1856] + ldr q2, [x2, #1856] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2880] + ldr q2, [x2, #2880] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3904] + ldr q2, [x2, #3904] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4928] + ldr q2, [x2, #4928] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #832] + ldr q1, [x1, #848] + ldr q2, [x2, #848] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1872] + ldr q2, [x2, #1872] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2896] + ldr q2, [x2, #2896] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3920] + ldr q2, [x2, #3920] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4944] + ldr q2, [x2, #4944] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #848] + ldr q1, [x1, #864] + ldr q2, [x2, #864] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1888] + ldr q2, [x2, #1888] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2912] + ldr q2, [x2, #2912] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3936] + ldr q2, [x2, #3936] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4960] + ldr q2, [x2, #4960] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #864] + ldr q1, [x1, #880] + ldr q2, [x2, #880] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1904] + ldr q2, [x2, #1904] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2928] + ldr q2, [x2, #2928] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3952] + ldr q2, [x2, #3952] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4976] + ldr q2, [x2, #4976] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #880] + ldr q1, [x1, #896] + ldr q2, [x2, #896] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1920] + ldr q2, [x2, #1920] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2944] + ldr q2, [x2, #2944] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3968] + ldr q2, [x2, #3968] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4992] + ldr q2, [x2, #4992] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #896] + ldr q1, [x1, #912] + ldr q2, [x2, #912] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1936] + ldr q2, [x2, #1936] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2960] + ldr q2, [x2, #2960] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3984] + ldr q2, [x2, #3984] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5008] + ldr q2, [x2, #5008] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #912] + ldr q1, [x1, #928] + ldr q2, [x2, #928] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1952] + ldr q2, [x2, #1952] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2976] + ldr q2, [x2, #2976] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4000] + ldr q2, [x2, #4000] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5024] + ldr q2, [x2, #5024] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #928] + ldr q1, [x1, #944] + ldr q2, [x2, #944] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1968] + ldr q2, [x2, #1968] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2992] + ldr q2, [x2, #2992] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4016] + ldr q2, [x2, #4016] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5040] + ldr q2, [x2, #5040] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #944] + ldr q1, [x1, #960] + ldr q2, [x2, #960] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1984] + ldr q2, [x2, #1984] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3008] + ldr q2, [x2, #3008] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4032] + ldr q2, [x2, #4032] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5056] + ldr q2, [x2, #5056] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #960] + ldr q1, [x1, #976] + ldr q2, [x2, #976] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2000] + ldr q2, [x2, #2000] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3024] + ldr q2, [x2, #3024] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4048] + ldr q2, [x2, #4048] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5072] + ldr q2, [x2, #5072] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #976] + ldr q1, [x1, #992] + ldr q2, [x2, #992] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2016] + ldr q2, [x2, #2016] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3040] + ldr q2, [x2, #3040] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4064] + ldr q2, [x2, #4064] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5088] + ldr q2, [x2, #5088] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #992] + ldr q1, [x1, #1008] + ldr q2, [x2, #1008] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2032] + ldr q2, [x2, #2032] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3056] + ldr q2, [x2, #3056] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4080] + ldr q2, [x2, #4080] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5104] + ldr q2, [x2, #5104] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #1008] + ret +#ifndef __APPLE__ + .size mldsa_mul_vec_5_neon,.-mldsa_mul_vec_5_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mldsa_mul_vec_7_neon +WC_ASM_ATT_HIDDEN(mldsa_mul_vec_7_neon) +.type mldsa_mul_vec_7_neon,@function +.align 2 +mldsa_mul_vec_7_neon: +#else +.section __TEXT,__text +.globl _mldsa_mul_vec_7_neon +WC_ASM_ATT_HIDDEN(_mldsa_mul_vec_7_neon) +.p2align 2 +_mldsa_mul_vec_7_neon: +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_aarch64_consts + add x3, x3, :lo12:L_mldsa_aarch64_consts +#else + adrp x3, L_mldsa_aarch64_consts@PAGE + add x3, x3, L_mldsa_aarch64_consts@PAGEOFF +#endif /* __APPLE__ */ + ldr q0, [x3] + ldr q1, [x1] + ldr q2, [x2] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1024] + ldr q2, [x2, #1024] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2048] + ldr q2, [x2, #2048] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3072] + ldr q2, [x2, #3072] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4096] + ldr q2, [x2, #4096] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5120] + ldr q2, [x2, #5120] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6144] + ldr q2, [x2, #6144] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0] + ldr q1, [x1, #16] + ldr q2, [x2, #16] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1040] + ldr q2, [x2, #1040] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2064] + ldr q2, [x2, #2064] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3088] + ldr q2, [x2, #3088] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4112] + ldr q2, [x2, #4112] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5136] + ldr q2, [x2, #5136] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6160] + ldr q2, [x2, #6160] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #16] + ldr q1, [x1, #32] + ldr q2, [x2, #32] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1056] + ldr q2, [x2, #1056] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2080] + ldr q2, [x2, #2080] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3104] + ldr q2, [x2, #3104] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4128] + ldr q2, [x2, #4128] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5152] + ldr q2, [x2, #5152] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6176] + ldr q2, [x2, #6176] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #32] + ldr q1, [x1, #48] + ldr q2, [x2, #48] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1072] + ldr q2, [x2, #1072] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2096] + ldr q2, [x2, #2096] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3120] + ldr q2, [x2, #3120] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4144] + ldr q2, [x2, #4144] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5168] + ldr q2, [x2, #5168] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6192] + ldr q2, [x2, #6192] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #48] + ldr q1, [x1, #64] + ldr q2, [x2, #64] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1088] + ldr q2, [x2, #1088] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2112] + ldr q2, [x2, #2112] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3136] + ldr q2, [x2, #3136] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4160] + ldr q2, [x2, #4160] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5184] + ldr q2, [x2, #5184] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6208] + ldr q2, [x2, #6208] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #64] + ldr q1, [x1, #80] + ldr q2, [x2, #80] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1104] + ldr q2, [x2, #1104] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2128] + ldr q2, [x2, #2128] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3152] + ldr q2, [x2, #3152] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4176] + ldr q2, [x2, #4176] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5200] + ldr q2, [x2, #5200] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6224] + ldr q2, [x2, #6224] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #80] + ldr q1, [x1, #96] + ldr q2, [x2, #96] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1120] + ldr q2, [x2, #1120] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2144] + ldr q2, [x2, #2144] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3168] + ldr q2, [x2, #3168] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4192] + ldr q2, [x2, #4192] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5216] + ldr q2, [x2, #5216] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6240] + ldr q2, [x2, #6240] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #96] + ldr q1, [x1, #112] + ldr q2, [x2, #112] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1136] + ldr q2, [x2, #1136] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2160] + ldr q2, [x2, #2160] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3184] + ldr q2, [x2, #3184] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4208] + ldr q2, [x2, #4208] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5232] + ldr q2, [x2, #5232] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6256] + ldr q2, [x2, #6256] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #112] + ldr q1, [x1, #128] + ldr q2, [x2, #128] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1152] + ldr q2, [x2, #1152] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2176] + ldr q2, [x2, #2176] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3200] + ldr q2, [x2, #3200] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4224] + ldr q2, [x2, #4224] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5248] + ldr q2, [x2, #5248] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6272] + ldr q2, [x2, #6272] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #128] + ldr q1, [x1, #144] + ldr q2, [x2, #144] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1168] + ldr q2, [x2, #1168] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2192] + ldr q2, [x2, #2192] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3216] + ldr q2, [x2, #3216] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4240] + ldr q2, [x2, #4240] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5264] + ldr q2, [x2, #5264] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6288] + ldr q2, [x2, #6288] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #144] + ldr q1, [x1, #160] + ldr q2, [x2, #160] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1184] + ldr q2, [x2, #1184] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2208] + ldr q2, [x2, #2208] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3232] + ldr q2, [x2, #3232] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4256] + ldr q2, [x2, #4256] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5280] + ldr q2, [x2, #5280] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6304] + ldr q2, [x2, #6304] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #160] + ldr q1, [x1, #176] + ldr q2, [x2, #176] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1200] + ldr q2, [x2, #1200] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2224] + ldr q2, [x2, #2224] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3248] + ldr q2, [x2, #3248] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4272] + ldr q2, [x2, #4272] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5296] + ldr q2, [x2, #5296] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6320] + ldr q2, [x2, #6320] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #176] + ldr q1, [x1, #192] + ldr q2, [x2, #192] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1216] + ldr q2, [x2, #1216] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2240] + ldr q2, [x2, #2240] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3264] + ldr q2, [x2, #3264] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4288] + ldr q2, [x2, #4288] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5312] + ldr q2, [x2, #5312] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6336] + ldr q2, [x2, #6336] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #192] + ldr q1, [x1, #208] + ldr q2, [x2, #208] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1232] + ldr q2, [x2, #1232] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2256] + ldr q2, [x2, #2256] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3280] + ldr q2, [x2, #3280] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4304] + ldr q2, [x2, #4304] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5328] + ldr q2, [x2, #5328] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6352] + ldr q2, [x2, #6352] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #208] + ldr q1, [x1, #224] + ldr q2, [x2, #224] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1248] + ldr q2, [x2, #1248] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2272] + ldr q2, [x2, #2272] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3296] + ldr q2, [x2, #3296] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4320] + ldr q2, [x2, #4320] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5344] + ldr q2, [x2, #5344] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6368] + ldr q2, [x2, #6368] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #224] + ldr q1, [x1, #240] + ldr q2, [x2, #240] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1264] + ldr q2, [x2, #1264] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2288] + ldr q2, [x2, #2288] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3312] + ldr q2, [x2, #3312] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4336] + ldr q2, [x2, #4336] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5360] + ldr q2, [x2, #5360] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6384] + ldr q2, [x2, #6384] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #240] + ldr q1, [x1, #256] + ldr q2, [x2, #256] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1280] + ldr q2, [x2, #1280] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2304] + ldr q2, [x2, #2304] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3328] + ldr q2, [x2, #3328] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4352] + ldr q2, [x2, #4352] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5376] + ldr q2, [x2, #5376] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6400] + ldr q2, [x2, #6400] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #256] + ldr q1, [x1, #272] + ldr q2, [x2, #272] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1296] + ldr q2, [x2, #1296] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2320] + ldr q2, [x2, #2320] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3344] + ldr q2, [x2, #3344] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4368] + ldr q2, [x2, #4368] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5392] + ldr q2, [x2, #5392] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6416] + ldr q2, [x2, #6416] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #272] + ldr q1, [x1, #288] + ldr q2, [x2, #288] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1312] + ldr q2, [x2, #1312] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2336] + ldr q2, [x2, #2336] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3360] + ldr q2, [x2, #3360] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4384] + ldr q2, [x2, #4384] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5408] + ldr q2, [x2, #5408] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6432] + ldr q2, [x2, #6432] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #288] + ldr q1, [x1, #304] + ldr q2, [x2, #304] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1328] + ldr q2, [x2, #1328] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2352] + ldr q2, [x2, #2352] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3376] + ldr q2, [x2, #3376] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4400] + ldr q2, [x2, #4400] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5424] + ldr q2, [x2, #5424] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6448] + ldr q2, [x2, #6448] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #304] + ldr q1, [x1, #320] + ldr q2, [x2, #320] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1344] + ldr q2, [x2, #1344] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2368] + ldr q2, [x2, #2368] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3392] + ldr q2, [x2, #3392] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4416] + ldr q2, [x2, #4416] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5440] + ldr q2, [x2, #5440] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6464] + ldr q2, [x2, #6464] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #320] + ldr q1, [x1, #336] + ldr q2, [x2, #336] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1360] + ldr q2, [x2, #1360] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2384] + ldr q2, [x2, #2384] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3408] + ldr q2, [x2, #3408] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4432] + ldr q2, [x2, #4432] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5456] + ldr q2, [x2, #5456] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6480] + ldr q2, [x2, #6480] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #336] + ldr q1, [x1, #352] + ldr q2, [x2, #352] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1376] + ldr q2, [x2, #1376] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2400] + ldr q2, [x2, #2400] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3424] + ldr q2, [x2, #3424] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4448] + ldr q2, [x2, #4448] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5472] + ldr q2, [x2, #5472] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6496] + ldr q2, [x2, #6496] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #352] + ldr q1, [x1, #368] + ldr q2, [x2, #368] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1392] + ldr q2, [x2, #1392] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2416] + ldr q2, [x2, #2416] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3440] + ldr q2, [x2, #3440] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4464] + ldr q2, [x2, #4464] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5488] + ldr q2, [x2, #5488] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6512] + ldr q2, [x2, #6512] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #368] + ldr q1, [x1, #384] + ldr q2, [x2, #384] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1408] + ldr q2, [x2, #1408] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2432] + ldr q2, [x2, #2432] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3456] + ldr q2, [x2, #3456] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4480] + ldr q2, [x2, #4480] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5504] + ldr q2, [x2, #5504] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6528] + ldr q2, [x2, #6528] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #384] + ldr q1, [x1, #400] + ldr q2, [x2, #400] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1424] + ldr q2, [x2, #1424] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2448] + ldr q2, [x2, #2448] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3472] + ldr q2, [x2, #3472] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4496] + ldr q2, [x2, #4496] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5520] + ldr q2, [x2, #5520] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6544] + ldr q2, [x2, #6544] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #400] + ldr q1, [x1, #416] + ldr q2, [x2, #416] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1440] + ldr q2, [x2, #1440] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2464] + ldr q2, [x2, #2464] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3488] + ldr q2, [x2, #3488] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4512] + ldr q2, [x2, #4512] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5536] + ldr q2, [x2, #5536] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6560] + ldr q2, [x2, #6560] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #416] + ldr q1, [x1, #432] + ldr q2, [x2, #432] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1456] + ldr q2, [x2, #1456] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2480] + ldr q2, [x2, #2480] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3504] + ldr q2, [x2, #3504] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4528] + ldr q2, [x2, #4528] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5552] + ldr q2, [x2, #5552] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6576] + ldr q2, [x2, #6576] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #432] + ldr q1, [x1, #448] + ldr q2, [x2, #448] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1472] + ldr q2, [x2, #1472] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2496] + ldr q2, [x2, #2496] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3520] + ldr q2, [x2, #3520] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4544] + ldr q2, [x2, #4544] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5568] + ldr q2, [x2, #5568] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6592] + ldr q2, [x2, #6592] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #448] + ldr q1, [x1, #464] + ldr q2, [x2, #464] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1488] + ldr q2, [x2, #1488] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2512] + ldr q2, [x2, #2512] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3536] + ldr q2, [x2, #3536] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4560] + ldr q2, [x2, #4560] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5584] + ldr q2, [x2, #5584] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6608] + ldr q2, [x2, #6608] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #464] + ldr q1, [x1, #480] + ldr q2, [x2, #480] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1504] + ldr q2, [x2, #1504] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2528] + ldr q2, [x2, #2528] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3552] + ldr q2, [x2, #3552] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4576] + ldr q2, [x2, #4576] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5600] + ldr q2, [x2, #5600] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6624] + ldr q2, [x2, #6624] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #480] + ldr q1, [x1, #496] + ldr q2, [x2, #496] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1520] + ldr q2, [x2, #1520] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2544] + ldr q2, [x2, #2544] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3568] + ldr q2, [x2, #3568] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4592] + ldr q2, [x2, #4592] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5616] + ldr q2, [x2, #5616] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6640] + ldr q2, [x2, #6640] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #496] + ldr q1, [x1, #512] + ldr q2, [x2, #512] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1536] + ldr q2, [x2, #1536] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2560] + ldr q2, [x2, #2560] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3584] + ldr q2, [x2, #3584] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4608] + ldr q2, [x2, #4608] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5632] + ldr q2, [x2, #5632] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6656] + ldr q2, [x2, #6656] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #512] + ldr q1, [x1, #528] + ldr q2, [x2, #528] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1552] + ldr q2, [x2, #1552] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2576] + ldr q2, [x2, #2576] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3600] + ldr q2, [x2, #3600] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4624] + ldr q2, [x2, #4624] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5648] + ldr q2, [x2, #5648] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6672] + ldr q2, [x2, #6672] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #528] + ldr q1, [x1, #544] + ldr q2, [x2, #544] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1568] + ldr q2, [x2, #1568] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2592] + ldr q2, [x2, #2592] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3616] + ldr q2, [x2, #3616] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4640] + ldr q2, [x2, #4640] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5664] + ldr q2, [x2, #5664] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6688] + ldr q2, [x2, #6688] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #544] + ldr q1, [x1, #560] + ldr q2, [x2, #560] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1584] + ldr q2, [x2, #1584] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2608] + ldr q2, [x2, #2608] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3632] + ldr q2, [x2, #3632] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4656] + ldr q2, [x2, #4656] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5680] + ldr q2, [x2, #5680] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6704] + ldr q2, [x2, #6704] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #560] + ldr q1, [x1, #576] + ldr q2, [x2, #576] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1600] + ldr q2, [x2, #1600] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2624] + ldr q2, [x2, #2624] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3648] + ldr q2, [x2, #3648] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4672] + ldr q2, [x2, #4672] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5696] + ldr q2, [x2, #5696] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6720] + ldr q2, [x2, #6720] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #576] + ldr q1, [x1, #592] + ldr q2, [x2, #592] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1616] + ldr q2, [x2, #1616] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2640] + ldr q2, [x2, #2640] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3664] + ldr q2, [x2, #3664] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4688] + ldr q2, [x2, #4688] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5712] + ldr q2, [x2, #5712] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6736] + ldr q2, [x2, #6736] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #592] + ldr q1, [x1, #608] + ldr q2, [x2, #608] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1632] + ldr q2, [x2, #1632] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2656] + ldr q2, [x2, #2656] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3680] + ldr q2, [x2, #3680] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4704] + ldr q2, [x2, #4704] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5728] + ldr q2, [x2, #5728] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6752] + ldr q2, [x2, #6752] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #608] + ldr q1, [x1, #624] + ldr q2, [x2, #624] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1648] + ldr q2, [x2, #1648] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2672] + ldr q2, [x2, #2672] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3696] + ldr q2, [x2, #3696] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4720] + ldr q2, [x2, #4720] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5744] + ldr q2, [x2, #5744] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6768] + ldr q2, [x2, #6768] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #624] + ldr q1, [x1, #640] + ldr q2, [x2, #640] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1664] + ldr q2, [x2, #1664] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2688] + ldr q2, [x2, #2688] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3712] + ldr q2, [x2, #3712] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4736] + ldr q2, [x2, #4736] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5760] + ldr q2, [x2, #5760] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6784] + ldr q2, [x2, #6784] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #640] + ldr q1, [x1, #656] + ldr q2, [x2, #656] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1680] + ldr q2, [x2, #1680] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2704] + ldr q2, [x2, #2704] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3728] + ldr q2, [x2, #3728] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4752] + ldr q2, [x2, #4752] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5776] + ldr q2, [x2, #5776] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6800] + ldr q2, [x2, #6800] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #656] + ldr q1, [x1, #672] + ldr q2, [x2, #672] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1696] + ldr q2, [x2, #1696] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2720] + ldr q2, [x2, #2720] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3744] + ldr q2, [x2, #3744] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4768] + ldr q2, [x2, #4768] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5792] + ldr q2, [x2, #5792] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6816] + ldr q2, [x2, #6816] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #672] + ldr q1, [x1, #688] + ldr q2, [x2, #688] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1712] + ldr q2, [x2, #1712] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2736] + ldr q2, [x2, #2736] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3760] + ldr q2, [x2, #3760] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4784] + ldr q2, [x2, #4784] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5808] + ldr q2, [x2, #5808] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6832] + ldr q2, [x2, #6832] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #688] + ldr q1, [x1, #704] + ldr q2, [x2, #704] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1728] + ldr q2, [x2, #1728] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2752] + ldr q2, [x2, #2752] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3776] + ldr q2, [x2, #3776] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4800] + ldr q2, [x2, #4800] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5824] + ldr q2, [x2, #5824] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6848] + ldr q2, [x2, #6848] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #704] + ldr q1, [x1, #720] + ldr q2, [x2, #720] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1744] + ldr q2, [x2, #1744] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2768] + ldr q2, [x2, #2768] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3792] + ldr q2, [x2, #3792] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4816] + ldr q2, [x2, #4816] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5840] + ldr q2, [x2, #5840] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6864] + ldr q2, [x2, #6864] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #720] + ldr q1, [x1, #736] + ldr q2, [x2, #736] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1760] + ldr q2, [x2, #1760] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2784] + ldr q2, [x2, #2784] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3808] + ldr q2, [x2, #3808] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4832] + ldr q2, [x2, #4832] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5856] + ldr q2, [x2, #5856] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6880] + ldr q2, [x2, #6880] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #736] + ldr q1, [x1, #752] + ldr q2, [x2, #752] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1776] + ldr q2, [x2, #1776] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2800] + ldr q2, [x2, #2800] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3824] + ldr q2, [x2, #3824] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4848] + ldr q2, [x2, #4848] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5872] + ldr q2, [x2, #5872] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6896] + ldr q2, [x2, #6896] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #752] + ldr q1, [x1, #768] + ldr q2, [x2, #768] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1792] + ldr q2, [x2, #1792] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2816] + ldr q2, [x2, #2816] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3840] + ldr q2, [x2, #3840] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4864] + ldr q2, [x2, #4864] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5888] + ldr q2, [x2, #5888] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6912] + ldr q2, [x2, #6912] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #768] + ldr q1, [x1, #784] + ldr q2, [x2, #784] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1808] + ldr q2, [x2, #1808] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2832] + ldr q2, [x2, #2832] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3856] + ldr q2, [x2, #3856] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4880] + ldr q2, [x2, #4880] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5904] + ldr q2, [x2, #5904] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6928] + ldr q2, [x2, #6928] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #784] + ldr q1, [x1, #800] + ldr q2, [x2, #800] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1824] + ldr q2, [x2, #1824] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2848] + ldr q2, [x2, #2848] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3872] + ldr q2, [x2, #3872] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4896] + ldr q2, [x2, #4896] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5920] + ldr q2, [x2, #5920] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6944] + ldr q2, [x2, #6944] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #800] + ldr q1, [x1, #816] + ldr q2, [x2, #816] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1840] + ldr q2, [x2, #1840] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2864] + ldr q2, [x2, #2864] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3888] + ldr q2, [x2, #3888] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4912] + ldr q2, [x2, #4912] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5936] + ldr q2, [x2, #5936] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6960] + ldr q2, [x2, #6960] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #816] + ldr q1, [x1, #832] + ldr q2, [x2, #832] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1856] + ldr q2, [x2, #1856] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2880] + ldr q2, [x2, #2880] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3904] + ldr q2, [x2, #3904] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4928] + ldr q2, [x2, #4928] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5952] + ldr q2, [x2, #5952] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6976] + ldr q2, [x2, #6976] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #832] + ldr q1, [x1, #848] + ldr q2, [x2, #848] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1872] + ldr q2, [x2, #1872] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2896] + ldr q2, [x2, #2896] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3920] + ldr q2, [x2, #3920] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4944] + ldr q2, [x2, #4944] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5968] + ldr q2, [x2, #5968] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6992] + ldr q2, [x2, #6992] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #848] + ldr q1, [x1, #864] + ldr q2, [x2, #864] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1888] + ldr q2, [x2, #1888] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2912] + ldr q2, [x2, #2912] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3936] + ldr q2, [x2, #3936] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4960] + ldr q2, [x2, #4960] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5984] + ldr q2, [x2, #5984] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #7008] + ldr q2, [x2, #7008] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #864] + ldr q1, [x1, #880] + ldr q2, [x2, #880] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1904] + ldr q2, [x2, #1904] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2928] + ldr q2, [x2, #2928] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3952] + ldr q2, [x2, #3952] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4976] + ldr q2, [x2, #4976] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6000] + ldr q2, [x2, #6000] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #7024] + ldr q2, [x2, #7024] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #880] + ldr q1, [x1, #896] + ldr q2, [x2, #896] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1920] + ldr q2, [x2, #1920] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2944] + ldr q2, [x2, #2944] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3968] + ldr q2, [x2, #3968] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4992] + ldr q2, [x2, #4992] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6016] + ldr q2, [x2, #6016] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #7040] + ldr q2, [x2, #7040] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #896] + ldr q1, [x1, #912] + ldr q2, [x2, #912] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1936] + ldr q2, [x2, #1936] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2960] + ldr q2, [x2, #2960] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3984] + ldr q2, [x2, #3984] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5008] + ldr q2, [x2, #5008] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6032] + ldr q2, [x2, #6032] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #7056] + ldr q2, [x2, #7056] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #912] + ldr q1, [x1, #928] + ldr q2, [x2, #928] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1952] + ldr q2, [x2, #1952] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2976] + ldr q2, [x2, #2976] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4000] + ldr q2, [x2, #4000] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5024] + ldr q2, [x2, #5024] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6048] + ldr q2, [x2, #6048] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #7072] + ldr q2, [x2, #7072] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #928] + ldr q1, [x1, #944] + ldr q2, [x2, #944] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1968] + ldr q2, [x2, #1968] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2992] + ldr q2, [x2, #2992] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4016] + ldr q2, [x2, #4016] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5040] + ldr q2, [x2, #5040] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6064] + ldr q2, [x2, #6064] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #7088] + ldr q2, [x2, #7088] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #944] + ldr q1, [x1, #960] + ldr q2, [x2, #960] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #1984] + ldr q2, [x2, #1984] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3008] + ldr q2, [x2, #3008] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4032] + ldr q2, [x2, #4032] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5056] + ldr q2, [x2, #5056] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6080] + ldr q2, [x2, #6080] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #7104] + ldr q2, [x2, #7104] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #960] + ldr q1, [x1, #976] + ldr q2, [x2, #976] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2000] + ldr q2, [x2, #2000] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3024] + ldr q2, [x2, #3024] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4048] + ldr q2, [x2, #4048] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5072] + ldr q2, [x2, #5072] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6096] + ldr q2, [x2, #6096] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #7120] + ldr q2, [x2, #7120] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #976] + ldr q1, [x1, #992] + ldr q2, [x2, #992] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2016] + ldr q2, [x2, #2016] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3040] + ldr q2, [x2, #3040] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4064] + ldr q2, [x2, #4064] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5088] + ldr q2, [x2, #5088] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6112] + ldr q2, [x2, #6112] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #7136] + ldr q2, [x2, #7136] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #992] + ldr q1, [x1, #1008] + ldr q2, [x2, #1008] + smull v3.2d, v1.2s, v2.2s + smull2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #2032] + ldr q2, [x2, #2032] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #3056] + ldr q2, [x2, #3056] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #4080] + ldr q2, [x2, #4080] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #5104] + ldr q2, [x2, #5104] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #6128] + ldr q2, [x2, #6128] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + ldr q1, [x1, #7152] + ldr q2, [x2, #7152] + smlal v3.2d, v1.2s, v2.2s + smlal2 v4.2d, v1.4s, v2.4s + uzp1 v5.4s, v3.4s, v4.4s + mul v5.4s, v5.4s, v0.s[1] + smull v6.2d, v5.2s, v0.s[0] + smull2 v7.2d, v5.4s, v0.s[0] + sub v3.2d, v3.2d, v6.2d + sub v4.2d, v4.2d, v7.2d + uzp2 v5.4s, v3.4s, v4.4s + str q5, [x0, #1008] + ret +#ifndef __APPLE__ + .size mldsa_mul_vec_7_neon,.-mldsa_mul_vec_7_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_aarch64_q, %object + .size L_mldsa_aarch64_q, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_aarch64_q: + .long 0x007fe001,0x007fe001,0x007fe001,0x007fe001 +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_eta_2_neon_idx, %object + .size L_mldsa_decode_eta_2_neon_idx, 192 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_eta_2_neon_idx: + .byte 0x00,0xff,0xff,0xff,0x00,0xff,0xff,0xff + .byte 0x00,0x01,0xff,0xff,0x01,0xff,0xff,0xff + .byte 0x01,0xff,0xff,0xff,0x01,0x02,0xff,0xff + .byte 0x02,0xff,0xff,0xff,0x02,0xff,0xff,0xff + .byte 0x01,0xff,0xff,0xff,0x01,0xff,0xff,0xff + .byte 0x01,0x02,0xff,0xff,0x02,0xff,0xff,0xff + .byte 0x02,0xff,0xff,0xff,0x02,0x03,0xff,0xff + .byte 0x03,0xff,0xff,0xff,0x03,0xff,0xff,0xff + .byte 0x04,0xff,0xff,0xff,0x04,0xff,0xff,0xff + .byte 0x04,0x05,0xff,0xff,0x05,0xff,0xff,0xff + .byte 0x05,0xff,0xff,0xff,0x05,0x06,0xff,0xff + .byte 0x06,0xff,0xff,0xff,0x06,0xff,0xff,0xff + .byte 0x07,0xff,0xff,0xff,0x07,0xff,0xff,0xff + .byte 0x07,0x08,0xff,0xff,0x08,0xff,0xff,0xff + .byte 0x08,0xff,0xff,0xff,0x08,0x09,0xff,0xff + .byte 0x09,0xff,0xff,0xff,0x09,0xff,0xff,0xff + .byte 0x0a,0xff,0xff,0xff,0x0a,0xff,0xff,0xff + .byte 0x0a,0x0b,0xff,0xff,0x0b,0xff,0xff,0xff + .byte 0x0b,0xff,0xff,0xff,0x0b,0x0c,0xff,0xff + .byte 0x0c,0xff,0xff,0xff,0x0c,0xff,0xff,0xff + .byte 0x0d,0xff,0xff,0xff,0x0d,0xff,0xff,0xff + .byte 0x0d,0x0e,0xff,0xff,0x0e,0xff,0xff,0xff + .byte 0x0e,0xff,0xff,0xff,0x0e,0x0f,0xff,0xff + .byte 0x0f,0xff,0xff,0xff,0x0f,0xff,0xff,0xff +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_eta_2_neon_shift, %object + .size L_mldsa_decode_eta_2_neon_shift, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_eta_2_neon_shift: + .long 0x00000000,0xfffffffd,0xfffffffa,0xffffffff + .long 0xfffffffc,0xfffffff9,0xfffffffe,0xfffffffb +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_eta_2_neon_mask, %object + .size L_mldsa_decode_eta_2_neon_mask, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_eta_2_neon_mask: + .long 0x00000007,0x00000007,0x00000007,0x00000007 +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_eta_2_neon_konst, %object + .size L_mldsa_decode_eta_2_neon_konst, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_eta_2_neon_konst: + .long 0x00000002,0x00000002,0x00000002,0x00000002 +#ifndef __APPLE__ +.text +.globl mldsa_decode_eta_2_neon +WC_ASM_ATT_HIDDEN(mldsa_decode_eta_2_neon) +.type mldsa_decode_eta_2_neon,@function +.align 2 +mldsa_decode_eta_2_neon: +#else +.section __TEXT,__text +.globl _mldsa_decode_eta_2_neon +WC_ASM_ATT_HIDDEN(_mldsa_decode_eta_2_neon) +.p2align 2 +_mldsa_decode_eta_2_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-32]! + add x29, sp, #0 + stp d8, d9, [x29, #16] +#ifndef __APPLE__ + adrp x2, L_mldsa_decode_eta_2_neon_idx + add x2, x2, :lo12:L_mldsa_decode_eta_2_neon_idx +#else + adrp x2, L_mldsa_decode_eta_2_neon_idx@PAGE + add x2, x2, L_mldsa_decode_eta_2_neon_idx@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_decode_eta_2_neon_shift + add x3, x3, :lo12:L_mldsa_decode_eta_2_neon_shift +#else + adrp x3, L_mldsa_decode_eta_2_neon_shift@PAGE + add x3, x3, L_mldsa_decode_eta_2_neon_shift@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x4, L_mldsa_decode_eta_2_neon_mask + add x4, x4, :lo12:L_mldsa_decode_eta_2_neon_mask +#else + adrp x4, L_mldsa_decode_eta_2_neon_mask@PAGE + add x4, x4, L_mldsa_decode_eta_2_neon_mask@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x5, L_mldsa_decode_eta_2_neon_konst + add x5, x5, :lo12:L_mldsa_decode_eta_2_neon_konst +#else + adrp x5, L_mldsa_decode_eta_2_neon_konst@PAGE + add x5, x5, L_mldsa_decode_eta_2_neon_konst@PAGEOFF +#endif /* __APPLE__ */ + ldr q1, [x2] + ldr q3, [x3] + ldr q2, [x2, #16] + ldr q4, [x3, #16] + ldr q5, [x4] + ldr q6, [x5] + mov x6, #27 +L_mldsa_decode_eta_2_neon_loop: + ldr q0, [x0] + tbl v7.16b, {v0.16b}, v1.16b + ushl v7.4s, v7.4s, v3.4s + and v7.16b, v7.16b, v5.16b + sub v7.4s, v6.4s, v7.4s + str q7, [x1] + tbl v8.16b, {v0.16b}, v2.16b + ushl v8.4s, v8.4s, v4.4s + and v8.16b, v8.16b, v5.16b + sub v8.4s, v6.4s, v8.4s + str q8, [x1, #16] + add x0, x0, #3 + add x1, x1, #32 + subs x6, x6, #1 + bne L_mldsa_decode_eta_2_neon_loop + sub x0, x0, #1 + ldr q0, [x0] + ldr q1, [x2, #32] + ldr q2, [x2, #48] + tbl v7.16b, {v0.16b}, v1.16b + ushl v7.4s, v7.4s, v3.4s + and v7.16b, v7.16b, v5.16b + sub v7.4s, v6.4s, v7.4s + str q7, [x1] + tbl v8.16b, {v0.16b}, v2.16b + ushl v8.4s, v8.4s, v4.4s + and v8.16b, v8.16b, v5.16b + sub v8.4s, v6.4s, v8.4s + str q8, [x1, #16] + add x1, x1, #32 + ldr q1, [x2, #64] + ldr q2, [x2, #80] + tbl v7.16b, {v0.16b}, v1.16b + ushl v7.4s, v7.4s, v3.4s + and v7.16b, v7.16b, v5.16b + sub v7.4s, v6.4s, v7.4s + str q7, [x1] + tbl v8.16b, {v0.16b}, v2.16b + ushl v8.4s, v8.4s, v4.4s + and v8.16b, v8.16b, v5.16b + sub v8.4s, v6.4s, v8.4s + str q8, [x1, #16] + add x1, x1, #32 + ldr q1, [x2, #96] + ldr q2, [x2, #112] + tbl v7.16b, {v0.16b}, v1.16b + ushl v7.4s, v7.4s, v3.4s + and v7.16b, v7.16b, v5.16b + sub v7.4s, v6.4s, v7.4s + str q7, [x1] + tbl v8.16b, {v0.16b}, v2.16b + ushl v8.4s, v8.4s, v4.4s + and v8.16b, v8.16b, v5.16b + sub v8.4s, v6.4s, v8.4s + str q8, [x1, #16] + add x1, x1, #32 + ldr q1, [x2, #128] + ldr q2, [x2, #144] + tbl v7.16b, {v0.16b}, v1.16b + ushl v7.4s, v7.4s, v3.4s + and v7.16b, v7.16b, v5.16b + sub v7.4s, v6.4s, v7.4s + str q7, [x1] + tbl v8.16b, {v0.16b}, v2.16b + ushl v8.4s, v8.4s, v4.4s + and v8.16b, v8.16b, v5.16b + sub v8.4s, v6.4s, v8.4s + str q8, [x1, #16] + add x1, x1, #32 + ldr q1, [x2, #160] + ldr q2, [x2, #176] + tbl v7.16b, {v0.16b}, v1.16b + ushl v7.4s, v7.4s, v3.4s + and v7.16b, v7.16b, v5.16b + sub v7.4s, v6.4s, v7.4s + str q7, [x1] + tbl v8.16b, {v0.16b}, v2.16b + ushl v8.4s, v8.4s, v4.4s + and v8.16b, v8.16b, v5.16b + sub v8.4s, v6.4s, v8.4s + str q8, [x1, #16] + ldp d8, d9, [x29, #16] + ldp x29, x30, [sp], #32 + ret +#ifndef __APPLE__ + .size mldsa_decode_eta_2_neon,.-mldsa_decode_eta_2_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_eta_4_neon_idx, %object + .size L_mldsa_decode_eta_4_neon_idx, 128 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_eta_4_neon_idx: + .byte 0x00,0xff,0xff,0xff,0x00,0xff,0xff,0xff + .byte 0x01,0xff,0xff,0xff,0x01,0xff,0xff,0xff + .byte 0x02,0xff,0xff,0xff,0x02,0xff,0xff,0xff + .byte 0x03,0xff,0xff,0xff,0x03,0xff,0xff,0xff + .byte 0x04,0xff,0xff,0xff,0x04,0xff,0xff,0xff + .byte 0x05,0xff,0xff,0xff,0x05,0xff,0xff,0xff + .byte 0x06,0xff,0xff,0xff,0x06,0xff,0xff,0xff + .byte 0x07,0xff,0xff,0xff,0x07,0xff,0xff,0xff + .byte 0x08,0xff,0xff,0xff,0x08,0xff,0xff,0xff + .byte 0x09,0xff,0xff,0xff,0x09,0xff,0xff,0xff + .byte 0x0a,0xff,0xff,0xff,0x0a,0xff,0xff,0xff + .byte 0x0b,0xff,0xff,0xff,0x0b,0xff,0xff,0xff + .byte 0x0c,0xff,0xff,0xff,0x0c,0xff,0xff,0xff + .byte 0x0d,0xff,0xff,0xff,0x0d,0xff,0xff,0xff + .byte 0x0e,0xff,0xff,0xff,0x0e,0xff,0xff,0xff + .byte 0x0f,0xff,0xff,0xff,0x0f,0xff,0xff,0xff +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_eta_4_neon_shift, %object + .size L_mldsa_decode_eta_4_neon_shift, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_eta_4_neon_shift: + .long 0x00000000,0xfffffffc,0x00000000,0xfffffffc +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_eta_4_neon_mask, %object + .size L_mldsa_decode_eta_4_neon_mask, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_eta_4_neon_mask: + .long 0x0000000f,0x0000000f,0x0000000f,0x0000000f +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_eta_4_neon_konst, %object + .size L_mldsa_decode_eta_4_neon_konst, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_eta_4_neon_konst: + .long 0x00000004,0x00000004,0x00000004,0x00000004 +#ifndef __APPLE__ +.text +.globl mldsa_decode_eta_4_neon +WC_ASM_ATT_HIDDEN(mldsa_decode_eta_4_neon) +.type mldsa_decode_eta_4_neon,@function +.align 2 +mldsa_decode_eta_4_neon: +#else +.section __TEXT,__text +.globl _mldsa_decode_eta_4_neon +WC_ASM_ATT_HIDDEN(_mldsa_decode_eta_4_neon) +.p2align 2 +_mldsa_decode_eta_4_neon: +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x2, L_mldsa_decode_eta_4_neon_idx + add x2, x2, :lo12:L_mldsa_decode_eta_4_neon_idx +#else + adrp x2, L_mldsa_decode_eta_4_neon_idx@PAGE + add x2, x2, L_mldsa_decode_eta_4_neon_idx@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_decode_eta_4_neon_shift + add x3, x3, :lo12:L_mldsa_decode_eta_4_neon_shift +#else + adrp x3, L_mldsa_decode_eta_4_neon_shift@PAGE + add x3, x3, L_mldsa_decode_eta_4_neon_shift@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x4, L_mldsa_decode_eta_4_neon_mask + add x4, x4, :lo12:L_mldsa_decode_eta_4_neon_mask +#else + adrp x4, L_mldsa_decode_eta_4_neon_mask@PAGE + add x4, x4, L_mldsa_decode_eta_4_neon_mask@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x5, L_mldsa_decode_eta_4_neon_konst + add x5, x5, :lo12:L_mldsa_decode_eta_4_neon_konst +#else + adrp x5, L_mldsa_decode_eta_4_neon_konst@PAGE + add x5, x5, L_mldsa_decode_eta_4_neon_konst@PAGEOFF +#endif /* __APPLE__ */ + ldr q1, [x2] + ldr q2, [x3] + ldr q3, [x4] + ldr q4, [x5] + mov x6, #57 +L_mldsa_decode_eta_4_neon_loop: + ldr q0, [x0] + tbl v5.16b, {v0.16b}, v1.16b + ushl v5.4s, v5.4s, v2.4s + and v5.16b, v5.16b, v3.16b + sub v5.4s, v4.4s, v5.4s + str q5, [x1] + add x0, x0, #2 + add x1, x1, #16 + subs x6, x6, #1 + bne L_mldsa_decode_eta_4_neon_loop + sub x0, x0, #2 + ldr q0, [x0] + ldr q1, [x2, #16] + tbl v5.16b, {v0.16b}, v1.16b + ushl v5.4s, v5.4s, v2.4s + and v5.16b, v5.16b, v3.16b + sub v5.4s, v4.4s, v5.4s + str q5, [x1] + add x1, x1, #16 + ldr q1, [x2, #32] + tbl v5.16b, {v0.16b}, v1.16b + ushl v5.4s, v5.4s, v2.4s + and v5.16b, v5.16b, v3.16b + sub v5.4s, v4.4s, v5.4s + str q5, [x1] + add x1, x1, #16 + ldr q1, [x2, #48] + tbl v5.16b, {v0.16b}, v1.16b + ushl v5.4s, v5.4s, v2.4s + and v5.16b, v5.16b, v3.16b + sub v5.4s, v4.4s, v5.4s + str q5, [x1] + add x1, x1, #16 + ldr q1, [x2, #64] + tbl v5.16b, {v0.16b}, v1.16b + ushl v5.4s, v5.4s, v2.4s + and v5.16b, v5.16b, v3.16b + sub v5.4s, v4.4s, v5.4s + str q5, [x1] + add x1, x1, #16 + ldr q1, [x2, #80] + tbl v5.16b, {v0.16b}, v1.16b + ushl v5.4s, v5.4s, v2.4s + and v5.16b, v5.16b, v3.16b + sub v5.4s, v4.4s, v5.4s + str q5, [x1] + add x1, x1, #16 + ldr q1, [x2, #96] + tbl v5.16b, {v0.16b}, v1.16b + ushl v5.4s, v5.4s, v2.4s + and v5.16b, v5.16b, v3.16b + sub v5.4s, v4.4s, v5.4s + str q5, [x1] + add x1, x1, #16 + ldr q1, [x2, #112] + tbl v5.16b, {v0.16b}, v1.16b + ushl v5.4s, v5.4s, v2.4s + and v5.16b, v5.16b, v3.16b + sub v5.4s, v4.4s, v5.4s + str q5, [x1] + ret +#ifndef __APPLE__ + .size mldsa_decode_eta_4_neon,.-mldsa_decode_eta_4_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_t1_neon_idx, %object + .size L_mldsa_decode_t1_neon_idx, 64 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_t1_neon_idx: + .byte 0x00,0x01,0xff,0xff,0x01,0x02,0xff,0xff + .byte 0x02,0x03,0xff,0xff,0x03,0x04,0xff,0xff + .byte 0x01,0x02,0xff,0xff,0x02,0x03,0xff,0xff + .byte 0x03,0x04,0xff,0xff,0x04,0x05,0xff,0xff + .byte 0x06,0x07,0xff,0xff,0x07,0x08,0xff,0xff + .byte 0x08,0x09,0xff,0xff,0x09,0x0a,0xff,0xff + .byte 0x0b,0x0c,0xff,0xff,0x0c,0x0d,0xff,0xff + .byte 0x0d,0x0e,0xff,0xff,0x0e,0x0f,0xff,0xff +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_t1_neon_shift, %object + .size L_mldsa_decode_t1_neon_shift, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_t1_neon_shift: + .long 0x00000000,0xfffffffe,0xfffffffc,0xfffffffa +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_t1_neon_mask, %object + .size L_mldsa_decode_t1_neon_mask, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_t1_neon_mask: + .long 0x000003ff,0x000003ff,0x000003ff,0x000003ff +#ifndef __APPLE__ +.text +.globl mldsa_decode_t1_neon +WC_ASM_ATT_HIDDEN(mldsa_decode_t1_neon) +.type mldsa_decode_t1_neon,@function +.align 2 +mldsa_decode_t1_neon: +#else +.section __TEXT,__text +.globl _mldsa_decode_t1_neon +WC_ASM_ATT_HIDDEN(_mldsa_decode_t1_neon) +.p2align 2 +_mldsa_decode_t1_neon: +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x2, L_mldsa_decode_t1_neon_idx + add x2, x2, :lo12:L_mldsa_decode_t1_neon_idx +#else + adrp x2, L_mldsa_decode_t1_neon_idx@PAGE + add x2, x2, L_mldsa_decode_t1_neon_idx@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_decode_t1_neon_shift + add x3, x3, :lo12:L_mldsa_decode_t1_neon_shift +#else + adrp x3, L_mldsa_decode_t1_neon_shift@PAGE + add x3, x3, L_mldsa_decode_t1_neon_shift@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x4, L_mldsa_decode_t1_neon_mask + add x4, x4, :lo12:L_mldsa_decode_t1_neon_mask +#else + adrp x4, L_mldsa_decode_t1_neon_mask@PAGE + add x4, x4, L_mldsa_decode_t1_neon_mask@PAGEOFF +#endif /* __APPLE__ */ + ldr q1, [x2] + ldr q2, [x3] + ldr q3, [x4] + mov x5, #61 +L_mldsa_decode_t1_neon_loop: + ldr q0, [x0] + tbl v4.16b, {v0.16b}, v1.16b + ushl v4.4s, v4.4s, v2.4s + and v4.16b, v4.16b, v3.16b + shl v4.4s, v4.4s, #13 + str q4, [x1] + add x0, x0, #5 + add x1, x1, #16 + subs x5, x5, #1 + bne L_mldsa_decode_t1_neon_loop + sub x0, x0, #1 + ldr q0, [x0] + ldr q1, [x2, #16] + tbl v4.16b, {v0.16b}, v1.16b + ushl v4.4s, v4.4s, v2.4s + and v4.16b, v4.16b, v3.16b + shl v4.4s, v4.4s, #13 + str q4, [x1] + add x1, x1, #16 + ldr q1, [x2, #32] + tbl v4.16b, {v0.16b}, v1.16b + ushl v4.4s, v4.4s, v2.4s + and v4.16b, v4.16b, v3.16b + shl v4.4s, v4.4s, #13 + str q4, [x1] + add x1, x1, #16 + ldr q1, [x2, #48] + tbl v4.16b, {v0.16b}, v1.16b + ushl v4.4s, v4.4s, v2.4s + and v4.16b, v4.16b, v3.16b + shl v4.4s, v4.4s, #13 + str q4, [x1] + ret +#ifndef __APPLE__ + .size mldsa_decode_t1_neon,.-mldsa_decode_t1_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_t0_neon_idx, %object + .size L_mldsa_decode_t0_neon_idx, 64 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_t0_neon_idx: + .byte 0x00,0x01,0xff,0xff,0x01,0x02,0x03,0xff + .byte 0x03,0x04,0xff,0xff,0x04,0x05,0x06,0xff + .byte 0x06,0x07,0x08,0xff,0x08,0x09,0xff,0xff + .byte 0x09,0x0a,0x0b,0xff,0x0b,0x0c,0xff,0xff + .byte 0x03,0x04,0xff,0xff,0x04,0x05,0x06,0xff + .byte 0x06,0x07,0xff,0xff,0x07,0x08,0x09,0xff + .byte 0x09,0x0a,0x0b,0xff,0x0b,0x0c,0xff,0xff + .byte 0x0c,0x0d,0x0e,0xff,0x0e,0x0f,0xff,0xff +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_t0_neon_shift, %object + .size L_mldsa_decode_t0_neon_shift, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_t0_neon_shift: + .long 0x00000000,0xfffffffb,0xfffffffe,0xfffffff9 + .long 0xfffffffc,0xffffffff,0xfffffffa,0xfffffffd +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_t0_neon_mask, %object + .size L_mldsa_decode_t0_neon_mask, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_t0_neon_mask: + .long 0x00001fff,0x00001fff,0x00001fff,0x00001fff +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_t0_neon_konst, %object + .size L_mldsa_decode_t0_neon_konst, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_t0_neon_konst: + .long 0x00001000,0x00001000,0x00001000,0x00001000 +#ifndef __APPLE__ +.text +.globl mldsa_decode_t0_neon +WC_ASM_ATT_HIDDEN(mldsa_decode_t0_neon) +.type mldsa_decode_t0_neon,@function +.align 2 +mldsa_decode_t0_neon: +#else +.section __TEXT,__text +.globl _mldsa_decode_t0_neon +WC_ASM_ATT_HIDDEN(_mldsa_decode_t0_neon) +.p2align 2 +_mldsa_decode_t0_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-32]! + add x29, sp, #0 + stp d8, d9, [x29, #16] +#ifndef __APPLE__ + adrp x2, L_mldsa_decode_t0_neon_idx + add x2, x2, :lo12:L_mldsa_decode_t0_neon_idx +#else + adrp x2, L_mldsa_decode_t0_neon_idx@PAGE + add x2, x2, L_mldsa_decode_t0_neon_idx@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_decode_t0_neon_shift + add x3, x3, :lo12:L_mldsa_decode_t0_neon_shift +#else + adrp x3, L_mldsa_decode_t0_neon_shift@PAGE + add x3, x3, L_mldsa_decode_t0_neon_shift@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x4, L_mldsa_decode_t0_neon_mask + add x4, x4, :lo12:L_mldsa_decode_t0_neon_mask +#else + adrp x4, L_mldsa_decode_t0_neon_mask@PAGE + add x4, x4, L_mldsa_decode_t0_neon_mask@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x5, L_mldsa_decode_t0_neon_konst + add x5, x5, :lo12:L_mldsa_decode_t0_neon_konst +#else + adrp x5, L_mldsa_decode_t0_neon_konst@PAGE + add x5, x5, L_mldsa_decode_t0_neon_konst@PAGEOFF +#endif /* __APPLE__ */ + ldr q1, [x2] + ldr q3, [x3] + ldr q2, [x2, #16] + ldr q4, [x3, #16] + ldr q5, [x4] + ldr q6, [x5] + mov x6, #31 +L_mldsa_decode_t0_neon_loop: + ldr q0, [x0] + tbl v7.16b, {v0.16b}, v1.16b + ushl v7.4s, v7.4s, v3.4s + and v7.16b, v7.16b, v5.16b + sub v7.4s, v6.4s, v7.4s + str q7, [x1] + tbl v8.16b, {v0.16b}, v2.16b + ushl v8.4s, v8.4s, v4.4s + and v8.16b, v8.16b, v5.16b + sub v8.4s, v6.4s, v8.4s + str q8, [x1, #16] + add x0, x0, #13 + add x1, x1, #32 + subs x6, x6, #1 + bne L_mldsa_decode_t0_neon_loop + sub x0, x0, #3 + ldr q0, [x0] + ldr q1, [x2, #32] + ldr q2, [x2, #48] + tbl v7.16b, {v0.16b}, v1.16b + ushl v7.4s, v7.4s, v3.4s + and v7.16b, v7.16b, v5.16b + sub v7.4s, v6.4s, v7.4s + str q7, [x1] + tbl v8.16b, {v0.16b}, v2.16b + ushl v8.4s, v8.4s, v4.4s + and v8.16b, v8.16b, v5.16b + sub v8.4s, v6.4s, v8.4s + str q8, [x1, #16] + ldp d8, d9, [x29, #16] + ldp x29, x30, [sp], #32 + ret +#ifndef __APPLE__ + .size mldsa_decode_t0_neon,.-mldsa_decode_t0_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_gamma1_17_neon_idx, %object + .size L_mldsa_decode_gamma1_17_neon_idx, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_gamma1_17_neon_idx: + .byte 0x00,0x01,0x02,0xff,0x02,0x03,0x04,0xff + .byte 0x04,0x05,0x06,0xff,0x06,0x07,0x08,0xff + .byte 0x07,0x08,0x09,0xff,0x09,0x0a,0x0b,0xff + .byte 0x0b,0x0c,0x0d,0xff,0x0d,0x0e,0x0f,0xff +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_gamma1_17_neon_shift, %object + .size L_mldsa_decode_gamma1_17_neon_shift, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_gamma1_17_neon_shift: + .long 0x00000000,0xfffffffe,0xfffffffc,0xfffffffa +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_gamma1_17_neon_mask, %object + .size L_mldsa_decode_gamma1_17_neon_mask, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_gamma1_17_neon_mask: + .long 0x0003ffff,0x0003ffff,0x0003ffff,0x0003ffff +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_gamma1_17_neon_konst, %object + .size L_mldsa_decode_gamma1_17_neon_konst, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_gamma1_17_neon_konst: + .long 0x00020000,0x00020000,0x00020000,0x00020000 +#ifndef __APPLE__ +.text +.globl mldsa_decode_gamma1_17_neon +WC_ASM_ATT_HIDDEN(mldsa_decode_gamma1_17_neon) +.type mldsa_decode_gamma1_17_neon,@function +.align 2 +mldsa_decode_gamma1_17_neon: +#else +.section __TEXT,__text +.globl _mldsa_decode_gamma1_17_neon +WC_ASM_ATT_HIDDEN(_mldsa_decode_gamma1_17_neon) +.p2align 2 +_mldsa_decode_gamma1_17_neon: +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x2, L_mldsa_decode_gamma1_17_neon_idx + add x2, x2, :lo12:L_mldsa_decode_gamma1_17_neon_idx +#else + adrp x2, L_mldsa_decode_gamma1_17_neon_idx@PAGE + add x2, x2, L_mldsa_decode_gamma1_17_neon_idx@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_decode_gamma1_17_neon_shift + add x3, x3, :lo12:L_mldsa_decode_gamma1_17_neon_shift +#else + adrp x3, L_mldsa_decode_gamma1_17_neon_shift@PAGE + add x3, x3, L_mldsa_decode_gamma1_17_neon_shift@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x4, L_mldsa_decode_gamma1_17_neon_mask + add x4, x4, :lo12:L_mldsa_decode_gamma1_17_neon_mask +#else + adrp x4, L_mldsa_decode_gamma1_17_neon_mask@PAGE + add x4, x4, L_mldsa_decode_gamma1_17_neon_mask@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x5, L_mldsa_decode_gamma1_17_neon_konst + add x5, x5, :lo12:L_mldsa_decode_gamma1_17_neon_konst +#else + adrp x5, L_mldsa_decode_gamma1_17_neon_konst@PAGE + add x5, x5, L_mldsa_decode_gamma1_17_neon_konst@PAGEOFF +#endif /* __APPLE__ */ + ldr q1, [x2] + ldr q2, [x3] + ldr q3, [x4] + ldr q4, [x5] + mov x6, #63 +L_mldsa_decode_gamma1_17_neon_loop: + ldr q0, [x0] + tbl v5.16b, {v0.16b}, v1.16b + ushl v5.4s, v5.4s, v2.4s + and v5.16b, v5.16b, v3.16b + sub v5.4s, v4.4s, v5.4s + str q5, [x1] + add x0, x0, #9 + add x1, x1, #16 + subs x6, x6, #1 + bne L_mldsa_decode_gamma1_17_neon_loop + sub x0, x0, #7 + ldr q0, [x0] + ldr q1, [x2, #16] + tbl v5.16b, {v0.16b}, v1.16b + ushl v5.4s, v5.4s, v2.4s + and v5.16b, v5.16b, v3.16b + sub v5.4s, v4.4s, v5.4s + str q5, [x1] + ret +#ifndef __APPLE__ + .size mldsa_decode_gamma1_17_neon,.-mldsa_decode_gamma1_17_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_gamma1_19_neon_idx, %object + .size L_mldsa_decode_gamma1_19_neon_idx, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_gamma1_19_neon_idx: + .byte 0x00,0x01,0x02,0xff,0x02,0x03,0x04,0xff + .byte 0x05,0x06,0x07,0xff,0x07,0x08,0x09,0xff + .byte 0x06,0x07,0x08,0xff,0x08,0x09,0x0a,0xff + .byte 0x0b,0x0c,0x0d,0xff,0x0d,0x0e,0x0f,0xff +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_gamma1_19_neon_shift, %object + .size L_mldsa_decode_gamma1_19_neon_shift, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_gamma1_19_neon_shift: + .long 0x00000000,0xfffffffc,0x00000000,0xfffffffc +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_gamma1_19_neon_mask, %object + .size L_mldsa_decode_gamma1_19_neon_mask, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_gamma1_19_neon_mask: + .long 0x000fffff,0x000fffff,0x000fffff,0x000fffff +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_decode_gamma1_19_neon_konst, %object + .size L_mldsa_decode_gamma1_19_neon_konst, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_decode_gamma1_19_neon_konst: + .long 0x00080000,0x00080000,0x00080000,0x00080000 +#ifndef __APPLE__ +.text +.globl mldsa_decode_gamma1_19_neon +WC_ASM_ATT_HIDDEN(mldsa_decode_gamma1_19_neon) +.type mldsa_decode_gamma1_19_neon,@function +.align 2 +mldsa_decode_gamma1_19_neon: +#else +.section __TEXT,__text +.globl _mldsa_decode_gamma1_19_neon +WC_ASM_ATT_HIDDEN(_mldsa_decode_gamma1_19_neon) +.p2align 2 +_mldsa_decode_gamma1_19_neon: +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x2, L_mldsa_decode_gamma1_19_neon_idx + add x2, x2, :lo12:L_mldsa_decode_gamma1_19_neon_idx +#else + adrp x2, L_mldsa_decode_gamma1_19_neon_idx@PAGE + add x2, x2, L_mldsa_decode_gamma1_19_neon_idx@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_decode_gamma1_19_neon_shift + add x3, x3, :lo12:L_mldsa_decode_gamma1_19_neon_shift +#else + adrp x3, L_mldsa_decode_gamma1_19_neon_shift@PAGE + add x3, x3, L_mldsa_decode_gamma1_19_neon_shift@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x4, L_mldsa_decode_gamma1_19_neon_mask + add x4, x4, :lo12:L_mldsa_decode_gamma1_19_neon_mask +#else + adrp x4, L_mldsa_decode_gamma1_19_neon_mask@PAGE + add x4, x4, L_mldsa_decode_gamma1_19_neon_mask@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x5, L_mldsa_decode_gamma1_19_neon_konst + add x5, x5, :lo12:L_mldsa_decode_gamma1_19_neon_konst +#else + adrp x5, L_mldsa_decode_gamma1_19_neon_konst@PAGE + add x5, x5, L_mldsa_decode_gamma1_19_neon_konst@PAGEOFF +#endif /* __APPLE__ */ + ldr q1, [x2] + ldr q2, [x3] + ldr q3, [x4] + ldr q4, [x5] + mov x6, #63 +L_mldsa_decode_gamma1_19_neon_loop: + ldr q0, [x0] + tbl v5.16b, {v0.16b}, v1.16b + ushl v5.4s, v5.4s, v2.4s + and v5.16b, v5.16b, v3.16b + sub v5.4s, v4.4s, v5.4s + str q5, [x1] + add x0, x0, #10 + add x1, x1, #16 + subs x6, x6, #1 + bne L_mldsa_decode_gamma1_19_neon_loop + sub x0, x0, #6 + ldr q0, [x0] + ldr q1, [x2, #16] + tbl v5.16b, {v0.16b}, v1.16b + ushl v5.4s, v5.4s, v2.4s + and v5.16b, v5.16b, v3.16b + sub v5.4s, v4.4s, v5.4s + str q5, [x1] + ret +#ifndef __APPLE__ + .size mldsa_decode_gamma1_19_neon,.-mldsa_decode_gamma1_19_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_encode_w1_88_neon_idx, %object + .size L_mldsa_encode_w1_88_neon_idx, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_encode_w1_88_neon_idx: + .byte 0x00,0x01,0x02,0x04,0x05,0x06,0x08,0x09 + .byte 0x0a,0x0c,0x0d,0x0e,0xff,0xff,0xff,0xff +#ifndef __APPLE__ +.text +.globl mldsa_encode_w1_88_neon +WC_ASM_ATT_HIDDEN(mldsa_encode_w1_88_neon) +.type mldsa_encode_w1_88_neon,@function +.align 2 +mldsa_encode_w1_88_neon: +#else +.section __TEXT,__text +.globl _mldsa_encode_w1_88_neon +WC_ASM_ATT_HIDDEN(_mldsa_encode_w1_88_neon) +.p2align 2 +_mldsa_encode_w1_88_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-32]! + add x29, sp, #0 + stp d8, d9, [x29, #16] +#ifndef __APPLE__ + adrp x2, L_mldsa_encode_w1_88_neon_idx + add x2, x2, :lo12:L_mldsa_encode_w1_88_neon_idx +#else + adrp x2, L_mldsa_encode_w1_88_neon_idx@PAGE + add x2, x2, L_mldsa_encode_w1_88_neon_idx@PAGEOFF +#endif /* __APPLE__ */ + ldr q8, [x2] + mov x3, #16 +L_mldsa_encode_w1_88_neon_loop: + ldr q0, [x0] + ldr q1, [x0, #16] + ldr q2, [x0, #32] + ldr q3, [x0, #48] + uzp1 v4.4s, v0.4s, v1.4s + uzp2 v7.4s, v0.4s, v1.4s + sli v4.4s, v7.4s, #6 + uzp1 v5.4s, v2.4s, v3.4s + uzp2 v7.4s, v2.4s, v3.4s + sli v5.4s, v7.4s, #6 + uzp1 v6.4s, v4.4s, v5.4s + uzp2 v7.4s, v4.4s, v5.4s + sli v6.4s, v7.4s, #12 + tbl v9.16b, {v6.16b}, v8.16b + str d9, [x1] + ext v9.16b, v9.16b, v9.16b, #8 + str s9, [x1, #8] + add x0, x0, #0x40 + add x1, x1, #12 + subs x3, x3, #1 + bne L_mldsa_encode_w1_88_neon_loop + ldp d8, d9, [x29, #16] + ldp x29, x30, [sp], #32 + ret +#ifndef __APPLE__ + .size mldsa_encode_w1_88_neon,.-mldsa_encode_w1_88_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_encode_w1_32_neon_idx, %object + .size L_mldsa_encode_w1_32_neon_idx, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_encode_w1_32_neon_idx: + .byte 0x00,0x04,0x08,0x0c,0x10,0x14,0x18,0x1c + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff +#ifndef __APPLE__ +.text +.globl mldsa_encode_w1_32_neon +WC_ASM_ATT_HIDDEN(mldsa_encode_w1_32_neon) +.type mldsa_encode_w1_32_neon,@function +.align 2 +mldsa_encode_w1_32_neon: +#else +.section __TEXT,__text +.globl _mldsa_encode_w1_32_neon +WC_ASM_ATT_HIDDEN(_mldsa_encode_w1_32_neon) +.p2align 2 +_mldsa_encode_w1_32_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-32]! + add x29, sp, #0 + stp d8, d9, [x29, #16] +#ifndef __APPLE__ + adrp x2, L_mldsa_encode_w1_32_neon_idx + add x2, x2, :lo12:L_mldsa_encode_w1_32_neon_idx +#else + adrp x2, L_mldsa_encode_w1_32_neon_idx@PAGE + add x2, x2, L_mldsa_encode_w1_32_neon_idx@PAGEOFF +#endif /* __APPLE__ */ + ldr q8, [x2] + mov x3, #16 +L_mldsa_encode_w1_32_neon_loop: + ldr q0, [x0] + ldr q1, [x0, #16] + ldr q2, [x0, #32] + ldr q3, [x0, #48] + uzp1 v4.4s, v0.4s, v1.4s + uzp2 v7.4s, v0.4s, v1.4s + sli v4.4s, v7.4s, #4 + uzp1 v5.4s, v2.4s, v3.4s + uzp2 v7.4s, v2.4s, v3.4s + sli v5.4s, v7.4s, #4 + tbl v9.16b, {v4.16b, v5.16b}, v8.16b + str d9, [x1] + add x0, x0, #0x40 + add x1, x1, #8 + subs x3, x3, #1 + bne L_mldsa_encode_w1_32_neon_loop + ldp d8, d9, [x29, #16] + ldp x29, x30, [sp], #32 + ret +#ifndef __APPLE__ + .size mldsa_encode_w1_32_neon,.-mldsa_encode_w1_32_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_encode_gamma1_17_neon_idx, %object + .size L_mldsa_encode_gamma1_17_neon_idx, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_encode_gamma1_17_neon_idx: + .byte 0x00,0x01,0x02,0xff,0x08,0x09,0x0a,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0x04,0x05,0x06,0xff,0x0c,0x0d + .byte 0x0e,0xff,0xff,0xff,0xff,0xff,0xff,0xff +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_encode_gamma1_17_neon_shift, %object + .size L_mldsa_encode_gamma1_17_neon_shift, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_encode_gamma1_17_neon_shift: + .long 0x00000000,0x00000002,0x00000004,0x00000006 +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_encode_gamma1_17_neon_gam, %object + .size L_mldsa_encode_gamma1_17_neon_gam, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_encode_gamma1_17_neon_gam: + .long 0x00020000,0x00020000,0x00020000,0x00020000 +#ifndef __APPLE__ +.text +.globl mldsa_encode_gamma1_17_neon +WC_ASM_ATT_HIDDEN(mldsa_encode_gamma1_17_neon) +.type mldsa_encode_gamma1_17_neon,@function +.align 2 +mldsa_encode_gamma1_17_neon: +#else +.section __TEXT,__text +.globl _mldsa_encode_gamma1_17_neon +WC_ASM_ATT_HIDDEN(_mldsa_encode_gamma1_17_neon) +.p2align 2 +_mldsa_encode_gamma1_17_neon: +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x2, L_mldsa_encode_gamma1_17_neon_idx + add x2, x2, :lo12:L_mldsa_encode_gamma1_17_neon_idx +#else + adrp x2, L_mldsa_encode_gamma1_17_neon_idx@PAGE + add x2, x2, L_mldsa_encode_gamma1_17_neon_idx@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_encode_gamma1_17_neon_shift + add x3, x3, :lo12:L_mldsa_encode_gamma1_17_neon_shift +#else + adrp x3, L_mldsa_encode_gamma1_17_neon_shift@PAGE + add x3, x3, L_mldsa_encode_gamma1_17_neon_shift@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x4, L_mldsa_encode_gamma1_17_neon_gam + add x4, x4, :lo12:L_mldsa_encode_gamma1_17_neon_gam +#else + adrp x4, L_mldsa_encode_gamma1_17_neon_gam@PAGE + add x4, x4, L_mldsa_encode_gamma1_17_neon_gam@PAGEOFF +#endif /* __APPLE__ */ + ldr q3, [x2] + ldr q4, [x2, #16] + ldr q1, [x3] + ldr q2, [x4] + mov x5, #0x40 +L_mldsa_encode_gamma1_17_neon_loop: + ldr q0, [x0] + sub v0.4s, v2.4s, v0.4s + ushl v0.4s, v0.4s, v1.4s + tbl v5.16b, {v0.16b}, v3.16b + tbl v6.16b, {v0.16b}, v4.16b + orr v5.16b, v5.16b, v6.16b + str d5, [x1] + ext v5.16b, v5.16b, v5.16b, #8 + str b5, [x1, #8] + add x0, x0, #16 + add x1, x1, #9 + subs x5, x5, #1 + bne L_mldsa_encode_gamma1_17_neon_loop + ret +#ifndef __APPLE__ + .size mldsa_encode_gamma1_17_neon,.-mldsa_encode_gamma1_17_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_encode_gamma1_19_neon_idx, %object + .size L_mldsa_encode_gamma1_19_neon_idx, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_encode_gamma1_19_neon_idx: + .byte 0x00,0x01,0x02,0xff,0xff,0x08,0x09,0x0a + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0x04,0x05,0x06,0xff,0xff,0x0c + .byte 0x0d,0x0e,0xff,0xff,0xff,0xff,0xff,0xff +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_encode_gamma1_19_neon_shift, %object + .size L_mldsa_encode_gamma1_19_neon_shift, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_encode_gamma1_19_neon_shift: + .long 0x00000000,0x00000004,0x00000000,0x00000004 +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_encode_gamma1_19_neon_gam, %object + .size L_mldsa_encode_gamma1_19_neon_gam, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_encode_gamma1_19_neon_gam: + .long 0x00080000,0x00080000,0x00080000,0x00080000 +#ifndef __APPLE__ +.text +.globl mldsa_encode_gamma1_19_neon +WC_ASM_ATT_HIDDEN(mldsa_encode_gamma1_19_neon) +.type mldsa_encode_gamma1_19_neon,@function +.align 2 +mldsa_encode_gamma1_19_neon: +#else +.section __TEXT,__text +.globl _mldsa_encode_gamma1_19_neon +WC_ASM_ATT_HIDDEN(_mldsa_encode_gamma1_19_neon) +.p2align 2 +_mldsa_encode_gamma1_19_neon: +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x2, L_mldsa_encode_gamma1_19_neon_idx + add x2, x2, :lo12:L_mldsa_encode_gamma1_19_neon_idx +#else + adrp x2, L_mldsa_encode_gamma1_19_neon_idx@PAGE + add x2, x2, L_mldsa_encode_gamma1_19_neon_idx@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x3, L_mldsa_encode_gamma1_19_neon_shift + add x3, x3, :lo12:L_mldsa_encode_gamma1_19_neon_shift +#else + adrp x3, L_mldsa_encode_gamma1_19_neon_shift@PAGE + add x3, x3, L_mldsa_encode_gamma1_19_neon_shift@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x4, L_mldsa_encode_gamma1_19_neon_gam + add x4, x4, :lo12:L_mldsa_encode_gamma1_19_neon_gam +#else + adrp x4, L_mldsa_encode_gamma1_19_neon_gam@PAGE + add x4, x4, L_mldsa_encode_gamma1_19_neon_gam@PAGEOFF +#endif /* __APPLE__ */ + ldr q3, [x2] + ldr q4, [x2, #16] + ldr q1, [x3] + ldr q2, [x4] + mov x5, #0x40 +L_mldsa_encode_gamma1_19_neon_loop: + ldr q0, [x0] + sub v0.4s, v2.4s, v0.4s + ushl v0.4s, v0.4s, v1.4s + tbl v5.16b, {v0.16b}, v3.16b + tbl v6.16b, {v0.16b}, v4.16b + orr v5.16b, v5.16b, v6.16b + str d5, [x1] + ext v5.16b, v5.16b, v5.16b, #8 + str h5, [x1, #8] + add x0, x0, #16 + add x1, x1, #10 + subs x5, x5, #1 + bne L_mldsa_encode_gamma1_19_neon_loop + ret +#ifndef __APPLE__ + .size mldsa_encode_gamma1_19_neon,.-mldsa_encode_gamma1_19_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mldsa_vec_check_low_neon +WC_ASM_ATT_HIDDEN(mldsa_vec_check_low_neon) +.type mldsa_vec_check_low_neon,@function +.align 2 +mldsa_vec_check_low_neon: +#else +.section __TEXT,__text +.globl _mldsa_vec_check_low_neon +WC_ASM_ATT_HIDDEN(_mldsa_vec_check_low_neon) +.p2align 2 +_mldsa_vec_check_low_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-64]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + dup v0.4s, w2 + eor v1.16b, v1.16b, v1.16b + eor v2.16b, v2.16b, v2.16b + eor v3.16b, v3.16b, v3.16b + eor v4.16b, v4.16b, v4.16b +L_mldsa_vec_check_low_outer: + mov x3, #16 +L_mldsa_vec_check_low_inner: + ldr q5, [x0] + ldr q6, [x0, #16] + ldr q7, [x0, #32] + ldr q8, [x0, #48] + sqabs v9.4s, v5.4s + cmge v9.4s, v9.4s, v0.4s + orr v1.16b, v1.16b, v9.16b + sqabs v10.4s, v6.4s + cmge v10.4s, v10.4s, v0.4s + orr v2.16b, v2.16b, v10.16b + sqabs v11.4s, v7.4s + cmge v11.4s, v11.4s, v0.4s + orr v3.16b, v3.16b, v11.16b + sqabs v12.4s, v8.4s + cmge v12.4s, v12.4s, v0.4s + orr v4.16b, v4.16b, v12.16b + add x0, x0, #0x40 + subs x3, x3, #1 + bne L_mldsa_vec_check_low_inner + subs w1, w1, #1 + bne L_mldsa_vec_check_low_outer + orr v1.16b, v1.16b, v2.16b + orr v3.16b, v3.16b, v4.16b + orr v1.16b, v1.16b, v3.16b + umaxv s1, v1.4s + mov w4, v1.s[0] + cmp w4, #0 + cset w4, eq + mov x0, x4 + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp x29, x30, [sp], #0x40 + ret +#ifndef __APPLE__ + .size mldsa_vec_check_low_neon,.-mldsa_vec_check_low_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mldsa_poly_add_neon +WC_ASM_ATT_HIDDEN(mldsa_poly_add_neon) +.type mldsa_poly_add_neon,@function +.align 2 +mldsa_poly_add_neon: +#else +.section __TEXT,__text +.globl _mldsa_poly_add_neon +WC_ASM_ATT_HIDDEN(_mldsa_poly_add_neon) +.p2align 2 +_mldsa_poly_add_neon: +#endif /* __APPLE__ */ + mov x2, #16 +L_mldsa_poly_add_neon_loop: + ldr q0, [x0] + ldr q4, [x1] + ldr q1, [x0, #16] + ldr q5, [x1, #16] + ldr q2, [x0, #32] + ldr q6, [x1, #32] + ldr q3, [x0, #48] + ldr q7, [x1, #48] + add v0.4s, v0.4s, v4.4s + add v1.4s, v1.4s, v5.4s + add v2.4s, v2.4s, v6.4s + add v3.4s, v3.4s, v7.4s + str q0, [x0] + str q1, [x0, #16] + str q2, [x0, #32] + str q3, [x0, #48] + add x0, x0, #0x40 + add x1, x1, #0x40 + subs x2, x2, #1 + bne L_mldsa_poly_add_neon_loop + ret +#ifndef __APPLE__ + .size mldsa_poly_add_neon,.-mldsa_poly_add_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl mldsa_poly_sub_neon +WC_ASM_ATT_HIDDEN(mldsa_poly_sub_neon) +.type mldsa_poly_sub_neon,@function +.align 2 +mldsa_poly_sub_neon: +#else +.section __TEXT,__text +.globl _mldsa_poly_sub_neon +WC_ASM_ATT_HIDDEN(_mldsa_poly_sub_neon) +.p2align 2 +_mldsa_poly_sub_neon: +#endif /* __APPLE__ */ + mov x2, #16 +L_mldsa_poly_sub_neon_loop: + ldr q0, [x0] + ldr q4, [x1] + ldr q1, [x0, #16] + ldr q5, [x1, #16] + ldr q2, [x0, #32] + ldr q6, [x1, #32] + ldr q3, [x0, #48] + ldr q7, [x1, #48] + sub v0.4s, v0.4s, v4.4s + sub v1.4s, v1.4s, v5.4s + sub v2.4s, v2.4s, v6.4s + sub v3.4s, v3.4s, v7.4s + str q0, [x0] + str q1, [x0, #16] + str q2, [x0, #32] + str q3, [x0, #48] + add x0, x0, #0x40 + add x1, x1, #0x40 + subs x2, x2, #1 + bne L_mldsa_poly_sub_neon_loop + ret +#ifndef __APPLE__ + .size mldsa_poly_sub_neon,.-mldsa_poly_sub_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_poly_make_pos_neon_q, %object + .size L_mldsa_poly_make_pos_neon_q, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_poly_make_pos_neon_q: + .long 0x007fe001,0x007fe001,0x007fe001,0x007fe001 +#ifndef __APPLE__ +.text +.globl mldsa_poly_make_pos_neon +WC_ASM_ATT_HIDDEN(mldsa_poly_make_pos_neon) +.type mldsa_poly_make_pos_neon,@function +.align 2 +mldsa_poly_make_pos_neon: +#else +.section __TEXT,__text +.globl _mldsa_poly_make_pos_neon +WC_ASM_ATT_HIDDEN(_mldsa_poly_make_pos_neon) +.p2align 2 +_mldsa_poly_make_pos_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-32]! + add x29, sp, #0 + stp d8, d9, [x29, #16] +#ifndef __APPLE__ + adrp x1, L_mldsa_poly_make_pos_neon_q + add x1, x1, :lo12:L_mldsa_poly_make_pos_neon_q +#else + adrp x1, L_mldsa_poly_make_pos_neon_q@PAGE + add x1, x1, L_mldsa_poly_make_pos_neon_q@PAGEOFF +#endif /* __APPLE__ */ + ldr q0, [x1] + mov x2, #16 +L_mldsa_poly_make_pos_neon_loop: + ldr q1, [x0] + ldr q2, [x0, #16] + ldr q3, [x0, #32] + ldr q4, [x0, #48] + sshr v5.4s, v1.4s, #31 + mls v1.4s, v5.4s, v0.4s + sshr v6.4s, v2.4s, #31 + mls v2.4s, v6.4s, v0.4s + sshr v7.4s, v3.4s, #31 + mls v3.4s, v7.4s, v0.4s + sshr v8.4s, v4.4s, #31 + mls v4.4s, v8.4s, v0.4s + str q1, [x0] + str q2, [x0, #16] + str q3, [x0, #32] + str q4, [x0, #48] + add x0, x0, #0x40 + subs x2, x2, #1 + bne L_mldsa_poly_make_pos_neon_loop + ldp d8, d9, [x29, #16] + ldp x29, x30, [sp], #32 + ret +#ifndef __APPLE__ + .size mldsa_poly_make_pos_neon,.-mldsa_poly_make_pos_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_poly_red_neon_q, %object + .size L_mldsa_poly_red_neon_q, 32 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_poly_red_neon_q: + .long 0x007fe001,0x007fe001,0x007fe001,0x007fe001 + .long 0x00400000,0x00400000,0x00400000,0x00400000 +#ifndef __APPLE__ +.text +.globl mldsa_poly_red_neon +WC_ASM_ATT_HIDDEN(mldsa_poly_red_neon) +.type mldsa_poly_red_neon,@function +.align 2 +mldsa_poly_red_neon: +#else +.section __TEXT,__text +.globl _mldsa_poly_red_neon +WC_ASM_ATT_HIDDEN(_mldsa_poly_red_neon) +.p2align 2 +_mldsa_poly_red_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-32]! + add x29, sp, #0 + stp d8, d9, [x29, #16] +#ifndef __APPLE__ + adrp x1, L_mldsa_poly_red_neon_q + add x1, x1, :lo12:L_mldsa_poly_red_neon_q +#else + adrp x1, L_mldsa_poly_red_neon_q@PAGE + add x1, x1, L_mldsa_poly_red_neon_q@PAGEOFF +#endif /* __APPLE__ */ + ldr q0, [x1] + ldr q1, [x1, #16] + mov x2, #16 +L_mldsa_poly_red_neon_loop: + ldr q2, [x0] + ldr q3, [x0, #16] + ldr q4, [x0, #32] + ldr q5, [x0, #48] + add v6.4s, v2.4s, v1.4s + sshr v6.4s, v6.4s, #23 + mls v2.4s, v6.4s, v0.4s + add v7.4s, v3.4s, v1.4s + sshr v7.4s, v7.4s, #23 + mls v3.4s, v7.4s, v0.4s + add v8.4s, v4.4s, v1.4s + sshr v8.4s, v8.4s, #23 + mls v4.4s, v8.4s, v0.4s + add v9.4s, v5.4s, v1.4s + sshr v9.4s, v9.4s, #23 + mls v5.4s, v9.4s, v0.4s + str q2, [x0] + str q3, [x0, #16] + str q4, [x0, #32] + str q5, [x0, #48] + add x0, x0, #0x40 + subs x2, x2, #1 + bne L_mldsa_poly_red_neon_loop + ldp d8, d9, [x29, #16] + ldp x29, x30, [sp], #32 + ret +#ifndef __APPLE__ + .size mldsa_poly_red_neon,.-mldsa_poly_red_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_extract_coeffs_eta2_neon_map, %object + .size L_mldsa_extract_coeffs_eta2_neon_map, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta2_neon_map: + .byte 0x02,0x01,0x00,0xff,0xfe,0x02,0x01,0x00 + .byte 0xff,0xfe,0x02,0x01,0x00,0xff,0xfe,0x00 +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_extract_coeffs_eta2_neon_lim, %object + .size L_mldsa_extract_coeffs_eta2_neon_lim, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta2_neon_lim: + .byte 0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f + .byte 0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f,0x0f +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_extract_coeffs_eta2_neon_bits, %object + .size L_mldsa_extract_coeffs_eta2_neon_bits, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta2_neon_bits: + .long 0x00000001,0x00000002,0x00000004,0x00000008 +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_extract_coeffs_eta2_neon_idx, %object + .size L_mldsa_extract_coeffs_eta2_neon_idx, 256 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta2_neon_idx: + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x08,0x09,0x0a,0x0b,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x08,0x09,0x0a,0x0b + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x08,0x09,0x0a,0x0b,0xff,0xff,0xff,0xff + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x08,0x09,0x0a,0x0b + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f +#ifndef __APPLE__ +.text +.globl mldsa_extract_coeffs_eta2_neon +WC_ASM_ATT_HIDDEN(mldsa_extract_coeffs_eta2_neon) +.type mldsa_extract_coeffs_eta2_neon,@function +.align 2 +mldsa_extract_coeffs_eta2_neon: +#else +.section __TEXT,__text +.globl _mldsa_extract_coeffs_eta2_neon +WC_ASM_ATT_HIDDEN(_mldsa_extract_coeffs_eta2_neon) +.p2align 2 +_mldsa_extract_coeffs_eta2_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x4, L_mldsa_extract_coeffs_eta2_neon_map + add x4, x4, :lo12:L_mldsa_extract_coeffs_eta2_neon_map +#else + adrp x4, L_mldsa_extract_coeffs_eta2_neon_map@PAGE + add x4, x4, L_mldsa_extract_coeffs_eta2_neon_map@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x5, L_mldsa_extract_coeffs_eta2_neon_lim + add x5, x5, :lo12:L_mldsa_extract_coeffs_eta2_neon_lim +#else + adrp x5, L_mldsa_extract_coeffs_eta2_neon_lim@PAGE + add x5, x5, L_mldsa_extract_coeffs_eta2_neon_lim@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x6, L_mldsa_extract_coeffs_eta2_neon_bits + add x6, x6, :lo12:L_mldsa_extract_coeffs_eta2_neon_bits +#else + adrp x6, L_mldsa_extract_coeffs_eta2_neon_bits@PAGE + add x6, x6, L_mldsa_extract_coeffs_eta2_neon_bits@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x7, L_mldsa_extract_coeffs_eta2_neon_idx + add x7, x7, :lo12:L_mldsa_extract_coeffs_eta2_neon_idx +#else + adrp x7, L_mldsa_extract_coeffs_eta2_neon_idx@PAGE + add x7, x7, L_mldsa_extract_coeffs_eta2_neon_idx@PAGEOFF +#endif /* __APPLE__ */ + ldr q2, [x4] + ldr q0, [x5] + ldr q1, [x6] + ldr w8, [x3] + add x2, x2, x8, lsl 2 +L_mldsa_extract_coeffs_eta2_neon_wide: + subs wzr, w1, #8 + blt L_mldsa_extract_coeffs_eta2_neon_tail + subs wzr, w8, #0xf0 + bgt L_mldsa_extract_coeffs_eta2_neon_tail + ldr d3, [x0] + movi v18.16b, #15 + ushr v5.16b, v3.16b, #4 + and v4.16b, v3.16b, v18.16b + zip1 v3.16b, v4.16b, v5.16b + tbl v6.16b, {v2.16b}, v3.16b + cmhi v8.16b, v0.16b, v3.16b + sxtl v7.8h, v6.8b + sxtl2 v18.8h, v6.16b + sxtl v10.4s, v7.4h + sxtl2 v11.4s, v7.8h + sxtl v12.4s, v18.4h + sxtl2 v13.4s, v18.8h + sxtl v9.8h, v8.8b + sxtl2 v18.8h, v8.16b + sxtl v14.4s, v9.4h + sxtl2 v15.4s, v9.8h + sxtl v16.4s, v18.4h + sxtl2 v17.4s, v18.8h + ushr v18.4s, v14.4s, #31 + addv s18, v18.4s + mov w10, v18.s[0] + and v18.16b, v14.16b, v1.16b + addv s18, v18.4s + mov w9, v18.s[0] + lsl x9, x9, #4 + ldr q18, [x7, x9] + tbl v10.16b, {v10.16b}, v18.16b + str q10, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + ushr v18.4s, v15.4s, #31 + addv s18, v18.4s + mov w10, v18.s[0] + and v18.16b, v15.16b, v1.16b + addv s18, v18.4s + mov w9, v18.s[0] + lsl x9, x9, #4 + ldr q18, [x7, x9] + tbl v11.16b, {v11.16b}, v18.16b + str q11, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + ushr v18.4s, v16.4s, #31 + addv s18, v18.4s + mov w10, v18.s[0] + and v18.16b, v16.16b, v1.16b + addv s18, v18.4s + mov w9, v18.s[0] + lsl x9, x9, #4 + ldr q18, [x7, x9] + tbl v12.16b, {v12.16b}, v18.16b + str q12, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + ushr v18.4s, v17.4s, #31 + addv s18, v18.4s + mov w10, v18.s[0] + and v18.16b, v17.16b, v1.16b + addv s18, v18.4s + mov w9, v18.s[0] + lsl x9, x9, #4 + ldr q18, [x7, x9] + tbl v13.16b, {v13.16b}, v18.16b + str q13, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + add x0, x0, #8 + sub w1, w1, #8 + b L_mldsa_extract_coeffs_eta2_neon_wide +L_mldsa_extract_coeffs_eta2_neon_tail: + subs wzr, w1, #0 + beq L_mldsa_extract_coeffs_eta2_neon_done + subs wzr, w8, #0x100 + beq L_mldsa_extract_coeffs_eta2_neon_done + ldrb w11, [x0] + add x0, x0, #1 + sub w1, w1, #1 + and w12, w11, #15 + subs wzr, w12, #15 + bcs L_mldsa_extract_coeffs_eta2_neon_skip_lo + ldrsb w12, [x4, x12] + str w12, [x2] + add x2, x2, #4 + add x8, x8, #1 + subs wzr, w8, #0x100 + beq L_mldsa_extract_coeffs_eta2_neon_done +L_mldsa_extract_coeffs_eta2_neon_skip_lo: + lsr w12, w11, #4 + subs wzr, w12, #15 + bcs L_mldsa_extract_coeffs_eta2_neon_skip_hi + ldrsb w12, [x4, x12] + str w12, [x2] + add x2, x2, #4 + add x8, x8, #1 +L_mldsa_extract_coeffs_eta2_neon_skip_hi: + b L_mldsa_extract_coeffs_eta2_neon_tail +L_mldsa_extract_coeffs_eta2_neon_done: + str w8, [x3] + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_extract_coeffs_eta2_neon,.-mldsa_extract_coeffs_eta2_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_extract_coeffs_eta4_neon_map, %object + .size L_mldsa_extract_coeffs_eta4_neon_map, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta4_neon_map: + .byte 0x04,0x03,0x02,0x01,0x00,0xff,0xfe,0xfd + .byte 0xfc,0x00,0x00,0x00,0x00,0x00,0x00,0x00 +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_extract_coeffs_eta4_neon_lim, %object + .size L_mldsa_extract_coeffs_eta4_neon_lim, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta4_neon_lim: + .byte 0x09,0x09,0x09,0x09,0x09,0x09,0x09,0x09 + .byte 0x09,0x09,0x09,0x09,0x09,0x09,0x09,0x09 +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_extract_coeffs_eta4_neon_bits, %object + .size L_mldsa_extract_coeffs_eta4_neon_bits, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta4_neon_bits: + .long 0x00000001,0x00000002,0x00000004,0x00000008 +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_extract_coeffs_eta4_neon_idx, %object + .size L_mldsa_extract_coeffs_eta4_neon_idx, 256 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_extract_coeffs_eta4_neon_idx: + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x08,0x09,0x0a,0x0b,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x08,0x09,0x0a,0x0b + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x08,0x09,0x0a,0x0b,0xff,0xff,0xff,0xff + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x08,0x09,0x0a,0x0b + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f +#ifndef __APPLE__ +.text +.globl mldsa_extract_coeffs_eta4_neon +WC_ASM_ATT_HIDDEN(mldsa_extract_coeffs_eta4_neon) +.type mldsa_extract_coeffs_eta4_neon,@function +.align 2 +mldsa_extract_coeffs_eta4_neon: +#else +.section __TEXT,__text +.globl _mldsa_extract_coeffs_eta4_neon +WC_ASM_ATT_HIDDEN(_mldsa_extract_coeffs_eta4_neon) +.p2align 2 +_mldsa_extract_coeffs_eta4_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] + stp d14, d15, [x29, #64] +#ifndef __APPLE__ + adrp x4, L_mldsa_extract_coeffs_eta4_neon_map + add x4, x4, :lo12:L_mldsa_extract_coeffs_eta4_neon_map +#else + adrp x4, L_mldsa_extract_coeffs_eta4_neon_map@PAGE + add x4, x4, L_mldsa_extract_coeffs_eta4_neon_map@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x5, L_mldsa_extract_coeffs_eta4_neon_lim + add x5, x5, :lo12:L_mldsa_extract_coeffs_eta4_neon_lim +#else + adrp x5, L_mldsa_extract_coeffs_eta4_neon_lim@PAGE + add x5, x5, L_mldsa_extract_coeffs_eta4_neon_lim@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x6, L_mldsa_extract_coeffs_eta4_neon_bits + add x6, x6, :lo12:L_mldsa_extract_coeffs_eta4_neon_bits +#else + adrp x6, L_mldsa_extract_coeffs_eta4_neon_bits@PAGE + add x6, x6, L_mldsa_extract_coeffs_eta4_neon_bits@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x7, L_mldsa_extract_coeffs_eta4_neon_idx + add x7, x7, :lo12:L_mldsa_extract_coeffs_eta4_neon_idx +#else + adrp x7, L_mldsa_extract_coeffs_eta4_neon_idx@PAGE + add x7, x7, L_mldsa_extract_coeffs_eta4_neon_idx@PAGEOFF +#endif /* __APPLE__ */ + ldr q2, [x4] + ldr q0, [x5] + ldr q1, [x6] + ldr w8, [x3] + add x2, x2, x8, lsl 2 +L_mldsa_extract_coeffs_eta4_neon_wide: + subs wzr, w1, #8 + blt L_mldsa_extract_coeffs_eta4_neon_tail + subs wzr, w8, #0xf0 + bgt L_mldsa_extract_coeffs_eta4_neon_tail + ldr d3, [x0] + movi v18.16b, #15 + ushr v5.16b, v3.16b, #4 + and v4.16b, v3.16b, v18.16b + zip1 v3.16b, v4.16b, v5.16b + tbl v6.16b, {v2.16b}, v3.16b + cmhi v8.16b, v0.16b, v3.16b + sxtl v7.8h, v6.8b + sxtl2 v18.8h, v6.16b + sxtl v10.4s, v7.4h + sxtl2 v11.4s, v7.8h + sxtl v12.4s, v18.4h + sxtl2 v13.4s, v18.8h + sxtl v9.8h, v8.8b + sxtl2 v18.8h, v8.16b + sxtl v14.4s, v9.4h + sxtl2 v15.4s, v9.8h + sxtl v16.4s, v18.4h + sxtl2 v17.4s, v18.8h + ushr v18.4s, v14.4s, #31 + addv s18, v18.4s + mov w10, v18.s[0] + and v18.16b, v14.16b, v1.16b + addv s18, v18.4s + mov w9, v18.s[0] + lsl x9, x9, #4 + ldr q18, [x7, x9] + tbl v10.16b, {v10.16b}, v18.16b + str q10, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + ushr v18.4s, v15.4s, #31 + addv s18, v18.4s + mov w10, v18.s[0] + and v18.16b, v15.16b, v1.16b + addv s18, v18.4s + mov w9, v18.s[0] + lsl x9, x9, #4 + ldr q18, [x7, x9] + tbl v11.16b, {v11.16b}, v18.16b + str q11, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + ushr v18.4s, v16.4s, #31 + addv s18, v18.4s + mov w10, v18.s[0] + and v18.16b, v16.16b, v1.16b + addv s18, v18.4s + mov w9, v18.s[0] + lsl x9, x9, #4 + ldr q18, [x7, x9] + tbl v12.16b, {v12.16b}, v18.16b + str q12, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + ushr v18.4s, v17.4s, #31 + addv s18, v18.4s + mov w10, v18.s[0] + and v18.16b, v17.16b, v1.16b + addv s18, v18.4s + mov w9, v18.s[0] + lsl x9, x9, #4 + ldr q18, [x7, x9] + tbl v13.16b, {v13.16b}, v18.16b + str q13, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + add x0, x0, #8 + sub w1, w1, #8 + b L_mldsa_extract_coeffs_eta4_neon_wide +L_mldsa_extract_coeffs_eta4_neon_tail: + subs wzr, w1, #0 + beq L_mldsa_extract_coeffs_eta4_neon_done + subs wzr, w8, #0x100 + beq L_mldsa_extract_coeffs_eta4_neon_done + ldrb w11, [x0] + add x0, x0, #1 + sub w1, w1, #1 + and w12, w11, #15 + subs wzr, w12, #9 + bcs L_mldsa_extract_coeffs_eta4_neon_skip_lo + ldrsb w12, [x4, x12] + str w12, [x2] + add x2, x2, #4 + add x8, x8, #1 + subs wzr, w8, #0x100 + beq L_mldsa_extract_coeffs_eta4_neon_done +L_mldsa_extract_coeffs_eta4_neon_skip_lo: + lsr w12, w11, #4 + subs wzr, w12, #9 + bcs L_mldsa_extract_coeffs_eta4_neon_skip_hi + ldrsb w12, [x4, x12] + str w12, [x2] + add x2, x2, #4 + add x8, x8, #1 +L_mldsa_extract_coeffs_eta4_neon_skip_hi: + b L_mldsa_extract_coeffs_eta4_neon_tail +L_mldsa_extract_coeffs_eta4_neon_done: + str w8, [x3] + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp d14, d15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret +#ifndef __APPLE__ + .size mldsa_extract_coeffs_eta4_neon,.-mldsa_extract_coeffs_eta4_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_rej_uniform_mask, %object + .size L_mldsa_rej_uniform_mask, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_rej_uniform_mask: + .long 0x007fffff,0x007fffff,0x007fffff,0x007fffff +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_rej_uniform_shuffle0, %object + .size L_mldsa_rej_uniform_shuffle0, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_rej_uniform_shuffle0: + .byte 0x00,0x01,0x02,0xff,0x03,0x04,0x05,0xff + .byte 0x06,0x07,0x08,0xff,0x09,0x0a,0x0b,0xff +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_rej_uniform_shuffle1, %object + .size L_mldsa_rej_uniform_shuffle1, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_rej_uniform_shuffle1: + .byte 0x0c,0x0d,0x0e,0xff,0x0f,0x10,0x11,0xff + .byte 0x12,0x13,0x14,0xff,0x15,0x16,0x17,0xff +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_rej_uniform_bits, %object + .size L_mldsa_rej_uniform_bits, 16 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_rej_uniform_bits: + .long 0x00000001,0x00000002,0x00000004,0x00000008 +#ifndef __APPLE__ + .text + .section .rodata + .type L_mldsa_rej_uniform_indices, %object + .size L_mldsa_rej_uniform_indices, 256 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 8-byte aligned, 64-bit aligned +#ifndef __APPLE__ + .align 3 +#else + .p2align 3 +#endif /* __APPLE__ */ +L_mldsa_rej_uniform_indices: + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x08,0x09,0x0a,0x0b,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x08,0x09,0x0a,0x0b + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x08,0x09,0x0a,0x0b,0xff,0xff,0xff,0xff + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f + .byte 0xff,0xff,0xff,0xff,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x08,0x09,0x0a,0x0b + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x04,0x05,0x06,0x07,0x08,0x09,0x0a,0x0b + .byte 0x0c,0x0d,0x0e,0x0f,0xff,0xff,0xff,0xff + .byte 0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07 + .byte 0x08,0x09,0x0a,0x0b,0x0c,0x0d,0x0e,0x0f +#ifndef __APPLE__ +.text +.globl mldsa_rej_uniform_neon +WC_ASM_ATT_HIDDEN(mldsa_rej_uniform_neon) +.type mldsa_rej_uniform_neon,@function +.align 2 +mldsa_rej_uniform_neon: +#else +.section __TEXT,__text +.globl _mldsa_rej_uniform_neon +WC_ASM_ATT_HIDDEN(_mldsa_rej_uniform_neon) +.p2align 2 +_mldsa_rej_uniform_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-64]! + add x29, sp, #0 + stp d8, d9, [x29, #16] + stp d10, d11, [x29, #32] + stp d12, d13, [x29, #48] +#ifndef __APPLE__ + adrp x4, L_mldsa_rej_uniform_mask + add x4, x4, :lo12:L_mldsa_rej_uniform_mask +#else + adrp x4, L_mldsa_rej_uniform_mask@PAGE + add x4, x4, L_mldsa_rej_uniform_mask@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x5, L_mldsa_aarch64_q + add x5, x5, :lo12:L_mldsa_aarch64_q +#else + adrp x5, L_mldsa_aarch64_q@PAGE + add x5, x5, L_mldsa_aarch64_q@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x6, L_mldsa_rej_uniform_bits + add x6, x6, :lo12:L_mldsa_rej_uniform_bits +#else + adrp x6, L_mldsa_rej_uniform_bits@PAGE + add x6, x6, L_mldsa_rej_uniform_bits@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x7, L_mldsa_rej_uniform_indices + add x7, x7, :lo12:L_mldsa_rej_uniform_indices +#else + adrp x7, L_mldsa_rej_uniform_indices@PAGE + add x7, x7, L_mldsa_rej_uniform_indices@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x8, L_mldsa_rej_uniform_shuffle0 + add x8, x8, :lo12:L_mldsa_rej_uniform_shuffle0 +#else + adrp x8, L_mldsa_rej_uniform_shuffle0@PAGE + add x8, x8, L_mldsa_rej_uniform_shuffle0@PAGEOFF +#endif /* __APPLE__ */ +#ifndef __APPLE__ + adrp x9, L_mldsa_rej_uniform_shuffle1 + add x9, x9, :lo12:L_mldsa_rej_uniform_shuffle1 +#else + adrp x9, L_mldsa_rej_uniform_shuffle1@PAGE + add x9, x9, L_mldsa_rej_uniform_shuffle1@PAGEOFF +#endif /* __APPLE__ */ + movz x15, #0xe001 + eor v1.16b, v1.16b, v1.16b + eor v12.16b, v12.16b, v12.16b + eor v13.16b, v13.16b, v13.16b + movk x15, #0x7f, lsl 16 + eor x14, x14, x14 + eor v10.16b, v10.16b, v10.16b + eor v11.16b, v11.16b, v11.16b + ldr q0, [x4] + ldr q3, [x5] + ldr q2, [x6] + ldr q4, [x8] + ldr q5, [x9] + subs wzr, w1, #0 + beq L_mldsa_rej_uniform_done + subs wzr, w1, #16 + blt L_mldsa_rej_uniform_loop_2 +L_mldsa_rej_uniform_loop_8: + ldr q6, [x2] + ldr d7, [x2, #16] + add x2, x2, #24 + tbl v8.16b, {v6.16b, v7.16b}, v4.16b + tbl v9.16b, {v6.16b, v7.16b}, v5.16b + and v6.16b, v8.16b, v0.16b + and v7.16b, v9.16b, v0.16b + cmgt v8.4s, v3.4s, v6.4s + cmgt v9.4s, v3.4s, v7.4s + ushr v12.4s, v8.4s, #31 + ushr v13.4s, v9.4s, #31 + addv s12, v12.4s + addv s13, v13.4s + mov x12, v12.d[0] + mov x13, v13.d[0] + and v10.16b, v8.16b, v2.16b + and v11.16b, v9.16b, v2.16b + addv s10, v10.4s + addv s11, v11.4s + mov w10, v10.s[0] + mov w11, v11.s[0] + lsl w10, w10, #4 + lsl w11, w11, #4 + ldr q10, [x7, x10] + ldr q11, [x7, x11] + tbl v8.16b, {v6.16b}, v10.16b + tbl v9.16b, {v7.16b}, v11.16b + str q8, [x0] + add x0, x0, x12, lsl 2 + add x14, x14, x12 + str q9, [x0] + add x0, x0, x13, lsl 2 + add x14, x14, x13 + subs w3, w3, #24 + beq L_mldsa_rej_uniform_done + sub w12, w1, w14 + subs x12, x12, #8 + blt L_mldsa_rej_uniform_loop_2 + b L_mldsa_rej_uniform_loop_8 +L_mldsa_rej_uniform_loop_2: + subs w12, w1, w14 + beq L_mldsa_rej_uniform_done + subs x12, x12, #2 + blt L_mldsa_rej_uniform_loop_lt_2 + ldr x4, [x2], #6 + lsr x5, x4, #24 + and x4, x4, #0x7fffff + and x5, x5, #0x7fffff + str w4, [x0] + add x6, x0, #4 + subs xzr, x4, x15 + csel x0, x6, x0, lt + cinc x14, x14, lt + str w5, [x0] + add x6, x0, #4 + subs xzr, x5, x15 + csel x0, x6, x0, lt + cinc x14, x14, lt + subs w3, w3, #6 + beq L_mldsa_rej_uniform_done + b L_mldsa_rej_uniform_loop_2 +L_mldsa_rej_uniform_loop_lt_2: + ldr x4, [x2], #6 + lsr x5, x4, #24 + and x4, x4, #0x7fffff + and x5, x5, #0x7fffff + str w4, [x0] + add x6, x0, #4 + subs xzr, x4, x15 + csel x0, x6, x0, lt + cinc x14, x14, lt + subs wzr, w1, w14 + beq L_mldsa_rej_uniform_done + str w5, [x0] + add x6, x0, #4 + subs xzr, x5, x15 + csel x0, x6, x0, lt + cinc x14, x14, lt + subs wzr, w1, w14 + beq L_mldsa_rej_uniform_done + subs w3, w3, #6 + beq L_mldsa_rej_uniform_done + b L_mldsa_rej_uniform_loop_lt_2 +L_mldsa_rej_uniform_done: + mov x0, x14 + ldp d8, d9, [x29, #16] + ldp d10, d11, [x29, #32] + ldp d12, d13, [x29, #48] + ldp x29, x30, [sp], #0x40 + ret +#ifndef __APPLE__ + .size mldsa_rej_uniform_neon,.-mldsa_rej_uniform_neon +#endif /* __APPLE__ */ +#endif /* WOLFSSL_HAVE_MLDSA */ +#endif /* __aarch64__ */ +#endif /* WOLFSSL_ARMASM */ + +#if defined(__linux__) && defined(__ELF__) +.section .note.GNU-stack,"",%progbits +#endif +#endif /* !WOLFSSL_ARMASM_INLINE */ diff --git a/wolfcrypt/src/port/arm/armv8-mldsa-asm.asm b/wolfcrypt/src/port/arm/armv8-mldsa-asm.asm new file mode 100644 index 00000000000..bf1d72c1df7 --- /dev/null +++ b/wolfcrypt/src/port/arm/armv8-mldsa-asm.asm @@ -0,0 +1,27751 @@ +; /* armv8-mldsa-asm +; * +; * Copyright (C) 2006-2026 wolfSSL Inc. +; * +; * This file is part of wolfSSL. +; * +; * wolfSSL is free software; you can redistribute it and/or modify +; * it under the terms of the GNU General Public License as published by +; * the Free Software Foundation; either version 3 of the License, or +; * (at your option) any later version. +; * +; * wolfSSL is distributed in the hope that it will be useful, +; * but WITHOUT ANY WARRANTY; without even the implied warranty of +; * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +; * GNU General Public License for more details. +; * +; * You should have received a copy of the GNU General Public License +; * along with this program; if not, write to the Free Software +; * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA +; */ + +; Generated using (from wolfssl): +; cd ../scripts +; ruby ./mldsa/mldsa.rb arm64 \ +; ../wolfssl/wolfcrypt/src/port/arm/armv8-mldsa-asm.asm + IF :DEF:WOLFSSL_HAVE_MLDSA + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_aarch64_consts + DCD 0x007fe001, 0x03802001, 0x00400000, 0x00000000 + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_aarch64_zetas + DCD 0xffff5c06, 0x000064f7, 0xffd83102, 0xfff81503 + DCD 0x00039e44, 0xfff42118, 0xfff2a128, 0x00071e24 + DCD 0x001bde2b, 0x0023e92b, 0xfffa84ad, 0xffe0147f + DCD 0x002f9a75, 0xffd3fb09, 0x002f7a49, 0x0028e527 + DCD 0x00299658, 0x000fa070, 0xffef85a4, 0x0036b788 + DCD 0xfff79d90, 0xffeeeaa0, 0x0027f968, 0xffdfd37b + DCD 0xffdfadd6, 0xffc51ae7, 0xffeaa4f7, 0xffcdfc98 + DCD 0x001ad035, 0xffffb422, 0x003d3201, 0x000445c5 + DCD 0x00294a67, 0x00017620, 0x002ef4cd, 0x0035dec5 + DCD 0xffe6a503, 0xffc9302c, 0xffd947d4, 0x003bbeaf + DCD 0xffc51585, 0xffd18e7c, 0x00368a96, 0xffd43e41 + DCD 0x00360400, 0xfffb6a4d, 0x0023d69c, 0xfff7c55d + DCD 0xffe6123d, 0xffe6ead6, 0x00357e1e, 0xffc5af59 + DCD 0x0035843f, 0xffdf5617, 0xffe7945c, 0x0038738c + DCD 0x000c63a8, 0x00081b9a, 0x000e8f76, 0x003b3853 + DCD 0x003b8534, 0xffd8fc30, 0x001f9d54, 0xffd54f2d + DCD 0xffc406e5, 0xffc406e5, 0xffe8ac81, 0xffe8ac81 + DCD 0xffc7e1cf, 0xffc7e1cf, 0xffd19819, 0xffd19819 + DCD 0xffe9d65d, 0xffe9d65d, 0x003509ee, 0x003509ee + DCD 0x002135c7, 0x002135c7, 0xffe7cfbb, 0xffe7cfbb + DCD 0xffeccf75, 0xffeccf75, 0x001d9772, 0x001d9772 + DCD 0xffc1b072, 0xffc1b072, 0xfff0bcf6, 0xfff0bcf6 + DCD 0xffcf5280, 0xffcf5280, 0xffcfd2ae, 0xffcfd2ae + DCD 0xffc890e0, 0xffc890e0, 0x0001efca, 0x0001efca + DCD 0x003410f2, 0x003410f2, 0xfff0fe85, 0xfff0fe85 + DCD 0x0020c638, 0x0020c638, 0x00296e9f, 0x00296e9f + DCD 0xffd2b7a3, 0xffd2b7a3, 0xffc7a44b, 0xffc7a44b + DCD 0xfff9ba6d, 0xfff9ba6d, 0xffda3409, 0xffda3409 + DCD 0xfff5c282, 0xfff5c282, 0xffed4113, 0xffed4113 + DCD 0xffffa63b, 0xffffa63b, 0xffec09f7, 0xffec09f7 + DCD 0xfffa2bdd, 0xfffa2bdd, 0x001495d4, 0x001495d4 + DCD 0x001c4563, 0x001c4563, 0xffea2c62, 0xffea2c62 + DCD 0xffccfbe9, 0xffccfbe9, 0x00040af0, 0x00040af0 + DCD 0x0007c417, 0x0007c417, 0x002f4588, 0x002f4588 + DCD 0x0000ad00, 0x0000ad00, 0xffef36be, 0xffef36be + DCD 0x000dcd44, 0x000dcd44, 0x003c675a, 0x003c675a + DCD 0xffc72bca, 0xffc72bca, 0xffffde7e, 0xffffde7e + DCD 0x00193948, 0x00193948, 0xffce69c0, 0xffce69c0 + DCD 0x0024756c, 0x0024756c, 0xfffcc7df, 0xfffcc7df + DCD 0x000b98a1, 0x000b98a1, 0xffebe808, 0xffebe808 + DCD 0x0002e46c, 0x0002e46c, 0xffc9c808, 0xffc9c808 + DCD 0x003036c2, 0x003036c2, 0xffe3bff6, 0xffe3bff6 + DCD 0xffdb3c93, 0xffdb3c93, 0xfffd4ae0, 0xfffd4ae0 + DCD 0x00141305, 0x00141305, 0x00147792, 0x00147792 + DCD 0x00139e25, 0x00139e25, 0xffe7d0e0, 0xffe7d0e0 + DCD 0xfff39944, 0xfff39944, 0xffea0802, 0xffea0802 + DCD 0xffd1eea2, 0xffd1eea2, 0xffc4c79c, 0xffc4c79c + DCD 0xffc8a057, 0xffc8a057, 0x003a97d9, 0x003a97d9 + DCD 0x001fea93, 0x0033ff5a, 0x002358d4, 0x003a41f8 + DCD 0xffccff72, 0x00223dfb, 0xffdaab9f, 0xffc9a422 + DCD 0x000412f5, 0x00252587, 0xffed24f0, 0x00359b5d + DCD 0xffca48a0, 0xffc6a2fc, 0xffedbb56, 0xffcf45de + DCD 0x000dbe5e, 0x001c5e1a, 0x000de0e6, 0x000c7f5a + DCD 0x00078f83, 0xffe7628a, 0xffff5704, 0xfff806fc + DCD 0xfff60021, 0xffd05af6, 0x001f0084, 0x0030ef86 + DCD 0xffc9b97d, 0xfff7fcd6, 0xfff44592, 0xffc921c2 + DCD 0x00053919, 0x0004610c, 0xffdacd41, 0x003eb01b + DCD 0x003472e7, 0xffcd003b, 0x001a7cc7, 0x00031924 + DCD 0x002b5ee5, 0x00291199, 0xffd87a3a, 0x00134d71 + DCD 0x003de11c, 0x00130984, 0x0025f051, 0x00185a46 + DCD 0xffc68518, 0x001314be, 0x00283891, 0xffc9db90 + DCD 0xffd25089, 0x001c853f, 0x001d0b4b, 0xffeff6a6 + DCD 0xffeba8be, 0x0012e11b, 0xffcd5e3e, 0xffea2d2f + DCD 0xfff91de4, 0x001406c7, 0x00327283, 0xffe20d6e + DCD 0xffec7953, 0x001d4099, 0xffd92578, 0xffeb05ad + DCD 0x0016e405, 0x000bdbe7, 0x00221de8, 0x0033f8cf + DCD 0xfff7b934, 0xffd4ca0c, 0xffe67ff8, 0xffe3d157 + DCD 0xffd8911b, 0xffc72c12, 0x000910d8, 0xffc65e1f + DCD 0xffe14658, 0x00251d8b, 0x002573b7, 0xfffd7c8f + DCD 0x001ddd98, 0x00336898, 0x0002d4bb, 0xffed93a7 + DCD 0xffcf6cbe, 0x00027c1c, 0x0018aa08, 0x002dfd71 + DCD 0x000c5ca5, 0x0019379a, 0xffc7a167, 0xffe48c3d + DCD 0xffd1a13c, 0x0035c539, 0x003b0115, 0x00041dc0 + DCD 0x0021c4f7, 0xfff11bf4, 0x001a35e7, 0x0007340e + DCD 0xfff97d45, 0x001a4cd0, 0xffe47cae, 0x001d2668 + DCD 0xffe68e98, 0xffef2633, 0xfffc05da, 0xffc57fdb + DCD 0xffd32764, 0xffdde1af, 0xfff993dd, 0xffdd1d09 + DCD 0x0002cc93, 0xfff11805, 0x00189c2a, 0xffc9e5a9 + DCD 0xfff78a50, 0x003bcf2c, 0xffff434e, 0xffeb36df + DCD 0x003c15ca, 0x00155e68, 0xfff316b6, 0x001e29ce + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_aarch64_zetas_qinv + DCD 0x00801c06, 0x6d1f44f7, 0x8cf87102, 0x8d187503 + DCD 0x61cc1e44, 0x58172118, 0x6017a128, 0x61cb9e24 + DCD 0x12613e2b, 0x93c9492b, 0xae1024ad, 0xbeeff47f + DCD 0x8cfe3a75, 0x1eb51b09, 0xeef89a49, 0x254dc527 + DCD 0x66f49658, 0x7c1da070, 0xaea405a4, 0x3327b788 + DCD 0x6ba99d90, 0x0d42eaa0, 0xeb54f968, 0x28cf337b + DCD 0x629a6dd6, 0xcba1fae7, 0xb50984f7, 0xd360fc98 + DCD 0x13a17035, 0x6d83f422, 0xa9fd5201, 0xba3ce5c5 + DCD 0x91f62a67, 0x9ec57620, 0xac4894cd, 0x6d8e7ec5 + DCD 0x5f070503, 0xbfceb02c, 0x8ed3c7d4, 0xdc919eaf + DCD 0xf3f5b585, 0xe3a10e7c, 0xde894a96, 0x6b1c5e41 + DCD 0xc0b60400, 0x7ac50a4d, 0x9cf7569c, 0xbe23655d + DCD 0x97adb23d, 0xca41aad6, 0x18f93e1e, 0x6d30cf59 + DCD 0x8d3d643f, 0x3b223617, 0x3473145c, 0x78a9f38c + DCD 0x588163a8, 0x9e7b5b9a, 0xeefd4f76, 0x89c59853 + DCD 0xa6e20534, 0xc75efc30, 0x99ca1d54, 0xc73aef2d + DCD 0xa220a6e5, 0xa220a6e5, 0xd8f8cc81, 0xd8f8cc81 + DCD 0x5081c1cf, 0x5081c1cf, 0x8a54b819, 0x8a54b819 + DCD 0x8035765d, 0x8035765d, 0x6272c9ee, 0x6272c9ee + DCD 0x5f5a15c7, 0x5f5a15c7, 0x085f2fbb, 0x085f2fbb + DCD 0xb35b6f75, 0xb35b6f75, 0xc20bd772, 0xc20bd772 + DCD 0xc4cff072, 0xc4cff072, 0x748f7cf6, 0x748f7cf6 + DCD 0xaa1f5280, 0xaa1f5280, 0x5b2592ae, 0x5b2592ae + DCD 0x21e490e0, 0x21e490e0, 0x80fb2fca, 0x80fb2fca + DCD 0xd15250f2, 0xd15250f2, 0xf1419e85, 0xf1419e85 + DCD 0xdce7c638, 0xdce7c638, 0x5a7d4e9f, 0x5a7d4e9f + DCD 0x114717a3, 0x114717a3, 0xfad1044b, 0xfad1044b + DCD 0xb4c75a6d, 0xb4c75a6d, 0x65db5409, 0x65db5409 + DCD 0x7f460282, 0x7f460282, 0x6a8fa113, 0x6a8fa113 + DCD 0xc347063b, 0xc347063b, 0x61aae9f7, 0x61aae9f7 + DCD 0xcaf5cbdd, 0xcaf5cbdd, 0xf8cf15d4, 0xf8cf15d4 + DCD 0x6348a563, 0x6348a563, 0x9c766c62, 0x9c766c62 + DCD 0x4eca1be9, 0x4eca1be9, 0xc9620af0, 0xc9620af0 + DCD 0x490aa417, 0x490aa417, 0x44e04588, 0x44e04588 + DCD 0x95a0ad00, 0x95a0ad00, 0x7fc6f6be, 0x7fc6f6be + DCD 0x27b64d44, 0x27b64d44, 0x4827a75a, 0x4827a75a + DCD 0x28406bca, 0x28406bca, 0xb4cf9e7e, 0xb4cf9e7e + DCD 0xa3423948, 0xa3423948, 0xed0669c0, 0xed0669c0 + DCD 0x88d1f56c, 0x88d1f56c, 0x2578a7df, 0x2578a7df + DCD 0xa69fb8a1, 0xa69fb8a1, 0x98ece808, 0x98ece808 + DCD 0xd690646c, 0xd690646c, 0x54cac808, 0x54cac808 + DCD 0xae0876c2, 0xae0876c2, 0x54e27ff6, 0x54e27ff6 + DCD 0x69ed9c93, 0x69ed9c93, 0xb9594ae0, 0xb9594ae0 + DCD 0x13f4b305, 0x13f4b305, 0x0e06b792, 0x0e06b792 + DCD 0xf5583e25, 0xf5583e25, 0x0a03d0e0, 0x0a03d0e0 + DCD 0xe11c1944, 0xe11c1944, 0x47ea4802, 0x47ea4802 + DCD 0x74a62ea2, 0x74a62ea2, 0x3ab8479c, 0x3ab8479c + DCD 0x44538057, 0x44538057, 0xcab5b7d9, 0xcab5b7d9 + DCD 0xfff24a93, 0x3b1f3f5a, 0x513dd8d4, 0x2c7941f8 + DCD 0xaebb3f72, 0x36619dfb, 0x01ce8b9f, 0xab4de422 + DCD 0xdbe2b2f5, 0xfd560587, 0xec8b24f0, 0x79213b5d + DCD 0x78de48a0, 0x462622fc, 0x64587b56, 0x718b05de + DCD 0x00d97e5e, 0xe6df9e1a, 0xe12aa0e6, 0x4af7bf5a + DCD 0x3c77ef83, 0xcf38a28a, 0x78dfd704, 0x72d786fc + DCD 0x337a2021, 0x682f1af6, 0xae2f8084, 0x7321af86 + DCD 0x6c79597d, 0xec92bcd6, 0x07a68592, 0x4b0161c2 + DCD 0x7ea85919, 0x3625e10c, 0xbd02ed41, 0x34c2101b + DCD 0xb71152e7, 0x6e54603b, 0x08335cc7, 0x61279924 + DCD 0x8d87fee5, 0xb9dc3199, 0xda1fba3a, 0x75416d71 + DCD 0x9e61611c, 0xaf438984, 0xd9b01051, 0x00611a46 + DCD 0xa4698518, 0xfbaad4be, 0x02ba5891, 0xb33bdb90 + DCD 0x29637089, 0xed44653f, 0x28066b4b, 0x43c4b6a6 + DCD 0x0e0368be, 0x3ab6411b, 0x84951e3e, 0x6a100d2f + DCD 0xc1b59de4, 0x396ce6c7, 0x1902d283, 0x428fcd6e + DCD 0x3196d953, 0xbfb06099, 0x48882578, 0x3e20a5ad + DCD 0xee178405, 0x2408bbe7, 0xefdf1de8, 0x53cdd8cf + DCD 0x2d1e3934, 0xc3164a0c, 0xb3e57ff8, 0x2a8eb157 + DCD 0xf07bf11b, 0x24496c12, 0x162410d8, 0x380a3e1f + DCD 0x5cac4658, 0x0a567d8b, 0xaf1c53b7, 0xa40f5c8f + DCD 0x4fd0dd98, 0x81466898, 0xe91a34bb, 0x7ae273a7 + DCD 0x86672cbe, 0xb185fc1c, 0x3159aa08, 0xcb5c1d71 + DCD 0xcd20fca5, 0xc20c779a, 0xdc748167, 0x66ec2c3d + DCD 0x85f9213c, 0x005ce539, 0x29dda115, 0xa3bc1dc0 + DCD 0x9940a4f7, 0xf96f9bf4, 0xef5715e7, 0x1788f40e + DCD 0xa1221d45, 0x21b44cd0, 0xf07a3cae, 0x10ea2668 + DCD 0xe5b98e98, 0x97358633, 0xfbb745da, 0x2e40dfdb + DCD 0x42bfa764, 0xa093c1af, 0xb7f533dd, 0x42fe3d09 + DCD 0x5c152c93, 0x3471b805, 0xa69ddc2a, 0x0bff05a9 + DCD 0x09418a50, 0x94214f2c, 0x7969034e, 0x734716df + DCD 0xc5f555ca, 0x17e25e68, 0xdfc9d6b6, 0x1657e9ce + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_aarch64_zetas_inv + DCD 0xffe1d632, 0x000ce94a, 0xffeaa198, 0xffc3ea36 + DCD 0x0014c921, 0x0000bcb2, 0xffc430d4, 0x000875b0 + DCD 0x00361a57, 0xffe763d6, 0x000ee7fb, 0xfffd336d + DCD 0x0022e2f7, 0x00066c23, 0x00221e51, 0x002cd89c + DCD 0x003a8025, 0x0003fa26, 0x0010d9cd, 0x00197168 + DCD 0xffe2d998, 0x001b8352, 0xffe5b330, 0x000682bb + DCD 0xfff8cbf2, 0xffe5ca19, 0x000ee40c, 0xffde3b09 + DCD 0xfffbe240, 0xffc4feeb, 0xffca3ac7, 0x002e5ec4 + DCD 0x001b73c3, 0x00385e99, 0xffe6c866, 0xfff3a35b + DCD 0xffd2028f, 0xffe755f8, 0xfffd83e4, 0x00309342 + DCD 0x00126c59, 0xfffd2b45, 0xffcc9768, 0xffe22268 + DCD 0x00028371, 0xffda8c49, 0xffdae275, 0x001eb9a8 + DCD 0x0039a1e1, 0xfff6ef28, 0x0038d3ee, 0x00276ee5 + DCD 0x001c2ea9, 0x00198008, 0x002b35f4, 0x000846cc + DCD 0xffcc0731, 0xffdde218, 0xfff42419, 0xffe91bfb + DCD 0x0014fa53, 0x0026da88, 0xffe2bf67, 0x001386ad + DCD 0x001df292, 0xffcd8d7d, 0xffebf939, 0x0006e21c + DCD 0x0015d2d1, 0x0032a1c2, 0xffed1ee5, 0x00145742 + DCD 0x0010095a, 0xffe2f4b5, 0xffe37ac1, 0x002daf77 + DCD 0x00362470, 0xffd7c76f, 0xffeceb42, 0x00397ae8 + DCD 0xffe7a5ba, 0xffda0faf, 0xffecf67c, 0xffc21ee4 + DCD 0xffecb28f, 0x002785c6, 0xffd6ee67, 0xffd4a11b + DCD 0xfffce6dc, 0xffe58339, 0x0032ffc5, 0xffcb8d19 + DCD 0xffc14fe5, 0x002532bf, 0xfffb9ef4, 0xfffac6e7 + DCD 0x0036de3e, 0x000bba6e, 0x0008032a, 0x00364683 + DCD 0xffcf107a, 0xffe0ff7c, 0x002fa50a, 0x0009ffdf + DCD 0x0007f904, 0x0000a8fc, 0x00189d76, 0xfff8707d + DCD 0xfff380a6, 0xfff21f1a, 0xffe3a1e6, 0xfff241a2 + DCD 0x0030ba22, 0x001244aa, 0x00395d04, 0x0035b760 + DCD 0xffca64a3, 0x0012db10, 0xffdada79, 0xfffbed0b + DCD 0x00365bde, 0x00255461, 0xffddc205, 0x0033008e + DCD 0xffc5be08, 0xffdca72c, 0xffcc00a6, 0xffe0156d + DCD 0xffc56827, 0xffc56827, 0x00375fa9, 0x00375fa9 + DCD 0x003b3864, 0x003b3864, 0x002e115e, 0x002e115e + DCD 0x0015f7fe, 0x0015f7fe, 0x000c66bc, 0x000c66bc + DCD 0x00182f20, 0x00182f20, 0xffec61db, 0xffec61db + DCD 0xffeb886e, 0xffeb886e, 0xffebecfb, 0xffebecfb + DCD 0x0002b520, 0x0002b520, 0x0024c36d, 0x0024c36d + DCD 0x001c400a, 0x001c400a, 0xffcfc93e, 0xffcfc93e + DCD 0x003637f8, 0x003637f8, 0xfffd1b94, 0xfffd1b94 + DCD 0x001417f8, 0x001417f8, 0xfff4675f, 0xfff4675f + DCD 0x00033821, 0x00033821, 0xffdb8a94, 0xffdb8a94 + DCD 0x00319640, 0x00319640, 0xffe6c6b8, 0xffe6c6b8 + DCD 0x00002182, 0x00002182, 0x0038d436, 0x0038d436 + DCD 0xffc398a6, 0xffc398a6, 0xfff232bc, 0xfff232bc + DCD 0x0010c942, 0x0010c942, 0xffff5300, 0xffff5300 + DCD 0xffd0ba78, 0xffd0ba78, 0xfff83be9, 0xfff83be9 + DCD 0xfffbf510, 0xfffbf510, 0x00330417, 0x00330417 + DCD 0x0015d39e, 0x0015d39e, 0xffe3ba9d, 0xffe3ba9d + DCD 0xffeb6a2c, 0xffeb6a2c, 0x0005d423, 0x0005d423 + DCD 0x0013f609, 0x0013f609, 0x000059c5, 0x000059c5 + DCD 0x0012beed, 0x0012beed, 0x000a3d7e, 0x000a3d7e + DCD 0x0025cbf7, 0x0025cbf7, 0x00064593, 0x00064593 + DCD 0x00385bb5, 0x00385bb5, 0x002d485d, 0x002d485d + DCD 0xffd69161, 0xffd69161, 0xffdf39c8, 0xffdf39c8 + DCD 0x000f017b, 0x000f017b, 0xffcbef0e, 0xffcbef0e + DCD 0xfffe1036, 0xfffe1036, 0x00376f20, 0x00376f20 + DCD 0x00302d52, 0x00302d52, 0x0030ad80, 0x0030ad80 + DCD 0x000f430a, 0x000f430a, 0x003e4f8e, 0x003e4f8e + DCD 0xffe2688e, 0xffe2688e, 0x0013308b, 0x0013308b + DCD 0x00183045, 0x00183045, 0xffdeca39, 0xffdeca39 + DCD 0xffcaf612, 0xffcaf612, 0x001629a3, 0x001629a3 + DCD 0x002e67e7, 0x002e67e7, 0x00381e31, 0x00381e31 + DCD 0x0017537f, 0x0017537f, 0x003bf91b, 0x003bf91b + DCD 0x002ab0d3, 0xffe062ac, 0x002703d0, 0xffc47acc + DCD 0xffc4c7ad, 0xfff1708a, 0xfff7e466, 0xfff39c58 + DCD 0xffc78c74, 0x00186ba4, 0x0020a9e9, 0xffca7bc1 + DCD 0x003a50a7, 0xffca81e2, 0x0019152a, 0x0019edc3 + DCD 0x00083aa3, 0xffdc2964, 0x000495b3, 0xffc9fc00 + DCD 0x002bc1bf, 0xffc9756a, 0x002e7184, 0x003aea7b + DCD 0xffc44151, 0x0026b82c, 0x0036cfd4, 0x00195afd + DCD 0xffca213b, 0xffd10b33, 0xfffe89e0, 0xffd6b599 + DCD 0xfffbba3b, 0xffc2cdff, 0x00004bde, 0xffe52fcb + DCD 0x00320368, 0x00155b09, 0x003ae519, 0x0020522a + DCD 0x00202c85, 0xffd80698, 0x00111560, 0x00086270 + DCD 0xffc94878, 0x00107a5c, 0xfff05f90, 0xffd669a8 + DCD 0xffd71ad9, 0xffd085b7, 0x002c04f7, 0xffd0658b + DCD 0x001feb81, 0x00057b53, 0xffdc16d5, 0xffe421d5 + DCD 0xfff8e1dc, 0x000d5ed8, 0x000bdee8, 0xfffc61bc + DCD 0x0007eafd, 0x0027cefe, 0x003caa21, 0x0000a3fa + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_aarch64_zetas_inv_qinv + DCD 0xe9a81632, 0x2036294a, 0xe81da198, 0x3a0aaa36 + DCD 0x8cb8e921, 0x8696fcb2, 0x6bdeb0d4, 0xf6be75b0 + DCD 0xf400fa57, 0x596223d6, 0xcb8e47fb, 0xa3ead36d + DCD 0xbd01c2f7, 0x480acc23, 0x5f6c3e51, 0xbd40589c + DCD 0xd1bf2025, 0x0448ba26, 0x68ca79cd, 0x1a467168 + DCD 0xef15d998, 0x0f85c352, 0xde4bb330, 0x5edde2bb + DCD 0xe8770bf2, 0x10a8ea19, 0x0690640c, 0x66bf5b09 + DCD 0x5c43e240, 0xd6225eeb, 0xffa31ac7, 0x7a06dec4 + DCD 0x9913d3c3, 0x238b7e99, 0x3df38866, 0x32df035b + DCD 0x34a3e28f, 0xcea655f8, 0x4e7a03e4, 0x7998d342 + DCD 0x851d8c59, 0x16e5cb45, 0x7eb99768, 0xb02f2268 + DCD 0x5bf0a371, 0x50e3ac49, 0xf5a98275, 0xa353b9a8 + DCD 0xc7f5c1e1, 0xe9dbef28, 0xdbb693ee, 0x0f840ee5 + DCD 0xd5714ea9, 0x4c1a8008, 0x3ce9b5f4, 0xd2e1c6cc + DCD 0xac322731, 0x1020e218, 0xdbf74419, 0x11e87bfb + DCD 0xc1df5a53, 0xb777da88, 0x404f9f67, 0xce6926ad + DCD 0xbd703292, 0xe6fd2d7d, 0xc6931939, 0x3e4a621c + DCD 0x95eff2d1, 0x7b6ae1c2, 0xc549bee5, 0xf1fc9742 + DCD 0xbc3b495a, 0xd7f994b5, 0x12bb9ac1, 0xd69c8f77 + DCD 0x4cc42470, 0xfd45a76f, 0x04552b42, 0x5b967ae8 + DCD 0xff9ee5ba, 0x264fefaf, 0x50bc767c, 0x619e9ee4 + DCD 0x8abe928f, 0x25e045c6, 0x4623ce67, 0x7278011b + DCD 0x9ed866dc, 0xf7cca339, 0x91ab9fc5, 0x48eead19 + DCD 0xcb3defe5, 0x42fd12bf, 0xc9da1ef4, 0x8157a6e7 + DCD 0xb4fe9e3e, 0xf8597a6e, 0x136d432a, 0x9386a683 + DCD 0x8cde507a, 0x51d07f7c, 0x97d0e50a, 0xcc85dfdf + DCD 0x8d287904, 0x872028fc, 0x30c75d76, 0xc388107d + DCD 0xb50840a6, 0x1ed55f1a, 0x192061e6, 0xff2681a2 + DCD 0x8e74fa22, 0x9ba784aa, 0xb9d9dd04, 0x8721b760 + DCD 0x86dec4a3, 0x1374db10, 0x02a9fa79, 0x241d4d0b + DCD 0x54b21bde, 0xfe317461, 0xc99e6205, 0x5144c08e + DCD 0xd386be08, 0xaec2272c, 0xc4e0c0a6, 0x000db56d + DCD 0x354a4827, 0x354a4827, 0xbbac7fa9, 0xbbac7fa9 + DCD 0xc547b864, 0xc547b864, 0x8b59d15e, 0x8b59d15e + DCD 0xb815b7fe, 0xb815b7fe, 0x1ee3e6bc, 0x1ee3e6bc + DCD 0xf5fc2f20, 0xf5fc2f20, 0x0aa7c1db, 0x0aa7c1db + DCD 0xf1f9486e, 0xf1f9486e, 0xec0b4cfb, 0xec0b4cfb + DCD 0x46a6b520, 0x46a6b520, 0x9612636d, 0x9612636d + DCD 0xab1d800a, 0xab1d800a, 0x51f7893e, 0x51f7893e + DCD 0xab3537f8, 0xab3537f8, 0x296f9b94, 0x296f9b94 + DCD 0x671317f8, 0x671317f8, 0x5960475f, 0x5960475f + DCD 0xda875821, 0xda875821, 0x772e0a94, 0x772e0a94 + DCD 0x12f99640, 0x12f99640, 0x5cbdc6b8, 0x5cbdc6b8 + DCD 0x4b306182, 0x4b306182, 0xd7bf9436, 0xd7bf9436 + DCD 0xb7d858a6, 0xb7d858a6, 0xd849b2bc, 0xd849b2bc + DCD 0x80390942, 0x80390942, 0x6a5f5300, 0x6a5f5300 + DCD 0xbb1fba78, 0xbb1fba78, 0xb6f55be9, 0xb6f55be9 + DCD 0x369df510, 0x369df510, 0xb135e417, 0xb135e417 + DCD 0x6389939e, 0x6389939e, 0x9cb75a9d, 0x9cb75a9d + DCD 0x0730ea2c, 0x0730ea2c, 0x350a3423, 0x350a3423 + DCD 0x9e551609, 0x9e551609, 0x3cb8f9c5, 0x3cb8f9c5 + DCD 0x95705eed, 0x95705eed, 0x80b9fd7e, 0x80b9fd7e + DCD 0x9a24abf7, 0x9a24abf7, 0x4b38a593, 0x4b38a593 + DCD 0x052efbb5, 0x052efbb5, 0xeeb8e85d, 0xeeb8e85d + DCD 0xa582b161, 0xa582b161, 0x231839c8, 0x231839c8 + DCD 0x0ebe617b, 0x0ebe617b, 0x2eadaf0e, 0x2eadaf0e + DCD 0x7f04d036, 0x7f04d036, 0xde1b6f20, 0xde1b6f20 + DCD 0xa4da6d52, 0xa4da6d52, 0x55e0ad80, 0x55e0ad80 + DCD 0x8b70830a, 0x8b70830a, 0x3b300f8e, 0x3b300f8e + DCD 0x3df4288e, 0x3df4288e, 0x4ca4908b, 0x4ca4908b + DCD 0xf7a0d045, 0xf7a0d045, 0xa0a5ea39, 0xa0a5ea39 + DCD 0x9d8d3612, 0x9d8d3612, 0x7fca89a3, 0x7fca89a3 + DCD 0x75ab47e7, 0x75ab47e7, 0xaf7e3e31, 0xaf7e3e31 + DCD 0x2707337f, 0x2707337f, 0x5ddf591b, 0x5ddf591b + DCD 0x38c510d3, 0x6635e2ac, 0x38a103d0, 0x591dfacc + DCD 0x763a67ad, 0x1102b08a, 0x6184a466, 0xa77e9c58 + DCD 0x87560c74, 0xcb8ceba4, 0xc4ddc9e9, 0x72c29bc1 + DCD 0x92cf30a7, 0xe706c1e2, 0x35be552a, 0x68524dc3 + DCD 0x41dc9aa3, 0x6308a964, 0x853af5b3, 0x3f49fc00 + DCD 0x94e3a1bf, 0x2176b56a, 0x1c5ef184, 0x0c0a4a7b + DCD 0x236e6151, 0x712c382c, 0x40314fd4, 0xa0f8fafd + DCD 0x9271813b, 0x53b76b33, 0x613a89e0, 0x6e09d599 + DCD 0x45c31a3b, 0x5602adff, 0x927c0bde, 0xec5e8fcb + DCD 0x2c9f0368, 0x4af67b09, 0x345e0519, 0x9d65922a + DCD 0xd730cc85, 0x14ab0698, 0xf2bd1560, 0x94566270 + DCD 0xccd84878, 0x515bfa5c, 0x83e25f90, 0x990b69a8 + DCD 0xdab23ad9, 0x110765b7, 0xe14ae4f7, 0x7301c58b + DCD 0x41100b81, 0x51efdb53, 0x6c36b6d5, 0xed9ec1d5 + DCD 0x9e3461dc, 0x9fe85ed8, 0xa7e8dee8, 0x9e33e1bc + DCD 0x72e78afd, 0x73078efe, 0x0900ca21, 0xff7fe3fa + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_ntt_neon +mldsa_ntt_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x1, L_mldsa_aarch64_zetas + add x1, x1, L_mldsa_aarch64_zetas + adrp x2, L_mldsa_aarch64_zetas_qinv + add x2, x2, L_mldsa_aarch64_zetas_qinv + adrp x3, L_mldsa_aarch64_consts + add x3, x3, L_mldsa_aarch64_consts + ldr Q4, [x3] + ldr Q5, [x0] + ldr Q6, [x0, #64] + ldr Q7, [x0, #128] + ldr Q8, [x0, #192] + ldr Q9, [x0, #256] + ldr Q10, [x0, #320] + ldr Q11, [x0, #384] + ldr Q12, [x0, #448] + ldr Q13, [x0, #512] + ldr Q14, [x0, #576] + ldr Q15, [x0, #640] + ldr Q16, [x0, #704] + ldr Q17, [x0, #768] + ldr Q18, [x0, #832] + ldr Q19, [x0, #896] + ldr Q20, [x0, #960] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0] + str Q6, [x0, #64] + str Q7, [x0, #128] + str Q8, [x0, #192] + str Q9, [x0, #256] + str Q10, [x0, #320] + str Q11, [x0, #384] + str Q12, [x0, #448] + str Q13, [x0, #512] + str Q14, [x0, #576] + str Q15, [x0, #640] + str Q16, [x0, #704] + str Q17, [x0, #768] + str Q18, [x0, #832] + str Q19, [x0, #896] + str Q20, [x0, #960] + ldr Q5, [x0, #16] + ldr Q6, [x0, #80] + ldr Q7, [x0, #144] + ldr Q8, [x0, #208] + ldr Q9, [x0, #272] + ldr Q10, [x0, #336] + ldr Q11, [x0, #400] + ldr Q12, [x0, #464] + ldr Q13, [x0, #528] + ldr Q14, [x0, #592] + ldr Q15, [x0, #656] + ldr Q16, [x0, #720] + ldr Q17, [x0, #784] + ldr Q18, [x0, #848] + ldr Q19, [x0, #912] + ldr Q20, [x0, #976] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0, #16] + str Q6, [x0, #80] + str Q7, [x0, #144] + str Q8, [x0, #208] + str Q9, [x0, #272] + str Q10, [x0, #336] + str Q11, [x0, #400] + str Q12, [x0, #464] + str Q13, [x0, #528] + str Q14, [x0, #592] + str Q15, [x0, #656] + str Q16, [x0, #720] + str Q17, [x0, #784] + str Q18, [x0, #848] + str Q19, [x0, #912] + str Q20, [x0, #976] + ldr Q5, [x0, #32] + ldr Q6, [x0, #96] + ldr Q7, [x0, #160] + ldr Q8, [x0, #224] + ldr Q9, [x0, #288] + ldr Q10, [x0, #352] + ldr Q11, [x0, #416] + ldr Q12, [x0, #480] + ldr Q13, [x0, #544] + ldr Q14, [x0, #608] + ldr Q15, [x0, #672] + ldr Q16, [x0, #736] + ldr Q17, [x0, #800] + ldr Q18, [x0, #864] + ldr Q19, [x0, #928] + ldr Q20, [x0, #992] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0, #32] + str Q6, [x0, #96] + str Q7, [x0, #160] + str Q8, [x0, #224] + str Q9, [x0, #288] + str Q10, [x0, #352] + str Q11, [x0, #416] + str Q12, [x0, #480] + str Q13, [x0, #544] + str Q14, [x0, #608] + str Q15, [x0, #672] + str Q16, [x0, #736] + str Q17, [x0, #800] + str Q18, [x0, #864] + str Q19, [x0, #928] + str Q20, [x0, #992] + ldr Q5, [x0, #48] + ldr Q6, [x0, #112] + ldr Q7, [x0, #176] + ldr Q8, [x0, #240] + ldr Q9, [x0, #304] + ldr Q10, [x0, #368] + ldr Q11, [x0, #432] + ldr Q12, [x0, #496] + ldr Q13, [x0, #560] + ldr Q14, [x0, #624] + ldr Q15, [x0, #688] + ldr Q16, [x0, #752] + ldr Q17, [x0, #816] + ldr Q18, [x0, #880] + ldr Q19, [x0, #944] + ldr Q20, [x0, #1008] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0, #48] + str Q6, [x0, #112] + str Q7, [x0, #176] + str Q8, [x0, #240] + str Q9, [x0, #304] + str Q10, [x0, #368] + str Q11, [x0, #432] + str Q12, [x0, #496] + str Q13, [x0, #560] + str Q14, [x0, #624] + str Q15, [x0, #688] + str Q16, [x0, #752] + str Q17, [x0, #816] + str Q18, [x0, #880] + str Q19, [x0, #944] + str Q20, [x0, #1008] + ldp Q5, Q6, [x0] + ldp Q7, Q8, [x0, #32] + ldp Q9, Q10, [x0, #64] + ldp Q11, Q12, [x0, #96] + ldp Q13, Q14, [x0, #128] + ldp Q15, Q16, [x0, #160] + ldp Q17, Q18, [x0, #192] + ldp Q19, Q20, [x0, #224] + ldr Q0, [x1, #64] + ldr Q1, [x2, #64] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #128] + ldp Q1, Q3, [x2, #128] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #256] + ldp Q1, Q3, [x2, #256] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #288] + ldp Q1, Q3, [x2, #288] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #320] + ldp Q1, Q3, [x2, #320] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #352] + ldp Q1, Q3, [x2, #352] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #768] + ldp Q1, Q3, [x2, #768] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #800] + ldp Q1, Q3, [x2, #800] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #832] + ldp Q1, Q3, [x2, #832] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #864] + ldp Q1, Q3, [x2, #864] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + stp Q5, Q6, [x0] + stp Q7, Q8, [x0, #32] + stp Q9, Q10, [x0, #64] + stp Q11, Q12, [x0, #96] + stp Q13, Q14, [x0, #128] + stp Q15, Q16, [x0, #160] + stp Q17, Q18, [x0, #192] + stp Q19, Q20, [x0, #224] + ldp Q5, Q6, [x0, #256] + ldp Q7, Q8, [x0, #288] + ldp Q9, Q10, [x0, #320] + ldp Q11, Q12, [x0, #352] + ldp Q13, Q14, [x0, #384] + ldp Q15, Q16, [x0, #416] + ldp Q17, Q18, [x0, #448] + ldp Q19, Q20, [x0, #480] + ldr Q0, [x1, #80] + ldr Q1, [x2, #80] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #160] + ldp Q1, Q3, [x2, #160] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #384] + ldp Q1, Q3, [x2, #384] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #416] + ldp Q1, Q3, [x2, #416] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #448] + ldp Q1, Q3, [x2, #448] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #480] + ldp Q1, Q3, [x2, #480] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #896] + ldp Q1, Q3, [x2, #896] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #928] + ldp Q1, Q3, [x2, #928] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #960] + ldp Q1, Q3, [x2, #960] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #992] + ldp Q1, Q3, [x2, #992] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + stp Q5, Q6, [x0, #256] + stp Q7, Q8, [x0, #288] + stp Q9, Q10, [x0, #320] + stp Q11, Q12, [x0, #352] + stp Q13, Q14, [x0, #384] + stp Q15, Q16, [x0, #416] + stp Q17, Q18, [x0, #448] + stp Q19, Q20, [x0, #480] + ldp Q5, Q6, [x0, #512] + ldp Q7, Q8, [x0, #544] + ldp Q9, Q10, [x0, #576] + ldp Q11, Q12, [x0, #608] + ldp Q13, Q14, [x0, #640] + ldp Q15, Q16, [x0, #672] + ldp Q17, Q18, [x0, #704] + ldp Q19, Q20, [x0, #736] + ldr Q0, [x1, #96] + ldr Q1, [x2, #96] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #192] + ldp Q1, Q3, [x2, #192] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #512] + ldp Q1, Q3, [x2, #512] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #544] + ldp Q1, Q3, [x2, #544] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #576] + ldp Q1, Q3, [x2, #576] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #608] + ldp Q1, Q3, [x2, #608] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldr Q0, [x1, #1024] + ldr Q2, [x1, #1040] + ldr Q1, [x2, #1024] + ldr Q3, [x2, #1040] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldr Q0, [x1, #1056] + ldr Q2, [x1, #1072] + ldr Q1, [x2, #1056] + ldr Q3, [x2, #1072] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldr Q0, [x1, #1088] + ldr Q2, [x1, #1104] + ldr Q1, [x2, #1088] + ldr Q3, [x2, #1104] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldr Q0, [x1, #1120] + ldr Q2, [x1, #1136] + ldr Q1, [x2, #1120] + ldr Q3, [x2, #1136] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + stp Q5, Q6, [x0, #512] + stp Q7, Q8, [x0, #544] + stp Q9, Q10, [x0, #576] + stp Q11, Q12, [x0, #608] + stp Q13, Q14, [x0, #640] + stp Q15, Q16, [x0, #672] + stp Q17, Q18, [x0, #704] + stp Q19, Q20, [x0, #736] + ldp Q5, Q6, [x0, #768] + ldp Q7, Q8, [x0, #800] + ldp Q9, Q10, [x0, #832] + ldp Q11, Q12, [x0, #864] + ldp Q13, Q14, [x0, #896] + ldp Q15, Q16, [x0, #928] + ldp Q17, Q18, [x0, #960] + ldp Q19, Q20, [x0, #992] + ldr Q0, [x1, #112] + ldr Q1, [x2, #112] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #224] + ldp Q1, Q3, [x2, #224] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #640] + ldp Q1, Q3, [x2, #640] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #672] + ldp Q1, Q3, [x2, #672] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #704] + ldp Q1, Q3, [x2, #704] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #736] + ldp Q1, Q3, [x2, #736] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldr Q0, [x1, #1152] + ldr Q2, [x1, #1168] + ldr Q1, [x2, #1152] + ldr Q3, [x2, #1168] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldr Q0, [x1, #1184] + ldr Q2, [x1, #1200] + ldr Q1, [x2, #1184] + ldr Q3, [x2, #1200] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldr Q0, [x1, #1216] + ldr Q2, [x1, #1232] + ldr Q1, [x2, #1216] + ldr Q3, [x2, #1232] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x1, #1264] + ldr Q1, [x2, #1248] + ldr Q3, [x2, #1264] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + stp Q5, Q6, [x0, #768] + stp Q7, Q8, [x0, #800] + stp Q9, Q10, [x0, #832] + stp Q11, Q12, [x0, #864] + stp Q13, Q14, [x0, #896] + stp Q15, Q16, [x0, #928] + stp Q17, Q18, [x0, #960] + stp Q19, Q20, [x0, #992] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_ntt_full_neon +mldsa_ntt_full_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x1, L_mldsa_aarch64_zetas + add x1, x1, L_mldsa_aarch64_zetas + adrp x2, L_mldsa_aarch64_zetas_qinv + add x2, x2, L_mldsa_aarch64_zetas_qinv + adrp x3, L_mldsa_aarch64_consts + add x3, x3, L_mldsa_aarch64_consts + ldr Q4, [x3] + ldr Q5, [x0] + ldr Q6, [x0, #64] + ldr Q7, [x0, #128] + ldr Q8, [x0, #192] + ldr Q9, [x0, #256] + ldr Q10, [x0, #320] + ldr Q11, [x0, #384] + ldr Q12, [x0, #448] + ldr Q13, [x0, #512] + ldr Q14, [x0, #576] + ldr Q15, [x0, #640] + ldr Q16, [x0, #704] + ldr Q17, [x0, #768] + ldr Q18, [x0, #832] + ldr Q19, [x0, #896] + ldr Q20, [x0, #960] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0] + str Q6, [x0, #64] + str Q7, [x0, #128] + str Q8, [x0, #192] + str Q9, [x0, #256] + str Q10, [x0, #320] + str Q11, [x0, #384] + str Q12, [x0, #448] + str Q13, [x0, #512] + str Q14, [x0, #576] + str Q15, [x0, #640] + str Q16, [x0, #704] + str Q17, [x0, #768] + str Q18, [x0, #832] + str Q19, [x0, #896] + str Q20, [x0, #960] + ldr Q5, [x0, #16] + ldr Q6, [x0, #80] + ldr Q7, [x0, #144] + ldr Q8, [x0, #208] + ldr Q9, [x0, #272] + ldr Q10, [x0, #336] + ldr Q11, [x0, #400] + ldr Q12, [x0, #464] + ldr Q13, [x0, #528] + ldr Q14, [x0, #592] + ldr Q15, [x0, #656] + ldr Q16, [x0, #720] + ldr Q17, [x0, #784] + ldr Q18, [x0, #848] + ldr Q19, [x0, #912] + ldr Q20, [x0, #976] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0, #16] + str Q6, [x0, #80] + str Q7, [x0, #144] + str Q8, [x0, #208] + str Q9, [x0, #272] + str Q10, [x0, #336] + str Q11, [x0, #400] + str Q12, [x0, #464] + str Q13, [x0, #528] + str Q14, [x0, #592] + str Q15, [x0, #656] + str Q16, [x0, #720] + str Q17, [x0, #784] + str Q18, [x0, #848] + str Q19, [x0, #912] + str Q20, [x0, #976] + ldr Q5, [x0, #32] + ldr Q6, [x0, #96] + ldr Q7, [x0, #160] + ldr Q8, [x0, #224] + ldr Q9, [x0, #288] + ldr Q10, [x0, #352] + ldr Q11, [x0, #416] + ldr Q12, [x0, #480] + ldr Q13, [x0, #544] + ldr Q14, [x0, #608] + ldr Q15, [x0, #672] + ldr Q16, [x0, #736] + ldr Q17, [x0, #800] + ldr Q18, [x0, #864] + ldr Q19, [x0, #928] + ldr Q20, [x0, #992] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0, #32] + str Q6, [x0, #96] + str Q7, [x0, #160] + str Q8, [x0, #224] + str Q9, [x0, #288] + str Q10, [x0, #352] + str Q11, [x0, #416] + str Q12, [x0, #480] + str Q13, [x0, #544] + str Q14, [x0, #608] + str Q15, [x0, #672] + str Q16, [x0, #736] + str Q17, [x0, #800] + str Q18, [x0, #864] + str Q19, [x0, #928] + str Q20, [x0, #992] + ldr Q5, [x0, #48] + ldr Q6, [x0, #112] + ldr Q7, [x0, #176] + ldr Q8, [x0, #240] + ldr Q9, [x0, #304] + ldr Q10, [x0, #368] + ldr Q11, [x0, #432] + ldr Q12, [x0, #496] + ldr Q13, [x0, #560] + ldr Q14, [x0, #624] + ldr Q15, [x0, #688] + ldr Q16, [x0, #752] + ldr Q17, [x0, #816] + ldr Q18, [x0, #880] + ldr Q19, [x0, #944] + ldr Q20, [x0, #1008] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0, #48] + str Q6, [x0, #112] + str Q7, [x0, #176] + str Q8, [x0, #240] + str Q9, [x0, #304] + str Q10, [x0, #368] + str Q11, [x0, #432] + str Q12, [x0, #496] + str Q13, [x0, #560] + str Q14, [x0, #624] + str Q15, [x0, #688] + str Q16, [x0, #752] + str Q17, [x0, #816] + str Q18, [x0, #880] + str Q19, [x0, #944] + str Q20, [x0, #1008] + ldp Q5, Q6, [x0] + ldp Q7, Q8, [x0, #32] + ldp Q9, Q10, [x0, #64] + ldp Q11, Q12, [x0, #96] + ldp Q13, Q14, [x0, #128] + ldp Q15, Q16, [x0, #160] + ldp Q17, Q18, [x0, #192] + ldp Q19, Q20, [x0, #224] + ldr Q0, [x1, #64] + ldr Q1, [x2, #64] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #128] + ldp Q1, Q3, [x2, #128] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #256] + ldp Q1, Q3, [x2, #256] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #288] + ldp Q1, Q3, [x2, #288] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #320] + ldp Q1, Q3, [x2, #320] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #352] + ldp Q1, Q3, [x2, #352] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #768] + ldp Q1, Q3, [x2, #768] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #800] + ldp Q1, Q3, [x2, #800] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #832] + ldp Q1, Q3, [x2, #832] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #864] + ldp Q1, Q3, [x2, #864] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + trn1 V29.4S, V5.4S, V6.4S + trn2 V30.4S, V5.4S, V6.4S + trn1 V5.2D, V29.2D, V30.2D + trn2 V6.2D, V29.2D, V30.2D + trn1 V29.4S, V7.4S, V8.4S + trn2 V30.4S, V7.4S, V8.4S + trn1 V7.2D, V29.2D, V30.2D + trn2 V8.2D, V29.2D, V30.2D + trn1 V29.4S, V9.4S, V10.4S + trn2 V30.4S, V9.4S, V10.4S + trn1 V9.2D, V29.2D, V30.2D + trn2 V10.2D, V29.2D, V30.2D + trn1 V29.4S, V11.4S, V12.4S + trn2 V30.4S, V11.4S, V12.4S + trn1 V11.2D, V29.2D, V30.2D + trn2 V12.2D, V29.2D, V30.2D + trn1 V29.4S, V13.4S, V14.4S + trn2 V30.4S, V13.4S, V14.4S + trn1 V13.2D, V29.2D, V30.2D + trn2 V14.2D, V29.2D, V30.2D + trn1 V29.4S, V15.4S, V16.4S + trn2 V30.4S, V15.4S, V16.4S + trn1 V15.2D, V29.2D, V30.2D + trn2 V16.2D, V29.2D, V30.2D + trn1 V29.4S, V17.4S, V18.4S + trn2 V30.4S, V17.4S, V18.4S + trn1 V17.2D, V29.2D, V30.2D + trn2 V18.2D, V29.2D, V30.2D + trn1 V29.4S, V19.4S, V20.4S + trn2 V30.4S, V19.4S, V20.4S + trn1 V19.2D, V29.2D, V30.2D + trn2 V20.2D, V29.2D, V30.2D + stp Q5, Q6, [x0] + stp Q7, Q8, [x0, #32] + stp Q9, Q10, [x0, #64] + stp Q11, Q12, [x0, #96] + stp Q13, Q14, [x0, #128] + stp Q15, Q16, [x0, #160] + stp Q17, Q18, [x0, #192] + stp Q19, Q20, [x0, #224] + ldp Q5, Q6, [x0, #256] + ldp Q7, Q8, [x0, #288] + ldp Q9, Q10, [x0, #320] + ldp Q11, Q12, [x0, #352] + ldp Q13, Q14, [x0, #384] + ldp Q15, Q16, [x0, #416] + ldp Q17, Q18, [x0, #448] + ldp Q19, Q20, [x0, #480] + ldr Q0, [x1, #80] + ldr Q1, [x2, #80] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #160] + ldp Q1, Q3, [x2, #160] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #384] + ldp Q1, Q3, [x2, #384] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #416] + ldp Q1, Q3, [x2, #416] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #448] + ldp Q1, Q3, [x2, #448] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #480] + ldp Q1, Q3, [x2, #480] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #896] + ldp Q1, Q3, [x2, #896] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #928] + ldp Q1, Q3, [x2, #928] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #960] + ldp Q1, Q3, [x2, #960] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #992] + ldp Q1, Q3, [x2, #992] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + trn1 V29.4S, V5.4S, V6.4S + trn2 V30.4S, V5.4S, V6.4S + trn1 V5.2D, V29.2D, V30.2D + trn2 V6.2D, V29.2D, V30.2D + trn1 V29.4S, V7.4S, V8.4S + trn2 V30.4S, V7.4S, V8.4S + trn1 V7.2D, V29.2D, V30.2D + trn2 V8.2D, V29.2D, V30.2D + trn1 V29.4S, V9.4S, V10.4S + trn2 V30.4S, V9.4S, V10.4S + trn1 V9.2D, V29.2D, V30.2D + trn2 V10.2D, V29.2D, V30.2D + trn1 V29.4S, V11.4S, V12.4S + trn2 V30.4S, V11.4S, V12.4S + trn1 V11.2D, V29.2D, V30.2D + trn2 V12.2D, V29.2D, V30.2D + trn1 V29.4S, V13.4S, V14.4S + trn2 V30.4S, V13.4S, V14.4S + trn1 V13.2D, V29.2D, V30.2D + trn2 V14.2D, V29.2D, V30.2D + trn1 V29.4S, V15.4S, V16.4S + trn2 V30.4S, V15.4S, V16.4S + trn1 V15.2D, V29.2D, V30.2D + trn2 V16.2D, V29.2D, V30.2D + trn1 V29.4S, V17.4S, V18.4S + trn2 V30.4S, V17.4S, V18.4S + trn1 V17.2D, V29.2D, V30.2D + trn2 V18.2D, V29.2D, V30.2D + trn1 V29.4S, V19.4S, V20.4S + trn2 V30.4S, V19.4S, V20.4S + trn1 V19.2D, V29.2D, V30.2D + trn2 V20.2D, V29.2D, V30.2D + stp Q5, Q6, [x0, #256] + stp Q7, Q8, [x0, #288] + stp Q9, Q10, [x0, #320] + stp Q11, Q12, [x0, #352] + stp Q13, Q14, [x0, #384] + stp Q15, Q16, [x0, #416] + stp Q17, Q18, [x0, #448] + stp Q19, Q20, [x0, #480] + ldp Q5, Q6, [x0, #512] + ldp Q7, Q8, [x0, #544] + ldp Q9, Q10, [x0, #576] + ldp Q11, Q12, [x0, #608] + ldp Q13, Q14, [x0, #640] + ldp Q15, Q16, [x0, #672] + ldp Q17, Q18, [x0, #704] + ldp Q19, Q20, [x0, #736] + ldr Q0, [x1, #96] + ldr Q1, [x2, #96] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #192] + ldp Q1, Q3, [x2, #192] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #512] + ldp Q1, Q3, [x2, #512] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #544] + ldp Q1, Q3, [x2, #544] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #576] + ldp Q1, Q3, [x2, #576] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #608] + ldp Q1, Q3, [x2, #608] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldr Q0, [x1, #1024] + ldr Q2, [x1, #1040] + ldr Q1, [x2, #1024] + ldr Q3, [x2, #1040] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldr Q0, [x1, #1056] + ldr Q2, [x1, #1072] + ldr Q1, [x2, #1056] + ldr Q3, [x2, #1072] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldr Q0, [x1, #1088] + ldr Q2, [x1, #1104] + ldr Q1, [x2, #1088] + ldr Q3, [x2, #1104] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldr Q0, [x1, #1120] + ldr Q2, [x1, #1136] + ldr Q1, [x2, #1120] + ldr Q3, [x2, #1136] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + trn1 V29.4S, V5.4S, V6.4S + trn2 V30.4S, V5.4S, V6.4S + trn1 V5.2D, V29.2D, V30.2D + trn2 V6.2D, V29.2D, V30.2D + trn1 V29.4S, V7.4S, V8.4S + trn2 V30.4S, V7.4S, V8.4S + trn1 V7.2D, V29.2D, V30.2D + trn2 V8.2D, V29.2D, V30.2D + trn1 V29.4S, V9.4S, V10.4S + trn2 V30.4S, V9.4S, V10.4S + trn1 V9.2D, V29.2D, V30.2D + trn2 V10.2D, V29.2D, V30.2D + trn1 V29.4S, V11.4S, V12.4S + trn2 V30.4S, V11.4S, V12.4S + trn1 V11.2D, V29.2D, V30.2D + trn2 V12.2D, V29.2D, V30.2D + trn1 V29.4S, V13.4S, V14.4S + trn2 V30.4S, V13.4S, V14.4S + trn1 V13.2D, V29.2D, V30.2D + trn2 V14.2D, V29.2D, V30.2D + trn1 V29.4S, V15.4S, V16.4S + trn2 V30.4S, V15.4S, V16.4S + trn1 V15.2D, V29.2D, V30.2D + trn2 V16.2D, V29.2D, V30.2D + trn1 V29.4S, V17.4S, V18.4S + trn2 V30.4S, V17.4S, V18.4S + trn1 V17.2D, V29.2D, V30.2D + trn2 V18.2D, V29.2D, V30.2D + trn1 V29.4S, V19.4S, V20.4S + trn2 V30.4S, V19.4S, V20.4S + trn1 V19.2D, V29.2D, V30.2D + trn2 V20.2D, V29.2D, V30.2D + stp Q5, Q6, [x0, #512] + stp Q7, Q8, [x0, #544] + stp Q9, Q10, [x0, #576] + stp Q11, Q12, [x0, #608] + stp Q13, Q14, [x0, #640] + stp Q15, Q16, [x0, #672] + stp Q17, Q18, [x0, #704] + stp Q19, Q20, [x0, #736] + ldp Q5, Q6, [x0, #768] + ldp Q7, Q8, [x0, #800] + ldp Q9, Q10, [x0, #832] + ldp Q11, Q12, [x0, #864] + ldp Q13, Q14, [x0, #896] + ldp Q15, Q16, [x0, #928] + ldp Q17, Q18, [x0, #960] + ldp Q19, Q20, [x0, #992] + ldr Q0, [x1, #112] + ldr Q1, [x2, #112] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #224] + ldp Q1, Q3, [x2, #224] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #640] + ldp Q1, Q3, [x2, #640] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #672] + ldp Q1, Q3, [x2, #672] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #704] + ldp Q1, Q3, [x2, #704] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #736] + ldp Q1, Q3, [x2, #736] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldr Q0, [x1, #1152] + ldr Q2, [x1, #1168] + ldr Q1, [x2, #1152] + ldr Q3, [x2, #1168] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V21.4S, V21.4S, V29.4S + sub V22.4S, V22.4S, V30.4S + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldr Q0, [x1, #1184] + ldr Q2, [x1, #1200] + ldr Q1, [x2, #1184] + ldr Q3, [x2, #1200] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V23.4S, V23.4S, V29.4S + sub V24.4S, V24.4S, V30.4S + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldr Q0, [x1, #1216] + ldr Q2, [x1, #1232] + ldr Q1, [x2, #1216] + ldr Q3, [x2, #1232] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V25.4S, V25.4S, V29.4S + sub V26.4S, V26.4S, V30.4S + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x1, #1264] + ldr Q1, [x2, #1248] + ldr Q3, [x2, #1264] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmulh V29.4S, V29.4S, V4.S[0] + sqrdmulh V30.4S, V30.4S, V4.S[0] + sub V27.4S, V27.4S, V29.4S + sub V28.4S, V28.4S, V30.4S + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + trn1 V29.4S, V5.4S, V6.4S + trn2 V30.4S, V5.4S, V6.4S + trn1 V5.2D, V29.2D, V30.2D + trn2 V6.2D, V29.2D, V30.2D + trn1 V29.4S, V7.4S, V8.4S + trn2 V30.4S, V7.4S, V8.4S + trn1 V7.2D, V29.2D, V30.2D + trn2 V8.2D, V29.2D, V30.2D + trn1 V29.4S, V9.4S, V10.4S + trn2 V30.4S, V9.4S, V10.4S + trn1 V9.2D, V29.2D, V30.2D + trn2 V10.2D, V29.2D, V30.2D + trn1 V29.4S, V11.4S, V12.4S + trn2 V30.4S, V11.4S, V12.4S + trn1 V11.2D, V29.2D, V30.2D + trn2 V12.2D, V29.2D, V30.2D + trn1 V29.4S, V13.4S, V14.4S + trn2 V30.4S, V13.4S, V14.4S + trn1 V13.2D, V29.2D, V30.2D + trn2 V14.2D, V29.2D, V30.2D + trn1 V29.4S, V15.4S, V16.4S + trn2 V30.4S, V15.4S, V16.4S + trn1 V15.2D, V29.2D, V30.2D + trn2 V16.2D, V29.2D, V30.2D + trn1 V29.4S, V17.4S, V18.4S + trn2 V30.4S, V17.4S, V18.4S + trn1 V17.2D, V29.2D, V30.2D + trn2 V18.2D, V29.2D, V30.2D + trn1 V29.4S, V19.4S, V20.4S + trn2 V30.4S, V19.4S, V20.4S + trn1 V19.2D, V29.2D, V30.2D + trn2 V20.2D, V29.2D, V30.2D + stp Q5, Q6, [x0, #768] + stp Q7, Q8, [x0, #800] + stp Q9, Q10, [x0, #832] + stp Q11, Q12, [x0, #864] + stp Q13, Q14, [x0, #896] + stp Q15, Q16, [x0, #928] + stp Q17, Q18, [x0, #960] + stp Q19, Q20, [x0, #992] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_invntt_neon +mldsa_invntt_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x1, L_mldsa_aarch64_zetas_inv + add x1, x1, L_mldsa_aarch64_zetas_inv + adrp x2, L_mldsa_aarch64_zetas_inv_qinv + add x2, x2, L_mldsa_aarch64_zetas_inv_qinv + adrp x3, L_mldsa_aarch64_consts + add x3, x3, L_mldsa_aarch64_consts + ldr Q4, [x3] + ldp Q5, Q6, [x0] + ldp Q7, Q8, [x0, #32] + ldp Q9, Q10, [x0, #64] + ldp Q11, Q12, [x0, #96] + ldp Q13, Q14, [x0, #128] + ldp Q15, Q16, [x0, #160] + ldp Q17, Q18, [x0, #192] + ldp Q19, Q20, [x0, #224] + ldp Q0, Q1, [x1] + ldp Q2, Q3, [x2] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #32] + ldp Q2, Q3, [x2, #32] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #64] + ldp Q2, Q3, [x2, #64] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #96] + ldp Q2, Q3, [x2, #96] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #512] + ldp Q2, Q3, [x2, #512] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #544] + ldp Q2, Q3, [x2, #544] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #576] + ldp Q2, Q3, [x2, #576] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #608] + ldp Q2, Q3, [x2, #608] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1024] + ldr Q1, [x1, #1040] + ldr Q2, [x2, #1024] + ldr Q3, [x2, #1040] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1152] + ldr Q2, [x2, #1152] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0] + stp Q7, Q8, [x0, #32] + stp Q9, Q10, [x0, #64] + stp Q11, Q12, [x0, #96] + stp Q13, Q14, [x0, #128] + stp Q15, Q16, [x0, #160] + stp Q17, Q18, [x0, #192] + stp Q19, Q20, [x0, #224] + ldp Q5, Q6, [x0, #256] + ldp Q7, Q8, [x0, #288] + ldp Q9, Q10, [x0, #320] + ldp Q11, Q12, [x0, #352] + ldp Q13, Q14, [x0, #384] + ldp Q15, Q16, [x0, #416] + ldp Q17, Q18, [x0, #448] + ldp Q19, Q20, [x0, #480] + ldp Q0, Q1, [x1, #128] + ldp Q2, Q3, [x2, #128] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #160] + ldp Q2, Q3, [x2, #160] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #192] + ldp Q2, Q3, [x2, #192] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #224] + ldp Q2, Q3, [x2, #224] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #640] + ldp Q2, Q3, [x2, #640] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #672] + ldp Q2, Q3, [x2, #672] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #704] + ldp Q2, Q3, [x2, #704] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #736] + ldp Q2, Q3, [x2, #736] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1056] + ldr Q1, [x1, #1072] + ldr Q2, [x2, #1056] + ldr Q3, [x2, #1072] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1168] + ldr Q2, [x2, #1168] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0, #256] + stp Q7, Q8, [x0, #288] + stp Q9, Q10, [x0, #320] + stp Q11, Q12, [x0, #352] + stp Q13, Q14, [x0, #384] + stp Q15, Q16, [x0, #416] + stp Q17, Q18, [x0, #448] + stp Q19, Q20, [x0, #480] + ldp Q5, Q6, [x0, #512] + ldp Q7, Q8, [x0, #544] + ldp Q9, Q10, [x0, #576] + ldp Q11, Q12, [x0, #608] + ldp Q13, Q14, [x0, #640] + ldp Q15, Q16, [x0, #672] + ldp Q17, Q18, [x0, #704] + ldp Q19, Q20, [x0, #736] + ldp Q0, Q1, [x1, #256] + ldp Q2, Q3, [x2, #256] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #288] + ldp Q2, Q3, [x2, #288] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #320] + ldp Q2, Q3, [x2, #320] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #352] + ldp Q2, Q3, [x2, #352] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #768] + ldp Q2, Q3, [x2, #768] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #800] + ldp Q2, Q3, [x2, #800] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #832] + ldp Q2, Q3, [x2, #832] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #864] + ldp Q2, Q3, [x2, #864] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1088] + ldr Q1, [x1, #1104] + ldr Q2, [x2, #1088] + ldr Q3, [x2, #1104] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1184] + ldr Q2, [x2, #1184] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0, #512] + stp Q7, Q8, [x0, #544] + stp Q9, Q10, [x0, #576] + stp Q11, Q12, [x0, #608] + stp Q13, Q14, [x0, #640] + stp Q15, Q16, [x0, #672] + stp Q17, Q18, [x0, #704] + stp Q19, Q20, [x0, #736] + ldp Q5, Q6, [x0, #768] + ldp Q7, Q8, [x0, #800] + ldp Q9, Q10, [x0, #832] + ldp Q11, Q12, [x0, #864] + ldp Q13, Q14, [x0, #896] + ldp Q15, Q16, [x0, #928] + ldp Q17, Q18, [x0, #960] + ldp Q19, Q20, [x0, #992] + ldp Q0, Q1, [x1, #384] + ldp Q2, Q3, [x2, #384] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #416] + ldp Q2, Q3, [x2, #416] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #448] + ldp Q2, Q3, [x2, #448] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #480] + ldp Q2, Q3, [x2, #480] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #896] + ldp Q2, Q3, [x2, #896] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #928] + ldp Q2, Q3, [x2, #928] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #960] + ldp Q2, Q3, [x2, #960] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #992] + ldp Q2, Q3, [x2, #992] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1120] + ldr Q1, [x1, #1136] + ldr Q2, [x2, #1120] + ldr Q3, [x2, #1136] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1200] + ldr Q2, [x2, #1200] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0, #768] + stp Q7, Q8, [x0, #800] + stp Q9, Q10, [x0, #832] + stp Q11, Q12, [x0, #864] + stp Q13, Q14, [x0, #896] + stp Q15, Q16, [x0, #928] + stp Q17, Q18, [x0, #960] + stp Q19, Q20, [x0, #992] + ldr Q5, [x0] + ldr Q6, [x0, #64] + ldr Q7, [x0, #128] + ldr Q8, [x0, #192] + ldr Q9, [x0, #256] + ldr Q10, [x0, #320] + ldr Q11, [x0, #384] + ldr Q12, [x0, #448] + ldr Q13, [x0, #512] + ldr Q14, [x0, #576] + ldr Q15, [x0, #640] + ldr Q16, [x0, #704] + ldr Q17, [x0, #768] + ldr Q18, [x0, #832] + ldr Q19, [x0, #896] + ldr Q20, [x0, #960] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V23.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V13.4S, V13.4S, V21.4S + sub V14.4S, V14.4S, V23.4S + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V5.4S, V5.4S, V21.4S + sub V6.4S, V6.4S, V22.4S + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V22.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V22.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V22.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0] + str Q6, [x0, #64] + str Q7, [x0, #128] + str Q8, [x0, #192] + str Q9, [x0, #256] + str Q10, [x0, #320] + str Q11, [x0, #384] + str Q12, [x0, #448] + str Q13, [x0, #512] + str Q14, [x0, #576] + str Q15, [x0, #640] + str Q16, [x0, #704] + str Q17, [x0, #768] + str Q18, [x0, #832] + str Q19, [x0, #896] + str Q20, [x0, #960] + ldr Q5, [x0, #16] + ldr Q6, [x0, #80] + ldr Q7, [x0, #144] + ldr Q8, [x0, #208] + ldr Q9, [x0, #272] + ldr Q10, [x0, #336] + ldr Q11, [x0, #400] + ldr Q12, [x0, #464] + ldr Q13, [x0, #528] + ldr Q14, [x0, #592] + ldr Q15, [x0, #656] + ldr Q16, [x0, #720] + ldr Q17, [x0, #784] + ldr Q18, [x0, #848] + ldr Q19, [x0, #912] + ldr Q20, [x0, #976] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V23.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V13.4S, V13.4S, V21.4S + sub V14.4S, V14.4S, V23.4S + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V5.4S, V5.4S, V21.4S + sub V6.4S, V6.4S, V22.4S + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V22.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V22.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V22.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0, #16] + str Q6, [x0, #80] + str Q7, [x0, #144] + str Q8, [x0, #208] + str Q9, [x0, #272] + str Q10, [x0, #336] + str Q11, [x0, #400] + str Q12, [x0, #464] + str Q13, [x0, #528] + str Q14, [x0, #592] + str Q15, [x0, #656] + str Q16, [x0, #720] + str Q17, [x0, #784] + str Q18, [x0, #848] + str Q19, [x0, #912] + str Q20, [x0, #976] + ldr Q5, [x0, #32] + ldr Q6, [x0, #96] + ldr Q7, [x0, #160] + ldr Q8, [x0, #224] + ldr Q9, [x0, #288] + ldr Q10, [x0, #352] + ldr Q11, [x0, #416] + ldr Q12, [x0, #480] + ldr Q13, [x0, #544] + ldr Q14, [x0, #608] + ldr Q15, [x0, #672] + ldr Q16, [x0, #736] + ldr Q17, [x0, #800] + ldr Q18, [x0, #864] + ldr Q19, [x0, #928] + ldr Q20, [x0, #992] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V23.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V13.4S, V13.4S, V21.4S + sub V14.4S, V14.4S, V23.4S + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V5.4S, V5.4S, V21.4S + sub V6.4S, V6.4S, V22.4S + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V22.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V22.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V22.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0, #32] + str Q6, [x0, #96] + str Q7, [x0, #160] + str Q8, [x0, #224] + str Q9, [x0, #288] + str Q10, [x0, #352] + str Q11, [x0, #416] + str Q12, [x0, #480] + str Q13, [x0, #544] + str Q14, [x0, #608] + str Q15, [x0, #672] + str Q16, [x0, #736] + str Q17, [x0, #800] + str Q18, [x0, #864] + str Q19, [x0, #928] + str Q20, [x0, #992] + ldr Q5, [x0, #48] + ldr Q6, [x0, #112] + ldr Q7, [x0, #176] + ldr Q8, [x0, #240] + ldr Q9, [x0, #304] + ldr Q10, [x0, #368] + ldr Q11, [x0, #432] + ldr Q12, [x0, #496] + ldr Q13, [x0, #560] + ldr Q14, [x0, #624] + ldr Q15, [x0, #688] + ldr Q16, [x0, #752] + ldr Q17, [x0, #816] + ldr Q18, [x0, #880] + ldr Q19, [x0, #944] + ldr Q20, [x0, #1008] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V23.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V13.4S, V13.4S, V21.4S + sub V14.4S, V14.4S, V23.4S + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V5.4S, V5.4S, V21.4S + sub V6.4S, V6.4S, V22.4S + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V22.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V22.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V22.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0, #48] + str Q6, [x0, #112] + str Q7, [x0, #176] + str Q8, [x0, #240] + str Q9, [x0, #304] + str Q10, [x0, #368] + str Q11, [x0, #432] + str Q12, [x0, #496] + str Q13, [x0, #560] + str Q14, [x0, #624] + str Q15, [x0, #688] + str Q16, [x0, #752] + str Q17, [x0, #816] + str Q18, [x0, #880] + str Q19, [x0, #944] + str Q20, [x0, #1008] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_invntt_full_neon +mldsa_invntt_full_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x1, L_mldsa_aarch64_zetas_inv + add x1, x1, L_mldsa_aarch64_zetas_inv + adrp x2, L_mldsa_aarch64_zetas_inv_qinv + add x2, x2, L_mldsa_aarch64_zetas_inv_qinv + adrp x3, L_mldsa_aarch64_consts + add x3, x3, L_mldsa_aarch64_consts + ldr Q4, [x3] + ldp Q5, Q6, [x0] + ldp Q7, Q8, [x0, #32] + ldp Q9, Q10, [x0, #64] + ldp Q11, Q12, [x0, #96] + ldp Q13, Q14, [x0, #128] + ldp Q15, Q16, [x0, #160] + ldp Q17, Q18, [x0, #192] + ldp Q19, Q20, [x0, #224] + trn1 V21.2D, V5.2D, V6.2D + trn2 V22.2D, V5.2D, V6.2D + trn1 V5.4S, V21.4S, V22.4S + trn2 V6.4S, V21.4S, V22.4S + trn1 V21.2D, V7.2D, V8.2D + trn2 V22.2D, V7.2D, V8.2D + trn1 V7.4S, V21.4S, V22.4S + trn2 V8.4S, V21.4S, V22.4S + trn1 V21.2D, V9.2D, V10.2D + trn2 V22.2D, V9.2D, V10.2D + trn1 V9.4S, V21.4S, V22.4S + trn2 V10.4S, V21.4S, V22.4S + trn1 V21.2D, V11.2D, V12.2D + trn2 V22.2D, V11.2D, V12.2D + trn1 V11.4S, V21.4S, V22.4S + trn2 V12.4S, V21.4S, V22.4S + trn1 V21.2D, V13.2D, V14.2D + trn2 V22.2D, V13.2D, V14.2D + trn1 V13.4S, V21.4S, V22.4S + trn2 V14.4S, V21.4S, V22.4S + trn1 V21.2D, V15.2D, V16.2D + trn2 V22.2D, V15.2D, V16.2D + trn1 V15.4S, V21.4S, V22.4S + trn2 V16.4S, V21.4S, V22.4S + trn1 V21.2D, V17.2D, V18.2D + trn2 V22.2D, V17.2D, V18.2D + trn1 V17.4S, V21.4S, V22.4S + trn2 V18.4S, V21.4S, V22.4S + trn1 V21.2D, V19.2D, V20.2D + trn2 V22.2D, V19.2D, V20.2D + trn1 V19.4S, V21.4S, V22.4S + trn2 V20.4S, V21.4S, V22.4S + ldp Q0, Q1, [x1] + ldp Q2, Q3, [x2] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #32] + ldp Q2, Q3, [x2, #32] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #64] + ldp Q2, Q3, [x2, #64] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #96] + ldp Q2, Q3, [x2, #96] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #512] + ldp Q2, Q3, [x2, #512] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #544] + ldp Q2, Q3, [x2, #544] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #576] + ldp Q2, Q3, [x2, #576] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #608] + ldp Q2, Q3, [x2, #608] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1024] + ldr Q1, [x1, #1040] + ldr Q2, [x2, #1024] + ldr Q3, [x2, #1040] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1152] + ldr Q2, [x2, #1152] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0] + stp Q7, Q8, [x0, #32] + stp Q9, Q10, [x0, #64] + stp Q11, Q12, [x0, #96] + stp Q13, Q14, [x0, #128] + stp Q15, Q16, [x0, #160] + stp Q17, Q18, [x0, #192] + stp Q19, Q20, [x0, #224] + ldp Q5, Q6, [x0, #256] + ldp Q7, Q8, [x0, #288] + ldp Q9, Q10, [x0, #320] + ldp Q11, Q12, [x0, #352] + ldp Q13, Q14, [x0, #384] + ldp Q15, Q16, [x0, #416] + ldp Q17, Q18, [x0, #448] + ldp Q19, Q20, [x0, #480] + trn1 V21.2D, V5.2D, V6.2D + trn2 V22.2D, V5.2D, V6.2D + trn1 V5.4S, V21.4S, V22.4S + trn2 V6.4S, V21.4S, V22.4S + trn1 V21.2D, V7.2D, V8.2D + trn2 V22.2D, V7.2D, V8.2D + trn1 V7.4S, V21.4S, V22.4S + trn2 V8.4S, V21.4S, V22.4S + trn1 V21.2D, V9.2D, V10.2D + trn2 V22.2D, V9.2D, V10.2D + trn1 V9.4S, V21.4S, V22.4S + trn2 V10.4S, V21.4S, V22.4S + trn1 V21.2D, V11.2D, V12.2D + trn2 V22.2D, V11.2D, V12.2D + trn1 V11.4S, V21.4S, V22.4S + trn2 V12.4S, V21.4S, V22.4S + trn1 V21.2D, V13.2D, V14.2D + trn2 V22.2D, V13.2D, V14.2D + trn1 V13.4S, V21.4S, V22.4S + trn2 V14.4S, V21.4S, V22.4S + trn1 V21.2D, V15.2D, V16.2D + trn2 V22.2D, V15.2D, V16.2D + trn1 V15.4S, V21.4S, V22.4S + trn2 V16.4S, V21.4S, V22.4S + trn1 V21.2D, V17.2D, V18.2D + trn2 V22.2D, V17.2D, V18.2D + trn1 V17.4S, V21.4S, V22.4S + trn2 V18.4S, V21.4S, V22.4S + trn1 V21.2D, V19.2D, V20.2D + trn2 V22.2D, V19.2D, V20.2D + trn1 V19.4S, V21.4S, V22.4S + trn2 V20.4S, V21.4S, V22.4S + ldp Q0, Q1, [x1, #128] + ldp Q2, Q3, [x2, #128] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #160] + ldp Q2, Q3, [x2, #160] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #192] + ldp Q2, Q3, [x2, #192] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #224] + ldp Q2, Q3, [x2, #224] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #640] + ldp Q2, Q3, [x2, #640] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #672] + ldp Q2, Q3, [x2, #672] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #704] + ldp Q2, Q3, [x2, #704] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #736] + ldp Q2, Q3, [x2, #736] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1056] + ldr Q1, [x1, #1072] + ldr Q2, [x2, #1056] + ldr Q3, [x2, #1072] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1168] + ldr Q2, [x2, #1168] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0, #256] + stp Q7, Q8, [x0, #288] + stp Q9, Q10, [x0, #320] + stp Q11, Q12, [x0, #352] + stp Q13, Q14, [x0, #384] + stp Q15, Q16, [x0, #416] + stp Q17, Q18, [x0, #448] + stp Q19, Q20, [x0, #480] + ldp Q5, Q6, [x0, #512] + ldp Q7, Q8, [x0, #544] + ldp Q9, Q10, [x0, #576] + ldp Q11, Q12, [x0, #608] + ldp Q13, Q14, [x0, #640] + ldp Q15, Q16, [x0, #672] + ldp Q17, Q18, [x0, #704] + ldp Q19, Q20, [x0, #736] + trn1 V21.2D, V5.2D, V6.2D + trn2 V22.2D, V5.2D, V6.2D + trn1 V5.4S, V21.4S, V22.4S + trn2 V6.4S, V21.4S, V22.4S + trn1 V21.2D, V7.2D, V8.2D + trn2 V22.2D, V7.2D, V8.2D + trn1 V7.4S, V21.4S, V22.4S + trn2 V8.4S, V21.4S, V22.4S + trn1 V21.2D, V9.2D, V10.2D + trn2 V22.2D, V9.2D, V10.2D + trn1 V9.4S, V21.4S, V22.4S + trn2 V10.4S, V21.4S, V22.4S + trn1 V21.2D, V11.2D, V12.2D + trn2 V22.2D, V11.2D, V12.2D + trn1 V11.4S, V21.4S, V22.4S + trn2 V12.4S, V21.4S, V22.4S + trn1 V21.2D, V13.2D, V14.2D + trn2 V22.2D, V13.2D, V14.2D + trn1 V13.4S, V21.4S, V22.4S + trn2 V14.4S, V21.4S, V22.4S + trn1 V21.2D, V15.2D, V16.2D + trn2 V22.2D, V15.2D, V16.2D + trn1 V15.4S, V21.4S, V22.4S + trn2 V16.4S, V21.4S, V22.4S + trn1 V21.2D, V17.2D, V18.2D + trn2 V22.2D, V17.2D, V18.2D + trn1 V17.4S, V21.4S, V22.4S + trn2 V18.4S, V21.4S, V22.4S + trn1 V21.2D, V19.2D, V20.2D + trn2 V22.2D, V19.2D, V20.2D + trn1 V19.4S, V21.4S, V22.4S + trn2 V20.4S, V21.4S, V22.4S + ldp Q0, Q1, [x1, #256] + ldp Q2, Q3, [x2, #256] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #288] + ldp Q2, Q3, [x2, #288] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #320] + ldp Q2, Q3, [x2, #320] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #352] + ldp Q2, Q3, [x2, #352] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #768] + ldp Q2, Q3, [x2, #768] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #800] + ldp Q2, Q3, [x2, #800] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #832] + ldp Q2, Q3, [x2, #832] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #864] + ldp Q2, Q3, [x2, #864] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1088] + ldr Q1, [x1, #1104] + ldr Q2, [x2, #1088] + ldr Q3, [x2, #1104] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1184] + ldr Q2, [x2, #1184] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0, #512] + stp Q7, Q8, [x0, #544] + stp Q9, Q10, [x0, #576] + stp Q11, Q12, [x0, #608] + stp Q13, Q14, [x0, #640] + stp Q15, Q16, [x0, #672] + stp Q17, Q18, [x0, #704] + stp Q19, Q20, [x0, #736] + ldp Q5, Q6, [x0, #768] + ldp Q7, Q8, [x0, #800] + ldp Q9, Q10, [x0, #832] + ldp Q11, Q12, [x0, #864] + ldp Q13, Q14, [x0, #896] + ldp Q15, Q16, [x0, #928] + ldp Q17, Q18, [x0, #960] + ldp Q19, Q20, [x0, #992] + trn1 V21.2D, V5.2D, V6.2D + trn2 V22.2D, V5.2D, V6.2D + trn1 V5.4S, V21.4S, V22.4S + trn2 V6.4S, V21.4S, V22.4S + trn1 V21.2D, V7.2D, V8.2D + trn2 V22.2D, V7.2D, V8.2D + trn1 V7.4S, V21.4S, V22.4S + trn2 V8.4S, V21.4S, V22.4S + trn1 V21.2D, V9.2D, V10.2D + trn2 V22.2D, V9.2D, V10.2D + trn1 V9.4S, V21.4S, V22.4S + trn2 V10.4S, V21.4S, V22.4S + trn1 V21.2D, V11.2D, V12.2D + trn2 V22.2D, V11.2D, V12.2D + trn1 V11.4S, V21.4S, V22.4S + trn2 V12.4S, V21.4S, V22.4S + trn1 V21.2D, V13.2D, V14.2D + trn2 V22.2D, V13.2D, V14.2D + trn1 V13.4S, V21.4S, V22.4S + trn2 V14.4S, V21.4S, V22.4S + trn1 V21.2D, V15.2D, V16.2D + trn2 V22.2D, V15.2D, V16.2D + trn1 V15.4S, V21.4S, V22.4S + trn2 V16.4S, V21.4S, V22.4S + trn1 V21.2D, V17.2D, V18.2D + trn2 V22.2D, V17.2D, V18.2D + trn1 V17.4S, V21.4S, V22.4S + trn2 V18.4S, V21.4S, V22.4S + trn1 V21.2D, V19.2D, V20.2D + trn2 V22.2D, V19.2D, V20.2D + trn1 V19.4S, V21.4S, V22.4S + trn2 V20.4S, V21.4S, V22.4S + ldp Q0, Q1, [x1, #384] + ldp Q2, Q3, [x2, #384] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #416] + ldp Q2, Q3, [x2, #416] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #448] + ldp Q2, Q3, [x2, #448] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #480] + ldp Q2, Q3, [x2, #480] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #896] + ldp Q2, Q3, [x2, #896] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #928] + ldp Q2, Q3, [x2, #928] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #960] + ldp Q2, Q3, [x2, #960] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #992] + ldp Q2, Q3, [x2, #992] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1120] + ldr Q1, [x1, #1136] + ldr Q2, [x2, #1120] + ldr Q3, [x2, #1136] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1200] + ldr Q2, [x2, #1200] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0, #768] + stp Q7, Q8, [x0, #800] + stp Q9, Q10, [x0, #832] + stp Q11, Q12, [x0, #864] + stp Q13, Q14, [x0, #896] + stp Q15, Q16, [x0, #928] + stp Q17, Q18, [x0, #960] + stp Q19, Q20, [x0, #992] + ldr Q5, [x0] + ldr Q6, [x0, #64] + ldr Q7, [x0, #128] + ldr Q8, [x0, #192] + ldr Q9, [x0, #256] + ldr Q10, [x0, #320] + ldr Q11, [x0, #384] + ldr Q12, [x0, #448] + ldr Q13, [x0, #512] + ldr Q14, [x0, #576] + ldr Q15, [x0, #640] + ldr Q16, [x0, #704] + ldr Q17, [x0, #768] + ldr Q18, [x0, #832] + ldr Q19, [x0, #896] + ldr Q20, [x0, #960] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V23.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V13.4S, V13.4S, V21.4S + sub V14.4S, V14.4S, V23.4S + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V5.4S, V5.4S, V21.4S + sub V6.4S, V6.4S, V22.4S + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V22.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V22.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V22.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0] + str Q6, [x0, #64] + str Q7, [x0, #128] + str Q8, [x0, #192] + str Q9, [x0, #256] + str Q10, [x0, #320] + str Q11, [x0, #384] + str Q12, [x0, #448] + str Q13, [x0, #512] + str Q14, [x0, #576] + str Q15, [x0, #640] + str Q16, [x0, #704] + str Q17, [x0, #768] + str Q18, [x0, #832] + str Q19, [x0, #896] + str Q20, [x0, #960] + ldr Q5, [x0, #16] + ldr Q6, [x0, #80] + ldr Q7, [x0, #144] + ldr Q8, [x0, #208] + ldr Q9, [x0, #272] + ldr Q10, [x0, #336] + ldr Q11, [x0, #400] + ldr Q12, [x0, #464] + ldr Q13, [x0, #528] + ldr Q14, [x0, #592] + ldr Q15, [x0, #656] + ldr Q16, [x0, #720] + ldr Q17, [x0, #784] + ldr Q18, [x0, #848] + ldr Q19, [x0, #912] + ldr Q20, [x0, #976] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V23.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V13.4S, V13.4S, V21.4S + sub V14.4S, V14.4S, V23.4S + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V5.4S, V5.4S, V21.4S + sub V6.4S, V6.4S, V22.4S + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V22.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V22.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V22.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0, #16] + str Q6, [x0, #80] + str Q7, [x0, #144] + str Q8, [x0, #208] + str Q9, [x0, #272] + str Q10, [x0, #336] + str Q11, [x0, #400] + str Q12, [x0, #464] + str Q13, [x0, #528] + str Q14, [x0, #592] + str Q15, [x0, #656] + str Q16, [x0, #720] + str Q17, [x0, #784] + str Q18, [x0, #848] + str Q19, [x0, #912] + str Q20, [x0, #976] + ldr Q5, [x0, #32] + ldr Q6, [x0, #96] + ldr Q7, [x0, #160] + ldr Q8, [x0, #224] + ldr Q9, [x0, #288] + ldr Q10, [x0, #352] + ldr Q11, [x0, #416] + ldr Q12, [x0, #480] + ldr Q13, [x0, #544] + ldr Q14, [x0, #608] + ldr Q15, [x0, #672] + ldr Q16, [x0, #736] + ldr Q17, [x0, #800] + ldr Q18, [x0, #864] + ldr Q19, [x0, #928] + ldr Q20, [x0, #992] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V23.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V13.4S, V13.4S, V21.4S + sub V14.4S, V14.4S, V23.4S + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V5.4S, V5.4S, V21.4S + sub V6.4S, V6.4S, V22.4S + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V22.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V22.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V22.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0, #32] + str Q6, [x0, #96] + str Q7, [x0, #160] + str Q8, [x0, #224] + str Q9, [x0, #288] + str Q10, [x0, #352] + str Q11, [x0, #416] + str Q12, [x0, #480] + str Q13, [x0, #544] + str Q14, [x0, #608] + str Q15, [x0, #672] + str Q16, [x0, #736] + str Q17, [x0, #800] + str Q18, [x0, #864] + str Q19, [x0, #928] + str Q20, [x0, #992] + ldr Q5, [x0, #48] + ldr Q6, [x0, #112] + ldr Q7, [x0, #176] + ldr Q8, [x0, #240] + ldr Q9, [x0, #304] + ldr Q10, [x0, #368] + ldr Q11, [x0, #432] + ldr Q12, [x0, #496] + ldr Q13, [x0, #560] + ldr Q14, [x0, #624] + ldr Q15, [x0, #688] + ldr Q16, [x0, #752] + ldr Q17, [x0, #816] + ldr Q18, [x0, #880] + ldr Q19, [x0, #944] + ldr Q20, [x0, #1008] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V6.4S, V6.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V10.4S, V10.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V14.4S, V14.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V18.4S, V18.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V23.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V23.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V23.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V13.4S, V13.4S, V21.4S + sub V14.4S, V14.4S, V23.4S + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V15.4S, V15.4S, V21.4S + sub V16.4S, V16.4S, V23.4S + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V17.4S, V17.4S, V21.4S + sub V18.4S, V18.4S, V23.4S + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V23.4S, V23.4S, V4.S[0] + sub V19.4S, V19.4S, V21.4S + sub V20.4S, V20.4S, V23.4S + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V5.4S, V5.4S, V21.4S + sub V6.4S, V6.4S, V22.4S + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V7.4S, V7.4S, V21.4S + sub V8.4S, V8.4S, V22.4S + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V9.4S, V9.4S, V21.4S + sub V10.4S, V10.4S, V22.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmulh V21.4S, V21.4S, V4.S[0] + sqrdmulh V22.4S, V22.4S, V4.S[0] + sub V11.4S, V11.4S, V21.4S + sub V12.4S, V12.4S, V22.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0, #48] + str Q6, [x0, #112] + str Q7, [x0, #176] + str Q8, [x0, #240] + str Q9, [x0, #304] + str Q10, [x0, #368] + str Q11, [x0, #432] + str Q12, [x0, #496] + str Q13, [x0, #560] + str Q14, [x0, #624] + str Q15, [x0, #688] + str Q16, [x0, #752] + str Q17, [x0, #816] + str Q18, [x0, #880] + str Q19, [x0, #944] + str Q20, [x0, #1008] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_mul_neon +mldsa_mul_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x3, L_mldsa_aarch64_consts + add x3, x3, L_mldsa_aarch64_consts + ldr Q0, [x3] + ldr Q1, [x1] + ldr Q5, [x2] + ldr Q2, [x1, #16] + ldr Q6, [x2, #16] + ldr Q3, [x1, #32] + ldr Q7, [x2, #32] + ldr Q4, [x1, #48] + ldr Q8, [x2, #48] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0] + str Q10, [x0, #16] + str Q11, [x0, #32] + str Q12, [x0, #48] + ldr Q1, [x1, #64] + ldr Q5, [x2, #64] + ldr Q2, [x1, #80] + ldr Q6, [x2, #80] + ldr Q3, [x1, #96] + ldr Q7, [x2, #96] + ldr Q4, [x1, #112] + ldr Q8, [x2, #112] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #64] + str Q10, [x0, #80] + str Q11, [x0, #96] + str Q12, [x0, #112] + ldr Q1, [x1, #128] + ldr Q5, [x2, #128] + ldr Q2, [x1, #144] + ldr Q6, [x2, #144] + ldr Q3, [x1, #160] + ldr Q7, [x2, #160] + ldr Q4, [x1, #176] + ldr Q8, [x2, #176] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #128] + str Q10, [x0, #144] + str Q11, [x0, #160] + str Q12, [x0, #176] + ldr Q1, [x1, #192] + ldr Q5, [x2, #192] + ldr Q2, [x1, #208] + ldr Q6, [x2, #208] + ldr Q3, [x1, #224] + ldr Q7, [x2, #224] + ldr Q4, [x1, #240] + ldr Q8, [x2, #240] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #192] + str Q10, [x0, #208] + str Q11, [x0, #224] + str Q12, [x0, #240] + ldr Q1, [x1, #256] + ldr Q5, [x2, #256] + ldr Q2, [x1, #272] + ldr Q6, [x2, #272] + ldr Q3, [x1, #288] + ldr Q7, [x2, #288] + ldr Q4, [x1, #304] + ldr Q8, [x2, #304] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #256] + str Q10, [x0, #272] + str Q11, [x0, #288] + str Q12, [x0, #304] + ldr Q1, [x1, #320] + ldr Q5, [x2, #320] + ldr Q2, [x1, #336] + ldr Q6, [x2, #336] + ldr Q3, [x1, #352] + ldr Q7, [x2, #352] + ldr Q4, [x1, #368] + ldr Q8, [x2, #368] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #320] + str Q10, [x0, #336] + str Q11, [x0, #352] + str Q12, [x0, #368] + ldr Q1, [x1, #384] + ldr Q5, [x2, #384] + ldr Q2, [x1, #400] + ldr Q6, [x2, #400] + ldr Q3, [x1, #416] + ldr Q7, [x2, #416] + ldr Q4, [x1, #432] + ldr Q8, [x2, #432] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #384] + str Q10, [x0, #400] + str Q11, [x0, #416] + str Q12, [x0, #432] + ldr Q1, [x1, #448] + ldr Q5, [x2, #448] + ldr Q2, [x1, #464] + ldr Q6, [x2, #464] + ldr Q3, [x1, #480] + ldr Q7, [x2, #480] + ldr Q4, [x1, #496] + ldr Q8, [x2, #496] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #448] + str Q10, [x0, #464] + str Q11, [x0, #480] + str Q12, [x0, #496] + ldr Q1, [x1, #512] + ldr Q5, [x2, #512] + ldr Q2, [x1, #528] + ldr Q6, [x2, #528] + ldr Q3, [x1, #544] + ldr Q7, [x2, #544] + ldr Q4, [x1, #560] + ldr Q8, [x2, #560] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #512] + str Q10, [x0, #528] + str Q11, [x0, #544] + str Q12, [x0, #560] + ldr Q1, [x1, #576] + ldr Q5, [x2, #576] + ldr Q2, [x1, #592] + ldr Q6, [x2, #592] + ldr Q3, [x1, #608] + ldr Q7, [x2, #608] + ldr Q4, [x1, #624] + ldr Q8, [x2, #624] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #576] + str Q10, [x0, #592] + str Q11, [x0, #608] + str Q12, [x0, #624] + ldr Q1, [x1, #640] + ldr Q5, [x2, #640] + ldr Q2, [x1, #656] + ldr Q6, [x2, #656] + ldr Q3, [x1, #672] + ldr Q7, [x2, #672] + ldr Q4, [x1, #688] + ldr Q8, [x2, #688] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #640] + str Q10, [x0, #656] + str Q11, [x0, #672] + str Q12, [x0, #688] + ldr Q1, [x1, #704] + ldr Q5, [x2, #704] + ldr Q2, [x1, #720] + ldr Q6, [x2, #720] + ldr Q3, [x1, #736] + ldr Q7, [x2, #736] + ldr Q4, [x1, #752] + ldr Q8, [x2, #752] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #704] + str Q10, [x0, #720] + str Q11, [x0, #736] + str Q12, [x0, #752] + ldr Q1, [x1, #768] + ldr Q5, [x2, #768] + ldr Q2, [x1, #784] + ldr Q6, [x2, #784] + ldr Q3, [x1, #800] + ldr Q7, [x2, #800] + ldr Q4, [x1, #816] + ldr Q8, [x2, #816] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #768] + str Q10, [x0, #784] + str Q11, [x0, #800] + str Q12, [x0, #816] + ldr Q1, [x1, #832] + ldr Q5, [x2, #832] + ldr Q2, [x1, #848] + ldr Q6, [x2, #848] + ldr Q3, [x1, #864] + ldr Q7, [x2, #864] + ldr Q4, [x1, #880] + ldr Q8, [x2, #880] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #832] + str Q10, [x0, #848] + str Q11, [x0, #864] + str Q12, [x0, #880] + ldr Q1, [x1, #896] + ldr Q5, [x2, #896] + ldr Q2, [x1, #912] + ldr Q6, [x2, #912] + ldr Q3, [x1, #928] + ldr Q7, [x2, #928] + ldr Q4, [x1, #944] + ldr Q8, [x2, #944] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #896] + str Q10, [x0, #912] + str Q11, [x0, #928] + str Q12, [x0, #944] + ldr Q1, [x1, #960] + ldr Q5, [x2, #960] + ldr Q2, [x1, #976] + ldr Q6, [x2, #976] + ldr Q3, [x1, #992] + ldr Q7, [x2, #992] + ldr Q4, [x1, #1008] + ldr Q8, [x2, #1008] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V9.4S, V9.4S, V17.4S + sub V10.4S, V10.4S, V18.4S + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmulh V17.4S, V17.4S, V0.S[0] + sqrdmulh V18.4S, V18.4S, V0.S[0] + sub V11.4S, V11.4S, V17.4S + sub V12.4S, V12.4S, V18.4S + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #960] + str Q10, [x0, #976] + str Q11, [x0, #992] + str Q12, [x0, #1008] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + IF :LNOT::DEF:WOLFSSL_AARCH64_NO_SQRDMLSH + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_ntt_sqrdmlsh_neon +mldsa_ntt_sqrdmlsh_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x1, L_mldsa_aarch64_zetas + add x1, x1, L_mldsa_aarch64_zetas + adrp x2, L_mldsa_aarch64_zetas_qinv + add x2, x2, L_mldsa_aarch64_zetas_qinv + adrp x3, L_mldsa_aarch64_consts + add x3, x3, L_mldsa_aarch64_consts + ldr Q4, [x3] + ldr Q5, [x0] + ldr Q6, [x0, #64] + ldr Q7, [x0, #128] + ldr Q8, [x0, #192] + ldr Q9, [x0, #256] + ldr Q10, [x0, #320] + ldr Q11, [x0, #384] + ldr Q12, [x0, #448] + ldr Q13, [x0, #512] + ldr Q14, [x0, #576] + ldr Q15, [x0, #640] + ldr Q16, [x0, #704] + ldr Q17, [x0, #768] + ldr Q18, [x0, #832] + ldr Q19, [x0, #896] + ldr Q20, [x0, #960] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0] + str Q6, [x0, #64] + str Q7, [x0, #128] + str Q8, [x0, #192] + str Q9, [x0, #256] + str Q10, [x0, #320] + str Q11, [x0, #384] + str Q12, [x0, #448] + str Q13, [x0, #512] + str Q14, [x0, #576] + str Q15, [x0, #640] + str Q16, [x0, #704] + str Q17, [x0, #768] + str Q18, [x0, #832] + str Q19, [x0, #896] + str Q20, [x0, #960] + ldr Q5, [x0, #16] + ldr Q6, [x0, #80] + ldr Q7, [x0, #144] + ldr Q8, [x0, #208] + ldr Q9, [x0, #272] + ldr Q10, [x0, #336] + ldr Q11, [x0, #400] + ldr Q12, [x0, #464] + ldr Q13, [x0, #528] + ldr Q14, [x0, #592] + ldr Q15, [x0, #656] + ldr Q16, [x0, #720] + ldr Q17, [x0, #784] + ldr Q18, [x0, #848] + ldr Q19, [x0, #912] + ldr Q20, [x0, #976] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0, #16] + str Q6, [x0, #80] + str Q7, [x0, #144] + str Q8, [x0, #208] + str Q9, [x0, #272] + str Q10, [x0, #336] + str Q11, [x0, #400] + str Q12, [x0, #464] + str Q13, [x0, #528] + str Q14, [x0, #592] + str Q15, [x0, #656] + str Q16, [x0, #720] + str Q17, [x0, #784] + str Q18, [x0, #848] + str Q19, [x0, #912] + str Q20, [x0, #976] + ldr Q5, [x0, #32] + ldr Q6, [x0, #96] + ldr Q7, [x0, #160] + ldr Q8, [x0, #224] + ldr Q9, [x0, #288] + ldr Q10, [x0, #352] + ldr Q11, [x0, #416] + ldr Q12, [x0, #480] + ldr Q13, [x0, #544] + ldr Q14, [x0, #608] + ldr Q15, [x0, #672] + ldr Q16, [x0, #736] + ldr Q17, [x0, #800] + ldr Q18, [x0, #864] + ldr Q19, [x0, #928] + ldr Q20, [x0, #992] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0, #32] + str Q6, [x0, #96] + str Q7, [x0, #160] + str Q8, [x0, #224] + str Q9, [x0, #288] + str Q10, [x0, #352] + str Q11, [x0, #416] + str Q12, [x0, #480] + str Q13, [x0, #544] + str Q14, [x0, #608] + str Q15, [x0, #672] + str Q16, [x0, #736] + str Q17, [x0, #800] + str Q18, [x0, #864] + str Q19, [x0, #928] + str Q20, [x0, #992] + ldr Q5, [x0, #48] + ldr Q6, [x0, #112] + ldr Q7, [x0, #176] + ldr Q8, [x0, #240] + ldr Q9, [x0, #304] + ldr Q10, [x0, #368] + ldr Q11, [x0, #432] + ldr Q12, [x0, #496] + ldr Q13, [x0, #560] + ldr Q14, [x0, #624] + ldr Q15, [x0, #688] + ldr Q16, [x0, #752] + ldr Q17, [x0, #816] + ldr Q18, [x0, #880] + ldr Q19, [x0, #944] + ldr Q20, [x0, #1008] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0, #48] + str Q6, [x0, #112] + str Q7, [x0, #176] + str Q8, [x0, #240] + str Q9, [x0, #304] + str Q10, [x0, #368] + str Q11, [x0, #432] + str Q12, [x0, #496] + str Q13, [x0, #560] + str Q14, [x0, #624] + str Q15, [x0, #688] + str Q16, [x0, #752] + str Q17, [x0, #816] + str Q18, [x0, #880] + str Q19, [x0, #944] + str Q20, [x0, #1008] + ldp Q5, Q6, [x0] + ldp Q7, Q8, [x0, #32] + ldp Q9, Q10, [x0, #64] + ldp Q11, Q12, [x0, #96] + ldp Q13, Q14, [x0, #128] + ldp Q15, Q16, [x0, #160] + ldp Q17, Q18, [x0, #192] + ldp Q19, Q20, [x0, #224] + ldr Q0, [x1, #64] + ldr Q1, [x2, #64] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #128] + ldp Q1, Q3, [x2, #128] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #256] + ldp Q1, Q3, [x2, #256] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #288] + ldp Q1, Q3, [x2, #288] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #320] + ldp Q1, Q3, [x2, #320] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #352] + ldp Q1, Q3, [x2, #352] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #768] + ldp Q1, Q3, [x2, #768] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #800] + ldp Q1, Q3, [x2, #800] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #832] + ldp Q1, Q3, [x2, #832] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #864] + ldp Q1, Q3, [x2, #864] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + stp Q5, Q6, [x0] + stp Q7, Q8, [x0, #32] + stp Q9, Q10, [x0, #64] + stp Q11, Q12, [x0, #96] + stp Q13, Q14, [x0, #128] + stp Q15, Q16, [x0, #160] + stp Q17, Q18, [x0, #192] + stp Q19, Q20, [x0, #224] + ldp Q5, Q6, [x0, #256] + ldp Q7, Q8, [x0, #288] + ldp Q9, Q10, [x0, #320] + ldp Q11, Q12, [x0, #352] + ldp Q13, Q14, [x0, #384] + ldp Q15, Q16, [x0, #416] + ldp Q17, Q18, [x0, #448] + ldp Q19, Q20, [x0, #480] + ldr Q0, [x1, #80] + ldr Q1, [x2, #80] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #160] + ldp Q1, Q3, [x2, #160] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #384] + ldp Q1, Q3, [x2, #384] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #416] + ldp Q1, Q3, [x2, #416] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #448] + ldp Q1, Q3, [x2, #448] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #480] + ldp Q1, Q3, [x2, #480] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #896] + ldp Q1, Q3, [x2, #896] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #928] + ldp Q1, Q3, [x2, #928] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #960] + ldp Q1, Q3, [x2, #960] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #992] + ldp Q1, Q3, [x2, #992] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + stp Q5, Q6, [x0, #256] + stp Q7, Q8, [x0, #288] + stp Q9, Q10, [x0, #320] + stp Q11, Q12, [x0, #352] + stp Q13, Q14, [x0, #384] + stp Q15, Q16, [x0, #416] + stp Q17, Q18, [x0, #448] + stp Q19, Q20, [x0, #480] + ldp Q5, Q6, [x0, #512] + ldp Q7, Q8, [x0, #544] + ldp Q9, Q10, [x0, #576] + ldp Q11, Q12, [x0, #608] + ldp Q13, Q14, [x0, #640] + ldp Q15, Q16, [x0, #672] + ldp Q17, Q18, [x0, #704] + ldp Q19, Q20, [x0, #736] + ldr Q0, [x1, #96] + ldr Q1, [x2, #96] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #192] + ldp Q1, Q3, [x2, #192] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #512] + ldp Q1, Q3, [x2, #512] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #544] + ldp Q1, Q3, [x2, #544] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #576] + ldp Q1, Q3, [x2, #576] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #608] + ldp Q1, Q3, [x2, #608] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldr Q0, [x1, #1024] + ldr Q2, [x1, #1040] + ldr Q1, [x2, #1024] + ldr Q3, [x2, #1040] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldr Q0, [x1, #1056] + ldr Q2, [x1, #1072] + ldr Q1, [x2, #1056] + ldr Q3, [x2, #1072] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldr Q0, [x1, #1088] + ldr Q2, [x1, #1104] + ldr Q1, [x2, #1088] + ldr Q3, [x2, #1104] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldr Q0, [x1, #1120] + ldr Q2, [x1, #1136] + ldr Q1, [x2, #1120] + ldr Q3, [x2, #1136] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + stp Q5, Q6, [x0, #512] + stp Q7, Q8, [x0, #544] + stp Q9, Q10, [x0, #576] + stp Q11, Q12, [x0, #608] + stp Q13, Q14, [x0, #640] + stp Q15, Q16, [x0, #672] + stp Q17, Q18, [x0, #704] + stp Q19, Q20, [x0, #736] + ldp Q5, Q6, [x0, #768] + ldp Q7, Q8, [x0, #800] + ldp Q9, Q10, [x0, #832] + ldp Q11, Q12, [x0, #864] + ldp Q13, Q14, [x0, #896] + ldp Q15, Q16, [x0, #928] + ldp Q17, Q18, [x0, #960] + ldp Q19, Q20, [x0, #992] + ldr Q0, [x1, #112] + ldr Q1, [x2, #112] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #224] + ldp Q1, Q3, [x2, #224] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #640] + ldp Q1, Q3, [x2, #640] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #672] + ldp Q1, Q3, [x2, #672] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #704] + ldp Q1, Q3, [x2, #704] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #736] + ldp Q1, Q3, [x2, #736] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldr Q0, [x1, #1152] + ldr Q2, [x1, #1168] + ldr Q1, [x2, #1152] + ldr Q3, [x2, #1168] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldr Q0, [x1, #1184] + ldr Q2, [x1, #1200] + ldr Q1, [x2, #1184] + ldr Q3, [x2, #1200] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldr Q0, [x1, #1216] + ldr Q2, [x1, #1232] + ldr Q1, [x2, #1216] + ldr Q3, [x2, #1232] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x1, #1264] + ldr Q1, [x2, #1248] + ldr Q3, [x2, #1264] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + stp Q5, Q6, [x0, #768] + stp Q7, Q8, [x0, #800] + stp Q9, Q10, [x0, #832] + stp Q11, Q12, [x0, #864] + stp Q13, Q14, [x0, #896] + stp Q15, Q16, [x0, #928] + stp Q17, Q18, [x0, #960] + stp Q19, Q20, [x0, #992] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_ntt_full_sqrdmlsh_neon +mldsa_ntt_full_sqrdmlsh_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x1, L_mldsa_aarch64_zetas + add x1, x1, L_mldsa_aarch64_zetas + adrp x2, L_mldsa_aarch64_zetas_qinv + add x2, x2, L_mldsa_aarch64_zetas_qinv + adrp x3, L_mldsa_aarch64_consts + add x3, x3, L_mldsa_aarch64_consts + ldr Q4, [x3] + ldr Q5, [x0] + ldr Q6, [x0, #64] + ldr Q7, [x0, #128] + ldr Q8, [x0, #192] + ldr Q9, [x0, #256] + ldr Q10, [x0, #320] + ldr Q11, [x0, #384] + ldr Q12, [x0, #448] + ldr Q13, [x0, #512] + ldr Q14, [x0, #576] + ldr Q15, [x0, #640] + ldr Q16, [x0, #704] + ldr Q17, [x0, #768] + ldr Q18, [x0, #832] + ldr Q19, [x0, #896] + ldr Q20, [x0, #960] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0] + str Q6, [x0, #64] + str Q7, [x0, #128] + str Q8, [x0, #192] + str Q9, [x0, #256] + str Q10, [x0, #320] + str Q11, [x0, #384] + str Q12, [x0, #448] + str Q13, [x0, #512] + str Q14, [x0, #576] + str Q15, [x0, #640] + str Q16, [x0, #704] + str Q17, [x0, #768] + str Q18, [x0, #832] + str Q19, [x0, #896] + str Q20, [x0, #960] + ldr Q5, [x0, #16] + ldr Q6, [x0, #80] + ldr Q7, [x0, #144] + ldr Q8, [x0, #208] + ldr Q9, [x0, #272] + ldr Q10, [x0, #336] + ldr Q11, [x0, #400] + ldr Q12, [x0, #464] + ldr Q13, [x0, #528] + ldr Q14, [x0, #592] + ldr Q15, [x0, #656] + ldr Q16, [x0, #720] + ldr Q17, [x0, #784] + ldr Q18, [x0, #848] + ldr Q19, [x0, #912] + ldr Q20, [x0, #976] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0, #16] + str Q6, [x0, #80] + str Q7, [x0, #144] + str Q8, [x0, #208] + str Q9, [x0, #272] + str Q10, [x0, #336] + str Q11, [x0, #400] + str Q12, [x0, #464] + str Q13, [x0, #528] + str Q14, [x0, #592] + str Q15, [x0, #656] + str Q16, [x0, #720] + str Q17, [x0, #784] + str Q18, [x0, #848] + str Q19, [x0, #912] + str Q20, [x0, #976] + ldr Q5, [x0, #32] + ldr Q6, [x0, #96] + ldr Q7, [x0, #160] + ldr Q8, [x0, #224] + ldr Q9, [x0, #288] + ldr Q10, [x0, #352] + ldr Q11, [x0, #416] + ldr Q12, [x0, #480] + ldr Q13, [x0, #544] + ldr Q14, [x0, #608] + ldr Q15, [x0, #672] + ldr Q16, [x0, #736] + ldr Q17, [x0, #800] + ldr Q18, [x0, #864] + ldr Q19, [x0, #928] + ldr Q20, [x0, #992] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0, #32] + str Q6, [x0, #96] + str Q7, [x0, #160] + str Q8, [x0, #224] + str Q9, [x0, #288] + str Q10, [x0, #352] + str Q11, [x0, #416] + str Q12, [x0, #480] + str Q13, [x0, #544] + str Q14, [x0, #608] + str Q15, [x0, #672] + str Q16, [x0, #736] + str Q17, [x0, #800] + str Q18, [x0, #864] + str Q19, [x0, #928] + str Q20, [x0, #992] + ldr Q5, [x0, #48] + ldr Q6, [x0, #112] + ldr Q7, [x0, #176] + ldr Q8, [x0, #240] + ldr Q9, [x0, #304] + ldr Q10, [x0, #368] + ldr Q11, [x0, #432] + ldr Q12, [x0, #496] + ldr Q13, [x0, #560] + ldr Q14, [x0, #624] + ldr Q15, [x0, #688] + ldr Q16, [x0, #752] + ldr Q17, [x0, #816] + ldr Q18, [x0, #880] + ldr Q19, [x0, #944] + ldr Q20, [x0, #1008] + ldr Q0, [x1] + ldr Q1, [x2] + mul V29.4S, V13.4S, V1.S[1] + mul V30.4S, V14.4S, V1.S[1] + sqrdmulh V21.4S, V13.4S, V0.S[1] + sqrdmulh V22.4S, V14.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V15.4S, V1.S[1] + mul V30.4S, V16.4S, V1.S[1] + sqrdmulh V23.4S, V15.4S, V0.S[1] + sqrdmulh V24.4S, V16.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[1] + mul V30.4S, V18.4S, V1.S[1] + sqrdmulh V25.4S, V17.4S, V0.S[1] + sqrdmulh V26.4S, V18.4S, V0.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[1] + mul V30.4S, V20.4S, V1.S[1] + sqrdmulh V27.4S, V19.4S, V0.S[1] + sqrdmulh V28.4S, V20.4S, V0.S[1] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V13.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V14.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V15.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V16.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V9.4S, V25.4S + add V9.4S, V9.4S, V25.4S + sub V18.4S, V10.4S, V26.4S + add V10.4S, V10.4S, V26.4S + sub V19.4S, V11.4S, V27.4S + add V11.4S, V11.4S, V27.4S + sub V20.4S, V12.4S, V28.4S + add V12.4S, V12.4S, V28.4S + mul V29.4S, V9.4S, V1.S[2] + mul V30.4S, V10.4S, V1.S[2] + sqrdmulh V21.4S, V9.4S, V0.S[2] + sqrdmulh V22.4S, V10.4S, V0.S[2] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[2] + sqrdmulh V23.4S, V11.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[2] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V17.4S, V1.S[3] + mul V30.4S, V18.4S, V1.S[3] + sqrdmulh V25.4S, V17.4S, V0.S[3] + sqrdmulh V26.4S, V18.4S, V0.S[3] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V9.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V10.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V7.4S, V23.4S + add V7.4S, V7.4S, V23.4S + sub V12.4S, V8.4S, V24.4S + add V8.4S, V8.4S, V24.4S + sub V17.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V18.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V15.4S, V27.4S + add V15.4S, V15.4S, V27.4S + sub V20.4S, V16.4S, V28.4S + add V16.4S, V16.4S, V28.4S + ldr Q0, [x1, #16] + ldr Q1, [x2, #16] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #32] + ldp Q1, Q3, [x2, #32] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + str Q5, [x0, #48] + str Q6, [x0, #112] + str Q7, [x0, #176] + str Q8, [x0, #240] + str Q9, [x0, #304] + str Q10, [x0, #368] + str Q11, [x0, #432] + str Q12, [x0, #496] + str Q13, [x0, #560] + str Q14, [x0, #624] + str Q15, [x0, #688] + str Q16, [x0, #752] + str Q17, [x0, #816] + str Q18, [x0, #880] + str Q19, [x0, #944] + str Q20, [x0, #1008] + ldp Q5, Q6, [x0] + ldp Q7, Q8, [x0, #32] + ldp Q9, Q10, [x0, #64] + ldp Q11, Q12, [x0, #96] + ldp Q13, Q14, [x0, #128] + ldp Q15, Q16, [x0, #160] + ldp Q17, Q18, [x0, #192] + ldp Q19, Q20, [x0, #224] + ldr Q0, [x1, #64] + ldr Q1, [x2, #64] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #128] + ldp Q1, Q3, [x2, #128] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #256] + ldp Q1, Q3, [x2, #256] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #288] + ldp Q1, Q3, [x2, #288] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #320] + ldp Q1, Q3, [x2, #320] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #352] + ldp Q1, Q3, [x2, #352] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #768] + ldp Q1, Q3, [x2, #768] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #800] + ldp Q1, Q3, [x2, #800] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #832] + ldp Q1, Q3, [x2, #832] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #864] + ldp Q1, Q3, [x2, #864] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + trn1 V29.4S, V5.4S, V6.4S + trn2 V30.4S, V5.4S, V6.4S + trn1 V5.2D, V29.2D, V30.2D + trn2 V6.2D, V29.2D, V30.2D + trn1 V29.4S, V7.4S, V8.4S + trn2 V30.4S, V7.4S, V8.4S + trn1 V7.2D, V29.2D, V30.2D + trn2 V8.2D, V29.2D, V30.2D + trn1 V29.4S, V9.4S, V10.4S + trn2 V30.4S, V9.4S, V10.4S + trn1 V9.2D, V29.2D, V30.2D + trn2 V10.2D, V29.2D, V30.2D + trn1 V29.4S, V11.4S, V12.4S + trn2 V30.4S, V11.4S, V12.4S + trn1 V11.2D, V29.2D, V30.2D + trn2 V12.2D, V29.2D, V30.2D + trn1 V29.4S, V13.4S, V14.4S + trn2 V30.4S, V13.4S, V14.4S + trn1 V13.2D, V29.2D, V30.2D + trn2 V14.2D, V29.2D, V30.2D + trn1 V29.4S, V15.4S, V16.4S + trn2 V30.4S, V15.4S, V16.4S + trn1 V15.2D, V29.2D, V30.2D + trn2 V16.2D, V29.2D, V30.2D + trn1 V29.4S, V17.4S, V18.4S + trn2 V30.4S, V17.4S, V18.4S + trn1 V17.2D, V29.2D, V30.2D + trn2 V18.2D, V29.2D, V30.2D + trn1 V29.4S, V19.4S, V20.4S + trn2 V30.4S, V19.4S, V20.4S + trn1 V19.2D, V29.2D, V30.2D + trn2 V20.2D, V29.2D, V30.2D + stp Q5, Q6, [x0] + stp Q7, Q8, [x0, #32] + stp Q9, Q10, [x0, #64] + stp Q11, Q12, [x0, #96] + stp Q13, Q14, [x0, #128] + stp Q15, Q16, [x0, #160] + stp Q17, Q18, [x0, #192] + stp Q19, Q20, [x0, #224] + ldp Q5, Q6, [x0, #256] + ldp Q7, Q8, [x0, #288] + ldp Q9, Q10, [x0, #320] + ldp Q11, Q12, [x0, #352] + ldp Q13, Q14, [x0, #384] + ldp Q15, Q16, [x0, #416] + ldp Q17, Q18, [x0, #448] + ldp Q19, Q20, [x0, #480] + ldr Q0, [x1, #80] + ldr Q1, [x2, #80] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #160] + ldp Q1, Q3, [x2, #160] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #384] + ldp Q1, Q3, [x2, #384] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #416] + ldp Q1, Q3, [x2, #416] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #448] + ldp Q1, Q3, [x2, #448] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #480] + ldp Q1, Q3, [x2, #480] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #896] + ldp Q1, Q3, [x2, #896] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #928] + ldp Q1, Q3, [x2, #928] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #960] + ldp Q1, Q3, [x2, #960] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #992] + ldp Q1, Q3, [x2, #992] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + trn1 V29.4S, V5.4S, V6.4S + trn2 V30.4S, V5.4S, V6.4S + trn1 V5.2D, V29.2D, V30.2D + trn2 V6.2D, V29.2D, V30.2D + trn1 V29.4S, V7.4S, V8.4S + trn2 V30.4S, V7.4S, V8.4S + trn1 V7.2D, V29.2D, V30.2D + trn2 V8.2D, V29.2D, V30.2D + trn1 V29.4S, V9.4S, V10.4S + trn2 V30.4S, V9.4S, V10.4S + trn1 V9.2D, V29.2D, V30.2D + trn2 V10.2D, V29.2D, V30.2D + trn1 V29.4S, V11.4S, V12.4S + trn2 V30.4S, V11.4S, V12.4S + trn1 V11.2D, V29.2D, V30.2D + trn2 V12.2D, V29.2D, V30.2D + trn1 V29.4S, V13.4S, V14.4S + trn2 V30.4S, V13.4S, V14.4S + trn1 V13.2D, V29.2D, V30.2D + trn2 V14.2D, V29.2D, V30.2D + trn1 V29.4S, V15.4S, V16.4S + trn2 V30.4S, V15.4S, V16.4S + trn1 V15.2D, V29.2D, V30.2D + trn2 V16.2D, V29.2D, V30.2D + trn1 V29.4S, V17.4S, V18.4S + trn2 V30.4S, V17.4S, V18.4S + trn1 V17.2D, V29.2D, V30.2D + trn2 V18.2D, V29.2D, V30.2D + trn1 V29.4S, V19.4S, V20.4S + trn2 V30.4S, V19.4S, V20.4S + trn1 V19.2D, V29.2D, V30.2D + trn2 V20.2D, V29.2D, V30.2D + stp Q5, Q6, [x0, #256] + stp Q7, Q8, [x0, #288] + stp Q9, Q10, [x0, #320] + stp Q11, Q12, [x0, #352] + stp Q13, Q14, [x0, #384] + stp Q15, Q16, [x0, #416] + stp Q17, Q18, [x0, #448] + stp Q19, Q20, [x0, #480] + ldp Q5, Q6, [x0, #512] + ldp Q7, Q8, [x0, #544] + ldp Q9, Q10, [x0, #576] + ldp Q11, Q12, [x0, #608] + ldp Q13, Q14, [x0, #640] + ldp Q15, Q16, [x0, #672] + ldp Q17, Q18, [x0, #704] + ldp Q19, Q20, [x0, #736] + ldr Q0, [x1, #96] + ldr Q1, [x2, #96] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #192] + ldp Q1, Q3, [x2, #192] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #512] + ldp Q1, Q3, [x2, #512] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #544] + ldp Q1, Q3, [x2, #544] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #576] + ldp Q1, Q3, [x2, #576] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #608] + ldp Q1, Q3, [x2, #608] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldr Q0, [x1, #1024] + ldr Q2, [x1, #1040] + ldr Q1, [x2, #1024] + ldr Q3, [x2, #1040] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldr Q0, [x1, #1056] + ldr Q2, [x1, #1072] + ldr Q1, [x2, #1056] + ldr Q3, [x2, #1072] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldr Q0, [x1, #1088] + ldr Q2, [x1, #1104] + ldr Q1, [x2, #1088] + ldr Q3, [x2, #1104] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldr Q0, [x1, #1120] + ldr Q2, [x1, #1136] + ldr Q1, [x2, #1120] + ldr Q3, [x2, #1136] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + trn1 V29.4S, V5.4S, V6.4S + trn2 V30.4S, V5.4S, V6.4S + trn1 V5.2D, V29.2D, V30.2D + trn2 V6.2D, V29.2D, V30.2D + trn1 V29.4S, V7.4S, V8.4S + trn2 V30.4S, V7.4S, V8.4S + trn1 V7.2D, V29.2D, V30.2D + trn2 V8.2D, V29.2D, V30.2D + trn1 V29.4S, V9.4S, V10.4S + trn2 V30.4S, V9.4S, V10.4S + trn1 V9.2D, V29.2D, V30.2D + trn2 V10.2D, V29.2D, V30.2D + trn1 V29.4S, V11.4S, V12.4S + trn2 V30.4S, V11.4S, V12.4S + trn1 V11.2D, V29.2D, V30.2D + trn2 V12.2D, V29.2D, V30.2D + trn1 V29.4S, V13.4S, V14.4S + trn2 V30.4S, V13.4S, V14.4S + trn1 V13.2D, V29.2D, V30.2D + trn2 V14.2D, V29.2D, V30.2D + trn1 V29.4S, V15.4S, V16.4S + trn2 V30.4S, V15.4S, V16.4S + trn1 V15.2D, V29.2D, V30.2D + trn2 V16.2D, V29.2D, V30.2D + trn1 V29.4S, V17.4S, V18.4S + trn2 V30.4S, V17.4S, V18.4S + trn1 V17.2D, V29.2D, V30.2D + trn2 V18.2D, V29.2D, V30.2D + trn1 V29.4S, V19.4S, V20.4S + trn2 V30.4S, V19.4S, V20.4S + trn1 V19.2D, V29.2D, V30.2D + trn2 V20.2D, V29.2D, V30.2D + stp Q5, Q6, [x0, #512] + stp Q7, Q8, [x0, #544] + stp Q9, Q10, [x0, #576] + stp Q11, Q12, [x0, #608] + stp Q13, Q14, [x0, #640] + stp Q15, Q16, [x0, #672] + stp Q17, Q18, [x0, #704] + stp Q19, Q20, [x0, #736] + ldp Q5, Q6, [x0, #768] + ldp Q7, Q8, [x0, #800] + ldp Q9, Q10, [x0, #832] + ldp Q11, Q12, [x0, #864] + ldp Q13, Q14, [x0, #896] + ldp Q15, Q16, [x0, #928] + ldp Q17, Q18, [x0, #960] + ldp Q19, Q20, [x0, #992] + ldr Q0, [x1, #112] + ldr Q1, [x2, #112] + mul V29.4S, V7.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[0] + sqrdmulh V21.4S, V7.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[0] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V11.4S, V1.S[1] + mul V30.4S, V12.4S, V1.S[1] + sqrdmulh V23.4S, V11.4S, V0.S[1] + sqrdmulh V24.4S, V12.4S, V0.S[1] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V15.4S, V1.S[2] + mul V30.4S, V16.4S, V1.S[2] + sqrdmulh V25.4S, V15.4S, V0.S[2] + sqrdmulh V26.4S, V16.4S, V0.S[2] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V19.4S, V1.S[3] + mul V30.4S, V20.4S, V1.S[3] + sqrdmulh V27.4S, V19.4S, V0.S[3] + sqrdmulh V28.4S, V20.4S, V0.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V7.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V6.4S, V22.4S + add V6.4S, V6.4S, V22.4S + sub V11.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V10.4S, V24.4S + add V10.4S, V10.4S, V24.4S + sub V15.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V14.4S, V26.4S + add V14.4S, V14.4S, V26.4S + sub V19.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V18.4S, V28.4S + add V18.4S, V18.4S, V28.4S + ldp Q0, Q2, [x1, #224] + ldp Q1, Q3, [x2, #224] + mul V29.4S, V6.4S, V1.S[0] + mul V30.4S, V8.4S, V1.S[1] + sqrdmulh V21.4S, V6.4S, V0.S[0] + sqrdmulh V22.4S, V8.4S, V0.S[1] + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + mul V29.4S, V10.4S, V1.S[2] + mul V30.4S, V12.4S, V1.S[3] + sqrdmulh V23.4S, V10.4S, V0.S[2] + sqrdmulh V24.4S, V12.4S, V0.S[3] + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + mul V29.4S, V14.4S, V3.S[0] + mul V30.4S, V16.4S, V3.S[1] + sqrdmulh V25.4S, V14.4S, V2.S[0] + sqrdmulh V26.4S, V16.4S, V2.S[1] + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + mul V29.4S, V18.4S, V3.S[2] + mul V30.4S, V20.4S, V3.S[3] + sqrdmulh V27.4S, V18.4S, V2.S[2] + sqrdmulh V28.4S, V20.4S, V2.S[3] + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldp Q0, Q2, [x1, #640] + ldp Q1, Q3, [x2, #640] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V29.2D, V6.2D + trn2 V8.2D, V30.2D, V8.2D + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldp Q0, Q2, [x1, #672] + ldp Q1, Q3, [x2, #672] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V29.2D, V10.2D + trn2 V12.2D, V30.2D, V12.2D + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldp Q0, Q2, [x1, #704] + ldp Q1, Q3, [x2, #704] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V29.2D, V14.2D + trn2 V16.2D, V30.2D, V16.2D + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldp Q0, Q2, [x1, #736] + ldp Q1, Q3, [x2, #736] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V29.2D, V18.2D + trn2 V20.2D, V30.2D, V20.2D + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + ldr Q0, [x1, #1152] + ldr Q2, [x1, #1168] + ldr Q1, [x2, #1152] + ldr Q3, [x2, #1168] + mov V29.16B, V5.16B + mov V30.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V29.4S, V6.4S + trn2 V8.4S, V30.4S, V8.4S + mul V29.4S, V6.4S, V1.4S + mul V30.4S, V8.4S, V3.4S + sqrdmulh V21.4S, V6.4S, V0.4S + sqrdmulh V22.4S, V8.4S, V2.4S + sqrdmlsh V21.4S, V29.4S, V4.S[0] + sqrdmlsh V22.4S, V30.4S, V4.S[0] + sshr V21.4S, V21.4S, #1 + sshr V22.4S, V22.4S, #1 + ldr Q0, [x1, #1184] + ldr Q2, [x1, #1200] + ldr Q1, [x2, #1184] + ldr Q3, [x2, #1200] + mov V29.16B, V9.16B + mov V30.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V29.4S, V10.4S + trn2 V12.4S, V30.4S, V12.4S + mul V29.4S, V10.4S, V1.4S + mul V30.4S, V12.4S, V3.4S + sqrdmulh V23.4S, V10.4S, V0.4S + sqrdmulh V24.4S, V12.4S, V2.4S + sqrdmlsh V23.4S, V29.4S, V4.S[0] + sqrdmlsh V24.4S, V30.4S, V4.S[0] + sshr V23.4S, V23.4S, #1 + sshr V24.4S, V24.4S, #1 + ldr Q0, [x1, #1216] + ldr Q2, [x1, #1232] + ldr Q1, [x2, #1216] + ldr Q3, [x2, #1232] + mov V29.16B, V13.16B + mov V30.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V29.4S, V14.4S + trn2 V16.4S, V30.4S, V16.4S + mul V29.4S, V14.4S, V1.4S + mul V30.4S, V16.4S, V3.4S + sqrdmulh V25.4S, V14.4S, V0.4S + sqrdmulh V26.4S, V16.4S, V2.4S + sqrdmlsh V25.4S, V29.4S, V4.S[0] + sqrdmlsh V26.4S, V30.4S, V4.S[0] + sshr V25.4S, V25.4S, #1 + sshr V26.4S, V26.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x1, #1264] + ldr Q1, [x2, #1248] + ldr Q3, [x2, #1264] + mov V29.16B, V17.16B + mov V30.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V29.4S, V18.4S + trn2 V20.4S, V30.4S, V20.4S + mul V29.4S, V18.4S, V1.4S + mul V30.4S, V20.4S, V3.4S + sqrdmulh V27.4S, V18.4S, V0.4S + sqrdmulh V28.4S, V20.4S, V2.4S + sqrdmlsh V27.4S, V29.4S, V4.S[0] + sqrdmlsh V28.4S, V30.4S, V4.S[0] + sshr V27.4S, V27.4S, #1 + sshr V28.4S, V28.4S, #1 + sub V6.4S, V5.4S, V21.4S + add V5.4S, V5.4S, V21.4S + sub V8.4S, V7.4S, V22.4S + add V7.4S, V7.4S, V22.4S + sub V10.4S, V9.4S, V23.4S + add V9.4S, V9.4S, V23.4S + sub V12.4S, V11.4S, V24.4S + add V11.4S, V11.4S, V24.4S + sub V14.4S, V13.4S, V25.4S + add V13.4S, V13.4S, V25.4S + sub V16.4S, V15.4S, V26.4S + add V15.4S, V15.4S, V26.4S + sub V18.4S, V17.4S, V27.4S + add V17.4S, V17.4S, V27.4S + sub V20.4S, V19.4S, V28.4S + add V19.4S, V19.4S, V28.4S + trn1 V29.4S, V5.4S, V6.4S + trn2 V30.4S, V5.4S, V6.4S + trn1 V5.2D, V29.2D, V30.2D + trn2 V6.2D, V29.2D, V30.2D + trn1 V29.4S, V7.4S, V8.4S + trn2 V30.4S, V7.4S, V8.4S + trn1 V7.2D, V29.2D, V30.2D + trn2 V8.2D, V29.2D, V30.2D + trn1 V29.4S, V9.4S, V10.4S + trn2 V30.4S, V9.4S, V10.4S + trn1 V9.2D, V29.2D, V30.2D + trn2 V10.2D, V29.2D, V30.2D + trn1 V29.4S, V11.4S, V12.4S + trn2 V30.4S, V11.4S, V12.4S + trn1 V11.2D, V29.2D, V30.2D + trn2 V12.2D, V29.2D, V30.2D + trn1 V29.4S, V13.4S, V14.4S + trn2 V30.4S, V13.4S, V14.4S + trn1 V13.2D, V29.2D, V30.2D + trn2 V14.2D, V29.2D, V30.2D + trn1 V29.4S, V15.4S, V16.4S + trn2 V30.4S, V15.4S, V16.4S + trn1 V15.2D, V29.2D, V30.2D + trn2 V16.2D, V29.2D, V30.2D + trn1 V29.4S, V17.4S, V18.4S + trn2 V30.4S, V17.4S, V18.4S + trn1 V17.2D, V29.2D, V30.2D + trn2 V18.2D, V29.2D, V30.2D + trn1 V29.4S, V19.4S, V20.4S + trn2 V30.4S, V19.4S, V20.4S + trn1 V19.2D, V29.2D, V30.2D + trn2 V20.2D, V29.2D, V30.2D + stp Q5, Q6, [x0, #768] + stp Q7, Q8, [x0, #800] + stp Q9, Q10, [x0, #832] + stp Q11, Q12, [x0, #864] + stp Q13, Q14, [x0, #896] + stp Q15, Q16, [x0, #928] + stp Q17, Q18, [x0, #960] + stp Q19, Q20, [x0, #992] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_invntt_sqrdmlsh_neon +mldsa_invntt_sqrdmlsh_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x1, L_mldsa_aarch64_zetas_inv + add x1, x1, L_mldsa_aarch64_zetas_inv + adrp x2, L_mldsa_aarch64_zetas_inv_qinv + add x2, x2, L_mldsa_aarch64_zetas_inv_qinv + adrp x3, L_mldsa_aarch64_consts + add x3, x3, L_mldsa_aarch64_consts + ldr Q4, [x3] + ldp Q5, Q6, [x0] + ldp Q7, Q8, [x0, #32] + ldp Q9, Q10, [x0, #64] + ldp Q11, Q12, [x0, #96] + ldp Q13, Q14, [x0, #128] + ldp Q15, Q16, [x0, #160] + ldp Q17, Q18, [x0, #192] + ldp Q19, Q20, [x0, #224] + ldp Q0, Q1, [x1] + ldp Q2, Q3, [x2] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #32] + ldp Q2, Q3, [x2, #32] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #64] + ldp Q2, Q3, [x2, #64] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #96] + ldp Q2, Q3, [x2, #96] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #512] + ldp Q2, Q3, [x2, #512] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #544] + ldp Q2, Q3, [x2, #544] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #576] + ldp Q2, Q3, [x2, #576] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #608] + ldp Q2, Q3, [x2, #608] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1024] + ldr Q1, [x1, #1040] + ldr Q2, [x2, #1024] + ldr Q3, [x2, #1040] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1152] + ldr Q2, [x2, #1152] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0] + stp Q7, Q8, [x0, #32] + stp Q9, Q10, [x0, #64] + stp Q11, Q12, [x0, #96] + stp Q13, Q14, [x0, #128] + stp Q15, Q16, [x0, #160] + stp Q17, Q18, [x0, #192] + stp Q19, Q20, [x0, #224] + ldp Q5, Q6, [x0, #256] + ldp Q7, Q8, [x0, #288] + ldp Q9, Q10, [x0, #320] + ldp Q11, Q12, [x0, #352] + ldp Q13, Q14, [x0, #384] + ldp Q15, Q16, [x0, #416] + ldp Q17, Q18, [x0, #448] + ldp Q19, Q20, [x0, #480] + ldp Q0, Q1, [x1, #128] + ldp Q2, Q3, [x2, #128] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #160] + ldp Q2, Q3, [x2, #160] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #192] + ldp Q2, Q3, [x2, #192] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #224] + ldp Q2, Q3, [x2, #224] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #640] + ldp Q2, Q3, [x2, #640] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #672] + ldp Q2, Q3, [x2, #672] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #704] + ldp Q2, Q3, [x2, #704] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #736] + ldp Q2, Q3, [x2, #736] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1056] + ldr Q1, [x1, #1072] + ldr Q2, [x2, #1056] + ldr Q3, [x2, #1072] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1168] + ldr Q2, [x2, #1168] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0, #256] + stp Q7, Q8, [x0, #288] + stp Q9, Q10, [x0, #320] + stp Q11, Q12, [x0, #352] + stp Q13, Q14, [x0, #384] + stp Q15, Q16, [x0, #416] + stp Q17, Q18, [x0, #448] + stp Q19, Q20, [x0, #480] + ldp Q5, Q6, [x0, #512] + ldp Q7, Q8, [x0, #544] + ldp Q9, Q10, [x0, #576] + ldp Q11, Q12, [x0, #608] + ldp Q13, Q14, [x0, #640] + ldp Q15, Q16, [x0, #672] + ldp Q17, Q18, [x0, #704] + ldp Q19, Q20, [x0, #736] + ldp Q0, Q1, [x1, #256] + ldp Q2, Q3, [x2, #256] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #288] + ldp Q2, Q3, [x2, #288] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #320] + ldp Q2, Q3, [x2, #320] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #352] + ldp Q2, Q3, [x2, #352] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #768] + ldp Q2, Q3, [x2, #768] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #800] + ldp Q2, Q3, [x2, #800] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #832] + ldp Q2, Q3, [x2, #832] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #864] + ldp Q2, Q3, [x2, #864] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1088] + ldr Q1, [x1, #1104] + ldr Q2, [x2, #1088] + ldr Q3, [x2, #1104] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1184] + ldr Q2, [x2, #1184] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0, #512] + stp Q7, Q8, [x0, #544] + stp Q9, Q10, [x0, #576] + stp Q11, Q12, [x0, #608] + stp Q13, Q14, [x0, #640] + stp Q15, Q16, [x0, #672] + stp Q17, Q18, [x0, #704] + stp Q19, Q20, [x0, #736] + ldp Q5, Q6, [x0, #768] + ldp Q7, Q8, [x0, #800] + ldp Q9, Q10, [x0, #832] + ldp Q11, Q12, [x0, #864] + ldp Q13, Q14, [x0, #896] + ldp Q15, Q16, [x0, #928] + ldp Q17, Q18, [x0, #960] + ldp Q19, Q20, [x0, #992] + ldp Q0, Q1, [x1, #384] + ldp Q2, Q3, [x2, #384] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #416] + ldp Q2, Q3, [x2, #416] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #448] + ldp Q2, Q3, [x2, #448] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #480] + ldp Q2, Q3, [x2, #480] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #896] + ldp Q2, Q3, [x2, #896] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #928] + ldp Q2, Q3, [x2, #928] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #960] + ldp Q2, Q3, [x2, #960] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #992] + ldp Q2, Q3, [x2, #992] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1120] + ldr Q1, [x1, #1136] + ldr Q2, [x2, #1120] + ldr Q3, [x2, #1136] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1200] + ldr Q2, [x2, #1200] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0, #768] + stp Q7, Q8, [x0, #800] + stp Q9, Q10, [x0, #832] + stp Q11, Q12, [x0, #864] + stp Q13, Q14, [x0, #896] + stp Q15, Q16, [x0, #928] + stp Q17, Q18, [x0, #960] + stp Q19, Q20, [x0, #992] + ldr Q5, [x0] + ldr Q6, [x0, #64] + ldr Q7, [x0, #128] + ldr Q8, [x0, #192] + ldr Q9, [x0, #256] + ldr Q10, [x0, #320] + ldr Q11, [x0, #384] + ldr Q12, [x0, #448] + ldr Q13, [x0, #512] + ldr Q14, [x0, #576] + ldr Q15, [x0, #640] + ldr Q16, [x0, #704] + ldr Q17, [x0, #768] + ldr Q18, [x0, #832] + ldr Q19, [x0, #896] + ldr Q20, [x0, #960] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V23.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmlsh V13.4S, V21.4S, V4.S[0] + sqrdmlsh V14.4S, V23.4S, V4.S[0] + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmlsh V5.4S, V21.4S, V4.S[0] + sqrdmlsh V6.4S, V22.4S, V4.S[0] + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V22.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V22.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V22.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0] + str Q6, [x0, #64] + str Q7, [x0, #128] + str Q8, [x0, #192] + str Q9, [x0, #256] + str Q10, [x0, #320] + str Q11, [x0, #384] + str Q12, [x0, #448] + str Q13, [x0, #512] + str Q14, [x0, #576] + str Q15, [x0, #640] + str Q16, [x0, #704] + str Q17, [x0, #768] + str Q18, [x0, #832] + str Q19, [x0, #896] + str Q20, [x0, #960] + ldr Q5, [x0, #16] + ldr Q6, [x0, #80] + ldr Q7, [x0, #144] + ldr Q8, [x0, #208] + ldr Q9, [x0, #272] + ldr Q10, [x0, #336] + ldr Q11, [x0, #400] + ldr Q12, [x0, #464] + ldr Q13, [x0, #528] + ldr Q14, [x0, #592] + ldr Q15, [x0, #656] + ldr Q16, [x0, #720] + ldr Q17, [x0, #784] + ldr Q18, [x0, #848] + ldr Q19, [x0, #912] + ldr Q20, [x0, #976] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V23.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmlsh V13.4S, V21.4S, V4.S[0] + sqrdmlsh V14.4S, V23.4S, V4.S[0] + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmlsh V5.4S, V21.4S, V4.S[0] + sqrdmlsh V6.4S, V22.4S, V4.S[0] + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V22.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V22.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V22.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0, #16] + str Q6, [x0, #80] + str Q7, [x0, #144] + str Q8, [x0, #208] + str Q9, [x0, #272] + str Q10, [x0, #336] + str Q11, [x0, #400] + str Q12, [x0, #464] + str Q13, [x0, #528] + str Q14, [x0, #592] + str Q15, [x0, #656] + str Q16, [x0, #720] + str Q17, [x0, #784] + str Q18, [x0, #848] + str Q19, [x0, #912] + str Q20, [x0, #976] + ldr Q5, [x0, #32] + ldr Q6, [x0, #96] + ldr Q7, [x0, #160] + ldr Q8, [x0, #224] + ldr Q9, [x0, #288] + ldr Q10, [x0, #352] + ldr Q11, [x0, #416] + ldr Q12, [x0, #480] + ldr Q13, [x0, #544] + ldr Q14, [x0, #608] + ldr Q15, [x0, #672] + ldr Q16, [x0, #736] + ldr Q17, [x0, #800] + ldr Q18, [x0, #864] + ldr Q19, [x0, #928] + ldr Q20, [x0, #992] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V23.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmlsh V13.4S, V21.4S, V4.S[0] + sqrdmlsh V14.4S, V23.4S, V4.S[0] + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmlsh V5.4S, V21.4S, V4.S[0] + sqrdmlsh V6.4S, V22.4S, V4.S[0] + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V22.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V22.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V22.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0, #32] + str Q6, [x0, #96] + str Q7, [x0, #160] + str Q8, [x0, #224] + str Q9, [x0, #288] + str Q10, [x0, #352] + str Q11, [x0, #416] + str Q12, [x0, #480] + str Q13, [x0, #544] + str Q14, [x0, #608] + str Q15, [x0, #672] + str Q16, [x0, #736] + str Q17, [x0, #800] + str Q18, [x0, #864] + str Q19, [x0, #928] + str Q20, [x0, #992] + ldr Q5, [x0, #48] + ldr Q6, [x0, #112] + ldr Q7, [x0, #176] + ldr Q8, [x0, #240] + ldr Q9, [x0, #304] + ldr Q10, [x0, #368] + ldr Q11, [x0, #432] + ldr Q12, [x0, #496] + ldr Q13, [x0, #560] + ldr Q14, [x0, #624] + ldr Q15, [x0, #688] + ldr Q16, [x0, #752] + ldr Q17, [x0, #816] + ldr Q18, [x0, #880] + ldr Q19, [x0, #944] + ldr Q20, [x0, #1008] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V23.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmlsh V13.4S, V21.4S, V4.S[0] + sqrdmlsh V14.4S, V23.4S, V4.S[0] + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmlsh V5.4S, V21.4S, V4.S[0] + sqrdmlsh V6.4S, V22.4S, V4.S[0] + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V22.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V22.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V22.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0, #48] + str Q6, [x0, #112] + str Q7, [x0, #176] + str Q8, [x0, #240] + str Q9, [x0, #304] + str Q10, [x0, #368] + str Q11, [x0, #432] + str Q12, [x0, #496] + str Q13, [x0, #560] + str Q14, [x0, #624] + str Q15, [x0, #688] + str Q16, [x0, #752] + str Q17, [x0, #816] + str Q18, [x0, #880] + str Q19, [x0, #944] + str Q20, [x0, #1008] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_invntt_full_sqrdmlsh_neon +mldsa_invntt_full_sqrdmlsh_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x1, L_mldsa_aarch64_zetas_inv + add x1, x1, L_mldsa_aarch64_zetas_inv + adrp x2, L_mldsa_aarch64_zetas_inv_qinv + add x2, x2, L_mldsa_aarch64_zetas_inv_qinv + adrp x3, L_mldsa_aarch64_consts + add x3, x3, L_mldsa_aarch64_consts + ldr Q4, [x3] + ldp Q5, Q6, [x0] + ldp Q7, Q8, [x0, #32] + ldp Q9, Q10, [x0, #64] + ldp Q11, Q12, [x0, #96] + ldp Q13, Q14, [x0, #128] + ldp Q15, Q16, [x0, #160] + ldp Q17, Q18, [x0, #192] + ldp Q19, Q20, [x0, #224] + trn1 V21.2D, V5.2D, V6.2D + trn2 V22.2D, V5.2D, V6.2D + trn1 V5.4S, V21.4S, V22.4S + trn2 V6.4S, V21.4S, V22.4S + trn1 V21.2D, V7.2D, V8.2D + trn2 V22.2D, V7.2D, V8.2D + trn1 V7.4S, V21.4S, V22.4S + trn2 V8.4S, V21.4S, V22.4S + trn1 V21.2D, V9.2D, V10.2D + trn2 V22.2D, V9.2D, V10.2D + trn1 V9.4S, V21.4S, V22.4S + trn2 V10.4S, V21.4S, V22.4S + trn1 V21.2D, V11.2D, V12.2D + trn2 V22.2D, V11.2D, V12.2D + trn1 V11.4S, V21.4S, V22.4S + trn2 V12.4S, V21.4S, V22.4S + trn1 V21.2D, V13.2D, V14.2D + trn2 V22.2D, V13.2D, V14.2D + trn1 V13.4S, V21.4S, V22.4S + trn2 V14.4S, V21.4S, V22.4S + trn1 V21.2D, V15.2D, V16.2D + trn2 V22.2D, V15.2D, V16.2D + trn1 V15.4S, V21.4S, V22.4S + trn2 V16.4S, V21.4S, V22.4S + trn1 V21.2D, V17.2D, V18.2D + trn2 V22.2D, V17.2D, V18.2D + trn1 V17.4S, V21.4S, V22.4S + trn2 V18.4S, V21.4S, V22.4S + trn1 V21.2D, V19.2D, V20.2D + trn2 V22.2D, V19.2D, V20.2D + trn1 V19.4S, V21.4S, V22.4S + trn2 V20.4S, V21.4S, V22.4S + ldp Q0, Q1, [x1] + ldp Q2, Q3, [x2] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #32] + ldp Q2, Q3, [x2, #32] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #64] + ldp Q2, Q3, [x2, #64] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #96] + ldp Q2, Q3, [x2, #96] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #512] + ldp Q2, Q3, [x2, #512] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #544] + ldp Q2, Q3, [x2, #544] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #576] + ldp Q2, Q3, [x2, #576] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #608] + ldp Q2, Q3, [x2, #608] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1024] + ldr Q1, [x1, #1040] + ldr Q2, [x2, #1024] + ldr Q3, [x2, #1040] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1152] + ldr Q2, [x2, #1152] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0] + stp Q7, Q8, [x0, #32] + stp Q9, Q10, [x0, #64] + stp Q11, Q12, [x0, #96] + stp Q13, Q14, [x0, #128] + stp Q15, Q16, [x0, #160] + stp Q17, Q18, [x0, #192] + stp Q19, Q20, [x0, #224] + ldp Q5, Q6, [x0, #256] + ldp Q7, Q8, [x0, #288] + ldp Q9, Q10, [x0, #320] + ldp Q11, Q12, [x0, #352] + ldp Q13, Q14, [x0, #384] + ldp Q15, Q16, [x0, #416] + ldp Q17, Q18, [x0, #448] + ldp Q19, Q20, [x0, #480] + trn1 V21.2D, V5.2D, V6.2D + trn2 V22.2D, V5.2D, V6.2D + trn1 V5.4S, V21.4S, V22.4S + trn2 V6.4S, V21.4S, V22.4S + trn1 V21.2D, V7.2D, V8.2D + trn2 V22.2D, V7.2D, V8.2D + trn1 V7.4S, V21.4S, V22.4S + trn2 V8.4S, V21.4S, V22.4S + trn1 V21.2D, V9.2D, V10.2D + trn2 V22.2D, V9.2D, V10.2D + trn1 V9.4S, V21.4S, V22.4S + trn2 V10.4S, V21.4S, V22.4S + trn1 V21.2D, V11.2D, V12.2D + trn2 V22.2D, V11.2D, V12.2D + trn1 V11.4S, V21.4S, V22.4S + trn2 V12.4S, V21.4S, V22.4S + trn1 V21.2D, V13.2D, V14.2D + trn2 V22.2D, V13.2D, V14.2D + trn1 V13.4S, V21.4S, V22.4S + trn2 V14.4S, V21.4S, V22.4S + trn1 V21.2D, V15.2D, V16.2D + trn2 V22.2D, V15.2D, V16.2D + trn1 V15.4S, V21.4S, V22.4S + trn2 V16.4S, V21.4S, V22.4S + trn1 V21.2D, V17.2D, V18.2D + trn2 V22.2D, V17.2D, V18.2D + trn1 V17.4S, V21.4S, V22.4S + trn2 V18.4S, V21.4S, V22.4S + trn1 V21.2D, V19.2D, V20.2D + trn2 V22.2D, V19.2D, V20.2D + trn1 V19.4S, V21.4S, V22.4S + trn2 V20.4S, V21.4S, V22.4S + ldp Q0, Q1, [x1, #128] + ldp Q2, Q3, [x2, #128] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #160] + ldp Q2, Q3, [x2, #160] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #192] + ldp Q2, Q3, [x2, #192] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #224] + ldp Q2, Q3, [x2, #224] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #640] + ldp Q2, Q3, [x2, #640] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #672] + ldp Q2, Q3, [x2, #672] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #704] + ldp Q2, Q3, [x2, #704] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #736] + ldp Q2, Q3, [x2, #736] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1056] + ldr Q1, [x1, #1072] + ldr Q2, [x2, #1056] + ldr Q3, [x2, #1072] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1168] + ldr Q2, [x2, #1168] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0, #256] + stp Q7, Q8, [x0, #288] + stp Q9, Q10, [x0, #320] + stp Q11, Q12, [x0, #352] + stp Q13, Q14, [x0, #384] + stp Q15, Q16, [x0, #416] + stp Q17, Q18, [x0, #448] + stp Q19, Q20, [x0, #480] + ldp Q5, Q6, [x0, #512] + ldp Q7, Q8, [x0, #544] + ldp Q9, Q10, [x0, #576] + ldp Q11, Q12, [x0, #608] + ldp Q13, Q14, [x0, #640] + ldp Q15, Q16, [x0, #672] + ldp Q17, Q18, [x0, #704] + ldp Q19, Q20, [x0, #736] + trn1 V21.2D, V5.2D, V6.2D + trn2 V22.2D, V5.2D, V6.2D + trn1 V5.4S, V21.4S, V22.4S + trn2 V6.4S, V21.4S, V22.4S + trn1 V21.2D, V7.2D, V8.2D + trn2 V22.2D, V7.2D, V8.2D + trn1 V7.4S, V21.4S, V22.4S + trn2 V8.4S, V21.4S, V22.4S + trn1 V21.2D, V9.2D, V10.2D + trn2 V22.2D, V9.2D, V10.2D + trn1 V9.4S, V21.4S, V22.4S + trn2 V10.4S, V21.4S, V22.4S + trn1 V21.2D, V11.2D, V12.2D + trn2 V22.2D, V11.2D, V12.2D + trn1 V11.4S, V21.4S, V22.4S + trn2 V12.4S, V21.4S, V22.4S + trn1 V21.2D, V13.2D, V14.2D + trn2 V22.2D, V13.2D, V14.2D + trn1 V13.4S, V21.4S, V22.4S + trn2 V14.4S, V21.4S, V22.4S + trn1 V21.2D, V15.2D, V16.2D + trn2 V22.2D, V15.2D, V16.2D + trn1 V15.4S, V21.4S, V22.4S + trn2 V16.4S, V21.4S, V22.4S + trn1 V21.2D, V17.2D, V18.2D + trn2 V22.2D, V17.2D, V18.2D + trn1 V17.4S, V21.4S, V22.4S + trn2 V18.4S, V21.4S, V22.4S + trn1 V21.2D, V19.2D, V20.2D + trn2 V22.2D, V19.2D, V20.2D + trn1 V19.4S, V21.4S, V22.4S + trn2 V20.4S, V21.4S, V22.4S + ldp Q0, Q1, [x1, #256] + ldp Q2, Q3, [x2, #256] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #288] + ldp Q2, Q3, [x2, #288] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #320] + ldp Q2, Q3, [x2, #320] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #352] + ldp Q2, Q3, [x2, #352] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #768] + ldp Q2, Q3, [x2, #768] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #800] + ldp Q2, Q3, [x2, #800] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #832] + ldp Q2, Q3, [x2, #832] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #864] + ldp Q2, Q3, [x2, #864] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1088] + ldr Q1, [x1, #1104] + ldr Q2, [x2, #1088] + ldr Q3, [x2, #1104] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1184] + ldr Q2, [x2, #1184] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0, #512] + stp Q7, Q8, [x0, #544] + stp Q9, Q10, [x0, #576] + stp Q11, Q12, [x0, #608] + stp Q13, Q14, [x0, #640] + stp Q15, Q16, [x0, #672] + stp Q17, Q18, [x0, #704] + stp Q19, Q20, [x0, #736] + ldp Q5, Q6, [x0, #768] + ldp Q7, Q8, [x0, #800] + ldp Q9, Q10, [x0, #832] + ldp Q11, Q12, [x0, #864] + ldp Q13, Q14, [x0, #896] + ldp Q15, Q16, [x0, #928] + ldp Q17, Q18, [x0, #960] + ldp Q19, Q20, [x0, #992] + trn1 V21.2D, V5.2D, V6.2D + trn2 V22.2D, V5.2D, V6.2D + trn1 V5.4S, V21.4S, V22.4S + trn2 V6.4S, V21.4S, V22.4S + trn1 V21.2D, V7.2D, V8.2D + trn2 V22.2D, V7.2D, V8.2D + trn1 V7.4S, V21.4S, V22.4S + trn2 V8.4S, V21.4S, V22.4S + trn1 V21.2D, V9.2D, V10.2D + trn2 V22.2D, V9.2D, V10.2D + trn1 V9.4S, V21.4S, V22.4S + trn2 V10.4S, V21.4S, V22.4S + trn1 V21.2D, V11.2D, V12.2D + trn2 V22.2D, V11.2D, V12.2D + trn1 V11.4S, V21.4S, V22.4S + trn2 V12.4S, V21.4S, V22.4S + trn1 V21.2D, V13.2D, V14.2D + trn2 V22.2D, V13.2D, V14.2D + trn1 V13.4S, V21.4S, V22.4S + trn2 V14.4S, V21.4S, V22.4S + trn1 V21.2D, V15.2D, V16.2D + trn2 V22.2D, V15.2D, V16.2D + trn1 V15.4S, V21.4S, V22.4S + trn2 V16.4S, V21.4S, V22.4S + trn1 V21.2D, V17.2D, V18.2D + trn2 V22.2D, V17.2D, V18.2D + trn1 V17.4S, V21.4S, V22.4S + trn2 V18.4S, V21.4S, V22.4S + trn1 V21.2D, V19.2D, V20.2D + trn2 V22.2D, V19.2D, V20.2D + trn1 V19.4S, V21.4S, V22.4S + trn2 V20.4S, V21.4S, V22.4S + ldp Q0, Q1, [x1, #384] + ldp Q2, Q3, [x2, #384] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #416] + ldp Q2, Q3, [x2, #416] + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #448] + ldp Q2, Q3, [x2, #448] + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #480] + ldp Q2, Q3, [x2, #480] + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldp Q0, Q1, [x1, #896] + ldp Q2, Q3, [x2, #896] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.4S, V5.4S, V6.4S + trn1 V7.4S, V7.4S, V8.4S + trn2 V6.4S, V21.4S, V6.4S + trn2 V8.4S, V22.4S, V8.4S + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V6.4S, V22.4S, V0.4S + sqrdmulh V8.4S, V24.4S, V1.4S + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + ldp Q0, Q1, [x1, #928] + ldp Q2, Q3, [x2, #928] + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.4S, V9.4S, V10.4S + trn1 V11.4S, V11.4S, V12.4S + trn2 V10.4S, V21.4S, V10.4S + trn2 V12.4S, V22.4S, V12.4S + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V10.4S, V22.4S, V0.4S + sqrdmulh V12.4S, V24.4S, V1.4S + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + ldp Q0, Q1, [x1, #960] + ldp Q2, Q3, [x2, #960] + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.4S, V13.4S, V14.4S + trn1 V15.4S, V15.4S, V16.4S + trn2 V14.4S, V21.4S, V14.4S + trn2 V16.4S, V22.4S, V16.4S + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V14.4S, V22.4S, V0.4S + sqrdmulh V16.4S, V24.4S, V1.4S + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + ldp Q0, Q1, [x1, #992] + ldp Q2, Q3, [x2, #992] + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.4S, V17.4S, V18.4S + trn1 V19.4S, V19.4S, V20.4S + trn2 V18.4S, V21.4S, V18.4S + trn2 V20.4S, V22.4S, V20.4S + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V2.4S + mul V23.4S, V24.4S, V3.4S + sqrdmulh V18.4S, V22.4S, V0.4S + sqrdmulh V20.4S, V24.4S, V1.4S + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1120] + ldr Q1, [x1, #1136] + ldr Q2, [x2, #1120] + ldr Q3, [x2, #1136] + mov V21.16B, V5.16B + mov V22.16B, V7.16B + trn1 V5.2D, V5.2D, V6.2D + trn1 V7.2D, V7.2D, V8.2D + trn2 V6.2D, V21.2D, V6.2D + trn2 V8.2D, V22.2D, V8.2D + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + mov V21.16B, V9.16B + mov V22.16B, V11.16B + trn1 V9.2D, V9.2D, V10.2D + trn1 V11.2D, V11.2D, V12.2D + trn2 V10.2D, V21.2D, V10.2D + trn2 V12.2D, V22.2D, V12.2D + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + mov V21.16B, V13.16B + mov V22.16B, V15.16B + trn1 V13.2D, V13.2D, V14.2D + trn1 V15.2D, V15.2D, V16.2D + trn2 V14.2D, V21.2D, V14.2D + trn2 V16.2D, V22.2D, V16.2D + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + mov V21.16B, V17.16B + mov V22.16B, V19.16B + trn1 V17.2D, V17.2D, V18.2D + trn1 V19.2D, V19.2D, V20.2D + trn2 V18.2D, V21.2D, V18.2D + trn2 V20.2D, V22.2D, V20.2D + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1200] + ldr Q2, [x2, #1200] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + stp Q5, Q6, [x0, #768] + stp Q7, Q8, [x0, #800] + stp Q9, Q10, [x0, #832] + stp Q11, Q12, [x0, #864] + stp Q13, Q14, [x0, #896] + stp Q15, Q16, [x0, #928] + stp Q17, Q18, [x0, #960] + stp Q19, Q20, [x0, #992] + ldr Q5, [x0] + ldr Q6, [x0, #64] + ldr Q7, [x0, #128] + ldr Q8, [x0, #192] + ldr Q9, [x0, #256] + ldr Q10, [x0, #320] + ldr Q11, [x0, #384] + ldr Q12, [x0, #448] + ldr Q13, [x0, #512] + ldr Q14, [x0, #576] + ldr Q15, [x0, #640] + ldr Q16, [x0, #704] + ldr Q17, [x0, #768] + ldr Q18, [x0, #832] + ldr Q19, [x0, #896] + ldr Q20, [x0, #960] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V23.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmlsh V13.4S, V21.4S, V4.S[0] + sqrdmlsh V14.4S, V23.4S, V4.S[0] + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmlsh V5.4S, V21.4S, V4.S[0] + sqrdmlsh V6.4S, V22.4S, V4.S[0] + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V22.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V22.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V22.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0] + str Q6, [x0, #64] + str Q7, [x0, #128] + str Q8, [x0, #192] + str Q9, [x0, #256] + str Q10, [x0, #320] + str Q11, [x0, #384] + str Q12, [x0, #448] + str Q13, [x0, #512] + str Q14, [x0, #576] + str Q15, [x0, #640] + str Q16, [x0, #704] + str Q17, [x0, #768] + str Q18, [x0, #832] + str Q19, [x0, #896] + str Q20, [x0, #960] + ldr Q5, [x0, #16] + ldr Q6, [x0, #80] + ldr Q7, [x0, #144] + ldr Q8, [x0, #208] + ldr Q9, [x0, #272] + ldr Q10, [x0, #336] + ldr Q11, [x0, #400] + ldr Q12, [x0, #464] + ldr Q13, [x0, #528] + ldr Q14, [x0, #592] + ldr Q15, [x0, #656] + ldr Q16, [x0, #720] + ldr Q17, [x0, #784] + ldr Q18, [x0, #848] + ldr Q19, [x0, #912] + ldr Q20, [x0, #976] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V23.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmlsh V13.4S, V21.4S, V4.S[0] + sqrdmlsh V14.4S, V23.4S, V4.S[0] + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmlsh V5.4S, V21.4S, V4.S[0] + sqrdmlsh V6.4S, V22.4S, V4.S[0] + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V22.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V22.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V22.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0, #16] + str Q6, [x0, #80] + str Q7, [x0, #144] + str Q8, [x0, #208] + str Q9, [x0, #272] + str Q10, [x0, #336] + str Q11, [x0, #400] + str Q12, [x0, #464] + str Q13, [x0, #528] + str Q14, [x0, #592] + str Q15, [x0, #656] + str Q16, [x0, #720] + str Q17, [x0, #784] + str Q18, [x0, #848] + str Q19, [x0, #912] + str Q20, [x0, #976] + ldr Q5, [x0, #32] + ldr Q6, [x0, #96] + ldr Q7, [x0, #160] + ldr Q8, [x0, #224] + ldr Q9, [x0, #288] + ldr Q10, [x0, #352] + ldr Q11, [x0, #416] + ldr Q12, [x0, #480] + ldr Q13, [x0, #544] + ldr Q14, [x0, #608] + ldr Q15, [x0, #672] + ldr Q16, [x0, #736] + ldr Q17, [x0, #800] + ldr Q18, [x0, #864] + ldr Q19, [x0, #928] + ldr Q20, [x0, #992] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V23.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmlsh V13.4S, V21.4S, V4.S[0] + sqrdmlsh V14.4S, V23.4S, V4.S[0] + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmlsh V5.4S, V21.4S, V4.S[0] + sqrdmlsh V6.4S, V22.4S, V4.S[0] + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V22.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V22.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V22.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0, #32] + str Q6, [x0, #96] + str Q7, [x0, #160] + str Q8, [x0, #224] + str Q9, [x0, #288] + str Q10, [x0, #352] + str Q11, [x0, #416] + str Q12, [x0, #480] + str Q13, [x0, #544] + str Q14, [x0, #608] + str Q15, [x0, #672] + str Q16, [x0, #736] + str Q17, [x0, #800] + str Q18, [x0, #864] + str Q19, [x0, #928] + str Q20, [x0, #992] + ldr Q5, [x0, #48] + ldr Q6, [x0, #112] + ldr Q7, [x0, #176] + ldr Q8, [x0, #240] + ldr Q9, [x0, #304] + ldr Q10, [x0, #368] + ldr Q11, [x0, #432] + ldr Q12, [x0, #496] + ldr Q13, [x0, #560] + ldr Q14, [x0, #624] + ldr Q15, [x0, #688] + ldr Q16, [x0, #752] + ldr Q17, [x0, #816] + ldr Q18, [x0, #880] + ldr Q19, [x0, #944] + ldr Q20, [x0, #1008] + ldr Q0, [x1, #1216] + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1216] + ldr Q3, [x2, #1232] + sub V22.4S, V5.4S, V6.4S + sub V24.4S, V7.4S, V8.4S + add V5.4S, V5.4S, V6.4S + add V7.4S, V7.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V6.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[1] + sqrdmlsh V6.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V6.4S, V6.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V10.4S + sub V24.4S, V11.4S, V12.4S + add V9.4S, V9.4S, V10.4S + add V11.4S, V11.4S, V12.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V10.4S, V22.4S, V0.S[2] + sqrdmulh V12.4S, V24.4S, V0.S[3] + sqrdmlsh V10.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V10.4S, V10.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V14.4S + sub V24.4S, V15.4S, V16.4S + add V13.4S, V13.4S, V14.4S + add V15.4S, V15.4S, V16.4S + mul V21.4S, V22.4S, V3.S[0] + mul V23.4S, V24.4S, V3.S[1] + sqrdmulh V14.4S, V22.4S, V1.S[0] + sqrdmulh V16.4S, V24.4S, V1.S[1] + sqrdmlsh V14.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V14.4S, V14.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V18.4S + sub V24.4S, V19.4S, V20.4S + add V17.4S, V17.4S, V18.4S + add V19.4S, V19.4S, V20.4S + mul V21.4S, V22.4S, V3.S[2] + mul V23.4S, V24.4S, V3.S[3] + sqrdmulh V18.4S, V22.4S, V1.S[2] + sqrdmulh V20.4S, V24.4S, V1.S[3] + sqrdmlsh V18.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V18.4S, V18.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1248] + ldr Q2, [x2, #1248] + sub V22.4S, V5.4S, V7.4S + sub V24.4S, V6.4S, V8.4S + add V5.4S, V5.4S, V7.4S + add V6.4S, V6.4S, V8.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V7.4S, V22.4S, V0.S[0] + sqrdmulh V8.4S, V24.4S, V0.S[0] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V23.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + sub V22.4S, V9.4S, V11.4S + sub V24.4S, V10.4S, V12.4S + add V9.4S, V9.4S, V11.4S + add V10.4S, V10.4S, V12.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V11.4S, V22.4S, V0.S[1] + sqrdmulh V12.4S, V24.4S, V0.S[1] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V15.4S + sub V24.4S, V14.4S, V16.4S + add V13.4S, V13.4S, V15.4S + add V14.4S, V14.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V17.4S, V19.4S + sub V24.4S, V18.4S, V20.4S + add V17.4S, V17.4S, V19.4S + add V18.4S, V18.4S, V20.4S + mul V21.4S, V22.4S, V2.S[3] + mul V23.4S, V24.4S, V2.S[3] + sqrdmulh V19.4S, V22.4S, V0.S[3] + sqrdmulh V20.4S, V24.4S, V0.S[3] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + ldr Q0, [x1, #1264] + ldr Q2, [x2, #1264] + sub V22.4S, V5.4S, V9.4S + sub V24.4S, V6.4S, V10.4S + add V5.4S, V5.4S, V9.4S + add V6.4S, V6.4S, V10.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V9.4S, V22.4S, V0.S[0] + sqrdmulh V10.4S, V24.4S, V0.S[0] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V23.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + sub V22.4S, V7.4S, V11.4S + sub V24.4S, V8.4S, V12.4S + add V7.4S, V7.4S, V11.4S + add V8.4S, V8.4S, V12.4S + mul V21.4S, V22.4S, V2.S[0] + mul V23.4S, V24.4S, V2.S[0] + sqrdmulh V11.4S, V22.4S, V0.S[0] + sqrdmulh V12.4S, V24.4S, V0.S[0] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V23.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + sub V22.4S, V13.4S, V17.4S + sub V24.4S, V14.4S, V18.4S + add V13.4S, V13.4S, V17.4S + add V14.4S, V14.4S, V18.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V17.4S, V22.4S, V0.S[1] + sqrdmulh V18.4S, V24.4S, V0.S[1] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V15.4S, V19.4S + sub V24.4S, V16.4S, V20.4S + add V15.4S, V15.4S, V19.4S + add V16.4S, V16.4S, V20.4S + mul V21.4S, V22.4S, V2.S[1] + mul V23.4S, V24.4S, V2.S[1] + sqrdmulh V19.4S, V22.4S, V0.S[1] + sqrdmulh V20.4S, V24.4S, V0.S[1] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + sub V22.4S, V5.4S, V13.4S + sub V24.4S, V6.4S, V14.4S + add V5.4S, V5.4S, V13.4S + add V6.4S, V6.4S, V14.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V13.4S, V22.4S, V0.S[2] + sqrdmulh V14.4S, V24.4S, V0.S[2] + sqrdmlsh V13.4S, V21.4S, V4.S[0] + sqrdmlsh V14.4S, V23.4S, V4.S[0] + sshr V13.4S, V13.4S, #1 + sshr V14.4S, V14.4S, #1 + sub V22.4S, V7.4S, V15.4S + sub V24.4S, V8.4S, V16.4S + add V7.4S, V7.4S, V15.4S + add V8.4S, V8.4S, V16.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V15.4S, V22.4S, V0.S[2] + sqrdmulh V16.4S, V24.4S, V0.S[2] + sqrdmlsh V15.4S, V21.4S, V4.S[0] + sqrdmlsh V16.4S, V23.4S, V4.S[0] + sshr V15.4S, V15.4S, #1 + sshr V16.4S, V16.4S, #1 + sub V22.4S, V9.4S, V17.4S + sub V24.4S, V10.4S, V18.4S + add V9.4S, V9.4S, V17.4S + add V10.4S, V10.4S, V18.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V17.4S, V22.4S, V0.S[2] + sqrdmulh V18.4S, V24.4S, V0.S[2] + sqrdmlsh V17.4S, V21.4S, V4.S[0] + sqrdmlsh V18.4S, V23.4S, V4.S[0] + sshr V17.4S, V17.4S, #1 + sshr V18.4S, V18.4S, #1 + sub V22.4S, V11.4S, V19.4S + sub V24.4S, V12.4S, V20.4S + add V11.4S, V11.4S, V19.4S + add V12.4S, V12.4S, V20.4S + mul V21.4S, V22.4S, V2.S[2] + mul V23.4S, V24.4S, V2.S[2] + sqrdmulh V19.4S, V22.4S, V0.S[2] + sqrdmulh V20.4S, V24.4S, V0.S[2] + sqrdmlsh V19.4S, V21.4S, V4.S[0] + sqrdmlsh V20.4S, V23.4S, V4.S[0] + sshr V19.4S, V19.4S, #1 + sshr V20.4S, V20.4S, #1 + mul V21.4S, V5.4S, V2.S[3] + mul V22.4S, V6.4S, V2.S[3] + sqrdmulh V5.4S, V5.4S, V0.S[3] + sqrdmulh V6.4S, V6.4S, V0.S[3] + sqrdmlsh V5.4S, V21.4S, V4.S[0] + sqrdmlsh V6.4S, V22.4S, V4.S[0] + sshr V5.4S, V5.4S, #1 + sshr V6.4S, V6.4S, #1 + mul V21.4S, V7.4S, V2.S[3] + mul V22.4S, V8.4S, V2.S[3] + sqrdmulh V7.4S, V7.4S, V0.S[3] + sqrdmulh V8.4S, V8.4S, V0.S[3] + sqrdmlsh V7.4S, V21.4S, V4.S[0] + sqrdmlsh V8.4S, V22.4S, V4.S[0] + sshr V7.4S, V7.4S, #1 + sshr V8.4S, V8.4S, #1 + mul V21.4S, V9.4S, V2.S[3] + mul V22.4S, V10.4S, V2.S[3] + sqrdmulh V9.4S, V9.4S, V0.S[3] + sqrdmulh V10.4S, V10.4S, V0.S[3] + sqrdmlsh V9.4S, V21.4S, V4.S[0] + sqrdmlsh V10.4S, V22.4S, V4.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V21.4S, V11.4S, V2.S[3] + mul V22.4S, V12.4S, V2.S[3] + sqrdmulh V11.4S, V11.4S, V0.S[3] + sqrdmulh V12.4S, V12.4S, V0.S[3] + sqrdmlsh V11.4S, V21.4S, V4.S[0] + sqrdmlsh V12.4S, V22.4S, V4.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q5, [x0, #48] + str Q6, [x0, #112] + str Q7, [x0, #176] + str Q8, [x0, #240] + str Q9, [x0, #304] + str Q10, [x0, #368] + str Q11, [x0, #432] + str Q12, [x0, #496] + str Q13, [x0, #560] + str Q14, [x0, #624] + str Q15, [x0, #688] + str Q16, [x0, #752] + str Q17, [x0, #816] + str Q18, [x0, #880] + str Q19, [x0, #944] + str Q20, [x0, #1008] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_mul_sqrdmlsh_neon +mldsa_mul_sqrdmlsh_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x3, L_mldsa_aarch64_consts + add x3, x3, L_mldsa_aarch64_consts + ldr Q0, [x3] + ldr Q1, [x1] + ldr Q5, [x2] + ldr Q2, [x1, #16] + ldr Q6, [x2, #16] + ldr Q3, [x1, #32] + ldr Q7, [x2, #32] + ldr Q4, [x1, #48] + ldr Q8, [x2, #48] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0] + str Q10, [x0, #16] + str Q11, [x0, #32] + str Q12, [x0, #48] + ldr Q1, [x1, #64] + ldr Q5, [x2, #64] + ldr Q2, [x1, #80] + ldr Q6, [x2, #80] + ldr Q3, [x1, #96] + ldr Q7, [x2, #96] + ldr Q4, [x1, #112] + ldr Q8, [x2, #112] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #64] + str Q10, [x0, #80] + str Q11, [x0, #96] + str Q12, [x0, #112] + ldr Q1, [x1, #128] + ldr Q5, [x2, #128] + ldr Q2, [x1, #144] + ldr Q6, [x2, #144] + ldr Q3, [x1, #160] + ldr Q7, [x2, #160] + ldr Q4, [x1, #176] + ldr Q8, [x2, #176] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #128] + str Q10, [x0, #144] + str Q11, [x0, #160] + str Q12, [x0, #176] + ldr Q1, [x1, #192] + ldr Q5, [x2, #192] + ldr Q2, [x1, #208] + ldr Q6, [x2, #208] + ldr Q3, [x1, #224] + ldr Q7, [x2, #224] + ldr Q4, [x1, #240] + ldr Q8, [x2, #240] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #192] + str Q10, [x0, #208] + str Q11, [x0, #224] + str Q12, [x0, #240] + ldr Q1, [x1, #256] + ldr Q5, [x2, #256] + ldr Q2, [x1, #272] + ldr Q6, [x2, #272] + ldr Q3, [x1, #288] + ldr Q7, [x2, #288] + ldr Q4, [x1, #304] + ldr Q8, [x2, #304] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #256] + str Q10, [x0, #272] + str Q11, [x0, #288] + str Q12, [x0, #304] + ldr Q1, [x1, #320] + ldr Q5, [x2, #320] + ldr Q2, [x1, #336] + ldr Q6, [x2, #336] + ldr Q3, [x1, #352] + ldr Q7, [x2, #352] + ldr Q4, [x1, #368] + ldr Q8, [x2, #368] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #320] + str Q10, [x0, #336] + str Q11, [x0, #352] + str Q12, [x0, #368] + ldr Q1, [x1, #384] + ldr Q5, [x2, #384] + ldr Q2, [x1, #400] + ldr Q6, [x2, #400] + ldr Q3, [x1, #416] + ldr Q7, [x2, #416] + ldr Q4, [x1, #432] + ldr Q8, [x2, #432] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #384] + str Q10, [x0, #400] + str Q11, [x0, #416] + str Q12, [x0, #432] + ldr Q1, [x1, #448] + ldr Q5, [x2, #448] + ldr Q2, [x1, #464] + ldr Q6, [x2, #464] + ldr Q3, [x1, #480] + ldr Q7, [x2, #480] + ldr Q4, [x1, #496] + ldr Q8, [x2, #496] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #448] + str Q10, [x0, #464] + str Q11, [x0, #480] + str Q12, [x0, #496] + ldr Q1, [x1, #512] + ldr Q5, [x2, #512] + ldr Q2, [x1, #528] + ldr Q6, [x2, #528] + ldr Q3, [x1, #544] + ldr Q7, [x2, #544] + ldr Q4, [x1, #560] + ldr Q8, [x2, #560] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #512] + str Q10, [x0, #528] + str Q11, [x0, #544] + str Q12, [x0, #560] + ldr Q1, [x1, #576] + ldr Q5, [x2, #576] + ldr Q2, [x1, #592] + ldr Q6, [x2, #592] + ldr Q3, [x1, #608] + ldr Q7, [x2, #608] + ldr Q4, [x1, #624] + ldr Q8, [x2, #624] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #576] + str Q10, [x0, #592] + str Q11, [x0, #608] + str Q12, [x0, #624] + ldr Q1, [x1, #640] + ldr Q5, [x2, #640] + ldr Q2, [x1, #656] + ldr Q6, [x2, #656] + ldr Q3, [x1, #672] + ldr Q7, [x2, #672] + ldr Q4, [x1, #688] + ldr Q8, [x2, #688] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #640] + str Q10, [x0, #656] + str Q11, [x0, #672] + str Q12, [x0, #688] + ldr Q1, [x1, #704] + ldr Q5, [x2, #704] + ldr Q2, [x1, #720] + ldr Q6, [x2, #720] + ldr Q3, [x1, #736] + ldr Q7, [x2, #736] + ldr Q4, [x1, #752] + ldr Q8, [x2, #752] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #704] + str Q10, [x0, #720] + str Q11, [x0, #736] + str Q12, [x0, #752] + ldr Q1, [x1, #768] + ldr Q5, [x2, #768] + ldr Q2, [x1, #784] + ldr Q6, [x2, #784] + ldr Q3, [x1, #800] + ldr Q7, [x2, #800] + ldr Q4, [x1, #816] + ldr Q8, [x2, #816] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #768] + str Q10, [x0, #784] + str Q11, [x0, #800] + str Q12, [x0, #816] + ldr Q1, [x1, #832] + ldr Q5, [x2, #832] + ldr Q2, [x1, #848] + ldr Q6, [x2, #848] + ldr Q3, [x1, #864] + ldr Q7, [x2, #864] + ldr Q4, [x1, #880] + ldr Q8, [x2, #880] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #832] + str Q10, [x0, #848] + str Q11, [x0, #864] + str Q12, [x0, #880] + ldr Q1, [x1, #896] + ldr Q5, [x2, #896] + ldr Q2, [x1, #912] + ldr Q6, [x2, #912] + ldr Q3, [x1, #928] + ldr Q7, [x2, #928] + ldr Q4, [x1, #944] + ldr Q8, [x2, #944] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #896] + str Q10, [x0, #912] + str Q11, [x0, #928] + str Q12, [x0, #944] + ldr Q1, [x1, #960] + ldr Q5, [x2, #960] + ldr Q2, [x1, #976] + ldr Q6, [x2, #976] + ldr Q3, [x1, #992] + ldr Q7, [x2, #992] + ldr Q4, [x1, #1008] + ldr Q8, [x2, #1008] + mul V13.4S, V5.4S, V0.S[1] + mul V14.4S, V6.4S, V0.S[1] + mul V17.4S, V1.4S, V13.4S + mul V18.4S, V2.4S, V14.4S + sqrdmulh V9.4S, V1.4S, V5.4S + sqrdmulh V10.4S, V2.4S, V6.4S + sqrdmlsh V9.4S, V17.4S, V0.S[0] + sqrdmlsh V10.4S, V18.4S, V0.S[0] + sshr V9.4S, V9.4S, #1 + sshr V10.4S, V10.4S, #1 + mul V15.4S, V7.4S, V0.S[1] + mul V16.4S, V8.4S, V0.S[1] + mul V17.4S, V3.4S, V15.4S + mul V18.4S, V4.4S, V16.4S + sqrdmulh V11.4S, V3.4S, V7.4S + sqrdmulh V12.4S, V4.4S, V8.4S + sqrdmlsh V11.4S, V17.4S, V0.S[0] + sqrdmlsh V12.4S, V18.4S, V0.S[0] + sshr V11.4S, V11.4S, #1 + sshr V12.4S, V12.4S, #1 + str Q9, [x0, #960] + str Q10, [x0, #976] + str Q11, [x0, #992] + str Q12, [x0, #1008] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + ENDIF + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decompose_q88_consts + DCD 0x0000002c, 0x0000002c, 0x0000002c, 0x0000002c + DCD 0x003fefd4, 0x003fefd4, 0x003fefd4, 0x003fefd4 + DCD 0x0002e800, 0x0002e800, 0x0002e800, 0x0002e800 + DCD 0x00017400, 0x00017400, 0x00017400, 0x00017400 + DCD 0x0000002b, 0x0000002b, 0x0000002b, 0x0000002b + DCD 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_decompose_q88_neon +mldsa_decompose_q88_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x3, L_mldsa_decompose_q88_consts + add x3, x3, L_mldsa_decompose_q88_consts + ldr Q0, [x3] + ldr Q1, [x3, #16] + ldr Q2, [x3, #32] + ldr Q3, [x3, #48] + ldr Q4, [x3, #64] + mov x4, #16 +L_mldsa_decompose_q88_loop + ldr Q5, [x0] + ldr Q6, [x0, #16] + ldr Q7, [x0, #32] + ldr Q8, [x0, #48] + mov V13.16B, V1.16B + mla V13.4S, V5.4S, V0.4S + sshr V13.4S, V13.4S, #23 + mov V9.16B, V5.16B + mls V9.4S, V13.4S, V2.4S + sub V17.4S, V3.4S, V9.4S + usra V13.4S, V17.4S, #31 + mov V9.16B, V5.16B + mls V9.4S, V13.4S, V2.4S + sub V17.4S, V4.4S, V13.4S + ushr V17.4S, V17.4S, #31 + sub V9.4S, V9.4S, V17.4S + sub V17.4S, V13.4S, V0.4S + ushr V17.4S, V17.4S, #31 + neg V17.4S, V17.4S + and V13.16B, V13.16B, V17.16B + mov V14.16B, V1.16B + mla V14.4S, V6.4S, V0.4S + sshr V14.4S, V14.4S, #23 + mov V10.16B, V6.16B + mls V10.4S, V14.4S, V2.4S + sub V18.4S, V3.4S, V10.4S + usra V14.4S, V18.4S, #31 + mov V10.16B, V6.16B + mls V10.4S, V14.4S, V2.4S + sub V18.4S, V4.4S, V14.4S + ushr V18.4S, V18.4S, #31 + sub V10.4S, V10.4S, V18.4S + sub V18.4S, V14.4S, V0.4S + ushr V18.4S, V18.4S, #31 + neg V18.4S, V18.4S + and V14.16B, V14.16B, V18.16B + mov V15.16B, V1.16B + mla V15.4S, V7.4S, V0.4S + sshr V15.4S, V15.4S, #23 + mov V11.16B, V7.16B + mls V11.4S, V15.4S, V2.4S + sub V19.4S, V3.4S, V11.4S + usra V15.4S, V19.4S, #31 + mov V11.16B, V7.16B + mls V11.4S, V15.4S, V2.4S + sub V19.4S, V4.4S, V15.4S + ushr V19.4S, V19.4S, #31 + sub V11.4S, V11.4S, V19.4S + sub V19.4S, V15.4S, V0.4S + ushr V19.4S, V19.4S, #31 + neg V19.4S, V19.4S + and V15.16B, V15.16B, V19.16B + mov V16.16B, V1.16B + mla V16.4S, V8.4S, V0.4S + sshr V16.4S, V16.4S, #23 + mov V12.16B, V8.16B + mls V12.4S, V16.4S, V2.4S + sub V20.4S, V3.4S, V12.4S + usra V16.4S, V20.4S, #31 + mov V12.16B, V8.16B + mls V12.4S, V16.4S, V2.4S + sub V20.4S, V4.4S, V16.4S + ushr V20.4S, V20.4S, #31 + sub V12.4S, V12.4S, V20.4S + sub V20.4S, V16.4S, V0.4S + ushr V20.4S, V20.4S, #31 + neg V20.4S, V20.4S + and V16.16B, V16.16B, V20.16B + str Q9, [x1] + str Q13, [x2] + str Q10, [x1, #16] + str Q14, [x2, #16] + str Q11, [x1, #32] + str Q15, [x2, #32] + str Q12, [x1, #48] + str Q16, [x2, #48] + add x0, x0, #0x40 + add x1, x1, #0x40 + add x2, x2, #0x40 + subs x4, x4, #1 + bne L_mldsa_decompose_q88_loop + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decompose_q32_consts + DCD 0x0003feff, 0x0003feff, 0x0003feff, 0x0003feff + DCD 0x0003ff00, 0x0003ff00, 0x0003ff00, 0x0003ff00 + DCD 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f + DCD 0x0007fe00, 0x0007fe00, 0x0007fe00, 0x0007fe00 + DCD 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_decompose_q32_neon +mldsa_decompose_q32_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x3, L_mldsa_decompose_q32_consts + add x3, x3, L_mldsa_decompose_q32_consts + ldr Q0, [x3] + ldr Q1, [x3, #16] + ldr Q2, [x3, #32] + ldr Q3, [x3, #48] + mov x4, #16 +L_mldsa_decompose_q32_loop + ldr Q4, [x0] + ldr Q5, [x0, #16] + ldr Q6, [x0, #32] + ldr Q7, [x0, #48] + add V12.4S, V4.4S, V0.4S + sshr V12.4S, V12.4S, #19 + mov V8.16B, V4.16B + mls V8.4S, V12.4S, V3.4S + sub V16.4S, V1.4S, V8.4S + usra V12.4S, V16.4S, #31 + mov V8.16B, V4.16B + mls V8.4S, V12.4S, V3.4S + sshr V16.4S, V12.4S, #4 + sub V8.4S, V8.4S, V16.4S + and V12.16B, V12.16B, V2.16B + add V13.4S, V5.4S, V0.4S + sshr V13.4S, V13.4S, #19 + mov V9.16B, V5.16B + mls V9.4S, V13.4S, V3.4S + sub V17.4S, V1.4S, V9.4S + usra V13.4S, V17.4S, #31 + mov V9.16B, V5.16B + mls V9.4S, V13.4S, V3.4S + sshr V17.4S, V13.4S, #4 + sub V9.4S, V9.4S, V17.4S + and V13.16B, V13.16B, V2.16B + add V14.4S, V6.4S, V0.4S + sshr V14.4S, V14.4S, #19 + mov V10.16B, V6.16B + mls V10.4S, V14.4S, V3.4S + sub V18.4S, V1.4S, V10.4S + usra V14.4S, V18.4S, #31 + mov V10.16B, V6.16B + mls V10.4S, V14.4S, V3.4S + sshr V18.4S, V14.4S, #4 + sub V10.4S, V10.4S, V18.4S + and V14.16B, V14.16B, V2.16B + add V15.4S, V7.4S, V0.4S + sshr V15.4S, V15.4S, #19 + mov V11.16B, V7.16B + mls V11.4S, V15.4S, V3.4S + sub V19.4S, V1.4S, V11.4S + usra V15.4S, V19.4S, #31 + mov V11.16B, V7.16B + mls V11.4S, V15.4S, V3.4S + sshr V19.4S, V15.4S, #4 + sub V11.4S, V11.4S, V19.4S + and V15.16B, V15.16B, V2.16B + str Q8, [x1] + str Q12, [x2] + str Q9, [x1, #16] + str Q13, [x2, #16] + str Q10, [x1, #32] + str Q14, [x2, #32] + str Q11, [x1, #48] + str Q15, [x2, #48] + add x0, x0, #0x40 + add x1, x1, #0x40 + add x2, x2, #0x40 + subs x4, x4, #1 + bne L_mldsa_decompose_q32_loop + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_use_hint_q88_consts + DCD 0x0000002c, 0x0000002c, 0x0000002c, 0x0000002c + DCD 0x003fefd4, 0x003fefd4, 0x003fefd4, 0x003fefd4 + DCD 0x0002e800, 0x0002e800, 0x0002e800, 0x0002e800 + DCD 0x00017400, 0x00017400, 0x00017400, 0x00017400 + DCD 0x0000002b, 0x0000002b, 0x0000002b, 0x0000002b + DCD 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_use_hint_q88_neon +mldsa_use_hint_q88_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x1, L_mldsa_use_hint_q88_consts + add x1, x1, L_mldsa_use_hint_q88_consts + ldr Q0, [x1] + ldr Q1, [x1, #16] + ldr Q2, [x1, #32] + ldr Q3, [x1, #48] + ldr Q4, [x1, #64] + ldr Q5, [x1, #80] + mov x2, #16 +L_mldsa_use_hint_q88_loop + ldr Q6, [x0] + ldr Q7, [x0, #16] + ldr Q8, [x0, #32] + ldr Q9, [x0, #48] + sshr V18.4S, V6.4S, #31 + and V18.16B, V18.16B, V5.16B + add V6.4S, V6.4S, V18.4S + sshr V19.4S, V7.4S, #31 + and V19.16B, V19.16B, V5.16B + add V7.4S, V7.4S, V19.4S + sshr V20.4S, V8.4S, #31 + and V20.16B, V20.16B, V5.16B + add V8.4S, V8.4S, V20.4S + sshr V21.4S, V9.4S, #31 + and V21.16B, V21.16B, V5.16B + add V9.4S, V9.4S, V21.4S + mov V14.16B, V1.16B + mla V14.4S, V6.4S, V0.4S + sshr V14.4S, V14.4S, #23 + mov V10.16B, V6.16B + mls V10.4S, V14.4S, V2.4S + sub V18.4S, V3.4S, V10.4S + usra V14.4S, V18.4S, #31 + mov V10.16B, V6.16B + mls V10.4S, V14.4S, V2.4S + sub V18.4S, V4.4S, V14.4S + ushr V18.4S, V18.4S, #31 + sub V10.4S, V10.4S, V18.4S + sub V18.4S, V14.4S, V0.4S + ushr V18.4S, V18.4S, #31 + neg V18.4S, V18.4S + and V14.16B, V14.16B, V18.16B + mov V15.16B, V1.16B + mla V15.4S, V7.4S, V0.4S + sshr V15.4S, V15.4S, #23 + mov V11.16B, V7.16B + mls V11.4S, V15.4S, V2.4S + sub V19.4S, V3.4S, V11.4S + usra V15.4S, V19.4S, #31 + mov V11.16B, V7.16B + mls V11.4S, V15.4S, V2.4S + sub V19.4S, V4.4S, V15.4S + ushr V19.4S, V19.4S, #31 + sub V11.4S, V11.4S, V19.4S + sub V19.4S, V15.4S, V0.4S + ushr V19.4S, V19.4S, #31 + neg V19.4S, V19.4S + and V15.16B, V15.16B, V19.16B + mov V16.16B, V1.16B + mla V16.4S, V8.4S, V0.4S + sshr V16.4S, V16.4S, #23 + mov V12.16B, V8.16B + mls V12.4S, V16.4S, V2.4S + sub V20.4S, V3.4S, V12.4S + usra V16.4S, V20.4S, #31 + mov V12.16B, V8.16B + mls V12.4S, V16.4S, V2.4S + sub V20.4S, V4.4S, V16.4S + ushr V20.4S, V20.4S, #31 + sub V12.4S, V12.4S, V20.4S + sub V20.4S, V16.4S, V0.4S + ushr V20.4S, V20.4S, #31 + neg V20.4S, V20.4S + and V16.16B, V16.16B, V20.16B + mov V17.16B, V1.16B + mla V17.4S, V9.4S, V0.4S + sshr V17.4S, V17.4S, #23 + mov V13.16B, V9.16B + mls V13.4S, V17.4S, V2.4S + sub V21.4S, V3.4S, V13.4S + usra V17.4S, V21.4S, #31 + mov V13.16B, V9.16B + mls V13.4S, V17.4S, V2.4S + sub V21.4S, V4.4S, V17.4S + ushr V21.4S, V21.4S, #31 + sub V13.4S, V13.4S, V21.4S + sub V21.4S, V17.4S, V0.4S + ushr V21.4S, V21.4S, #31 + neg V21.4S, V21.4S + and V17.16B, V17.16B, V21.16B + str Q14, [x0] + str Q15, [x0, #16] + str Q16, [x0, #32] + str Q17, [x0, #48] + add x0, x0, #0x40 + subs x2, x2, #1 + bne L_mldsa_use_hint_q88_loop + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_use_hint_q32_consts + DCD 0x0003feff, 0x0003feff, 0x0003feff, 0x0003feff + DCD 0x0003ff00, 0x0003ff00, 0x0003ff00, 0x0003ff00 + DCD 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f + DCD 0x0007fe00, 0x0007fe00, 0x0007fe00, 0x0007fe00 + DCD 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_use_hint_q32_neon +mldsa_use_hint_q32_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x1, L_mldsa_use_hint_q32_consts + add x1, x1, L_mldsa_use_hint_q32_consts + ldr Q0, [x1] + ldr Q1, [x1, #16] + ldr Q2, [x1, #32] + ldr Q3, [x1, #48] + ldr Q4, [x1, #64] + mov x2, #16 +L_mldsa_use_hint_q32_loop + ldr Q5, [x0] + ldr Q6, [x0, #16] + ldr Q7, [x0, #32] + ldr Q8, [x0, #48] + sshr V17.4S, V5.4S, #31 + and V17.16B, V17.16B, V4.16B + add V5.4S, V5.4S, V17.4S + sshr V18.4S, V6.4S, #31 + and V18.16B, V18.16B, V4.16B + add V6.4S, V6.4S, V18.4S + sshr V19.4S, V7.4S, #31 + and V19.16B, V19.16B, V4.16B + add V7.4S, V7.4S, V19.4S + sshr V20.4S, V8.4S, #31 + and V20.16B, V20.16B, V4.16B + add V8.4S, V8.4S, V20.4S + add V13.4S, V5.4S, V0.4S + sshr V13.4S, V13.4S, #19 + mov V9.16B, V5.16B + mls V9.4S, V13.4S, V3.4S + sub V17.4S, V1.4S, V9.4S + usra V13.4S, V17.4S, #31 + mov V9.16B, V5.16B + mls V9.4S, V13.4S, V3.4S + sshr V17.4S, V13.4S, #4 + sub V9.4S, V9.4S, V17.4S + and V13.16B, V13.16B, V2.16B + add V14.4S, V6.4S, V0.4S + sshr V14.4S, V14.4S, #19 + mov V10.16B, V6.16B + mls V10.4S, V14.4S, V3.4S + sub V18.4S, V1.4S, V10.4S + usra V14.4S, V18.4S, #31 + mov V10.16B, V6.16B + mls V10.4S, V14.4S, V3.4S + sshr V18.4S, V14.4S, #4 + sub V10.4S, V10.4S, V18.4S + and V14.16B, V14.16B, V2.16B + add V15.4S, V7.4S, V0.4S + sshr V15.4S, V15.4S, #19 + mov V11.16B, V7.16B + mls V11.4S, V15.4S, V3.4S + sub V19.4S, V1.4S, V11.4S + usra V15.4S, V19.4S, #31 + mov V11.16B, V7.16B + mls V11.4S, V15.4S, V3.4S + sshr V19.4S, V15.4S, #4 + sub V11.4S, V11.4S, V19.4S + and V15.16B, V15.16B, V2.16B + add V16.4S, V8.4S, V0.4S + sshr V16.4S, V16.4S, #19 + mov V12.16B, V8.16B + mls V12.4S, V16.4S, V3.4S + sub V20.4S, V1.4S, V12.4S + usra V16.4S, V20.4S, #31 + mov V12.16B, V8.16B + mls V12.4S, V16.4S, V3.4S + sshr V20.4S, V16.4S, #4 + sub V12.4S, V12.4S, V20.4S + and V16.16B, V16.16B, V2.16B + str Q13, [x0] + str Q14, [x0, #16] + str Q15, [x0, #32] + str Q16, [x0, #48] + add x0, x0, #0x40 + subs x2, x2, #1 + bne L_mldsa_use_hint_q32_loop + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_make_hint_bits + DCD 0x00000001, 0x00000002, 0x00000004, 0x00000008 + DCD 0x00000010, 0x00000020, 0x00000040, 0x00000080 + DCD 0x00000100, 0x00000200, 0x00000400, 0x00000800 + DCD 0x00001000, 0x00002000, 0x00004000, 0x00008000 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_make_hint_neon +mldsa_make_hint_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x6, L_mldsa_make_hint_bits + add x6, x6, L_mldsa_make_hint_bits + dup V0.4S, w5 + neg V1.4S, V0.4S + ldr Q2, [x6] + ldr Q3, [x6, #16] + ldr Q4, [x6, #32] + ldr Q5, [x6, #48] + eor x7, x7, x7 +L_mldsa_make_hint_loop + ldr Q6, [x0] + ldr Q10, [x1] + ldr Q7, [x0, #16] + ldr Q11, [x1, #16] + ldr Q8, [x0, #32] + ldr Q12, [x1, #32] + ldr Q9, [x0, #48] + ldr Q13, [x1, #48] + cmgt V14.4S, V6.4S, V0.4S + cmgt V18.4S, V1.4S, V6.4S + cmeq V22.4S, V6.4S, V1.4S + cmeq V10.4S, V10.4S, #0 + bic V22.16B, V22.16B, V10.16B + orr V14.16B, V14.16B, V18.16B + orr V14.16B, V14.16B, V22.16B + and V14.16B, V14.16B, V2.16B + cmgt V15.4S, V7.4S, V0.4S + cmgt V19.4S, V1.4S, V7.4S + cmeq V23.4S, V7.4S, V1.4S + cmeq V11.4S, V11.4S, #0 + bic V23.16B, V23.16B, V11.16B + orr V15.16B, V15.16B, V19.16B + orr V15.16B, V15.16B, V23.16B + and V15.16B, V15.16B, V3.16B + cmgt V16.4S, V8.4S, V0.4S + cmgt V20.4S, V1.4S, V8.4S + cmeq V24.4S, V8.4S, V1.4S + cmeq V12.4S, V12.4S, #0 + bic V24.16B, V24.16B, V12.16B + orr V16.16B, V16.16B, V20.16B + orr V16.16B, V16.16B, V24.16B + and V16.16B, V16.16B, V4.16B + cmgt V17.4S, V9.4S, V0.4S + cmgt V21.4S, V1.4S, V9.4S + cmeq V25.4S, V9.4S, V1.4S + cmeq V13.4S, V13.4S, #0 + bic V25.16B, V25.16B, V13.16B + orr V17.16B, V17.16B, V21.16B + orr V17.16B, V17.16B, V25.16B + and V17.16B, V17.16B, V5.16B + orr V14.16B, V14.16B, V15.16B + orr V16.16B, V16.16B, V17.16B + orr V14.16B, V14.16B, V16.16B + addv S14, V14.4S + mov w8, V14.S[0] + cbz w8, L_mldsa_make_hint_next +L_mldsa_make_hint_bit + rbit w9, w8 + clz w9, w9 + add w9, w9, w7 + strb w9, [x2] + add x2, x2, #1 + add w3, w3, #1 + subs wzr, w3, w4 + bgt L_mldsa_make_hint_fail + sub w10, w8, #1 + and w8, w8, w10 + cbnz w8, L_mldsa_make_hint_bit +L_mldsa_make_hint_next + add x0, x0, #0x40 + add x1, x1, #0x40 + add x7, x7, #16 + subs wzr, w7, #0x100 + blt L_mldsa_make_hint_loop + mov x0, x3 + b L_mldsa_make_hint_done +L_mldsa_make_hint_fail + movn x0, #0 +L_mldsa_make_hint_done + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_mul_vec_4_neon +mldsa_mul_vec_4_neon PROC + adrp x3, L_mldsa_aarch64_consts + add x3, x3, L_mldsa_aarch64_consts + ldr Q0, [x3] + ldr Q1, [x1] + ldr Q2, [x2] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1024] + ldr Q2, [x2, #1024] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2048] + ldr Q2, [x2, #2048] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3072] + ldr Q2, [x2, #3072] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0] + ldr Q1, [x1, #16] + ldr Q2, [x2, #16] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1040] + ldr Q2, [x2, #1040] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2064] + ldr Q2, [x2, #2064] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3088] + ldr Q2, [x2, #3088] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #16] + ldr Q1, [x1, #32] + ldr Q2, [x2, #32] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1056] + ldr Q2, [x2, #1056] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2080] + ldr Q2, [x2, #2080] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3104] + ldr Q2, [x2, #3104] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #32] + ldr Q1, [x1, #48] + ldr Q2, [x2, #48] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1072] + ldr Q2, [x2, #1072] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2096] + ldr Q2, [x2, #2096] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3120] + ldr Q2, [x2, #3120] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #48] + ldr Q1, [x1, #64] + ldr Q2, [x2, #64] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1088] + ldr Q2, [x2, #1088] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2112] + ldr Q2, [x2, #2112] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3136] + ldr Q2, [x2, #3136] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #64] + ldr Q1, [x1, #80] + ldr Q2, [x2, #80] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1104] + ldr Q2, [x2, #1104] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2128] + ldr Q2, [x2, #2128] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3152] + ldr Q2, [x2, #3152] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #80] + ldr Q1, [x1, #96] + ldr Q2, [x2, #96] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1120] + ldr Q2, [x2, #1120] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2144] + ldr Q2, [x2, #2144] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3168] + ldr Q2, [x2, #3168] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #96] + ldr Q1, [x1, #112] + ldr Q2, [x2, #112] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1136] + ldr Q2, [x2, #1136] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2160] + ldr Q2, [x2, #2160] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3184] + ldr Q2, [x2, #3184] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #112] + ldr Q1, [x1, #128] + ldr Q2, [x2, #128] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1152] + ldr Q2, [x2, #1152] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2176] + ldr Q2, [x2, #2176] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3200] + ldr Q2, [x2, #3200] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #128] + ldr Q1, [x1, #144] + ldr Q2, [x2, #144] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1168] + ldr Q2, [x2, #1168] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2192] + ldr Q2, [x2, #2192] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3216] + ldr Q2, [x2, #3216] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #144] + ldr Q1, [x1, #160] + ldr Q2, [x2, #160] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1184] + ldr Q2, [x2, #1184] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2208] + ldr Q2, [x2, #2208] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3232] + ldr Q2, [x2, #3232] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #160] + ldr Q1, [x1, #176] + ldr Q2, [x2, #176] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1200] + ldr Q2, [x2, #1200] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2224] + ldr Q2, [x2, #2224] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3248] + ldr Q2, [x2, #3248] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #176] + ldr Q1, [x1, #192] + ldr Q2, [x2, #192] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1216] + ldr Q2, [x2, #1216] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2240] + ldr Q2, [x2, #2240] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3264] + ldr Q2, [x2, #3264] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #192] + ldr Q1, [x1, #208] + ldr Q2, [x2, #208] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1232] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2256] + ldr Q2, [x2, #2256] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3280] + ldr Q2, [x2, #3280] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #208] + ldr Q1, [x1, #224] + ldr Q2, [x2, #224] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1248] + ldr Q2, [x2, #1248] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2272] + ldr Q2, [x2, #2272] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3296] + ldr Q2, [x2, #3296] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #224] + ldr Q1, [x1, #240] + ldr Q2, [x2, #240] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1264] + ldr Q2, [x2, #1264] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2288] + ldr Q2, [x2, #2288] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3312] + ldr Q2, [x2, #3312] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #240] + ldr Q1, [x1, #256] + ldr Q2, [x2, #256] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1280] + ldr Q2, [x2, #1280] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2304] + ldr Q2, [x2, #2304] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3328] + ldr Q2, [x2, #3328] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #256] + ldr Q1, [x1, #272] + ldr Q2, [x2, #272] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1296] + ldr Q2, [x2, #1296] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2320] + ldr Q2, [x2, #2320] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3344] + ldr Q2, [x2, #3344] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #272] + ldr Q1, [x1, #288] + ldr Q2, [x2, #288] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1312] + ldr Q2, [x2, #1312] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2336] + ldr Q2, [x2, #2336] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3360] + ldr Q2, [x2, #3360] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #288] + ldr Q1, [x1, #304] + ldr Q2, [x2, #304] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1328] + ldr Q2, [x2, #1328] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2352] + ldr Q2, [x2, #2352] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3376] + ldr Q2, [x2, #3376] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #304] + ldr Q1, [x1, #320] + ldr Q2, [x2, #320] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1344] + ldr Q2, [x2, #1344] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2368] + ldr Q2, [x2, #2368] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3392] + ldr Q2, [x2, #3392] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #320] + ldr Q1, [x1, #336] + ldr Q2, [x2, #336] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1360] + ldr Q2, [x2, #1360] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2384] + ldr Q2, [x2, #2384] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3408] + ldr Q2, [x2, #3408] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #336] + ldr Q1, [x1, #352] + ldr Q2, [x2, #352] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1376] + ldr Q2, [x2, #1376] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2400] + ldr Q2, [x2, #2400] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3424] + ldr Q2, [x2, #3424] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #352] + ldr Q1, [x1, #368] + ldr Q2, [x2, #368] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1392] + ldr Q2, [x2, #1392] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2416] + ldr Q2, [x2, #2416] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3440] + ldr Q2, [x2, #3440] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #368] + ldr Q1, [x1, #384] + ldr Q2, [x2, #384] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1408] + ldr Q2, [x2, #1408] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2432] + ldr Q2, [x2, #2432] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3456] + ldr Q2, [x2, #3456] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #384] + ldr Q1, [x1, #400] + ldr Q2, [x2, #400] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1424] + ldr Q2, [x2, #1424] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2448] + ldr Q2, [x2, #2448] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3472] + ldr Q2, [x2, #3472] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #400] + ldr Q1, [x1, #416] + ldr Q2, [x2, #416] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1440] + ldr Q2, [x2, #1440] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2464] + ldr Q2, [x2, #2464] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3488] + ldr Q2, [x2, #3488] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #416] + ldr Q1, [x1, #432] + ldr Q2, [x2, #432] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1456] + ldr Q2, [x2, #1456] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2480] + ldr Q2, [x2, #2480] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3504] + ldr Q2, [x2, #3504] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #432] + ldr Q1, [x1, #448] + ldr Q2, [x2, #448] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1472] + ldr Q2, [x2, #1472] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2496] + ldr Q2, [x2, #2496] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3520] + ldr Q2, [x2, #3520] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #448] + ldr Q1, [x1, #464] + ldr Q2, [x2, #464] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1488] + ldr Q2, [x2, #1488] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2512] + ldr Q2, [x2, #2512] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3536] + ldr Q2, [x2, #3536] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #464] + ldr Q1, [x1, #480] + ldr Q2, [x2, #480] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1504] + ldr Q2, [x2, #1504] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2528] + ldr Q2, [x2, #2528] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3552] + ldr Q2, [x2, #3552] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #480] + ldr Q1, [x1, #496] + ldr Q2, [x2, #496] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1520] + ldr Q2, [x2, #1520] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2544] + ldr Q2, [x2, #2544] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3568] + ldr Q2, [x2, #3568] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #496] + ldr Q1, [x1, #512] + ldr Q2, [x2, #512] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1536] + ldr Q2, [x2, #1536] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2560] + ldr Q2, [x2, #2560] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3584] + ldr Q2, [x2, #3584] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #512] + ldr Q1, [x1, #528] + ldr Q2, [x2, #528] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1552] + ldr Q2, [x2, #1552] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2576] + ldr Q2, [x2, #2576] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3600] + ldr Q2, [x2, #3600] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #528] + ldr Q1, [x1, #544] + ldr Q2, [x2, #544] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1568] + ldr Q2, [x2, #1568] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2592] + ldr Q2, [x2, #2592] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3616] + ldr Q2, [x2, #3616] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #544] + ldr Q1, [x1, #560] + ldr Q2, [x2, #560] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1584] + ldr Q2, [x2, #1584] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2608] + ldr Q2, [x2, #2608] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3632] + ldr Q2, [x2, #3632] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #560] + ldr Q1, [x1, #576] + ldr Q2, [x2, #576] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1600] + ldr Q2, [x2, #1600] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2624] + ldr Q2, [x2, #2624] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3648] + ldr Q2, [x2, #3648] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #576] + ldr Q1, [x1, #592] + ldr Q2, [x2, #592] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1616] + ldr Q2, [x2, #1616] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2640] + ldr Q2, [x2, #2640] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3664] + ldr Q2, [x2, #3664] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #592] + ldr Q1, [x1, #608] + ldr Q2, [x2, #608] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1632] + ldr Q2, [x2, #1632] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2656] + ldr Q2, [x2, #2656] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3680] + ldr Q2, [x2, #3680] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #608] + ldr Q1, [x1, #624] + ldr Q2, [x2, #624] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1648] + ldr Q2, [x2, #1648] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2672] + ldr Q2, [x2, #2672] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3696] + ldr Q2, [x2, #3696] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #624] + ldr Q1, [x1, #640] + ldr Q2, [x2, #640] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1664] + ldr Q2, [x2, #1664] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2688] + ldr Q2, [x2, #2688] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3712] + ldr Q2, [x2, #3712] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #640] + ldr Q1, [x1, #656] + ldr Q2, [x2, #656] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1680] + ldr Q2, [x2, #1680] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2704] + ldr Q2, [x2, #2704] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3728] + ldr Q2, [x2, #3728] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #656] + ldr Q1, [x1, #672] + ldr Q2, [x2, #672] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1696] + ldr Q2, [x2, #1696] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2720] + ldr Q2, [x2, #2720] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3744] + ldr Q2, [x2, #3744] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #672] + ldr Q1, [x1, #688] + ldr Q2, [x2, #688] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1712] + ldr Q2, [x2, #1712] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2736] + ldr Q2, [x2, #2736] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3760] + ldr Q2, [x2, #3760] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #688] + ldr Q1, [x1, #704] + ldr Q2, [x2, #704] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1728] + ldr Q2, [x2, #1728] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2752] + ldr Q2, [x2, #2752] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3776] + ldr Q2, [x2, #3776] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #704] + ldr Q1, [x1, #720] + ldr Q2, [x2, #720] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1744] + ldr Q2, [x2, #1744] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2768] + ldr Q2, [x2, #2768] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3792] + ldr Q2, [x2, #3792] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #720] + ldr Q1, [x1, #736] + ldr Q2, [x2, #736] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1760] + ldr Q2, [x2, #1760] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2784] + ldr Q2, [x2, #2784] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3808] + ldr Q2, [x2, #3808] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #736] + ldr Q1, [x1, #752] + ldr Q2, [x2, #752] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1776] + ldr Q2, [x2, #1776] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2800] + ldr Q2, [x2, #2800] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3824] + ldr Q2, [x2, #3824] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #752] + ldr Q1, [x1, #768] + ldr Q2, [x2, #768] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1792] + ldr Q2, [x2, #1792] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2816] + ldr Q2, [x2, #2816] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3840] + ldr Q2, [x2, #3840] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #768] + ldr Q1, [x1, #784] + ldr Q2, [x2, #784] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1808] + ldr Q2, [x2, #1808] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2832] + ldr Q2, [x2, #2832] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3856] + ldr Q2, [x2, #3856] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #784] + ldr Q1, [x1, #800] + ldr Q2, [x2, #800] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1824] + ldr Q2, [x2, #1824] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2848] + ldr Q2, [x2, #2848] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3872] + ldr Q2, [x2, #3872] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #800] + ldr Q1, [x1, #816] + ldr Q2, [x2, #816] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1840] + ldr Q2, [x2, #1840] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2864] + ldr Q2, [x2, #2864] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3888] + ldr Q2, [x2, #3888] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #816] + ldr Q1, [x1, #832] + ldr Q2, [x2, #832] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1856] + ldr Q2, [x2, #1856] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2880] + ldr Q2, [x2, #2880] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3904] + ldr Q2, [x2, #3904] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #832] + ldr Q1, [x1, #848] + ldr Q2, [x2, #848] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1872] + ldr Q2, [x2, #1872] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2896] + ldr Q2, [x2, #2896] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3920] + ldr Q2, [x2, #3920] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #848] + ldr Q1, [x1, #864] + ldr Q2, [x2, #864] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1888] + ldr Q2, [x2, #1888] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2912] + ldr Q2, [x2, #2912] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3936] + ldr Q2, [x2, #3936] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #864] + ldr Q1, [x1, #880] + ldr Q2, [x2, #880] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1904] + ldr Q2, [x2, #1904] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2928] + ldr Q2, [x2, #2928] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3952] + ldr Q2, [x2, #3952] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #880] + ldr Q1, [x1, #896] + ldr Q2, [x2, #896] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1920] + ldr Q2, [x2, #1920] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2944] + ldr Q2, [x2, #2944] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3968] + ldr Q2, [x2, #3968] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #896] + ldr Q1, [x1, #912] + ldr Q2, [x2, #912] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1936] + ldr Q2, [x2, #1936] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2960] + ldr Q2, [x2, #2960] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3984] + ldr Q2, [x2, #3984] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #912] + ldr Q1, [x1, #928] + ldr Q2, [x2, #928] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1952] + ldr Q2, [x2, #1952] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2976] + ldr Q2, [x2, #2976] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4000] + ldr Q2, [x2, #4000] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #928] + ldr Q1, [x1, #944] + ldr Q2, [x2, #944] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1968] + ldr Q2, [x2, #1968] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2992] + ldr Q2, [x2, #2992] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4016] + ldr Q2, [x2, #4016] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #944] + ldr Q1, [x1, #960] + ldr Q2, [x2, #960] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1984] + ldr Q2, [x2, #1984] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3008] + ldr Q2, [x2, #3008] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4032] + ldr Q2, [x2, #4032] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #960] + ldr Q1, [x1, #976] + ldr Q2, [x2, #976] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2000] + ldr Q2, [x2, #2000] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3024] + ldr Q2, [x2, #3024] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4048] + ldr Q2, [x2, #4048] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #976] + ldr Q1, [x1, #992] + ldr Q2, [x2, #992] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2016] + ldr Q2, [x2, #2016] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3040] + ldr Q2, [x2, #3040] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4064] + ldr Q2, [x2, #4064] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #992] + ldr Q1, [x1, #1008] + ldr Q2, [x2, #1008] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2032] + ldr Q2, [x2, #2032] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3056] + ldr Q2, [x2, #3056] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4080] + ldr Q2, [x2, #4080] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #1008] + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_mul_vec_5_neon +mldsa_mul_vec_5_neon PROC + adrp x3, L_mldsa_aarch64_consts + add x3, x3, L_mldsa_aarch64_consts + ldr Q0, [x3] + ldr Q1, [x1] + ldr Q2, [x2] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1024] + ldr Q2, [x2, #1024] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2048] + ldr Q2, [x2, #2048] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3072] + ldr Q2, [x2, #3072] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4096] + ldr Q2, [x2, #4096] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0] + ldr Q1, [x1, #16] + ldr Q2, [x2, #16] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1040] + ldr Q2, [x2, #1040] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2064] + ldr Q2, [x2, #2064] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3088] + ldr Q2, [x2, #3088] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4112] + ldr Q2, [x2, #4112] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #16] + ldr Q1, [x1, #32] + ldr Q2, [x2, #32] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1056] + ldr Q2, [x2, #1056] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2080] + ldr Q2, [x2, #2080] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3104] + ldr Q2, [x2, #3104] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4128] + ldr Q2, [x2, #4128] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #32] + ldr Q1, [x1, #48] + ldr Q2, [x2, #48] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1072] + ldr Q2, [x2, #1072] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2096] + ldr Q2, [x2, #2096] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3120] + ldr Q2, [x2, #3120] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4144] + ldr Q2, [x2, #4144] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #48] + ldr Q1, [x1, #64] + ldr Q2, [x2, #64] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1088] + ldr Q2, [x2, #1088] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2112] + ldr Q2, [x2, #2112] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3136] + ldr Q2, [x2, #3136] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4160] + ldr Q2, [x2, #4160] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #64] + ldr Q1, [x1, #80] + ldr Q2, [x2, #80] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1104] + ldr Q2, [x2, #1104] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2128] + ldr Q2, [x2, #2128] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3152] + ldr Q2, [x2, #3152] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4176] + ldr Q2, [x2, #4176] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #80] + ldr Q1, [x1, #96] + ldr Q2, [x2, #96] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1120] + ldr Q2, [x2, #1120] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2144] + ldr Q2, [x2, #2144] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3168] + ldr Q2, [x2, #3168] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4192] + ldr Q2, [x2, #4192] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #96] + ldr Q1, [x1, #112] + ldr Q2, [x2, #112] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1136] + ldr Q2, [x2, #1136] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2160] + ldr Q2, [x2, #2160] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3184] + ldr Q2, [x2, #3184] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4208] + ldr Q2, [x2, #4208] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #112] + ldr Q1, [x1, #128] + ldr Q2, [x2, #128] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1152] + ldr Q2, [x2, #1152] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2176] + ldr Q2, [x2, #2176] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3200] + ldr Q2, [x2, #3200] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4224] + ldr Q2, [x2, #4224] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #128] + ldr Q1, [x1, #144] + ldr Q2, [x2, #144] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1168] + ldr Q2, [x2, #1168] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2192] + ldr Q2, [x2, #2192] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3216] + ldr Q2, [x2, #3216] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4240] + ldr Q2, [x2, #4240] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #144] + ldr Q1, [x1, #160] + ldr Q2, [x2, #160] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1184] + ldr Q2, [x2, #1184] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2208] + ldr Q2, [x2, #2208] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3232] + ldr Q2, [x2, #3232] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4256] + ldr Q2, [x2, #4256] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #160] + ldr Q1, [x1, #176] + ldr Q2, [x2, #176] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1200] + ldr Q2, [x2, #1200] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2224] + ldr Q2, [x2, #2224] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3248] + ldr Q2, [x2, #3248] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4272] + ldr Q2, [x2, #4272] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #176] + ldr Q1, [x1, #192] + ldr Q2, [x2, #192] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1216] + ldr Q2, [x2, #1216] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2240] + ldr Q2, [x2, #2240] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3264] + ldr Q2, [x2, #3264] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4288] + ldr Q2, [x2, #4288] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #192] + ldr Q1, [x1, #208] + ldr Q2, [x2, #208] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1232] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2256] + ldr Q2, [x2, #2256] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3280] + ldr Q2, [x2, #3280] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4304] + ldr Q2, [x2, #4304] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #208] + ldr Q1, [x1, #224] + ldr Q2, [x2, #224] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1248] + ldr Q2, [x2, #1248] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2272] + ldr Q2, [x2, #2272] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3296] + ldr Q2, [x2, #3296] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4320] + ldr Q2, [x2, #4320] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #224] + ldr Q1, [x1, #240] + ldr Q2, [x2, #240] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1264] + ldr Q2, [x2, #1264] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2288] + ldr Q2, [x2, #2288] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3312] + ldr Q2, [x2, #3312] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4336] + ldr Q2, [x2, #4336] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #240] + ldr Q1, [x1, #256] + ldr Q2, [x2, #256] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1280] + ldr Q2, [x2, #1280] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2304] + ldr Q2, [x2, #2304] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3328] + ldr Q2, [x2, #3328] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4352] + ldr Q2, [x2, #4352] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #256] + ldr Q1, [x1, #272] + ldr Q2, [x2, #272] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1296] + ldr Q2, [x2, #1296] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2320] + ldr Q2, [x2, #2320] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3344] + ldr Q2, [x2, #3344] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4368] + ldr Q2, [x2, #4368] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #272] + ldr Q1, [x1, #288] + ldr Q2, [x2, #288] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1312] + ldr Q2, [x2, #1312] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2336] + ldr Q2, [x2, #2336] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3360] + ldr Q2, [x2, #3360] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4384] + ldr Q2, [x2, #4384] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #288] + ldr Q1, [x1, #304] + ldr Q2, [x2, #304] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1328] + ldr Q2, [x2, #1328] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2352] + ldr Q2, [x2, #2352] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3376] + ldr Q2, [x2, #3376] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4400] + ldr Q2, [x2, #4400] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #304] + ldr Q1, [x1, #320] + ldr Q2, [x2, #320] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1344] + ldr Q2, [x2, #1344] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2368] + ldr Q2, [x2, #2368] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3392] + ldr Q2, [x2, #3392] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4416] + ldr Q2, [x2, #4416] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #320] + ldr Q1, [x1, #336] + ldr Q2, [x2, #336] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1360] + ldr Q2, [x2, #1360] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2384] + ldr Q2, [x2, #2384] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3408] + ldr Q2, [x2, #3408] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4432] + ldr Q2, [x2, #4432] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #336] + ldr Q1, [x1, #352] + ldr Q2, [x2, #352] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1376] + ldr Q2, [x2, #1376] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2400] + ldr Q2, [x2, #2400] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3424] + ldr Q2, [x2, #3424] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4448] + ldr Q2, [x2, #4448] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #352] + ldr Q1, [x1, #368] + ldr Q2, [x2, #368] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1392] + ldr Q2, [x2, #1392] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2416] + ldr Q2, [x2, #2416] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3440] + ldr Q2, [x2, #3440] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4464] + ldr Q2, [x2, #4464] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #368] + ldr Q1, [x1, #384] + ldr Q2, [x2, #384] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1408] + ldr Q2, [x2, #1408] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2432] + ldr Q2, [x2, #2432] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3456] + ldr Q2, [x2, #3456] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4480] + ldr Q2, [x2, #4480] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #384] + ldr Q1, [x1, #400] + ldr Q2, [x2, #400] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1424] + ldr Q2, [x2, #1424] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2448] + ldr Q2, [x2, #2448] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3472] + ldr Q2, [x2, #3472] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4496] + ldr Q2, [x2, #4496] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #400] + ldr Q1, [x1, #416] + ldr Q2, [x2, #416] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1440] + ldr Q2, [x2, #1440] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2464] + ldr Q2, [x2, #2464] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3488] + ldr Q2, [x2, #3488] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4512] + ldr Q2, [x2, #4512] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #416] + ldr Q1, [x1, #432] + ldr Q2, [x2, #432] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1456] + ldr Q2, [x2, #1456] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2480] + ldr Q2, [x2, #2480] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3504] + ldr Q2, [x2, #3504] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4528] + ldr Q2, [x2, #4528] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #432] + ldr Q1, [x1, #448] + ldr Q2, [x2, #448] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1472] + ldr Q2, [x2, #1472] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2496] + ldr Q2, [x2, #2496] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3520] + ldr Q2, [x2, #3520] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4544] + ldr Q2, [x2, #4544] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #448] + ldr Q1, [x1, #464] + ldr Q2, [x2, #464] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1488] + ldr Q2, [x2, #1488] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2512] + ldr Q2, [x2, #2512] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3536] + ldr Q2, [x2, #3536] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4560] + ldr Q2, [x2, #4560] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #464] + ldr Q1, [x1, #480] + ldr Q2, [x2, #480] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1504] + ldr Q2, [x2, #1504] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2528] + ldr Q2, [x2, #2528] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3552] + ldr Q2, [x2, #3552] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4576] + ldr Q2, [x2, #4576] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #480] + ldr Q1, [x1, #496] + ldr Q2, [x2, #496] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1520] + ldr Q2, [x2, #1520] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2544] + ldr Q2, [x2, #2544] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3568] + ldr Q2, [x2, #3568] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4592] + ldr Q2, [x2, #4592] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #496] + ldr Q1, [x1, #512] + ldr Q2, [x2, #512] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1536] + ldr Q2, [x2, #1536] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2560] + ldr Q2, [x2, #2560] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3584] + ldr Q2, [x2, #3584] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4608] + ldr Q2, [x2, #4608] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #512] + ldr Q1, [x1, #528] + ldr Q2, [x2, #528] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1552] + ldr Q2, [x2, #1552] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2576] + ldr Q2, [x2, #2576] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3600] + ldr Q2, [x2, #3600] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4624] + ldr Q2, [x2, #4624] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #528] + ldr Q1, [x1, #544] + ldr Q2, [x2, #544] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1568] + ldr Q2, [x2, #1568] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2592] + ldr Q2, [x2, #2592] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3616] + ldr Q2, [x2, #3616] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4640] + ldr Q2, [x2, #4640] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #544] + ldr Q1, [x1, #560] + ldr Q2, [x2, #560] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1584] + ldr Q2, [x2, #1584] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2608] + ldr Q2, [x2, #2608] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3632] + ldr Q2, [x2, #3632] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4656] + ldr Q2, [x2, #4656] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #560] + ldr Q1, [x1, #576] + ldr Q2, [x2, #576] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1600] + ldr Q2, [x2, #1600] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2624] + ldr Q2, [x2, #2624] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3648] + ldr Q2, [x2, #3648] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4672] + ldr Q2, [x2, #4672] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #576] + ldr Q1, [x1, #592] + ldr Q2, [x2, #592] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1616] + ldr Q2, [x2, #1616] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2640] + ldr Q2, [x2, #2640] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3664] + ldr Q2, [x2, #3664] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4688] + ldr Q2, [x2, #4688] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #592] + ldr Q1, [x1, #608] + ldr Q2, [x2, #608] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1632] + ldr Q2, [x2, #1632] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2656] + ldr Q2, [x2, #2656] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3680] + ldr Q2, [x2, #3680] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4704] + ldr Q2, [x2, #4704] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #608] + ldr Q1, [x1, #624] + ldr Q2, [x2, #624] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1648] + ldr Q2, [x2, #1648] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2672] + ldr Q2, [x2, #2672] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3696] + ldr Q2, [x2, #3696] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4720] + ldr Q2, [x2, #4720] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #624] + ldr Q1, [x1, #640] + ldr Q2, [x2, #640] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1664] + ldr Q2, [x2, #1664] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2688] + ldr Q2, [x2, #2688] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3712] + ldr Q2, [x2, #3712] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4736] + ldr Q2, [x2, #4736] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #640] + ldr Q1, [x1, #656] + ldr Q2, [x2, #656] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1680] + ldr Q2, [x2, #1680] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2704] + ldr Q2, [x2, #2704] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3728] + ldr Q2, [x2, #3728] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4752] + ldr Q2, [x2, #4752] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #656] + ldr Q1, [x1, #672] + ldr Q2, [x2, #672] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1696] + ldr Q2, [x2, #1696] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2720] + ldr Q2, [x2, #2720] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3744] + ldr Q2, [x2, #3744] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4768] + ldr Q2, [x2, #4768] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #672] + ldr Q1, [x1, #688] + ldr Q2, [x2, #688] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1712] + ldr Q2, [x2, #1712] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2736] + ldr Q2, [x2, #2736] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3760] + ldr Q2, [x2, #3760] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4784] + ldr Q2, [x2, #4784] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #688] + ldr Q1, [x1, #704] + ldr Q2, [x2, #704] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1728] + ldr Q2, [x2, #1728] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2752] + ldr Q2, [x2, #2752] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3776] + ldr Q2, [x2, #3776] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4800] + ldr Q2, [x2, #4800] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #704] + ldr Q1, [x1, #720] + ldr Q2, [x2, #720] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1744] + ldr Q2, [x2, #1744] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2768] + ldr Q2, [x2, #2768] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3792] + ldr Q2, [x2, #3792] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4816] + ldr Q2, [x2, #4816] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #720] + ldr Q1, [x1, #736] + ldr Q2, [x2, #736] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1760] + ldr Q2, [x2, #1760] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2784] + ldr Q2, [x2, #2784] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3808] + ldr Q2, [x2, #3808] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4832] + ldr Q2, [x2, #4832] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #736] + ldr Q1, [x1, #752] + ldr Q2, [x2, #752] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1776] + ldr Q2, [x2, #1776] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2800] + ldr Q2, [x2, #2800] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3824] + ldr Q2, [x2, #3824] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4848] + ldr Q2, [x2, #4848] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #752] + ldr Q1, [x1, #768] + ldr Q2, [x2, #768] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1792] + ldr Q2, [x2, #1792] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2816] + ldr Q2, [x2, #2816] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3840] + ldr Q2, [x2, #3840] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4864] + ldr Q2, [x2, #4864] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #768] + ldr Q1, [x1, #784] + ldr Q2, [x2, #784] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1808] + ldr Q2, [x2, #1808] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2832] + ldr Q2, [x2, #2832] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3856] + ldr Q2, [x2, #3856] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4880] + ldr Q2, [x2, #4880] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #784] + ldr Q1, [x1, #800] + ldr Q2, [x2, #800] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1824] + ldr Q2, [x2, #1824] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2848] + ldr Q2, [x2, #2848] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3872] + ldr Q2, [x2, #3872] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4896] + ldr Q2, [x2, #4896] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #800] + ldr Q1, [x1, #816] + ldr Q2, [x2, #816] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1840] + ldr Q2, [x2, #1840] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2864] + ldr Q2, [x2, #2864] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3888] + ldr Q2, [x2, #3888] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4912] + ldr Q2, [x2, #4912] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #816] + ldr Q1, [x1, #832] + ldr Q2, [x2, #832] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1856] + ldr Q2, [x2, #1856] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2880] + ldr Q2, [x2, #2880] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3904] + ldr Q2, [x2, #3904] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4928] + ldr Q2, [x2, #4928] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #832] + ldr Q1, [x1, #848] + ldr Q2, [x2, #848] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1872] + ldr Q2, [x2, #1872] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2896] + ldr Q2, [x2, #2896] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3920] + ldr Q2, [x2, #3920] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4944] + ldr Q2, [x2, #4944] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #848] + ldr Q1, [x1, #864] + ldr Q2, [x2, #864] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1888] + ldr Q2, [x2, #1888] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2912] + ldr Q2, [x2, #2912] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3936] + ldr Q2, [x2, #3936] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4960] + ldr Q2, [x2, #4960] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #864] + ldr Q1, [x1, #880] + ldr Q2, [x2, #880] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1904] + ldr Q2, [x2, #1904] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2928] + ldr Q2, [x2, #2928] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3952] + ldr Q2, [x2, #3952] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4976] + ldr Q2, [x2, #4976] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #880] + ldr Q1, [x1, #896] + ldr Q2, [x2, #896] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1920] + ldr Q2, [x2, #1920] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2944] + ldr Q2, [x2, #2944] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3968] + ldr Q2, [x2, #3968] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4992] + ldr Q2, [x2, #4992] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #896] + ldr Q1, [x1, #912] + ldr Q2, [x2, #912] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1936] + ldr Q2, [x2, #1936] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2960] + ldr Q2, [x2, #2960] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3984] + ldr Q2, [x2, #3984] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5008] + ldr Q2, [x2, #5008] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #912] + ldr Q1, [x1, #928] + ldr Q2, [x2, #928] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1952] + ldr Q2, [x2, #1952] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2976] + ldr Q2, [x2, #2976] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4000] + ldr Q2, [x2, #4000] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5024] + ldr Q2, [x2, #5024] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #928] + ldr Q1, [x1, #944] + ldr Q2, [x2, #944] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1968] + ldr Q2, [x2, #1968] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2992] + ldr Q2, [x2, #2992] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4016] + ldr Q2, [x2, #4016] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5040] + ldr Q2, [x2, #5040] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #944] + ldr Q1, [x1, #960] + ldr Q2, [x2, #960] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1984] + ldr Q2, [x2, #1984] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3008] + ldr Q2, [x2, #3008] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4032] + ldr Q2, [x2, #4032] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5056] + ldr Q2, [x2, #5056] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #960] + ldr Q1, [x1, #976] + ldr Q2, [x2, #976] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2000] + ldr Q2, [x2, #2000] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3024] + ldr Q2, [x2, #3024] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4048] + ldr Q2, [x2, #4048] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5072] + ldr Q2, [x2, #5072] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #976] + ldr Q1, [x1, #992] + ldr Q2, [x2, #992] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2016] + ldr Q2, [x2, #2016] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3040] + ldr Q2, [x2, #3040] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4064] + ldr Q2, [x2, #4064] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5088] + ldr Q2, [x2, #5088] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #992] + ldr Q1, [x1, #1008] + ldr Q2, [x2, #1008] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2032] + ldr Q2, [x2, #2032] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3056] + ldr Q2, [x2, #3056] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4080] + ldr Q2, [x2, #4080] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5104] + ldr Q2, [x2, #5104] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #1008] + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_mul_vec_7_neon +mldsa_mul_vec_7_neon PROC + adrp x3, L_mldsa_aarch64_consts + add x3, x3, L_mldsa_aarch64_consts + ldr Q0, [x3] + ldr Q1, [x1] + ldr Q2, [x2] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1024] + ldr Q2, [x2, #1024] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2048] + ldr Q2, [x2, #2048] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3072] + ldr Q2, [x2, #3072] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4096] + ldr Q2, [x2, #4096] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5120] + ldr Q2, [x2, #5120] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6144] + ldr Q2, [x2, #6144] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0] + ldr Q1, [x1, #16] + ldr Q2, [x2, #16] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1040] + ldr Q2, [x2, #1040] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2064] + ldr Q2, [x2, #2064] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3088] + ldr Q2, [x2, #3088] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4112] + ldr Q2, [x2, #4112] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5136] + ldr Q2, [x2, #5136] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6160] + ldr Q2, [x2, #6160] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #16] + ldr Q1, [x1, #32] + ldr Q2, [x2, #32] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1056] + ldr Q2, [x2, #1056] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2080] + ldr Q2, [x2, #2080] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3104] + ldr Q2, [x2, #3104] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4128] + ldr Q2, [x2, #4128] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5152] + ldr Q2, [x2, #5152] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6176] + ldr Q2, [x2, #6176] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #32] + ldr Q1, [x1, #48] + ldr Q2, [x2, #48] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1072] + ldr Q2, [x2, #1072] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2096] + ldr Q2, [x2, #2096] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3120] + ldr Q2, [x2, #3120] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4144] + ldr Q2, [x2, #4144] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5168] + ldr Q2, [x2, #5168] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6192] + ldr Q2, [x2, #6192] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #48] + ldr Q1, [x1, #64] + ldr Q2, [x2, #64] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1088] + ldr Q2, [x2, #1088] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2112] + ldr Q2, [x2, #2112] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3136] + ldr Q2, [x2, #3136] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4160] + ldr Q2, [x2, #4160] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5184] + ldr Q2, [x2, #5184] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6208] + ldr Q2, [x2, #6208] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #64] + ldr Q1, [x1, #80] + ldr Q2, [x2, #80] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1104] + ldr Q2, [x2, #1104] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2128] + ldr Q2, [x2, #2128] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3152] + ldr Q2, [x2, #3152] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4176] + ldr Q2, [x2, #4176] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5200] + ldr Q2, [x2, #5200] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6224] + ldr Q2, [x2, #6224] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #80] + ldr Q1, [x1, #96] + ldr Q2, [x2, #96] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1120] + ldr Q2, [x2, #1120] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2144] + ldr Q2, [x2, #2144] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3168] + ldr Q2, [x2, #3168] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4192] + ldr Q2, [x2, #4192] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5216] + ldr Q2, [x2, #5216] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6240] + ldr Q2, [x2, #6240] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #96] + ldr Q1, [x1, #112] + ldr Q2, [x2, #112] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1136] + ldr Q2, [x2, #1136] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2160] + ldr Q2, [x2, #2160] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3184] + ldr Q2, [x2, #3184] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4208] + ldr Q2, [x2, #4208] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5232] + ldr Q2, [x2, #5232] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6256] + ldr Q2, [x2, #6256] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #112] + ldr Q1, [x1, #128] + ldr Q2, [x2, #128] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1152] + ldr Q2, [x2, #1152] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2176] + ldr Q2, [x2, #2176] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3200] + ldr Q2, [x2, #3200] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4224] + ldr Q2, [x2, #4224] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5248] + ldr Q2, [x2, #5248] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6272] + ldr Q2, [x2, #6272] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #128] + ldr Q1, [x1, #144] + ldr Q2, [x2, #144] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1168] + ldr Q2, [x2, #1168] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2192] + ldr Q2, [x2, #2192] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3216] + ldr Q2, [x2, #3216] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4240] + ldr Q2, [x2, #4240] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5264] + ldr Q2, [x2, #5264] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6288] + ldr Q2, [x2, #6288] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #144] + ldr Q1, [x1, #160] + ldr Q2, [x2, #160] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1184] + ldr Q2, [x2, #1184] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2208] + ldr Q2, [x2, #2208] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3232] + ldr Q2, [x2, #3232] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4256] + ldr Q2, [x2, #4256] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5280] + ldr Q2, [x2, #5280] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6304] + ldr Q2, [x2, #6304] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #160] + ldr Q1, [x1, #176] + ldr Q2, [x2, #176] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1200] + ldr Q2, [x2, #1200] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2224] + ldr Q2, [x2, #2224] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3248] + ldr Q2, [x2, #3248] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4272] + ldr Q2, [x2, #4272] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5296] + ldr Q2, [x2, #5296] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6320] + ldr Q2, [x2, #6320] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #176] + ldr Q1, [x1, #192] + ldr Q2, [x2, #192] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1216] + ldr Q2, [x2, #1216] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2240] + ldr Q2, [x2, #2240] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3264] + ldr Q2, [x2, #3264] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4288] + ldr Q2, [x2, #4288] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5312] + ldr Q2, [x2, #5312] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6336] + ldr Q2, [x2, #6336] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #192] + ldr Q1, [x1, #208] + ldr Q2, [x2, #208] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1232] + ldr Q2, [x2, #1232] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2256] + ldr Q2, [x2, #2256] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3280] + ldr Q2, [x2, #3280] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4304] + ldr Q2, [x2, #4304] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5328] + ldr Q2, [x2, #5328] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6352] + ldr Q2, [x2, #6352] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #208] + ldr Q1, [x1, #224] + ldr Q2, [x2, #224] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1248] + ldr Q2, [x2, #1248] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2272] + ldr Q2, [x2, #2272] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3296] + ldr Q2, [x2, #3296] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4320] + ldr Q2, [x2, #4320] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5344] + ldr Q2, [x2, #5344] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6368] + ldr Q2, [x2, #6368] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #224] + ldr Q1, [x1, #240] + ldr Q2, [x2, #240] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1264] + ldr Q2, [x2, #1264] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2288] + ldr Q2, [x2, #2288] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3312] + ldr Q2, [x2, #3312] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4336] + ldr Q2, [x2, #4336] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5360] + ldr Q2, [x2, #5360] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6384] + ldr Q2, [x2, #6384] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #240] + ldr Q1, [x1, #256] + ldr Q2, [x2, #256] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1280] + ldr Q2, [x2, #1280] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2304] + ldr Q2, [x2, #2304] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3328] + ldr Q2, [x2, #3328] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4352] + ldr Q2, [x2, #4352] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5376] + ldr Q2, [x2, #5376] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6400] + ldr Q2, [x2, #6400] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #256] + ldr Q1, [x1, #272] + ldr Q2, [x2, #272] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1296] + ldr Q2, [x2, #1296] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2320] + ldr Q2, [x2, #2320] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3344] + ldr Q2, [x2, #3344] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4368] + ldr Q2, [x2, #4368] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5392] + ldr Q2, [x2, #5392] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6416] + ldr Q2, [x2, #6416] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #272] + ldr Q1, [x1, #288] + ldr Q2, [x2, #288] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1312] + ldr Q2, [x2, #1312] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2336] + ldr Q2, [x2, #2336] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3360] + ldr Q2, [x2, #3360] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4384] + ldr Q2, [x2, #4384] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5408] + ldr Q2, [x2, #5408] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6432] + ldr Q2, [x2, #6432] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #288] + ldr Q1, [x1, #304] + ldr Q2, [x2, #304] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1328] + ldr Q2, [x2, #1328] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2352] + ldr Q2, [x2, #2352] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3376] + ldr Q2, [x2, #3376] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4400] + ldr Q2, [x2, #4400] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5424] + ldr Q2, [x2, #5424] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6448] + ldr Q2, [x2, #6448] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #304] + ldr Q1, [x1, #320] + ldr Q2, [x2, #320] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1344] + ldr Q2, [x2, #1344] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2368] + ldr Q2, [x2, #2368] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3392] + ldr Q2, [x2, #3392] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4416] + ldr Q2, [x2, #4416] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5440] + ldr Q2, [x2, #5440] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6464] + ldr Q2, [x2, #6464] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #320] + ldr Q1, [x1, #336] + ldr Q2, [x2, #336] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1360] + ldr Q2, [x2, #1360] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2384] + ldr Q2, [x2, #2384] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3408] + ldr Q2, [x2, #3408] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4432] + ldr Q2, [x2, #4432] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5456] + ldr Q2, [x2, #5456] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6480] + ldr Q2, [x2, #6480] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #336] + ldr Q1, [x1, #352] + ldr Q2, [x2, #352] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1376] + ldr Q2, [x2, #1376] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2400] + ldr Q2, [x2, #2400] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3424] + ldr Q2, [x2, #3424] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4448] + ldr Q2, [x2, #4448] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5472] + ldr Q2, [x2, #5472] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6496] + ldr Q2, [x2, #6496] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #352] + ldr Q1, [x1, #368] + ldr Q2, [x2, #368] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1392] + ldr Q2, [x2, #1392] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2416] + ldr Q2, [x2, #2416] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3440] + ldr Q2, [x2, #3440] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4464] + ldr Q2, [x2, #4464] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5488] + ldr Q2, [x2, #5488] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6512] + ldr Q2, [x2, #6512] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #368] + ldr Q1, [x1, #384] + ldr Q2, [x2, #384] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1408] + ldr Q2, [x2, #1408] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2432] + ldr Q2, [x2, #2432] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3456] + ldr Q2, [x2, #3456] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4480] + ldr Q2, [x2, #4480] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5504] + ldr Q2, [x2, #5504] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6528] + ldr Q2, [x2, #6528] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #384] + ldr Q1, [x1, #400] + ldr Q2, [x2, #400] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1424] + ldr Q2, [x2, #1424] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2448] + ldr Q2, [x2, #2448] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3472] + ldr Q2, [x2, #3472] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4496] + ldr Q2, [x2, #4496] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5520] + ldr Q2, [x2, #5520] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6544] + ldr Q2, [x2, #6544] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #400] + ldr Q1, [x1, #416] + ldr Q2, [x2, #416] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1440] + ldr Q2, [x2, #1440] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2464] + ldr Q2, [x2, #2464] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3488] + ldr Q2, [x2, #3488] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4512] + ldr Q2, [x2, #4512] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5536] + ldr Q2, [x2, #5536] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6560] + ldr Q2, [x2, #6560] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #416] + ldr Q1, [x1, #432] + ldr Q2, [x2, #432] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1456] + ldr Q2, [x2, #1456] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2480] + ldr Q2, [x2, #2480] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3504] + ldr Q2, [x2, #3504] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4528] + ldr Q2, [x2, #4528] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5552] + ldr Q2, [x2, #5552] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6576] + ldr Q2, [x2, #6576] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #432] + ldr Q1, [x1, #448] + ldr Q2, [x2, #448] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1472] + ldr Q2, [x2, #1472] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2496] + ldr Q2, [x2, #2496] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3520] + ldr Q2, [x2, #3520] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4544] + ldr Q2, [x2, #4544] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5568] + ldr Q2, [x2, #5568] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6592] + ldr Q2, [x2, #6592] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #448] + ldr Q1, [x1, #464] + ldr Q2, [x2, #464] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1488] + ldr Q2, [x2, #1488] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2512] + ldr Q2, [x2, #2512] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3536] + ldr Q2, [x2, #3536] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4560] + ldr Q2, [x2, #4560] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5584] + ldr Q2, [x2, #5584] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6608] + ldr Q2, [x2, #6608] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #464] + ldr Q1, [x1, #480] + ldr Q2, [x2, #480] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1504] + ldr Q2, [x2, #1504] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2528] + ldr Q2, [x2, #2528] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3552] + ldr Q2, [x2, #3552] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4576] + ldr Q2, [x2, #4576] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5600] + ldr Q2, [x2, #5600] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6624] + ldr Q2, [x2, #6624] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #480] + ldr Q1, [x1, #496] + ldr Q2, [x2, #496] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1520] + ldr Q2, [x2, #1520] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2544] + ldr Q2, [x2, #2544] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3568] + ldr Q2, [x2, #3568] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4592] + ldr Q2, [x2, #4592] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5616] + ldr Q2, [x2, #5616] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6640] + ldr Q2, [x2, #6640] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #496] + ldr Q1, [x1, #512] + ldr Q2, [x2, #512] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1536] + ldr Q2, [x2, #1536] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2560] + ldr Q2, [x2, #2560] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3584] + ldr Q2, [x2, #3584] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4608] + ldr Q2, [x2, #4608] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5632] + ldr Q2, [x2, #5632] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6656] + ldr Q2, [x2, #6656] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #512] + ldr Q1, [x1, #528] + ldr Q2, [x2, #528] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1552] + ldr Q2, [x2, #1552] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2576] + ldr Q2, [x2, #2576] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3600] + ldr Q2, [x2, #3600] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4624] + ldr Q2, [x2, #4624] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5648] + ldr Q2, [x2, #5648] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6672] + ldr Q2, [x2, #6672] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #528] + ldr Q1, [x1, #544] + ldr Q2, [x2, #544] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1568] + ldr Q2, [x2, #1568] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2592] + ldr Q2, [x2, #2592] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3616] + ldr Q2, [x2, #3616] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4640] + ldr Q2, [x2, #4640] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5664] + ldr Q2, [x2, #5664] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6688] + ldr Q2, [x2, #6688] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #544] + ldr Q1, [x1, #560] + ldr Q2, [x2, #560] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1584] + ldr Q2, [x2, #1584] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2608] + ldr Q2, [x2, #2608] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3632] + ldr Q2, [x2, #3632] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4656] + ldr Q2, [x2, #4656] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5680] + ldr Q2, [x2, #5680] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6704] + ldr Q2, [x2, #6704] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #560] + ldr Q1, [x1, #576] + ldr Q2, [x2, #576] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1600] + ldr Q2, [x2, #1600] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2624] + ldr Q2, [x2, #2624] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3648] + ldr Q2, [x2, #3648] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4672] + ldr Q2, [x2, #4672] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5696] + ldr Q2, [x2, #5696] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6720] + ldr Q2, [x2, #6720] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #576] + ldr Q1, [x1, #592] + ldr Q2, [x2, #592] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1616] + ldr Q2, [x2, #1616] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2640] + ldr Q2, [x2, #2640] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3664] + ldr Q2, [x2, #3664] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4688] + ldr Q2, [x2, #4688] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5712] + ldr Q2, [x2, #5712] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6736] + ldr Q2, [x2, #6736] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #592] + ldr Q1, [x1, #608] + ldr Q2, [x2, #608] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1632] + ldr Q2, [x2, #1632] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2656] + ldr Q2, [x2, #2656] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3680] + ldr Q2, [x2, #3680] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4704] + ldr Q2, [x2, #4704] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5728] + ldr Q2, [x2, #5728] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6752] + ldr Q2, [x2, #6752] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #608] + ldr Q1, [x1, #624] + ldr Q2, [x2, #624] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1648] + ldr Q2, [x2, #1648] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2672] + ldr Q2, [x2, #2672] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3696] + ldr Q2, [x2, #3696] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4720] + ldr Q2, [x2, #4720] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5744] + ldr Q2, [x2, #5744] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6768] + ldr Q2, [x2, #6768] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #624] + ldr Q1, [x1, #640] + ldr Q2, [x2, #640] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1664] + ldr Q2, [x2, #1664] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2688] + ldr Q2, [x2, #2688] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3712] + ldr Q2, [x2, #3712] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4736] + ldr Q2, [x2, #4736] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5760] + ldr Q2, [x2, #5760] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6784] + ldr Q2, [x2, #6784] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #640] + ldr Q1, [x1, #656] + ldr Q2, [x2, #656] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1680] + ldr Q2, [x2, #1680] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2704] + ldr Q2, [x2, #2704] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3728] + ldr Q2, [x2, #3728] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4752] + ldr Q2, [x2, #4752] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5776] + ldr Q2, [x2, #5776] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6800] + ldr Q2, [x2, #6800] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #656] + ldr Q1, [x1, #672] + ldr Q2, [x2, #672] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1696] + ldr Q2, [x2, #1696] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2720] + ldr Q2, [x2, #2720] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3744] + ldr Q2, [x2, #3744] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4768] + ldr Q2, [x2, #4768] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5792] + ldr Q2, [x2, #5792] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6816] + ldr Q2, [x2, #6816] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #672] + ldr Q1, [x1, #688] + ldr Q2, [x2, #688] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1712] + ldr Q2, [x2, #1712] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2736] + ldr Q2, [x2, #2736] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3760] + ldr Q2, [x2, #3760] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4784] + ldr Q2, [x2, #4784] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5808] + ldr Q2, [x2, #5808] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6832] + ldr Q2, [x2, #6832] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #688] + ldr Q1, [x1, #704] + ldr Q2, [x2, #704] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1728] + ldr Q2, [x2, #1728] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2752] + ldr Q2, [x2, #2752] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3776] + ldr Q2, [x2, #3776] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4800] + ldr Q2, [x2, #4800] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5824] + ldr Q2, [x2, #5824] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6848] + ldr Q2, [x2, #6848] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #704] + ldr Q1, [x1, #720] + ldr Q2, [x2, #720] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1744] + ldr Q2, [x2, #1744] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2768] + ldr Q2, [x2, #2768] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3792] + ldr Q2, [x2, #3792] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4816] + ldr Q2, [x2, #4816] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5840] + ldr Q2, [x2, #5840] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6864] + ldr Q2, [x2, #6864] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #720] + ldr Q1, [x1, #736] + ldr Q2, [x2, #736] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1760] + ldr Q2, [x2, #1760] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2784] + ldr Q2, [x2, #2784] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3808] + ldr Q2, [x2, #3808] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4832] + ldr Q2, [x2, #4832] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5856] + ldr Q2, [x2, #5856] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6880] + ldr Q2, [x2, #6880] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #736] + ldr Q1, [x1, #752] + ldr Q2, [x2, #752] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1776] + ldr Q2, [x2, #1776] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2800] + ldr Q2, [x2, #2800] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3824] + ldr Q2, [x2, #3824] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4848] + ldr Q2, [x2, #4848] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5872] + ldr Q2, [x2, #5872] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6896] + ldr Q2, [x2, #6896] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #752] + ldr Q1, [x1, #768] + ldr Q2, [x2, #768] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1792] + ldr Q2, [x2, #1792] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2816] + ldr Q2, [x2, #2816] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3840] + ldr Q2, [x2, #3840] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4864] + ldr Q2, [x2, #4864] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5888] + ldr Q2, [x2, #5888] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6912] + ldr Q2, [x2, #6912] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #768] + ldr Q1, [x1, #784] + ldr Q2, [x2, #784] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1808] + ldr Q2, [x2, #1808] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2832] + ldr Q2, [x2, #2832] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3856] + ldr Q2, [x2, #3856] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4880] + ldr Q2, [x2, #4880] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5904] + ldr Q2, [x2, #5904] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6928] + ldr Q2, [x2, #6928] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #784] + ldr Q1, [x1, #800] + ldr Q2, [x2, #800] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1824] + ldr Q2, [x2, #1824] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2848] + ldr Q2, [x2, #2848] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3872] + ldr Q2, [x2, #3872] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4896] + ldr Q2, [x2, #4896] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5920] + ldr Q2, [x2, #5920] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6944] + ldr Q2, [x2, #6944] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #800] + ldr Q1, [x1, #816] + ldr Q2, [x2, #816] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1840] + ldr Q2, [x2, #1840] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2864] + ldr Q2, [x2, #2864] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3888] + ldr Q2, [x2, #3888] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4912] + ldr Q2, [x2, #4912] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5936] + ldr Q2, [x2, #5936] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6960] + ldr Q2, [x2, #6960] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #816] + ldr Q1, [x1, #832] + ldr Q2, [x2, #832] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1856] + ldr Q2, [x2, #1856] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2880] + ldr Q2, [x2, #2880] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3904] + ldr Q2, [x2, #3904] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4928] + ldr Q2, [x2, #4928] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5952] + ldr Q2, [x2, #5952] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6976] + ldr Q2, [x2, #6976] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #832] + ldr Q1, [x1, #848] + ldr Q2, [x2, #848] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1872] + ldr Q2, [x2, #1872] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2896] + ldr Q2, [x2, #2896] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3920] + ldr Q2, [x2, #3920] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4944] + ldr Q2, [x2, #4944] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5968] + ldr Q2, [x2, #5968] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6992] + ldr Q2, [x2, #6992] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #848] + ldr Q1, [x1, #864] + ldr Q2, [x2, #864] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1888] + ldr Q2, [x2, #1888] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2912] + ldr Q2, [x2, #2912] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3936] + ldr Q2, [x2, #3936] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4960] + ldr Q2, [x2, #4960] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5984] + ldr Q2, [x2, #5984] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #7008] + ldr Q2, [x2, #7008] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #864] + ldr Q1, [x1, #880] + ldr Q2, [x2, #880] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1904] + ldr Q2, [x2, #1904] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2928] + ldr Q2, [x2, #2928] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3952] + ldr Q2, [x2, #3952] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4976] + ldr Q2, [x2, #4976] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6000] + ldr Q2, [x2, #6000] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #7024] + ldr Q2, [x2, #7024] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #880] + ldr Q1, [x1, #896] + ldr Q2, [x2, #896] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1920] + ldr Q2, [x2, #1920] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2944] + ldr Q2, [x2, #2944] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3968] + ldr Q2, [x2, #3968] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4992] + ldr Q2, [x2, #4992] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6016] + ldr Q2, [x2, #6016] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #7040] + ldr Q2, [x2, #7040] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #896] + ldr Q1, [x1, #912] + ldr Q2, [x2, #912] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1936] + ldr Q2, [x2, #1936] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2960] + ldr Q2, [x2, #2960] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3984] + ldr Q2, [x2, #3984] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5008] + ldr Q2, [x2, #5008] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6032] + ldr Q2, [x2, #6032] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #7056] + ldr Q2, [x2, #7056] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #912] + ldr Q1, [x1, #928] + ldr Q2, [x2, #928] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1952] + ldr Q2, [x2, #1952] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2976] + ldr Q2, [x2, #2976] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4000] + ldr Q2, [x2, #4000] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5024] + ldr Q2, [x2, #5024] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6048] + ldr Q2, [x2, #6048] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #7072] + ldr Q2, [x2, #7072] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #928] + ldr Q1, [x1, #944] + ldr Q2, [x2, #944] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1968] + ldr Q2, [x2, #1968] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2992] + ldr Q2, [x2, #2992] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4016] + ldr Q2, [x2, #4016] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5040] + ldr Q2, [x2, #5040] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6064] + ldr Q2, [x2, #6064] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #7088] + ldr Q2, [x2, #7088] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #944] + ldr Q1, [x1, #960] + ldr Q2, [x2, #960] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #1984] + ldr Q2, [x2, #1984] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3008] + ldr Q2, [x2, #3008] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4032] + ldr Q2, [x2, #4032] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5056] + ldr Q2, [x2, #5056] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6080] + ldr Q2, [x2, #6080] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #7104] + ldr Q2, [x2, #7104] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #960] + ldr Q1, [x1, #976] + ldr Q2, [x2, #976] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2000] + ldr Q2, [x2, #2000] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3024] + ldr Q2, [x2, #3024] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4048] + ldr Q2, [x2, #4048] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5072] + ldr Q2, [x2, #5072] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6096] + ldr Q2, [x2, #6096] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #7120] + ldr Q2, [x2, #7120] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #976] + ldr Q1, [x1, #992] + ldr Q2, [x2, #992] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2016] + ldr Q2, [x2, #2016] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3040] + ldr Q2, [x2, #3040] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4064] + ldr Q2, [x2, #4064] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5088] + ldr Q2, [x2, #5088] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6112] + ldr Q2, [x2, #6112] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #7136] + ldr Q2, [x2, #7136] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #992] + ldr Q1, [x1, #1008] + ldr Q2, [x2, #1008] + smull V3.2D, V1.2S, V2.2S + smull2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #2032] + ldr Q2, [x2, #2032] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #3056] + ldr Q2, [x2, #3056] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #4080] + ldr Q2, [x2, #4080] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #5104] + ldr Q2, [x2, #5104] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #6128] + ldr Q2, [x2, #6128] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + ldr Q1, [x1, #7152] + ldr Q2, [x2, #7152] + smlal V3.2D, V1.2S, V2.2S + smlal2 V4.2D, V1.4S, V2.4S + uzp1 V5.4S, V3.4S, V4.4S + mul V5.4S, V5.4S, V0.S[1] + smull V6.2D, V5.2S, V0.S[0] + smull2 V7.2D, V5.4S, V0.S[0] + sub V3.2D, V3.2D, V6.2D + sub V4.2D, V4.2D, V7.2D + uzp2 V5.4S, V3.4S, V4.4S + str Q5, [x0, #1008] + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_aarch64_q + DCD 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001 + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_eta_2_neon_idx + DCB 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff + DCB 0x01, 0xff, 0xff, 0xff, 0x01, 0x02, 0xff, 0xff + DCB 0x02, 0xff, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff + DCB 0x01, 0xff, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff + DCB 0x01, 0x02, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff + DCB 0x02, 0xff, 0xff, 0xff, 0x02, 0x03, 0xff, 0xff + DCB 0x03, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff + DCB 0x04, 0xff, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff + DCB 0x04, 0x05, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff + DCB 0x05, 0xff, 0xff, 0xff, 0x05, 0x06, 0xff, 0xff + DCB 0x06, 0xff, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff + DCB 0x07, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff + DCB 0x07, 0x08, 0xff, 0xff, 0x08, 0xff, 0xff, 0xff + DCB 0x08, 0xff, 0xff, 0xff, 0x08, 0x09, 0xff, 0xff + DCB 0x09, 0xff, 0xff, 0xff, 0x09, 0xff, 0xff, 0xff + DCB 0x0a, 0xff, 0xff, 0xff, 0x0a, 0xff, 0xff, 0xff + DCB 0x0a, 0x0b, 0xff, 0xff, 0x0b, 0xff, 0xff, 0xff + DCB 0x0b, 0xff, 0xff, 0xff, 0x0b, 0x0c, 0xff, 0xff + DCB 0x0c, 0xff, 0xff, 0xff, 0x0c, 0xff, 0xff, 0xff + DCB 0x0d, 0xff, 0xff, 0xff, 0x0d, 0xff, 0xff, 0xff + DCB 0x0d, 0x0e, 0xff, 0xff, 0x0e, 0xff, 0xff, 0xff + DCB 0x0e, 0xff, 0xff, 0xff, 0x0e, 0x0f, 0xff, 0xff + DCB 0x0f, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_eta_2_neon_shift + DCD 0x00000000, 0xfffffffd, 0xfffffffa, 0xffffffff + DCD 0xfffffffc, 0xfffffff9, 0xfffffffe, 0xfffffffb + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_eta_2_neon_mask + DCD 0x00000007, 0x00000007, 0x00000007, 0x00000007 + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_eta_2_neon_konst + DCD 0x00000002, 0x00000002, 0x00000002, 0x00000002 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_decode_eta_2_neon +mldsa_decode_eta_2_neon PROC + stp x29, x30, [sp, #-32]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + adrp x2, L_mldsa_decode_eta_2_neon_idx + add x2, x2, L_mldsa_decode_eta_2_neon_idx + adrp x3, L_mldsa_decode_eta_2_neon_shift + add x3, x3, L_mldsa_decode_eta_2_neon_shift + adrp x4, L_mldsa_decode_eta_2_neon_mask + add x4, x4, L_mldsa_decode_eta_2_neon_mask + adrp x5, L_mldsa_decode_eta_2_neon_konst + add x5, x5, L_mldsa_decode_eta_2_neon_konst + ldr Q1, [x2] + ldr Q3, [x3] + ldr Q2, [x2, #16] + ldr Q4, [x3, #16] + ldr Q5, [x4] + ldr Q6, [x5] + mov x6, #27 +L_mldsa_decode_eta_2_neon_loop + ldr Q0, [x0] + tbl V7.16B, {V0.16B}, V1.16B + ushl V7.4S, V7.4S, V3.4S + and V7.16B, V7.16B, V5.16B + sub V7.4S, V6.4S, V7.4S + str Q7, [x1] + tbl V8.16B, {V0.16B}, V2.16B + ushl V8.4S, V8.4S, V4.4S + and V8.16B, V8.16B, V5.16B + sub V8.4S, V6.4S, V8.4S + str Q8, [x1, #16] + add x0, x0, #3 + add x1, x1, #32 + subs x6, x6, #1 + bne L_mldsa_decode_eta_2_neon_loop + sub x0, x0, #1 + ldr Q0, [x0] + ldr Q1, [x2, #32] + ldr Q2, [x2, #48] + tbl V7.16B, {V0.16B}, V1.16B + ushl V7.4S, V7.4S, V3.4S + and V7.16B, V7.16B, V5.16B + sub V7.4S, V6.4S, V7.4S + str Q7, [x1] + tbl V8.16B, {V0.16B}, V2.16B + ushl V8.4S, V8.4S, V4.4S + and V8.16B, V8.16B, V5.16B + sub V8.4S, V6.4S, V8.4S + str Q8, [x1, #16] + add x1, x1, #32 + ldr Q1, [x2, #64] + ldr Q2, [x2, #80] + tbl V7.16B, {V0.16B}, V1.16B + ushl V7.4S, V7.4S, V3.4S + and V7.16B, V7.16B, V5.16B + sub V7.4S, V6.4S, V7.4S + str Q7, [x1] + tbl V8.16B, {V0.16B}, V2.16B + ushl V8.4S, V8.4S, V4.4S + and V8.16B, V8.16B, V5.16B + sub V8.4S, V6.4S, V8.4S + str Q8, [x1, #16] + add x1, x1, #32 + ldr Q1, [x2, #96] + ldr Q2, [x2, #112] + tbl V7.16B, {V0.16B}, V1.16B + ushl V7.4S, V7.4S, V3.4S + and V7.16B, V7.16B, V5.16B + sub V7.4S, V6.4S, V7.4S + str Q7, [x1] + tbl V8.16B, {V0.16B}, V2.16B + ushl V8.4S, V8.4S, V4.4S + and V8.16B, V8.16B, V5.16B + sub V8.4S, V6.4S, V8.4S + str Q8, [x1, #16] + add x1, x1, #32 + ldr Q1, [x2, #128] + ldr Q2, [x2, #144] + tbl V7.16B, {V0.16B}, V1.16B + ushl V7.4S, V7.4S, V3.4S + and V7.16B, V7.16B, V5.16B + sub V7.4S, V6.4S, V7.4S + str Q7, [x1] + tbl V8.16B, {V0.16B}, V2.16B + ushl V8.4S, V8.4S, V4.4S + and V8.16B, V8.16B, V5.16B + sub V8.4S, V6.4S, V8.4S + str Q8, [x1, #16] + add x1, x1, #32 + ldr Q1, [x2, #160] + ldr Q2, [x2, #176] + tbl V7.16B, {V0.16B}, V1.16B + ushl V7.4S, V7.4S, V3.4S + and V7.16B, V7.16B, V5.16B + sub V7.4S, V6.4S, V7.4S + str Q7, [x1] + tbl V8.16B, {V0.16B}, V2.16B + ushl V8.4S, V8.4S, V4.4S + and V8.16B, V8.16B, V5.16B + sub V8.4S, V6.4S, V8.4S + str Q8, [x1, #16] + ldp D8, D9, [x29, #16] + ldp x29, x30, [sp], #32 + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_eta_4_neon_idx + DCB 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff + DCB 0x01, 0xff, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff + DCB 0x02, 0xff, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff + DCB 0x03, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff + DCB 0x04, 0xff, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff + DCB 0x05, 0xff, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff + DCB 0x06, 0xff, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff + DCB 0x07, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff + DCB 0x08, 0xff, 0xff, 0xff, 0x08, 0xff, 0xff, 0xff + DCB 0x09, 0xff, 0xff, 0xff, 0x09, 0xff, 0xff, 0xff + DCB 0x0a, 0xff, 0xff, 0xff, 0x0a, 0xff, 0xff, 0xff + DCB 0x0b, 0xff, 0xff, 0xff, 0x0b, 0xff, 0xff, 0xff + DCB 0x0c, 0xff, 0xff, 0xff, 0x0c, 0xff, 0xff, 0xff + DCB 0x0d, 0xff, 0xff, 0xff, 0x0d, 0xff, 0xff, 0xff + DCB 0x0e, 0xff, 0xff, 0xff, 0x0e, 0xff, 0xff, 0xff + DCB 0x0f, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_eta_4_neon_shift + DCD 0x00000000, 0xfffffffc, 0x00000000, 0xfffffffc + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_eta_4_neon_mask + DCD 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_eta_4_neon_konst + DCD 0x00000004, 0x00000004, 0x00000004, 0x00000004 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_decode_eta_4_neon +mldsa_decode_eta_4_neon PROC + adrp x2, L_mldsa_decode_eta_4_neon_idx + add x2, x2, L_mldsa_decode_eta_4_neon_idx + adrp x3, L_mldsa_decode_eta_4_neon_shift + add x3, x3, L_mldsa_decode_eta_4_neon_shift + adrp x4, L_mldsa_decode_eta_4_neon_mask + add x4, x4, L_mldsa_decode_eta_4_neon_mask + adrp x5, L_mldsa_decode_eta_4_neon_konst + add x5, x5, L_mldsa_decode_eta_4_neon_konst + ldr Q1, [x2] + ldr Q2, [x3] + ldr Q3, [x4] + ldr Q4, [x5] + mov x6, #57 +L_mldsa_decode_eta_4_neon_loop + ldr Q0, [x0] + tbl V5.16B, {V0.16B}, V1.16B + ushl V5.4S, V5.4S, V2.4S + and V5.16B, V5.16B, V3.16B + sub V5.4S, V4.4S, V5.4S + str Q5, [x1] + add x0, x0, #2 + add x1, x1, #16 + subs x6, x6, #1 + bne L_mldsa_decode_eta_4_neon_loop + sub x0, x0, #2 + ldr Q0, [x0] + ldr Q1, [x2, #16] + tbl V5.16B, {V0.16B}, V1.16B + ushl V5.4S, V5.4S, V2.4S + and V5.16B, V5.16B, V3.16B + sub V5.4S, V4.4S, V5.4S + str Q5, [x1] + add x1, x1, #16 + ldr Q1, [x2, #32] + tbl V5.16B, {V0.16B}, V1.16B + ushl V5.4S, V5.4S, V2.4S + and V5.16B, V5.16B, V3.16B + sub V5.4S, V4.4S, V5.4S + str Q5, [x1] + add x1, x1, #16 + ldr Q1, [x2, #48] + tbl V5.16B, {V0.16B}, V1.16B + ushl V5.4S, V5.4S, V2.4S + and V5.16B, V5.16B, V3.16B + sub V5.4S, V4.4S, V5.4S + str Q5, [x1] + add x1, x1, #16 + ldr Q1, [x2, #64] + tbl V5.16B, {V0.16B}, V1.16B + ushl V5.4S, V5.4S, V2.4S + and V5.16B, V5.16B, V3.16B + sub V5.4S, V4.4S, V5.4S + str Q5, [x1] + add x1, x1, #16 + ldr Q1, [x2, #80] + tbl V5.16B, {V0.16B}, V1.16B + ushl V5.4S, V5.4S, V2.4S + and V5.16B, V5.16B, V3.16B + sub V5.4S, V4.4S, V5.4S + str Q5, [x1] + add x1, x1, #16 + ldr Q1, [x2, #96] + tbl V5.16B, {V0.16B}, V1.16B + ushl V5.4S, V5.4S, V2.4S + and V5.16B, V5.16B, V3.16B + sub V5.4S, V4.4S, V5.4S + str Q5, [x1] + add x1, x1, #16 + ldr Q1, [x2, #112] + tbl V5.16B, {V0.16B}, V1.16B + ushl V5.4S, V5.4S, V2.4S + and V5.16B, V5.16B, V3.16B + sub V5.4S, V4.4S, V5.4S + str Q5, [x1] + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_t1_neon_idx + DCB 0x00, 0x01, 0xff, 0xff, 0x01, 0x02, 0xff, 0xff + DCB 0x02, 0x03, 0xff, 0xff, 0x03, 0x04, 0xff, 0xff + DCB 0x01, 0x02, 0xff, 0xff, 0x02, 0x03, 0xff, 0xff + DCB 0x03, 0x04, 0xff, 0xff, 0x04, 0x05, 0xff, 0xff + DCB 0x06, 0x07, 0xff, 0xff, 0x07, 0x08, 0xff, 0xff + DCB 0x08, 0x09, 0xff, 0xff, 0x09, 0x0a, 0xff, 0xff + DCB 0x0b, 0x0c, 0xff, 0xff, 0x0c, 0x0d, 0xff, 0xff + DCB 0x0d, 0x0e, 0xff, 0xff, 0x0e, 0x0f, 0xff, 0xff + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_t1_neon_shift + DCD 0x00000000, 0xfffffffe, 0xfffffffc, 0xfffffffa + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_t1_neon_mask + DCD 0x000003ff, 0x000003ff, 0x000003ff, 0x000003ff + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_decode_t1_neon +mldsa_decode_t1_neon PROC + adrp x2, L_mldsa_decode_t1_neon_idx + add x2, x2, L_mldsa_decode_t1_neon_idx + adrp x3, L_mldsa_decode_t1_neon_shift + add x3, x3, L_mldsa_decode_t1_neon_shift + adrp x4, L_mldsa_decode_t1_neon_mask + add x4, x4, L_mldsa_decode_t1_neon_mask + ldr Q1, [x2] + ldr Q2, [x3] + ldr Q3, [x4] + mov x5, #61 +L_mldsa_decode_t1_neon_loop + ldr Q0, [x0] + tbl V4.16B, {V0.16B}, V1.16B + ushl V4.4S, V4.4S, V2.4S + and V4.16B, V4.16B, V3.16B + shl V4.4S, V4.4S, #13 + str Q4, [x1] + add x0, x0, #5 + add x1, x1, #16 + subs x5, x5, #1 + bne L_mldsa_decode_t1_neon_loop + sub x0, x0, #1 + ldr Q0, [x0] + ldr Q1, [x2, #16] + tbl V4.16B, {V0.16B}, V1.16B + ushl V4.4S, V4.4S, V2.4S + and V4.16B, V4.16B, V3.16B + shl V4.4S, V4.4S, #13 + str Q4, [x1] + add x1, x1, #16 + ldr Q1, [x2, #32] + tbl V4.16B, {V0.16B}, V1.16B + ushl V4.4S, V4.4S, V2.4S + and V4.16B, V4.16B, V3.16B + shl V4.4S, V4.4S, #13 + str Q4, [x1] + add x1, x1, #16 + ldr Q1, [x2, #48] + tbl V4.16B, {V0.16B}, V1.16B + ushl V4.4S, V4.4S, V2.4S + and V4.16B, V4.16B, V3.16B + shl V4.4S, V4.4S, #13 + str Q4, [x1] + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_t0_neon_idx + DCB 0x00, 0x01, 0xff, 0xff, 0x01, 0x02, 0x03, 0xff + DCB 0x03, 0x04, 0xff, 0xff, 0x04, 0x05, 0x06, 0xff + DCB 0x06, 0x07, 0x08, 0xff, 0x08, 0x09, 0xff, 0xff + DCB 0x09, 0x0a, 0x0b, 0xff, 0x0b, 0x0c, 0xff, 0xff + DCB 0x03, 0x04, 0xff, 0xff, 0x04, 0x05, 0x06, 0xff + DCB 0x06, 0x07, 0xff, 0xff, 0x07, 0x08, 0x09, 0xff + DCB 0x09, 0x0a, 0x0b, 0xff, 0x0b, 0x0c, 0xff, 0xff + DCB 0x0c, 0x0d, 0x0e, 0xff, 0x0e, 0x0f, 0xff, 0xff + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_t0_neon_shift + DCD 0x00000000, 0xfffffffb, 0xfffffffe, 0xfffffff9 + DCD 0xfffffffc, 0xffffffff, 0xfffffffa, 0xfffffffd + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_t0_neon_mask + DCD 0x00001fff, 0x00001fff, 0x00001fff, 0x00001fff + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_t0_neon_konst + DCD 0x00001000, 0x00001000, 0x00001000, 0x00001000 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_decode_t0_neon +mldsa_decode_t0_neon PROC + stp x29, x30, [sp, #-32]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + adrp x2, L_mldsa_decode_t0_neon_idx + add x2, x2, L_mldsa_decode_t0_neon_idx + adrp x3, L_mldsa_decode_t0_neon_shift + add x3, x3, L_mldsa_decode_t0_neon_shift + adrp x4, L_mldsa_decode_t0_neon_mask + add x4, x4, L_mldsa_decode_t0_neon_mask + adrp x5, L_mldsa_decode_t0_neon_konst + add x5, x5, L_mldsa_decode_t0_neon_konst + ldr Q1, [x2] + ldr Q3, [x3] + ldr Q2, [x2, #16] + ldr Q4, [x3, #16] + ldr Q5, [x4] + ldr Q6, [x5] + mov x6, #31 +L_mldsa_decode_t0_neon_loop + ldr Q0, [x0] + tbl V7.16B, {V0.16B}, V1.16B + ushl V7.4S, V7.4S, V3.4S + and V7.16B, V7.16B, V5.16B + sub V7.4S, V6.4S, V7.4S + str Q7, [x1] + tbl V8.16B, {V0.16B}, V2.16B + ushl V8.4S, V8.4S, V4.4S + and V8.16B, V8.16B, V5.16B + sub V8.4S, V6.4S, V8.4S + str Q8, [x1, #16] + add x0, x0, #13 + add x1, x1, #32 + subs x6, x6, #1 + bne L_mldsa_decode_t0_neon_loop + sub x0, x0, #3 + ldr Q0, [x0] + ldr Q1, [x2, #32] + ldr Q2, [x2, #48] + tbl V7.16B, {V0.16B}, V1.16B + ushl V7.4S, V7.4S, V3.4S + and V7.16B, V7.16B, V5.16B + sub V7.4S, V6.4S, V7.4S + str Q7, [x1] + tbl V8.16B, {V0.16B}, V2.16B + ushl V8.4S, V8.4S, V4.4S + and V8.16B, V8.16B, V5.16B + sub V8.4S, V6.4S, V8.4S + str Q8, [x1, #16] + ldp D8, D9, [x29, #16] + ldp x29, x30, [sp], #32 + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_gamma1_17_neon_idx + DCB 0x00, 0x01, 0x02, 0xff, 0x02, 0x03, 0x04, 0xff + DCB 0x04, 0x05, 0x06, 0xff, 0x06, 0x07, 0x08, 0xff + DCB 0x07, 0x08, 0x09, 0xff, 0x09, 0x0a, 0x0b, 0xff + DCB 0x0b, 0x0c, 0x0d, 0xff, 0x0d, 0x0e, 0x0f, 0xff + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_gamma1_17_neon_shift + DCD 0x00000000, 0xfffffffe, 0xfffffffc, 0xfffffffa + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_gamma1_17_neon_mask + DCD 0x0003ffff, 0x0003ffff, 0x0003ffff, 0x0003ffff + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_gamma1_17_neon_konst + DCD 0x00020000, 0x00020000, 0x00020000, 0x00020000 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_decode_gamma1_17_neon +mldsa_decode_gamma1_17_neon PROC + adrp x2, L_mldsa_decode_gamma1_17_neon_idx + add x2, x2, L_mldsa_decode_gamma1_17_neon_idx + adrp x3, L_mldsa_decode_gamma1_17_neon_shift + add x3, x3, L_mldsa_decode_gamma1_17_neon_shift + adrp x4, L_mldsa_decode_gamma1_17_neon_mask + add x4, x4, L_mldsa_decode_gamma1_17_neon_mask + adrp x5, L_mldsa_decode_gamma1_17_neon_konst + add x5, x5, L_mldsa_decode_gamma1_17_neon_konst + ldr Q1, [x2] + ldr Q2, [x3] + ldr Q3, [x4] + ldr Q4, [x5] + mov x6, #63 +L_mldsa_decode_gamma1_17_neon_loop + ldr Q0, [x0] + tbl V5.16B, {V0.16B}, V1.16B + ushl V5.4S, V5.4S, V2.4S + and V5.16B, V5.16B, V3.16B + sub V5.4S, V4.4S, V5.4S + str Q5, [x1] + add x0, x0, #9 + add x1, x1, #16 + subs x6, x6, #1 + bne L_mldsa_decode_gamma1_17_neon_loop + sub x0, x0, #7 + ldr Q0, [x0] + ldr Q1, [x2, #16] + tbl V5.16B, {V0.16B}, V1.16B + ushl V5.4S, V5.4S, V2.4S + and V5.16B, V5.16B, V3.16B + sub V5.4S, V4.4S, V5.4S + str Q5, [x1] + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_gamma1_19_neon_idx + DCB 0x00, 0x01, 0x02, 0xff, 0x02, 0x03, 0x04, 0xff + DCB 0x05, 0x06, 0x07, 0xff, 0x07, 0x08, 0x09, 0xff + DCB 0x06, 0x07, 0x08, 0xff, 0x08, 0x09, 0x0a, 0xff + DCB 0x0b, 0x0c, 0x0d, 0xff, 0x0d, 0x0e, 0x0f, 0xff + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_gamma1_19_neon_shift + DCD 0x00000000, 0xfffffffc, 0x00000000, 0xfffffffc + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_gamma1_19_neon_mask + DCD 0x000fffff, 0x000fffff, 0x000fffff, 0x000fffff + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_decode_gamma1_19_neon_konst + DCD 0x00080000, 0x00080000, 0x00080000, 0x00080000 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_decode_gamma1_19_neon +mldsa_decode_gamma1_19_neon PROC + adrp x2, L_mldsa_decode_gamma1_19_neon_idx + add x2, x2, L_mldsa_decode_gamma1_19_neon_idx + adrp x3, L_mldsa_decode_gamma1_19_neon_shift + add x3, x3, L_mldsa_decode_gamma1_19_neon_shift + adrp x4, L_mldsa_decode_gamma1_19_neon_mask + add x4, x4, L_mldsa_decode_gamma1_19_neon_mask + adrp x5, L_mldsa_decode_gamma1_19_neon_konst + add x5, x5, L_mldsa_decode_gamma1_19_neon_konst + ldr Q1, [x2] + ldr Q2, [x3] + ldr Q3, [x4] + ldr Q4, [x5] + mov x6, #63 +L_mldsa_decode_gamma1_19_neon_loop + ldr Q0, [x0] + tbl V5.16B, {V0.16B}, V1.16B + ushl V5.4S, V5.4S, V2.4S + and V5.16B, V5.16B, V3.16B + sub V5.4S, V4.4S, V5.4S + str Q5, [x1] + add x0, x0, #10 + add x1, x1, #16 + subs x6, x6, #1 + bne L_mldsa_decode_gamma1_19_neon_loop + sub x0, x0, #6 + ldr Q0, [x0] + ldr Q1, [x2, #16] + tbl V5.16B, {V0.16B}, V1.16B + ushl V5.4S, V5.4S, V2.4S + and V5.16B, V5.16B, V3.16B + sub V5.4S, V4.4S, V5.4S + str Q5, [x1] + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_encode_w1_88_neon_idx + DCB 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09 + DCB 0x0a, 0x0c, 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_encode_w1_88_neon +mldsa_encode_w1_88_neon PROC + stp x29, x30, [sp, #-32]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + adrp x2, L_mldsa_encode_w1_88_neon_idx + add x2, x2, L_mldsa_encode_w1_88_neon_idx + ldr Q8, [x2] + mov x3, #16 +L_mldsa_encode_w1_88_neon_loop + ldr Q0, [x0] + ldr Q1, [x0, #16] + ldr Q2, [x0, #32] + ldr Q3, [x0, #48] + uzp1 V4.4S, V0.4S, V1.4S + uzp2 V7.4S, V0.4S, V1.4S + sli V4.4S, V7.4S, #6 + uzp1 V5.4S, V2.4S, V3.4S + uzp2 V7.4S, V2.4S, V3.4S + sli V5.4S, V7.4S, #6 + uzp1 V6.4S, V4.4S, V5.4S + uzp2 V7.4S, V4.4S, V5.4S + sli V6.4S, V7.4S, #12 + tbl V9.16B, {V6.16B}, V8.16B + str D9, [x1] + ext8 V9.16B, V9.16B, V9.16B, #8 + str S9, [x1, #8] + add x0, x0, #0x40 + add x1, x1, #12 + subs x3, x3, #1 + bne L_mldsa_encode_w1_88_neon_loop + ldp D8, D9, [x29, #16] + ldp x29, x30, [sp], #32 + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_encode_w1_32_neon_idx + DCB 0x00, 0x04, 0x08, 0x0c, 0x10, 0x14, 0x18, 0x1c + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_encode_w1_32_neon +mldsa_encode_w1_32_neon PROC + stp x29, x30, [sp, #-32]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + adrp x2, L_mldsa_encode_w1_32_neon_idx + add x2, x2, L_mldsa_encode_w1_32_neon_idx + ldr Q8, [x2] + mov x3, #16 +L_mldsa_encode_w1_32_neon_loop + ldr Q0, [x0] + ldr Q1, [x0, #16] + ldr Q2, [x0, #32] + ldr Q3, [x0, #48] + uzp1 V4.4S, V0.4S, V1.4S + uzp2 V7.4S, V0.4S, V1.4S + sli V4.4S, V7.4S, #4 + uzp1 V5.4S, V2.4S, V3.4S + uzp2 V7.4S, V2.4S, V3.4S + sli V5.4S, V7.4S, #4 + tbl V9.16B, {V4.16B, V5.16B}, V8.16B + str D9, [x1] + add x0, x0, #0x40 + add x1, x1, #8 + subs x3, x3, #1 + bne L_mldsa_encode_w1_32_neon_loop + ldp D8, D9, [x29, #16] + ldp x29, x30, [sp], #32 + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_encode_gamma1_17_neon_idx + DCB 0x00, 0x01, 0x02, 0xff, 0x08, 0x09, 0x0a, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0x04, 0x05, 0x06, 0xff, 0x0c, 0x0d + DCB 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_encode_gamma1_17_neon_shift + DCD 0x00000000, 0x00000002, 0x00000004, 0x00000006 + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_encode_gamma1_17_neon_gam + DCD 0x00020000, 0x00020000, 0x00020000, 0x00020000 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_encode_gamma1_17_neon +mldsa_encode_gamma1_17_neon PROC + adrp x2, L_mldsa_encode_gamma1_17_neon_idx + add x2, x2, L_mldsa_encode_gamma1_17_neon_idx + adrp x3, L_mldsa_encode_gamma1_17_neon_shift + add x3, x3, L_mldsa_encode_gamma1_17_neon_shift + adrp x4, L_mldsa_encode_gamma1_17_neon_gam + add x4, x4, L_mldsa_encode_gamma1_17_neon_gam + ldr Q3, [x2] + ldr Q4, [x2, #16] + ldr Q1, [x3] + ldr Q2, [x4] + mov x5, #0x40 +L_mldsa_encode_gamma1_17_neon_loop + ldr Q0, [x0] + sub V0.4S, V2.4S, V0.4S + ushl V0.4S, V0.4S, V1.4S + tbl V5.16B, {V0.16B}, V3.16B + tbl V6.16B, {V0.16B}, V4.16B + orr V5.16B, V5.16B, V6.16B + str D5, [x1] + ext8 V5.16B, V5.16B, V5.16B, #8 + str B5, [x1, #8] + add x0, x0, #16 + add x1, x1, #9 + subs x5, x5, #1 + bne L_mldsa_encode_gamma1_17_neon_loop + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_encode_gamma1_19_neon_idx + DCB 0x00, 0x01, 0x02, 0xff, 0xff, 0x08, 0x09, 0x0a + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0x04, 0x05, 0x06, 0xff, 0xff, 0x0c + DCB 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_encode_gamma1_19_neon_shift + DCD 0x00000000, 0x00000004, 0x00000000, 0x00000004 + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_encode_gamma1_19_neon_gam + DCD 0x00080000, 0x00080000, 0x00080000, 0x00080000 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_encode_gamma1_19_neon +mldsa_encode_gamma1_19_neon PROC + adrp x2, L_mldsa_encode_gamma1_19_neon_idx + add x2, x2, L_mldsa_encode_gamma1_19_neon_idx + adrp x3, L_mldsa_encode_gamma1_19_neon_shift + add x3, x3, L_mldsa_encode_gamma1_19_neon_shift + adrp x4, L_mldsa_encode_gamma1_19_neon_gam + add x4, x4, L_mldsa_encode_gamma1_19_neon_gam + ldr Q3, [x2] + ldr Q4, [x2, #16] + ldr Q1, [x3] + ldr Q2, [x4] + mov x5, #0x40 +L_mldsa_encode_gamma1_19_neon_loop + ldr Q0, [x0] + sub V0.4S, V2.4S, V0.4S + ushl V0.4S, V0.4S, V1.4S + tbl V5.16B, {V0.16B}, V3.16B + tbl V6.16B, {V0.16B}, V4.16B + orr V5.16B, V5.16B, V6.16B + str D5, [x1] + ext8 V5.16B, V5.16B, V5.16B, #8 + str H5, [x1, #8] + add x0, x0, #16 + add x1, x1, #10 + subs x5, x5, #1 + bne L_mldsa_encode_gamma1_19_neon_loop + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_vec_check_low_neon +mldsa_vec_check_low_neon PROC + stp x29, x30, [sp, #-64]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + dup V0.4S, w2 + eor V1.16B, V1.16B, V1.16B + eor V2.16B, V2.16B, V2.16B + eor V3.16B, V3.16B, V3.16B + eor V4.16B, V4.16B, V4.16B +L_mldsa_vec_check_low_outer + mov x3, #16 +L_mldsa_vec_check_low_inner + ldr Q5, [x0] + ldr Q6, [x0, #16] + ldr Q7, [x0, #32] + ldr Q8, [x0, #48] + sqabs V9.4S, V5.4S + cmge V9.4S, V9.4S, V0.4S + orr V1.16B, V1.16B, V9.16B + sqabs V10.4S, V6.4S + cmge V10.4S, V10.4S, V0.4S + orr V2.16B, V2.16B, V10.16B + sqabs V11.4S, V7.4S + cmge V11.4S, V11.4S, V0.4S + orr V3.16B, V3.16B, V11.16B + sqabs V12.4S, V8.4S + cmge V12.4S, V12.4S, V0.4S + orr V4.16B, V4.16B, V12.16B + add x0, x0, #0x40 + subs x3, x3, #1 + bne L_mldsa_vec_check_low_inner + subs w1, w1, #1 + bne L_mldsa_vec_check_low_outer + orr V1.16B, V1.16B, V2.16B + orr V3.16B, V3.16B, V4.16B + orr V1.16B, V1.16B, V3.16B + umaxv S1, V1.4S + mov w4, V1.S[0] + cmp w4, #0 + cset w4, eq + mov x0, x4 + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp x29, x30, [sp], #0x40 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_poly_add_neon +mldsa_poly_add_neon PROC + mov x2, #16 +L_mldsa_poly_add_neon_loop + ldr Q0, [x0] + ldr Q4, [x1] + ldr Q1, [x0, #16] + ldr Q5, [x1, #16] + ldr Q2, [x0, #32] + ldr Q6, [x1, #32] + ldr Q3, [x0, #48] + ldr Q7, [x1, #48] + add V0.4S, V0.4S, V4.4S + add V1.4S, V1.4S, V5.4S + add V2.4S, V2.4S, V6.4S + add V3.4S, V3.4S, V7.4S + str Q0, [x0] + str Q1, [x0, #16] + str Q2, [x0, #32] + str Q3, [x0, #48] + add x0, x0, #0x40 + add x1, x1, #0x40 + subs x2, x2, #1 + bne L_mldsa_poly_add_neon_loop + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_poly_sub_neon +mldsa_poly_sub_neon PROC + mov x2, #16 +L_mldsa_poly_sub_neon_loop + ldr Q0, [x0] + ldr Q4, [x1] + ldr Q1, [x0, #16] + ldr Q5, [x1, #16] + ldr Q2, [x0, #32] + ldr Q6, [x1, #32] + ldr Q3, [x0, #48] + ldr Q7, [x1, #48] + sub V0.4S, V0.4S, V4.4S + sub V1.4S, V1.4S, V5.4S + sub V2.4S, V2.4S, V6.4S + sub V3.4S, V3.4S, V7.4S + str Q0, [x0] + str Q1, [x0, #16] + str Q2, [x0, #32] + str Q3, [x0, #48] + add x0, x0, #0x40 + add x1, x1, #0x40 + subs x2, x2, #1 + bne L_mldsa_poly_sub_neon_loop + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_poly_make_pos_neon_q + DCD 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_poly_make_pos_neon +mldsa_poly_make_pos_neon PROC + stp x29, x30, [sp, #-32]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + adrp x1, L_mldsa_poly_make_pos_neon_q + add x1, x1, L_mldsa_poly_make_pos_neon_q + ldr Q0, [x1] + mov x2, #16 +L_mldsa_poly_make_pos_neon_loop + ldr Q1, [x0] + ldr Q2, [x0, #16] + ldr Q3, [x0, #32] + ldr Q4, [x0, #48] + sshr V5.4S, V1.4S, #31 + mls V1.4S, V5.4S, V0.4S + sshr V6.4S, V2.4S, #31 + mls V2.4S, V6.4S, V0.4S + sshr V7.4S, V3.4S, #31 + mls V3.4S, V7.4S, V0.4S + sshr V8.4S, V4.4S, #31 + mls V4.4S, V8.4S, V0.4S + str Q1, [x0] + str Q2, [x0, #16] + str Q3, [x0, #32] + str Q4, [x0, #48] + add x0, x0, #0x40 + subs x2, x2, #1 + bne L_mldsa_poly_make_pos_neon_loop + ldp D8, D9, [x29, #16] + ldp x29, x30, [sp], #32 + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_poly_red_neon_q + DCD 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001 + DCD 0x00400000, 0x00400000, 0x00400000, 0x00400000 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_poly_red_neon +mldsa_poly_red_neon PROC + stp x29, x30, [sp, #-32]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + adrp x1, L_mldsa_poly_red_neon_q + add x1, x1, L_mldsa_poly_red_neon_q + ldr Q0, [x1] + ldr Q1, [x1, #16] + mov x2, #16 +L_mldsa_poly_red_neon_loop + ldr Q2, [x0] + ldr Q3, [x0, #16] + ldr Q4, [x0, #32] + ldr Q5, [x0, #48] + add V6.4S, V2.4S, V1.4S + sshr V6.4S, V6.4S, #23 + mls V2.4S, V6.4S, V0.4S + add V7.4S, V3.4S, V1.4S + sshr V7.4S, V7.4S, #23 + mls V3.4S, V7.4S, V0.4S + add V8.4S, V4.4S, V1.4S + sshr V8.4S, V8.4S, #23 + mls V4.4S, V8.4S, V0.4S + add V9.4S, V5.4S, V1.4S + sshr V9.4S, V9.4S, #23 + mls V5.4S, V9.4S, V0.4S + str Q2, [x0] + str Q3, [x0, #16] + str Q4, [x0, #32] + str Q5, [x0, #48] + add x0, x0, #0x40 + subs x2, x2, #1 + bne L_mldsa_poly_red_neon_loop + ldp D8, D9, [x29, #16] + ldp x29, x30, [sp], #32 + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_extract_coeffs_eta2_neon_map + DCB 0x02, 0x01, 0x00, 0xff, 0xfe, 0x02, 0x01, 0x00 + DCB 0xff, 0xfe, 0x02, 0x01, 0x00, 0xff, 0xfe, 0x00 + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_extract_coeffs_eta2_neon_lim + DCB 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f + DCB 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_extract_coeffs_eta2_neon_bits + DCD 0x00000001, 0x00000002, 0x00000004, 0x00000008 + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_extract_coeffs_eta2_neon_idx + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x04, 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + DCB 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff + DCB 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x0c, 0x0d, 0x0e, 0x0f + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x04, 0x05, 0x06, 0x07, 0x0c, 0x0d, 0x0e, 0x0f + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + DCB 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff + DCB 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b + DCB 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff + DCB 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b + DCB 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + DCB 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_extract_coeffs_eta2_neon +mldsa_extract_coeffs_eta2_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x4, L_mldsa_extract_coeffs_eta2_neon_map + add x4, x4, L_mldsa_extract_coeffs_eta2_neon_map + adrp x5, L_mldsa_extract_coeffs_eta2_neon_lim + add x5, x5, L_mldsa_extract_coeffs_eta2_neon_lim + adrp x6, L_mldsa_extract_coeffs_eta2_neon_bits + add x6, x6, L_mldsa_extract_coeffs_eta2_neon_bits + adrp x7, L_mldsa_extract_coeffs_eta2_neon_idx + add x7, x7, L_mldsa_extract_coeffs_eta2_neon_idx + ldr Q2, [x4] + ldr Q0, [x5] + ldr Q1, [x6] + ldr w8, [x3] + add x2, x2, x8, lsl 2 +L_mldsa_extract_coeffs_eta2_neon_wide + subs wzr, w1, #8 + blt L_mldsa_extract_coeffs_eta2_neon_tail + subs wzr, w8, #0xf0 + bgt L_mldsa_extract_coeffs_eta2_neon_tail + ldr D3, [x0] + movi V18.16B, #15 + ushr V5.16B, V3.16B, #4 + and V4.16B, V3.16B, V18.16B + zip1 V3.16B, V4.16B, V5.16B + tbl V6.16B, {V2.16B}, V3.16B + cmhi V8.16B, V0.16B, V3.16B + sxtl V7.8H, V6.8B + sxtl2 V18.8H, V6.16B + sxtl V10.4S, V7.4H + sxtl2 V11.4S, V7.8H + sxtl V12.4S, V18.4H + sxtl2 V13.4S, V18.8H + sxtl V9.8H, V8.8B + sxtl2 V18.8H, V8.16B + sxtl V14.4S, V9.4H + sxtl2 V15.4S, V9.8H + sxtl V16.4S, V18.4H + sxtl2 V17.4S, V18.8H + ushr V18.4S, V14.4S, #31 + addv S18, V18.4S + mov w10, V18.S[0] + and V18.16B, V14.16B, V1.16B + addv S18, V18.4S + mov w9, V18.S[0] + lsl x9, x9, #4 + ldr Q18, [x7, x9] + tbl V10.16B, {V10.16B}, V18.16B + str Q10, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + ushr V18.4S, V15.4S, #31 + addv S18, V18.4S + mov w10, V18.S[0] + and V18.16B, V15.16B, V1.16B + addv S18, V18.4S + mov w9, V18.S[0] + lsl x9, x9, #4 + ldr Q18, [x7, x9] + tbl V11.16B, {V11.16B}, V18.16B + str Q11, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + ushr V18.4S, V16.4S, #31 + addv S18, V18.4S + mov w10, V18.S[0] + and V18.16B, V16.16B, V1.16B + addv S18, V18.4S + mov w9, V18.S[0] + lsl x9, x9, #4 + ldr Q18, [x7, x9] + tbl V12.16B, {V12.16B}, V18.16B + str Q12, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + ushr V18.4S, V17.4S, #31 + addv S18, V18.4S + mov w10, V18.S[0] + and V18.16B, V17.16B, V1.16B + addv S18, V18.4S + mov w9, V18.S[0] + lsl x9, x9, #4 + ldr Q18, [x7, x9] + tbl V13.16B, {V13.16B}, V18.16B + str Q13, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + add x0, x0, #8 + sub w1, w1, #8 + b L_mldsa_extract_coeffs_eta2_neon_wide +L_mldsa_extract_coeffs_eta2_neon_tail + subs wzr, w1, #0 + beq L_mldsa_extract_coeffs_eta2_neon_done + subs wzr, w8, #0x100 + beq L_mldsa_extract_coeffs_eta2_neon_done + ldrb w11, [x0] + add x0, x0, #1 + sub w1, w1, #1 + and w12, w11, #15 + subs wzr, w12, #15 + bcs L_mldsa_extract_coeffs_eta2_neon_skip_lo + ldrsb w12, [x4, x12] + str w12, [x2] + add x2, x2, #4 + add x8, x8, #1 + subs wzr, w8, #0x100 + beq L_mldsa_extract_coeffs_eta2_neon_done +L_mldsa_extract_coeffs_eta2_neon_skip_lo + lsr w12, w11, #4 + subs wzr, w12, #15 + bcs L_mldsa_extract_coeffs_eta2_neon_skip_hi + ldrsb w12, [x4, x12] + str w12, [x2] + add x2, x2, #4 + add x8, x8, #1 +L_mldsa_extract_coeffs_eta2_neon_skip_hi + b L_mldsa_extract_coeffs_eta2_neon_tail +L_mldsa_extract_coeffs_eta2_neon_done + str w8, [x3] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_extract_coeffs_eta4_neon_map + DCB 0x04, 0x03, 0x02, 0x01, 0x00, 0xff, 0xfe, 0xfd + DCB 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_extract_coeffs_eta4_neon_lim + DCB 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09 + DCB 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09 + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_extract_coeffs_eta4_neon_bits + DCD 0x00000001, 0x00000002, 0x00000004, 0x00000008 + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_extract_coeffs_eta4_neon_idx + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x04, 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + DCB 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff + DCB 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x0c, 0x0d, 0x0e, 0x0f + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x04, 0x05, 0x06, 0x07, 0x0c, 0x0d, 0x0e, 0x0f + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + DCB 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff + DCB 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b + DCB 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff + DCB 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b + DCB 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + DCB 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_extract_coeffs_eta4_neon +mldsa_extract_coeffs_eta4_neon PROC + stp x29, x30, [sp, #-80]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + stp D14, D15, [x29, #64] + adrp x4, L_mldsa_extract_coeffs_eta4_neon_map + add x4, x4, L_mldsa_extract_coeffs_eta4_neon_map + adrp x5, L_mldsa_extract_coeffs_eta4_neon_lim + add x5, x5, L_mldsa_extract_coeffs_eta4_neon_lim + adrp x6, L_mldsa_extract_coeffs_eta4_neon_bits + add x6, x6, L_mldsa_extract_coeffs_eta4_neon_bits + adrp x7, L_mldsa_extract_coeffs_eta4_neon_idx + add x7, x7, L_mldsa_extract_coeffs_eta4_neon_idx + ldr Q2, [x4] + ldr Q0, [x5] + ldr Q1, [x6] + ldr w8, [x3] + add x2, x2, x8, lsl 2 +L_mldsa_extract_coeffs_eta4_neon_wide + subs wzr, w1, #8 + blt L_mldsa_extract_coeffs_eta4_neon_tail + subs wzr, w8, #0xf0 + bgt L_mldsa_extract_coeffs_eta4_neon_tail + ldr D3, [x0] + movi V18.16B, #15 + ushr V5.16B, V3.16B, #4 + and V4.16B, V3.16B, V18.16B + zip1 V3.16B, V4.16B, V5.16B + tbl V6.16B, {V2.16B}, V3.16B + cmhi V8.16B, V0.16B, V3.16B + sxtl V7.8H, V6.8B + sxtl2 V18.8H, V6.16B + sxtl V10.4S, V7.4H + sxtl2 V11.4S, V7.8H + sxtl V12.4S, V18.4H + sxtl2 V13.4S, V18.8H + sxtl V9.8H, V8.8B + sxtl2 V18.8H, V8.16B + sxtl V14.4S, V9.4H + sxtl2 V15.4S, V9.8H + sxtl V16.4S, V18.4H + sxtl2 V17.4S, V18.8H + ushr V18.4S, V14.4S, #31 + addv S18, V18.4S + mov w10, V18.S[0] + and V18.16B, V14.16B, V1.16B + addv S18, V18.4S + mov w9, V18.S[0] + lsl x9, x9, #4 + ldr Q18, [x7, x9] + tbl V10.16B, {V10.16B}, V18.16B + str Q10, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + ushr V18.4S, V15.4S, #31 + addv S18, V18.4S + mov w10, V18.S[0] + and V18.16B, V15.16B, V1.16B + addv S18, V18.4S + mov w9, V18.S[0] + lsl x9, x9, #4 + ldr Q18, [x7, x9] + tbl V11.16B, {V11.16B}, V18.16B + str Q11, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + ushr V18.4S, V16.4S, #31 + addv S18, V18.4S + mov w10, V18.S[0] + and V18.16B, V16.16B, V1.16B + addv S18, V18.4S + mov w9, V18.S[0] + lsl x9, x9, #4 + ldr Q18, [x7, x9] + tbl V12.16B, {V12.16B}, V18.16B + str Q12, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + ushr V18.4S, V17.4S, #31 + addv S18, V18.4S + mov w10, V18.S[0] + and V18.16B, V17.16B, V1.16B + addv S18, V18.4S + mov w9, V18.S[0] + lsl x9, x9, #4 + ldr Q18, [x7, x9] + tbl V13.16B, {V13.16B}, V18.16B + str Q13, [x2] + add x2, x2, x10, lsl 2 + add x8, x8, x10 + add x0, x0, #8 + sub w1, w1, #8 + b L_mldsa_extract_coeffs_eta4_neon_wide +L_mldsa_extract_coeffs_eta4_neon_tail + subs wzr, w1, #0 + beq L_mldsa_extract_coeffs_eta4_neon_done + subs wzr, w8, #0x100 + beq L_mldsa_extract_coeffs_eta4_neon_done + ldrb w11, [x0] + add x0, x0, #1 + sub w1, w1, #1 + and w12, w11, #15 + subs wzr, w12, #9 + bcs L_mldsa_extract_coeffs_eta4_neon_skip_lo + ldrsb w12, [x4, x12] + str w12, [x2] + add x2, x2, #4 + add x8, x8, #1 + subs wzr, w8, #0x100 + beq L_mldsa_extract_coeffs_eta4_neon_done +L_mldsa_extract_coeffs_eta4_neon_skip_lo + lsr w12, w11, #4 + subs wzr, w12, #9 + bcs L_mldsa_extract_coeffs_eta4_neon_skip_hi + ldrsb w12, [x4, x12] + str w12, [x2] + add x2, x2, #4 + add x8, x8, #1 +L_mldsa_extract_coeffs_eta4_neon_skip_hi + b L_mldsa_extract_coeffs_eta4_neon_tail +L_mldsa_extract_coeffs_eta4_neon_done + str w8, [x3] + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp D14, D15, [x29, #64] + ldp x29, x30, [sp], #0x50 + ret + ENDP + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_rej_uniform_mask + DCD 0x007fffff, 0x007fffff, 0x007fffff, 0x007fffff + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_rej_uniform_shuffle0 + DCB 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff + DCB 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_rej_uniform_shuffle1 + DCB 0x0c, 0x0d, 0x0e, 0xff, 0x0f, 0x10, 0x11, 0xff + DCB 0x12, 0x13, 0x14, 0xff, 0x15, 0x16, 0x17, 0xff + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_rej_uniform_bits + DCD 0x00000001, 0x00000002, 0x00000004, 0x00000008 + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 8 +L_mldsa_rej_uniform_indices + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x04, 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + DCB 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff + DCB 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x0c, 0x0d, 0x0e, 0x0f + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x04, 0x05, 0x06, 0x07, 0x0c, 0x0d, 0x0e, 0x0f + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + DCB 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff + DCB 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + DCB 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b + DCB 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff + DCB 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b + DCB 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff + DCB 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 + DCB 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT mldsa_rej_uniform_neon +mldsa_rej_uniform_neon PROC + stp x29, x30, [sp, #-64]! + add x29, sp, #0 + stp D8, D9, [x29, #16] + stp D10, D11, [x29, #32] + stp D12, D13, [x29, #48] + adrp x4, L_mldsa_rej_uniform_mask + add x4, x4, L_mldsa_rej_uniform_mask + adrp x5, L_mldsa_aarch64_q + add x5, x5, L_mldsa_aarch64_q + adrp x6, L_mldsa_rej_uniform_bits + add x6, x6, L_mldsa_rej_uniform_bits + adrp x7, L_mldsa_rej_uniform_indices + add x7, x7, L_mldsa_rej_uniform_indices + adrp x8, L_mldsa_rej_uniform_shuffle0 + add x8, x8, L_mldsa_rej_uniform_shuffle0 + adrp x9, L_mldsa_rej_uniform_shuffle1 + add x9, x9, L_mldsa_rej_uniform_shuffle1 + movz x15, #0xe001 + eor V1.16B, V1.16B, V1.16B + eor V12.16B, V12.16B, V12.16B + eor V13.16B, V13.16B, V13.16B + movk x15, #0x7f, lsl 16 + eor x14, x14, x14 + eor V10.16B, V10.16B, V10.16B + eor V11.16B, V11.16B, V11.16B + ldr Q0, [x4] + ldr Q3, [x5] + ldr Q2, [x6] + ldr Q4, [x8] + ldr Q5, [x9] + subs wzr, w1, #0 + beq L_mldsa_rej_uniform_done + subs wzr, w1, #16 + blt L_mldsa_rej_uniform_loop_2 +L_mldsa_rej_uniform_loop_8 + ldr Q6, [x2] + ldr D7, [x2, #16] + add x2, x2, #24 + tbl V8.16B, {V6.16B, V7.16B}, V4.16B + tbl V9.16B, {V6.16B, V7.16B}, V5.16B + and V6.16B, V8.16B, V0.16B + and V7.16B, V9.16B, V0.16B + cmgt V8.4S, V3.4S, V6.4S + cmgt V9.4S, V3.4S, V7.4S + ushr V12.4S, V8.4S, #31 + ushr V13.4S, V9.4S, #31 + addv S12, V12.4S + addv S13, V13.4S + mov x12, V12.D[0] + mov x13, V13.D[0] + and V10.16B, V8.16B, V2.16B + and V11.16B, V9.16B, V2.16B + addv S10, V10.4S + addv S11, V11.4S + mov w10, V10.S[0] + mov w11, V11.S[0] + lsl w10, w10, #4 + lsl w11, w11, #4 + ldr Q10, [x7, x10] + ldr Q11, [x7, x11] + tbl V8.16B, {V6.16B}, V10.16B + tbl V9.16B, {V7.16B}, V11.16B + str Q8, [x0] + add x0, x0, x12, lsl 2 + add x14, x14, x12 + str Q9, [x0] + add x0, x0, x13, lsl 2 + add x14, x14, x13 + subs w3, w3, #24 + beq L_mldsa_rej_uniform_done + sub w12, w1, w14 + subs x12, x12, #8 + blt L_mldsa_rej_uniform_loop_2 + b L_mldsa_rej_uniform_loop_8 +L_mldsa_rej_uniform_loop_2 + subs w12, w1, w14 + beq L_mldsa_rej_uniform_done + subs x12, x12, #2 + blt L_mldsa_rej_uniform_loop_lt_2 + ldr x4, [x2], #6 + lsr x5, x4, #24 + and x4, x4, #0x7fffff + and x5, x5, #0x7fffff + str w4, [x0] + add x6, x0, #4 + subs xzr, x4, x15 + csel x0, x6, x0, lt + cinc x14, x14, lt + str w5, [x0] + add x6, x0, #4 + subs xzr, x5, x15 + csel x0, x6, x0, lt + cinc x14, x14, lt + subs w3, w3, #6 + beq L_mldsa_rej_uniform_done + b L_mldsa_rej_uniform_loop_2 +L_mldsa_rej_uniform_loop_lt_2 + ldr x4, [x2], #6 + lsr x5, x4, #24 + and x4, x4, #0x7fffff + and x5, x5, #0x7fffff + str w4, [x0] + add x6, x0, #4 + subs xzr, x4, x15 + csel x0, x6, x0, lt + cinc x14, x14, lt + subs wzr, w1, w14 + beq L_mldsa_rej_uniform_done + str w5, [x0] + add x6, x0, #4 + subs xzr, x5, x15 + csel x0, x6, x0, lt + cinc x14, x14, lt + subs wzr, w1, w14 + beq L_mldsa_rej_uniform_done + subs w3, w3, #6 + beq L_mldsa_rej_uniform_done + b L_mldsa_rej_uniform_loop_lt_2 +L_mldsa_rej_uniform_done + mov x0, x14 + ldp D8, D9, [x29, #16] + ldp D10, D11, [x29, #32] + ldp D12, D13, [x29, #48] + ldp x29, x30, [sp], #0x40 + ret + ENDP + ENDIF + END diff --git a/wolfcrypt/src/port/arm/armv8-mldsa-asm_c.c b/wolfcrypt/src/port/arm/armv8-mldsa-asm_c.c new file mode 100644 index 00000000000..20a470513d3 --- /dev/null +++ b/wolfcrypt/src/port/arm/armv8-mldsa-asm_c.c @@ -0,0 +1,27660 @@ +/* armv8-mldsa-asm + * + * Copyright (C) 2006-2026 wolfSSL Inc. + * + * This file is part of wolfSSL. + * + * wolfSSL is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * wolfSSL is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1335, USA + */ + +/* Generated using (from wolfssl): + * cd ../scripts + * ruby ./mldsa/mldsa.rb arm64 \ + * ../wolfssl/wolfcrypt/src/port/arm/armv8-mldsa-asm.c + */ + +#define WC_FIPS_LL_CRYPTO +#define _WC_BUILDING_ARMV8_MLDSA_ASM_C + +#include +#include + +#ifdef WOLFSSL_ARMASM +#ifdef __aarch64__ +#ifdef WOLFSSL_ARMASM_INLINE +#include + +#ifdef WOLFSSL_HAVE_MLDSA +XALIGNED(8) static const word32 L_mldsa_aarch64_consts[] = { + 0x007fe001, 0x03802001, 0x00400000, 0x00000000, +}; + +XALIGNED(8) static const word32 L_mldsa_aarch64_zetas[] = { + 0xffff5c06, 0x000064f7, 0xffd83102, 0xfff81503, + 0x00039e44, 0xfff42118, 0xfff2a128, 0x00071e24, + 0x001bde2b, 0x0023e92b, 0xfffa84ad, 0xffe0147f, + 0x002f9a75, 0xffd3fb09, 0x002f7a49, 0x0028e527, + 0x00299658, 0x000fa070, 0xffef85a4, 0x0036b788, + 0xfff79d90, 0xffeeeaa0, 0x0027f968, 0xffdfd37b, + 0xffdfadd6, 0xffc51ae7, 0xffeaa4f7, 0xffcdfc98, + 0x001ad035, 0xffffb422, 0x003d3201, 0x000445c5, + 0x00294a67, 0x00017620, 0x002ef4cd, 0x0035dec5, + 0xffe6a503, 0xffc9302c, 0xffd947d4, 0x003bbeaf, + 0xffc51585, 0xffd18e7c, 0x00368a96, 0xffd43e41, + 0x00360400, 0xfffb6a4d, 0x0023d69c, 0xfff7c55d, + 0xffe6123d, 0xffe6ead6, 0x00357e1e, 0xffc5af59, + 0x0035843f, 0xffdf5617, 0xffe7945c, 0x0038738c, + 0x000c63a8, 0x00081b9a, 0x000e8f76, 0x003b3853, + 0x003b8534, 0xffd8fc30, 0x001f9d54, 0xffd54f2d, + 0xffc406e5, 0xffc406e5, 0xffe8ac81, 0xffe8ac81, + 0xffc7e1cf, 0xffc7e1cf, 0xffd19819, 0xffd19819, + 0xffe9d65d, 0xffe9d65d, 0x003509ee, 0x003509ee, + 0x002135c7, 0x002135c7, 0xffe7cfbb, 0xffe7cfbb, + 0xffeccf75, 0xffeccf75, 0x001d9772, 0x001d9772, + 0xffc1b072, 0xffc1b072, 0xfff0bcf6, 0xfff0bcf6, + 0xffcf5280, 0xffcf5280, 0xffcfd2ae, 0xffcfd2ae, + 0xffc890e0, 0xffc890e0, 0x0001efca, 0x0001efca, + 0x003410f2, 0x003410f2, 0xfff0fe85, 0xfff0fe85, + 0x0020c638, 0x0020c638, 0x00296e9f, 0x00296e9f, + 0xffd2b7a3, 0xffd2b7a3, 0xffc7a44b, 0xffc7a44b, + 0xfff9ba6d, 0xfff9ba6d, 0xffda3409, 0xffda3409, + 0xfff5c282, 0xfff5c282, 0xffed4113, 0xffed4113, + 0xffffa63b, 0xffffa63b, 0xffec09f7, 0xffec09f7, + 0xfffa2bdd, 0xfffa2bdd, 0x001495d4, 0x001495d4, + 0x001c4563, 0x001c4563, 0xffea2c62, 0xffea2c62, + 0xffccfbe9, 0xffccfbe9, 0x00040af0, 0x00040af0, + 0x0007c417, 0x0007c417, 0x002f4588, 0x002f4588, + 0x0000ad00, 0x0000ad00, 0xffef36be, 0xffef36be, + 0x000dcd44, 0x000dcd44, 0x003c675a, 0x003c675a, + 0xffc72bca, 0xffc72bca, 0xffffde7e, 0xffffde7e, + 0x00193948, 0x00193948, 0xffce69c0, 0xffce69c0, + 0x0024756c, 0x0024756c, 0xfffcc7df, 0xfffcc7df, + 0x000b98a1, 0x000b98a1, 0xffebe808, 0xffebe808, + 0x0002e46c, 0x0002e46c, 0xffc9c808, 0xffc9c808, + 0x003036c2, 0x003036c2, 0xffe3bff6, 0xffe3bff6, + 0xffdb3c93, 0xffdb3c93, 0xfffd4ae0, 0xfffd4ae0, + 0x00141305, 0x00141305, 0x00147792, 0x00147792, + 0x00139e25, 0x00139e25, 0xffe7d0e0, 0xffe7d0e0, + 0xfff39944, 0xfff39944, 0xffea0802, 0xffea0802, + 0xffd1eea2, 0xffd1eea2, 0xffc4c79c, 0xffc4c79c, + 0xffc8a057, 0xffc8a057, 0x003a97d9, 0x003a97d9, + 0x001fea93, 0x0033ff5a, 0x002358d4, 0x003a41f8, + 0xffccff72, 0x00223dfb, 0xffdaab9f, 0xffc9a422, + 0x000412f5, 0x00252587, 0xffed24f0, 0x00359b5d, + 0xffca48a0, 0xffc6a2fc, 0xffedbb56, 0xffcf45de, + 0x000dbe5e, 0x001c5e1a, 0x000de0e6, 0x000c7f5a, + 0x00078f83, 0xffe7628a, 0xffff5704, 0xfff806fc, + 0xfff60021, 0xffd05af6, 0x001f0084, 0x0030ef86, + 0xffc9b97d, 0xfff7fcd6, 0xfff44592, 0xffc921c2, + 0x00053919, 0x0004610c, 0xffdacd41, 0x003eb01b, + 0x003472e7, 0xffcd003b, 0x001a7cc7, 0x00031924, + 0x002b5ee5, 0x00291199, 0xffd87a3a, 0x00134d71, + 0x003de11c, 0x00130984, 0x0025f051, 0x00185a46, + 0xffc68518, 0x001314be, 0x00283891, 0xffc9db90, + 0xffd25089, 0x001c853f, 0x001d0b4b, 0xffeff6a6, + 0xffeba8be, 0x0012e11b, 0xffcd5e3e, 0xffea2d2f, + 0xfff91de4, 0x001406c7, 0x00327283, 0xffe20d6e, + 0xffec7953, 0x001d4099, 0xffd92578, 0xffeb05ad, + 0x0016e405, 0x000bdbe7, 0x00221de8, 0x0033f8cf, + 0xfff7b934, 0xffd4ca0c, 0xffe67ff8, 0xffe3d157, + 0xffd8911b, 0xffc72c12, 0x000910d8, 0xffc65e1f, + 0xffe14658, 0x00251d8b, 0x002573b7, 0xfffd7c8f, + 0x001ddd98, 0x00336898, 0x0002d4bb, 0xffed93a7, + 0xffcf6cbe, 0x00027c1c, 0x0018aa08, 0x002dfd71, + 0x000c5ca5, 0x0019379a, 0xffc7a167, 0xffe48c3d, + 0xffd1a13c, 0x0035c539, 0x003b0115, 0x00041dc0, + 0x0021c4f7, 0xfff11bf4, 0x001a35e7, 0x0007340e, + 0xfff97d45, 0x001a4cd0, 0xffe47cae, 0x001d2668, + 0xffe68e98, 0xffef2633, 0xfffc05da, 0xffc57fdb, + 0xffd32764, 0xffdde1af, 0xfff993dd, 0xffdd1d09, + 0x0002cc93, 0xfff11805, 0x00189c2a, 0xffc9e5a9, + 0xfff78a50, 0x003bcf2c, 0xffff434e, 0xffeb36df, + 0x003c15ca, 0x00155e68, 0xfff316b6, 0x001e29ce, +}; + +XALIGNED(8) static const word32 L_mldsa_aarch64_zetas_qinv[] = { + 0x00801c06, 0x6d1f44f7, 0x8cf87102, 0x8d187503, + 0x61cc1e44, 0x58172118, 0x6017a128, 0x61cb9e24, + 0x12613e2b, 0x93c9492b, 0xae1024ad, 0xbeeff47f, + 0x8cfe3a75, 0x1eb51b09, 0xeef89a49, 0x254dc527, + 0x66f49658, 0x7c1da070, 0xaea405a4, 0x3327b788, + 0x6ba99d90, 0x0d42eaa0, 0xeb54f968, 0x28cf337b, + 0x629a6dd6, 0xcba1fae7, 0xb50984f7, 0xd360fc98, + 0x13a17035, 0x6d83f422, 0xa9fd5201, 0xba3ce5c5, + 0x91f62a67, 0x9ec57620, 0xac4894cd, 0x6d8e7ec5, + 0x5f070503, 0xbfceb02c, 0x8ed3c7d4, 0xdc919eaf, + 0xf3f5b585, 0xe3a10e7c, 0xde894a96, 0x6b1c5e41, + 0xc0b60400, 0x7ac50a4d, 0x9cf7569c, 0xbe23655d, + 0x97adb23d, 0xca41aad6, 0x18f93e1e, 0x6d30cf59, + 0x8d3d643f, 0x3b223617, 0x3473145c, 0x78a9f38c, + 0x588163a8, 0x9e7b5b9a, 0xeefd4f76, 0x89c59853, + 0xa6e20534, 0xc75efc30, 0x99ca1d54, 0xc73aef2d, + 0xa220a6e5, 0xa220a6e5, 0xd8f8cc81, 0xd8f8cc81, + 0x5081c1cf, 0x5081c1cf, 0x8a54b819, 0x8a54b819, + 0x8035765d, 0x8035765d, 0x6272c9ee, 0x6272c9ee, + 0x5f5a15c7, 0x5f5a15c7, 0x085f2fbb, 0x085f2fbb, + 0xb35b6f75, 0xb35b6f75, 0xc20bd772, 0xc20bd772, + 0xc4cff072, 0xc4cff072, 0x748f7cf6, 0x748f7cf6, + 0xaa1f5280, 0xaa1f5280, 0x5b2592ae, 0x5b2592ae, + 0x21e490e0, 0x21e490e0, 0x80fb2fca, 0x80fb2fca, + 0xd15250f2, 0xd15250f2, 0xf1419e85, 0xf1419e85, + 0xdce7c638, 0xdce7c638, 0x5a7d4e9f, 0x5a7d4e9f, + 0x114717a3, 0x114717a3, 0xfad1044b, 0xfad1044b, + 0xb4c75a6d, 0xb4c75a6d, 0x65db5409, 0x65db5409, + 0x7f460282, 0x7f460282, 0x6a8fa113, 0x6a8fa113, + 0xc347063b, 0xc347063b, 0x61aae9f7, 0x61aae9f7, + 0xcaf5cbdd, 0xcaf5cbdd, 0xf8cf15d4, 0xf8cf15d4, + 0x6348a563, 0x6348a563, 0x9c766c62, 0x9c766c62, + 0x4eca1be9, 0x4eca1be9, 0xc9620af0, 0xc9620af0, + 0x490aa417, 0x490aa417, 0x44e04588, 0x44e04588, + 0x95a0ad00, 0x95a0ad00, 0x7fc6f6be, 0x7fc6f6be, + 0x27b64d44, 0x27b64d44, 0x4827a75a, 0x4827a75a, + 0x28406bca, 0x28406bca, 0xb4cf9e7e, 0xb4cf9e7e, + 0xa3423948, 0xa3423948, 0xed0669c0, 0xed0669c0, + 0x88d1f56c, 0x88d1f56c, 0x2578a7df, 0x2578a7df, + 0xa69fb8a1, 0xa69fb8a1, 0x98ece808, 0x98ece808, + 0xd690646c, 0xd690646c, 0x54cac808, 0x54cac808, + 0xae0876c2, 0xae0876c2, 0x54e27ff6, 0x54e27ff6, + 0x69ed9c93, 0x69ed9c93, 0xb9594ae0, 0xb9594ae0, + 0x13f4b305, 0x13f4b305, 0x0e06b792, 0x0e06b792, + 0xf5583e25, 0xf5583e25, 0x0a03d0e0, 0x0a03d0e0, + 0xe11c1944, 0xe11c1944, 0x47ea4802, 0x47ea4802, + 0x74a62ea2, 0x74a62ea2, 0x3ab8479c, 0x3ab8479c, + 0x44538057, 0x44538057, 0xcab5b7d9, 0xcab5b7d9, + 0xfff24a93, 0x3b1f3f5a, 0x513dd8d4, 0x2c7941f8, + 0xaebb3f72, 0x36619dfb, 0x01ce8b9f, 0xab4de422, + 0xdbe2b2f5, 0xfd560587, 0xec8b24f0, 0x79213b5d, + 0x78de48a0, 0x462622fc, 0x64587b56, 0x718b05de, + 0x00d97e5e, 0xe6df9e1a, 0xe12aa0e6, 0x4af7bf5a, + 0x3c77ef83, 0xcf38a28a, 0x78dfd704, 0x72d786fc, + 0x337a2021, 0x682f1af6, 0xae2f8084, 0x7321af86, + 0x6c79597d, 0xec92bcd6, 0x07a68592, 0x4b0161c2, + 0x7ea85919, 0x3625e10c, 0xbd02ed41, 0x34c2101b, + 0xb71152e7, 0x6e54603b, 0x08335cc7, 0x61279924, + 0x8d87fee5, 0xb9dc3199, 0xda1fba3a, 0x75416d71, + 0x9e61611c, 0xaf438984, 0xd9b01051, 0x00611a46, + 0xa4698518, 0xfbaad4be, 0x02ba5891, 0xb33bdb90, + 0x29637089, 0xed44653f, 0x28066b4b, 0x43c4b6a6, + 0x0e0368be, 0x3ab6411b, 0x84951e3e, 0x6a100d2f, + 0xc1b59de4, 0x396ce6c7, 0x1902d283, 0x428fcd6e, + 0x3196d953, 0xbfb06099, 0x48882578, 0x3e20a5ad, + 0xee178405, 0x2408bbe7, 0xefdf1de8, 0x53cdd8cf, + 0x2d1e3934, 0xc3164a0c, 0xb3e57ff8, 0x2a8eb157, + 0xf07bf11b, 0x24496c12, 0x162410d8, 0x380a3e1f, + 0x5cac4658, 0x0a567d8b, 0xaf1c53b7, 0xa40f5c8f, + 0x4fd0dd98, 0x81466898, 0xe91a34bb, 0x7ae273a7, + 0x86672cbe, 0xb185fc1c, 0x3159aa08, 0xcb5c1d71, + 0xcd20fca5, 0xc20c779a, 0xdc748167, 0x66ec2c3d, + 0x85f9213c, 0x005ce539, 0x29dda115, 0xa3bc1dc0, + 0x9940a4f7, 0xf96f9bf4, 0xef5715e7, 0x1788f40e, + 0xa1221d45, 0x21b44cd0, 0xf07a3cae, 0x10ea2668, + 0xe5b98e98, 0x97358633, 0xfbb745da, 0x2e40dfdb, + 0x42bfa764, 0xa093c1af, 0xb7f533dd, 0x42fe3d09, + 0x5c152c93, 0x3471b805, 0xa69ddc2a, 0x0bff05a9, + 0x09418a50, 0x94214f2c, 0x7969034e, 0x734716df, + 0xc5f555ca, 0x17e25e68, 0xdfc9d6b6, 0x1657e9ce, +}; + +XALIGNED(8) static const word32 L_mldsa_aarch64_zetas_inv[] = { + 0xffe1d632, 0x000ce94a, 0xffeaa198, 0xffc3ea36, + 0x0014c921, 0x0000bcb2, 0xffc430d4, 0x000875b0, + 0x00361a57, 0xffe763d6, 0x000ee7fb, 0xfffd336d, + 0x0022e2f7, 0x00066c23, 0x00221e51, 0x002cd89c, + 0x003a8025, 0x0003fa26, 0x0010d9cd, 0x00197168, + 0xffe2d998, 0x001b8352, 0xffe5b330, 0x000682bb, + 0xfff8cbf2, 0xffe5ca19, 0x000ee40c, 0xffde3b09, + 0xfffbe240, 0xffc4feeb, 0xffca3ac7, 0x002e5ec4, + 0x001b73c3, 0x00385e99, 0xffe6c866, 0xfff3a35b, + 0xffd2028f, 0xffe755f8, 0xfffd83e4, 0x00309342, + 0x00126c59, 0xfffd2b45, 0xffcc9768, 0xffe22268, + 0x00028371, 0xffda8c49, 0xffdae275, 0x001eb9a8, + 0x0039a1e1, 0xfff6ef28, 0x0038d3ee, 0x00276ee5, + 0x001c2ea9, 0x00198008, 0x002b35f4, 0x000846cc, + 0xffcc0731, 0xffdde218, 0xfff42419, 0xffe91bfb, + 0x0014fa53, 0x0026da88, 0xffe2bf67, 0x001386ad, + 0x001df292, 0xffcd8d7d, 0xffebf939, 0x0006e21c, + 0x0015d2d1, 0x0032a1c2, 0xffed1ee5, 0x00145742, + 0x0010095a, 0xffe2f4b5, 0xffe37ac1, 0x002daf77, + 0x00362470, 0xffd7c76f, 0xffeceb42, 0x00397ae8, + 0xffe7a5ba, 0xffda0faf, 0xffecf67c, 0xffc21ee4, + 0xffecb28f, 0x002785c6, 0xffd6ee67, 0xffd4a11b, + 0xfffce6dc, 0xffe58339, 0x0032ffc5, 0xffcb8d19, + 0xffc14fe5, 0x002532bf, 0xfffb9ef4, 0xfffac6e7, + 0x0036de3e, 0x000bba6e, 0x0008032a, 0x00364683, + 0xffcf107a, 0xffe0ff7c, 0x002fa50a, 0x0009ffdf, + 0x0007f904, 0x0000a8fc, 0x00189d76, 0xfff8707d, + 0xfff380a6, 0xfff21f1a, 0xffe3a1e6, 0xfff241a2, + 0x0030ba22, 0x001244aa, 0x00395d04, 0x0035b760, + 0xffca64a3, 0x0012db10, 0xffdada79, 0xfffbed0b, + 0x00365bde, 0x00255461, 0xffddc205, 0x0033008e, + 0xffc5be08, 0xffdca72c, 0xffcc00a6, 0xffe0156d, + 0xffc56827, 0xffc56827, 0x00375fa9, 0x00375fa9, + 0x003b3864, 0x003b3864, 0x002e115e, 0x002e115e, + 0x0015f7fe, 0x0015f7fe, 0x000c66bc, 0x000c66bc, + 0x00182f20, 0x00182f20, 0xffec61db, 0xffec61db, + 0xffeb886e, 0xffeb886e, 0xffebecfb, 0xffebecfb, + 0x0002b520, 0x0002b520, 0x0024c36d, 0x0024c36d, + 0x001c400a, 0x001c400a, 0xffcfc93e, 0xffcfc93e, + 0x003637f8, 0x003637f8, 0xfffd1b94, 0xfffd1b94, + 0x001417f8, 0x001417f8, 0xfff4675f, 0xfff4675f, + 0x00033821, 0x00033821, 0xffdb8a94, 0xffdb8a94, + 0x00319640, 0x00319640, 0xffe6c6b8, 0xffe6c6b8, + 0x00002182, 0x00002182, 0x0038d436, 0x0038d436, + 0xffc398a6, 0xffc398a6, 0xfff232bc, 0xfff232bc, + 0x0010c942, 0x0010c942, 0xffff5300, 0xffff5300, + 0xffd0ba78, 0xffd0ba78, 0xfff83be9, 0xfff83be9, + 0xfffbf510, 0xfffbf510, 0x00330417, 0x00330417, + 0x0015d39e, 0x0015d39e, 0xffe3ba9d, 0xffe3ba9d, + 0xffeb6a2c, 0xffeb6a2c, 0x0005d423, 0x0005d423, + 0x0013f609, 0x0013f609, 0x000059c5, 0x000059c5, + 0x0012beed, 0x0012beed, 0x000a3d7e, 0x000a3d7e, + 0x0025cbf7, 0x0025cbf7, 0x00064593, 0x00064593, + 0x00385bb5, 0x00385bb5, 0x002d485d, 0x002d485d, + 0xffd69161, 0xffd69161, 0xffdf39c8, 0xffdf39c8, + 0x000f017b, 0x000f017b, 0xffcbef0e, 0xffcbef0e, + 0xfffe1036, 0xfffe1036, 0x00376f20, 0x00376f20, + 0x00302d52, 0x00302d52, 0x0030ad80, 0x0030ad80, + 0x000f430a, 0x000f430a, 0x003e4f8e, 0x003e4f8e, + 0xffe2688e, 0xffe2688e, 0x0013308b, 0x0013308b, + 0x00183045, 0x00183045, 0xffdeca39, 0xffdeca39, + 0xffcaf612, 0xffcaf612, 0x001629a3, 0x001629a3, + 0x002e67e7, 0x002e67e7, 0x00381e31, 0x00381e31, + 0x0017537f, 0x0017537f, 0x003bf91b, 0x003bf91b, + 0x002ab0d3, 0xffe062ac, 0x002703d0, 0xffc47acc, + 0xffc4c7ad, 0xfff1708a, 0xfff7e466, 0xfff39c58, + 0xffc78c74, 0x00186ba4, 0x0020a9e9, 0xffca7bc1, + 0x003a50a7, 0xffca81e2, 0x0019152a, 0x0019edc3, + 0x00083aa3, 0xffdc2964, 0x000495b3, 0xffc9fc00, + 0x002bc1bf, 0xffc9756a, 0x002e7184, 0x003aea7b, + 0xffc44151, 0x0026b82c, 0x0036cfd4, 0x00195afd, + 0xffca213b, 0xffd10b33, 0xfffe89e0, 0xffd6b599, + 0xfffbba3b, 0xffc2cdff, 0x00004bde, 0xffe52fcb, + 0x00320368, 0x00155b09, 0x003ae519, 0x0020522a, + 0x00202c85, 0xffd80698, 0x00111560, 0x00086270, + 0xffc94878, 0x00107a5c, 0xfff05f90, 0xffd669a8, + 0xffd71ad9, 0xffd085b7, 0x002c04f7, 0xffd0658b, + 0x001feb81, 0x00057b53, 0xffdc16d5, 0xffe421d5, + 0xfff8e1dc, 0x000d5ed8, 0x000bdee8, 0xfffc61bc, + 0x0007eafd, 0x0027cefe, 0x003caa21, 0x0000a3fa, +}; + +XALIGNED(8) static const word32 L_mldsa_aarch64_zetas_inv_qinv[] = { + 0xe9a81632, 0x2036294a, 0xe81da198, 0x3a0aaa36, + 0x8cb8e921, 0x8696fcb2, 0x6bdeb0d4, 0xf6be75b0, + 0xf400fa57, 0x596223d6, 0xcb8e47fb, 0xa3ead36d, + 0xbd01c2f7, 0x480acc23, 0x5f6c3e51, 0xbd40589c, + 0xd1bf2025, 0x0448ba26, 0x68ca79cd, 0x1a467168, + 0xef15d998, 0x0f85c352, 0xde4bb330, 0x5edde2bb, + 0xe8770bf2, 0x10a8ea19, 0x0690640c, 0x66bf5b09, + 0x5c43e240, 0xd6225eeb, 0xffa31ac7, 0x7a06dec4, + 0x9913d3c3, 0x238b7e99, 0x3df38866, 0x32df035b, + 0x34a3e28f, 0xcea655f8, 0x4e7a03e4, 0x7998d342, + 0x851d8c59, 0x16e5cb45, 0x7eb99768, 0xb02f2268, + 0x5bf0a371, 0x50e3ac49, 0xf5a98275, 0xa353b9a8, + 0xc7f5c1e1, 0xe9dbef28, 0xdbb693ee, 0x0f840ee5, + 0xd5714ea9, 0x4c1a8008, 0x3ce9b5f4, 0xd2e1c6cc, + 0xac322731, 0x1020e218, 0xdbf74419, 0x11e87bfb, + 0xc1df5a53, 0xb777da88, 0x404f9f67, 0xce6926ad, + 0xbd703292, 0xe6fd2d7d, 0xc6931939, 0x3e4a621c, + 0x95eff2d1, 0x7b6ae1c2, 0xc549bee5, 0xf1fc9742, + 0xbc3b495a, 0xd7f994b5, 0x12bb9ac1, 0xd69c8f77, + 0x4cc42470, 0xfd45a76f, 0x04552b42, 0x5b967ae8, + 0xff9ee5ba, 0x264fefaf, 0x50bc767c, 0x619e9ee4, + 0x8abe928f, 0x25e045c6, 0x4623ce67, 0x7278011b, + 0x9ed866dc, 0xf7cca339, 0x91ab9fc5, 0x48eead19, + 0xcb3defe5, 0x42fd12bf, 0xc9da1ef4, 0x8157a6e7, + 0xb4fe9e3e, 0xf8597a6e, 0x136d432a, 0x9386a683, + 0x8cde507a, 0x51d07f7c, 0x97d0e50a, 0xcc85dfdf, + 0x8d287904, 0x872028fc, 0x30c75d76, 0xc388107d, + 0xb50840a6, 0x1ed55f1a, 0x192061e6, 0xff2681a2, + 0x8e74fa22, 0x9ba784aa, 0xb9d9dd04, 0x8721b760, + 0x86dec4a3, 0x1374db10, 0x02a9fa79, 0x241d4d0b, + 0x54b21bde, 0xfe317461, 0xc99e6205, 0x5144c08e, + 0xd386be08, 0xaec2272c, 0xc4e0c0a6, 0x000db56d, + 0x354a4827, 0x354a4827, 0xbbac7fa9, 0xbbac7fa9, + 0xc547b864, 0xc547b864, 0x8b59d15e, 0x8b59d15e, + 0xb815b7fe, 0xb815b7fe, 0x1ee3e6bc, 0x1ee3e6bc, + 0xf5fc2f20, 0xf5fc2f20, 0x0aa7c1db, 0x0aa7c1db, + 0xf1f9486e, 0xf1f9486e, 0xec0b4cfb, 0xec0b4cfb, + 0x46a6b520, 0x46a6b520, 0x9612636d, 0x9612636d, + 0xab1d800a, 0xab1d800a, 0x51f7893e, 0x51f7893e, + 0xab3537f8, 0xab3537f8, 0x296f9b94, 0x296f9b94, + 0x671317f8, 0x671317f8, 0x5960475f, 0x5960475f, + 0xda875821, 0xda875821, 0x772e0a94, 0x772e0a94, + 0x12f99640, 0x12f99640, 0x5cbdc6b8, 0x5cbdc6b8, + 0x4b306182, 0x4b306182, 0xd7bf9436, 0xd7bf9436, + 0xb7d858a6, 0xb7d858a6, 0xd849b2bc, 0xd849b2bc, + 0x80390942, 0x80390942, 0x6a5f5300, 0x6a5f5300, + 0xbb1fba78, 0xbb1fba78, 0xb6f55be9, 0xb6f55be9, + 0x369df510, 0x369df510, 0xb135e417, 0xb135e417, + 0x6389939e, 0x6389939e, 0x9cb75a9d, 0x9cb75a9d, + 0x0730ea2c, 0x0730ea2c, 0x350a3423, 0x350a3423, + 0x9e551609, 0x9e551609, 0x3cb8f9c5, 0x3cb8f9c5, + 0x95705eed, 0x95705eed, 0x80b9fd7e, 0x80b9fd7e, + 0x9a24abf7, 0x9a24abf7, 0x4b38a593, 0x4b38a593, + 0x052efbb5, 0x052efbb5, 0xeeb8e85d, 0xeeb8e85d, + 0xa582b161, 0xa582b161, 0x231839c8, 0x231839c8, + 0x0ebe617b, 0x0ebe617b, 0x2eadaf0e, 0x2eadaf0e, + 0x7f04d036, 0x7f04d036, 0xde1b6f20, 0xde1b6f20, + 0xa4da6d52, 0xa4da6d52, 0x55e0ad80, 0x55e0ad80, + 0x8b70830a, 0x8b70830a, 0x3b300f8e, 0x3b300f8e, + 0x3df4288e, 0x3df4288e, 0x4ca4908b, 0x4ca4908b, + 0xf7a0d045, 0xf7a0d045, 0xa0a5ea39, 0xa0a5ea39, + 0x9d8d3612, 0x9d8d3612, 0x7fca89a3, 0x7fca89a3, + 0x75ab47e7, 0x75ab47e7, 0xaf7e3e31, 0xaf7e3e31, + 0x2707337f, 0x2707337f, 0x5ddf591b, 0x5ddf591b, + 0x38c510d3, 0x6635e2ac, 0x38a103d0, 0x591dfacc, + 0x763a67ad, 0x1102b08a, 0x6184a466, 0xa77e9c58, + 0x87560c74, 0xcb8ceba4, 0xc4ddc9e9, 0x72c29bc1, + 0x92cf30a7, 0xe706c1e2, 0x35be552a, 0x68524dc3, + 0x41dc9aa3, 0x6308a964, 0x853af5b3, 0x3f49fc00, + 0x94e3a1bf, 0x2176b56a, 0x1c5ef184, 0x0c0a4a7b, + 0x236e6151, 0x712c382c, 0x40314fd4, 0xa0f8fafd, + 0x9271813b, 0x53b76b33, 0x613a89e0, 0x6e09d599, + 0x45c31a3b, 0x5602adff, 0x927c0bde, 0xec5e8fcb, + 0x2c9f0368, 0x4af67b09, 0x345e0519, 0x9d65922a, + 0xd730cc85, 0x14ab0698, 0xf2bd1560, 0x94566270, + 0xccd84878, 0x515bfa5c, 0x83e25f90, 0x990b69a8, + 0xdab23ad9, 0x110765b7, 0xe14ae4f7, 0x7301c58b, + 0x41100b81, 0x51efdb53, 0x6c36b6d5, 0xed9ec1d5, + 0x9e3461dc, 0x9fe85ed8, 0xa7e8dee8, 0x9e33e1bc, + 0x72e78afd, 0x73078efe, 0x0900ca21, 0xff7fe3fa, +}; + +void mldsa_ntt_neon(sword32* r) +{ + const word32* zetas = L_mldsa_aarch64_zetas; + const word32* qinv = L_mldsa_aarch64_zetas_qinv; + const word32* consts = L_mldsa_aarch64_consts; + __asm__ __volatile__ ( + "ldr q4, [%[consts]]\n\t" + "ldr q5, [%x[r]]\n\t" + "ldr q6, [%x[r], #64]\n\t" + "ldr q7, [%x[r], #128]\n\t" + "ldr q8, [%x[r], #192]\n\t" + "ldr q9, [%x[r], #256]\n\t" + "ldr q10, [%x[r], #320]\n\t" + "ldr q11, [%x[r], #384]\n\t" + "ldr q12, [%x[r], #448]\n\t" + "ldr q13, [%x[r], #512]\n\t" + "ldr q14, [%x[r], #576]\n\t" + "ldr q15, [%x[r], #640]\n\t" + "ldr q16, [%x[r], #704]\n\t" + "ldr q17, [%x[r], #768]\n\t" + "ldr q18, [%x[r], #832]\n\t" + "ldr q19, [%x[r], #896]\n\t" + "ldr q20, [%x[r], #960]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r]]\n\t" + "str q6, [%x[r], #64]\n\t" + "str q7, [%x[r], #128]\n\t" + "str q8, [%x[r], #192]\n\t" + "str q9, [%x[r], #256]\n\t" + "str q10, [%x[r], #320]\n\t" + "str q11, [%x[r], #384]\n\t" + "str q12, [%x[r], #448]\n\t" + "str q13, [%x[r], #512]\n\t" + "str q14, [%x[r], #576]\n\t" + "str q15, [%x[r], #640]\n\t" + "str q16, [%x[r], #704]\n\t" + "str q17, [%x[r], #768]\n\t" + "str q18, [%x[r], #832]\n\t" + "str q19, [%x[r], #896]\n\t" + "str q20, [%x[r], #960]\n\t" + "ldr q5, [%x[r], #16]\n\t" + "ldr q6, [%x[r], #80]\n\t" + "ldr q7, [%x[r], #144]\n\t" + "ldr q8, [%x[r], #208]\n\t" + "ldr q9, [%x[r], #272]\n\t" + "ldr q10, [%x[r], #336]\n\t" + "ldr q11, [%x[r], #400]\n\t" + "ldr q12, [%x[r], #464]\n\t" + "ldr q13, [%x[r], #528]\n\t" + "ldr q14, [%x[r], #592]\n\t" + "ldr q15, [%x[r], #656]\n\t" + "ldr q16, [%x[r], #720]\n\t" + "ldr q17, [%x[r], #784]\n\t" + "ldr q18, [%x[r], #848]\n\t" + "ldr q19, [%x[r], #912]\n\t" + "ldr q20, [%x[r], #976]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r], #16]\n\t" + "str q6, [%x[r], #80]\n\t" + "str q7, [%x[r], #144]\n\t" + "str q8, [%x[r], #208]\n\t" + "str q9, [%x[r], #272]\n\t" + "str q10, [%x[r], #336]\n\t" + "str q11, [%x[r], #400]\n\t" + "str q12, [%x[r], #464]\n\t" + "str q13, [%x[r], #528]\n\t" + "str q14, [%x[r], #592]\n\t" + "str q15, [%x[r], #656]\n\t" + "str q16, [%x[r], #720]\n\t" + "str q17, [%x[r], #784]\n\t" + "str q18, [%x[r], #848]\n\t" + "str q19, [%x[r], #912]\n\t" + "str q20, [%x[r], #976]\n\t" + "ldr q5, [%x[r], #32]\n\t" + "ldr q6, [%x[r], #96]\n\t" + "ldr q7, [%x[r], #160]\n\t" + "ldr q8, [%x[r], #224]\n\t" + "ldr q9, [%x[r], #288]\n\t" + "ldr q10, [%x[r], #352]\n\t" + "ldr q11, [%x[r], #416]\n\t" + "ldr q12, [%x[r], #480]\n\t" + "ldr q13, [%x[r], #544]\n\t" + "ldr q14, [%x[r], #608]\n\t" + "ldr q15, [%x[r], #672]\n\t" + "ldr q16, [%x[r], #736]\n\t" + "ldr q17, [%x[r], #800]\n\t" + "ldr q18, [%x[r], #864]\n\t" + "ldr q19, [%x[r], #928]\n\t" + "ldr q20, [%x[r], #992]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r], #32]\n\t" + "str q6, [%x[r], #96]\n\t" + "str q7, [%x[r], #160]\n\t" + "str q8, [%x[r], #224]\n\t" + "str q9, [%x[r], #288]\n\t" + "str q10, [%x[r], #352]\n\t" + "str q11, [%x[r], #416]\n\t" + "str q12, [%x[r], #480]\n\t" + "str q13, [%x[r], #544]\n\t" + "str q14, [%x[r], #608]\n\t" + "str q15, [%x[r], #672]\n\t" + "str q16, [%x[r], #736]\n\t" + "str q17, [%x[r], #800]\n\t" + "str q18, [%x[r], #864]\n\t" + "str q19, [%x[r], #928]\n\t" + "str q20, [%x[r], #992]\n\t" + "ldr q5, [%x[r], #48]\n\t" + "ldr q6, [%x[r], #112]\n\t" + "ldr q7, [%x[r], #176]\n\t" + "ldr q8, [%x[r], #240]\n\t" + "ldr q9, [%x[r], #304]\n\t" + "ldr q10, [%x[r], #368]\n\t" + "ldr q11, [%x[r], #432]\n\t" + "ldr q12, [%x[r], #496]\n\t" + "ldr q13, [%x[r], #560]\n\t" + "ldr q14, [%x[r], #624]\n\t" + "ldr q15, [%x[r], #688]\n\t" + "ldr q16, [%x[r], #752]\n\t" + "ldr q17, [%x[r], #816]\n\t" + "ldr q18, [%x[r], #880]\n\t" + "ldr q19, [%x[r], #944]\n\t" + "ldr q20, [%x[r], #1008]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r], #48]\n\t" + "str q6, [%x[r], #112]\n\t" + "str q7, [%x[r], #176]\n\t" + "str q8, [%x[r], #240]\n\t" + "str q9, [%x[r], #304]\n\t" + "str q10, [%x[r], #368]\n\t" + "str q11, [%x[r], #432]\n\t" + "str q12, [%x[r], #496]\n\t" + "str q13, [%x[r], #560]\n\t" + "str q14, [%x[r], #624]\n\t" + "str q15, [%x[r], #688]\n\t" + "str q16, [%x[r], #752]\n\t" + "str q17, [%x[r], #816]\n\t" + "str q18, [%x[r], #880]\n\t" + "str q19, [%x[r], #944]\n\t" + "str q20, [%x[r], #1008]\n\t" + "ldp q5, q6, [%x[r]]\n\t" + "ldp q7, q8, [%x[r], #32]\n\t" + "ldp q9, q10, [%x[r], #64]\n\t" + "ldp q11, q12, [%x[r], #96]\n\t" + "ldp q13, q14, [%x[r], #128]\n\t" + "ldp q15, q16, [%x[r], #160]\n\t" + "ldp q17, q18, [%x[r], #192]\n\t" + "ldp q19, q20, [%x[r], #224]\n\t" + "ldr q0, [%[zetas], #64]\n\t" + "ldr q1, [%[qinv], #64]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #128]\n\t" + "ldp q1, q3, [%[qinv], #128]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #256]\n\t" + "ldp q1, q3, [%[qinv], #256]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #288]\n\t" + "ldp q1, q3, [%[qinv], #288]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #320]\n\t" + "ldp q1, q3, [%[qinv], #320]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #352]\n\t" + "ldp q1, q3, [%[qinv], #352]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #768]\n\t" + "ldp q1, q3, [%[qinv], #768]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #800]\n\t" + "ldp q1, q3, [%[qinv], #800]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #832]\n\t" + "ldp q1, q3, [%[qinv], #832]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #864]\n\t" + "ldp q1, q3, [%[qinv], #864]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "stp q5, q6, [%x[r]]\n\t" + "stp q7, q8, [%x[r], #32]\n\t" + "stp q9, q10, [%x[r], #64]\n\t" + "stp q11, q12, [%x[r], #96]\n\t" + "stp q13, q14, [%x[r], #128]\n\t" + "stp q15, q16, [%x[r], #160]\n\t" + "stp q17, q18, [%x[r], #192]\n\t" + "stp q19, q20, [%x[r], #224]\n\t" + "ldp q5, q6, [%x[r], #256]\n\t" + "ldp q7, q8, [%x[r], #288]\n\t" + "ldp q9, q10, [%x[r], #320]\n\t" + "ldp q11, q12, [%x[r], #352]\n\t" + "ldp q13, q14, [%x[r], #384]\n\t" + "ldp q15, q16, [%x[r], #416]\n\t" + "ldp q17, q18, [%x[r], #448]\n\t" + "ldp q19, q20, [%x[r], #480]\n\t" + "ldr q0, [%[zetas], #80]\n\t" + "ldr q1, [%[qinv], #80]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #160]\n\t" + "ldp q1, q3, [%[qinv], #160]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #384]\n\t" + "ldp q1, q3, [%[qinv], #384]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #416]\n\t" + "ldp q1, q3, [%[qinv], #416]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #448]\n\t" + "ldp q1, q3, [%[qinv], #448]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #480]\n\t" + "ldp q1, q3, [%[qinv], #480]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #896]\n\t" + "ldp q1, q3, [%[qinv], #896]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #928]\n\t" + "ldp q1, q3, [%[qinv], #928]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #960]\n\t" + "ldp q1, q3, [%[qinv], #960]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #992]\n\t" + "ldp q1, q3, [%[qinv], #992]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "stp q5, q6, [%x[r], #256]\n\t" + "stp q7, q8, [%x[r], #288]\n\t" + "stp q9, q10, [%x[r], #320]\n\t" + "stp q11, q12, [%x[r], #352]\n\t" + "stp q13, q14, [%x[r], #384]\n\t" + "stp q15, q16, [%x[r], #416]\n\t" + "stp q17, q18, [%x[r], #448]\n\t" + "stp q19, q20, [%x[r], #480]\n\t" + "ldp q5, q6, [%x[r], #512]\n\t" + "ldp q7, q8, [%x[r], #544]\n\t" + "ldp q9, q10, [%x[r], #576]\n\t" + "ldp q11, q12, [%x[r], #608]\n\t" + "ldp q13, q14, [%x[r], #640]\n\t" + "ldp q15, q16, [%x[r], #672]\n\t" + "ldp q17, q18, [%x[r], #704]\n\t" + "ldp q19, q20, [%x[r], #736]\n\t" + "ldr q0, [%[zetas], #96]\n\t" + "ldr q1, [%[qinv], #96]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #192]\n\t" + "ldp q1, q3, [%[qinv], #192]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #512]\n\t" + "ldp q1, q3, [%[qinv], #512]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #544]\n\t" + "ldp q1, q3, [%[qinv], #544]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #576]\n\t" + "ldp q1, q3, [%[qinv], #576]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #608]\n\t" + "ldp q1, q3, [%[qinv], #608]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #1024]\n\t" + "ldr q2, [%[zetas], #1040]\n\t" + "ldr q1, [%[qinv], #1024]\n\t" + "ldr q3, [%[qinv], #1040]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldr q0, [%[zetas], #1056]\n\t" + "ldr q2, [%[zetas], #1072]\n\t" + "ldr q1, [%[qinv], #1056]\n\t" + "ldr q3, [%[qinv], #1072]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldr q0, [%[zetas], #1088]\n\t" + "ldr q2, [%[zetas], #1104]\n\t" + "ldr q1, [%[qinv], #1088]\n\t" + "ldr q3, [%[qinv], #1104]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldr q0, [%[zetas], #1120]\n\t" + "ldr q2, [%[zetas], #1136]\n\t" + "ldr q1, [%[qinv], #1120]\n\t" + "ldr q3, [%[qinv], #1136]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "stp q5, q6, [%x[r], #512]\n\t" + "stp q7, q8, [%x[r], #544]\n\t" + "stp q9, q10, [%x[r], #576]\n\t" + "stp q11, q12, [%x[r], #608]\n\t" + "stp q13, q14, [%x[r], #640]\n\t" + "stp q15, q16, [%x[r], #672]\n\t" + "stp q17, q18, [%x[r], #704]\n\t" + "stp q19, q20, [%x[r], #736]\n\t" + "ldp q5, q6, [%x[r], #768]\n\t" + "ldp q7, q8, [%x[r], #800]\n\t" + "ldp q9, q10, [%x[r], #832]\n\t" + "ldp q11, q12, [%x[r], #864]\n\t" + "ldp q13, q14, [%x[r], #896]\n\t" + "ldp q15, q16, [%x[r], #928]\n\t" + "ldp q17, q18, [%x[r], #960]\n\t" + "ldp q19, q20, [%x[r], #992]\n\t" + "ldr q0, [%[zetas], #112]\n\t" + "ldr q1, [%[qinv], #112]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #224]\n\t" + "ldp q1, q3, [%[qinv], #224]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #640]\n\t" + "ldp q1, q3, [%[qinv], #640]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #672]\n\t" + "ldp q1, q3, [%[qinv], #672]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #704]\n\t" + "ldp q1, q3, [%[qinv], #704]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #736]\n\t" + "ldp q1, q3, [%[qinv], #736]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #1152]\n\t" + "ldr q2, [%[zetas], #1168]\n\t" + "ldr q1, [%[qinv], #1152]\n\t" + "ldr q3, [%[qinv], #1168]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldr q0, [%[zetas], #1184]\n\t" + "ldr q2, [%[zetas], #1200]\n\t" + "ldr q1, [%[qinv], #1184]\n\t" + "ldr q3, [%[qinv], #1200]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldr q0, [%[zetas], #1216]\n\t" + "ldr q2, [%[zetas], #1232]\n\t" + "ldr q1, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldr q0, [%[zetas], #1248]\n\t" + "ldr q2, [%[zetas], #1264]\n\t" + "ldr q1, [%[qinv], #1248]\n\t" + "ldr q3, [%[qinv], #1264]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "stp q5, q6, [%x[r], #768]\n\t" + "stp q7, q8, [%x[r], #800]\n\t" + "stp q9, q10, [%x[r], #832]\n\t" + "stp q11, q12, [%x[r], #864]\n\t" + "stp q13, q14, [%x[r], #896]\n\t" + "stp q15, q16, [%x[r], #928]\n\t" + "stp q17, q18, [%x[r], #960]\n\t" + "stp q19, q20, [%x[r], #992]\n\t" + : [r] "+r" (r) + : [zetas] "r" (zetas), [qinv] "r" (qinv), [consts] "r" (consts) + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", + "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", + "v28", "v29", "v30" + ); +} + +void mldsa_ntt_full_neon(sword32* r) +{ + const word32* zetas = L_mldsa_aarch64_zetas; + const word32* qinv = L_mldsa_aarch64_zetas_qinv; + const word32* consts = L_mldsa_aarch64_consts; + __asm__ __volatile__ ( + "ldr q4, [%[consts]]\n\t" + "ldr q5, [%x[r]]\n\t" + "ldr q6, [%x[r], #64]\n\t" + "ldr q7, [%x[r], #128]\n\t" + "ldr q8, [%x[r], #192]\n\t" + "ldr q9, [%x[r], #256]\n\t" + "ldr q10, [%x[r], #320]\n\t" + "ldr q11, [%x[r], #384]\n\t" + "ldr q12, [%x[r], #448]\n\t" + "ldr q13, [%x[r], #512]\n\t" + "ldr q14, [%x[r], #576]\n\t" + "ldr q15, [%x[r], #640]\n\t" + "ldr q16, [%x[r], #704]\n\t" + "ldr q17, [%x[r], #768]\n\t" + "ldr q18, [%x[r], #832]\n\t" + "ldr q19, [%x[r], #896]\n\t" + "ldr q20, [%x[r], #960]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r]]\n\t" + "str q6, [%x[r], #64]\n\t" + "str q7, [%x[r], #128]\n\t" + "str q8, [%x[r], #192]\n\t" + "str q9, [%x[r], #256]\n\t" + "str q10, [%x[r], #320]\n\t" + "str q11, [%x[r], #384]\n\t" + "str q12, [%x[r], #448]\n\t" + "str q13, [%x[r], #512]\n\t" + "str q14, [%x[r], #576]\n\t" + "str q15, [%x[r], #640]\n\t" + "str q16, [%x[r], #704]\n\t" + "str q17, [%x[r], #768]\n\t" + "str q18, [%x[r], #832]\n\t" + "str q19, [%x[r], #896]\n\t" + "str q20, [%x[r], #960]\n\t" + "ldr q5, [%x[r], #16]\n\t" + "ldr q6, [%x[r], #80]\n\t" + "ldr q7, [%x[r], #144]\n\t" + "ldr q8, [%x[r], #208]\n\t" + "ldr q9, [%x[r], #272]\n\t" + "ldr q10, [%x[r], #336]\n\t" + "ldr q11, [%x[r], #400]\n\t" + "ldr q12, [%x[r], #464]\n\t" + "ldr q13, [%x[r], #528]\n\t" + "ldr q14, [%x[r], #592]\n\t" + "ldr q15, [%x[r], #656]\n\t" + "ldr q16, [%x[r], #720]\n\t" + "ldr q17, [%x[r], #784]\n\t" + "ldr q18, [%x[r], #848]\n\t" + "ldr q19, [%x[r], #912]\n\t" + "ldr q20, [%x[r], #976]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r], #16]\n\t" + "str q6, [%x[r], #80]\n\t" + "str q7, [%x[r], #144]\n\t" + "str q8, [%x[r], #208]\n\t" + "str q9, [%x[r], #272]\n\t" + "str q10, [%x[r], #336]\n\t" + "str q11, [%x[r], #400]\n\t" + "str q12, [%x[r], #464]\n\t" + "str q13, [%x[r], #528]\n\t" + "str q14, [%x[r], #592]\n\t" + "str q15, [%x[r], #656]\n\t" + "str q16, [%x[r], #720]\n\t" + "str q17, [%x[r], #784]\n\t" + "str q18, [%x[r], #848]\n\t" + "str q19, [%x[r], #912]\n\t" + "str q20, [%x[r], #976]\n\t" + "ldr q5, [%x[r], #32]\n\t" + "ldr q6, [%x[r], #96]\n\t" + "ldr q7, [%x[r], #160]\n\t" + "ldr q8, [%x[r], #224]\n\t" + "ldr q9, [%x[r], #288]\n\t" + "ldr q10, [%x[r], #352]\n\t" + "ldr q11, [%x[r], #416]\n\t" + "ldr q12, [%x[r], #480]\n\t" + "ldr q13, [%x[r], #544]\n\t" + "ldr q14, [%x[r], #608]\n\t" + "ldr q15, [%x[r], #672]\n\t" + "ldr q16, [%x[r], #736]\n\t" + "ldr q17, [%x[r], #800]\n\t" + "ldr q18, [%x[r], #864]\n\t" + "ldr q19, [%x[r], #928]\n\t" + "ldr q20, [%x[r], #992]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r], #32]\n\t" + "str q6, [%x[r], #96]\n\t" + "str q7, [%x[r], #160]\n\t" + "str q8, [%x[r], #224]\n\t" + "str q9, [%x[r], #288]\n\t" + "str q10, [%x[r], #352]\n\t" + "str q11, [%x[r], #416]\n\t" + "str q12, [%x[r], #480]\n\t" + "str q13, [%x[r], #544]\n\t" + "str q14, [%x[r], #608]\n\t" + "str q15, [%x[r], #672]\n\t" + "str q16, [%x[r], #736]\n\t" + "str q17, [%x[r], #800]\n\t" + "str q18, [%x[r], #864]\n\t" + "str q19, [%x[r], #928]\n\t" + "str q20, [%x[r], #992]\n\t" + "ldr q5, [%x[r], #48]\n\t" + "ldr q6, [%x[r], #112]\n\t" + "ldr q7, [%x[r], #176]\n\t" + "ldr q8, [%x[r], #240]\n\t" + "ldr q9, [%x[r], #304]\n\t" + "ldr q10, [%x[r], #368]\n\t" + "ldr q11, [%x[r], #432]\n\t" + "ldr q12, [%x[r], #496]\n\t" + "ldr q13, [%x[r], #560]\n\t" + "ldr q14, [%x[r], #624]\n\t" + "ldr q15, [%x[r], #688]\n\t" + "ldr q16, [%x[r], #752]\n\t" + "ldr q17, [%x[r], #816]\n\t" + "ldr q18, [%x[r], #880]\n\t" + "ldr q19, [%x[r], #944]\n\t" + "ldr q20, [%x[r], #1008]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r], #48]\n\t" + "str q6, [%x[r], #112]\n\t" + "str q7, [%x[r], #176]\n\t" + "str q8, [%x[r], #240]\n\t" + "str q9, [%x[r], #304]\n\t" + "str q10, [%x[r], #368]\n\t" + "str q11, [%x[r], #432]\n\t" + "str q12, [%x[r], #496]\n\t" + "str q13, [%x[r], #560]\n\t" + "str q14, [%x[r], #624]\n\t" + "str q15, [%x[r], #688]\n\t" + "str q16, [%x[r], #752]\n\t" + "str q17, [%x[r], #816]\n\t" + "str q18, [%x[r], #880]\n\t" + "str q19, [%x[r], #944]\n\t" + "str q20, [%x[r], #1008]\n\t" + "ldp q5, q6, [%x[r]]\n\t" + "ldp q7, q8, [%x[r], #32]\n\t" + "ldp q9, q10, [%x[r], #64]\n\t" + "ldp q11, q12, [%x[r], #96]\n\t" + "ldp q13, q14, [%x[r], #128]\n\t" + "ldp q15, q16, [%x[r], #160]\n\t" + "ldp q17, q18, [%x[r], #192]\n\t" + "ldp q19, q20, [%x[r], #224]\n\t" + "ldr q0, [%[zetas], #64]\n\t" + "ldr q1, [%[qinv], #64]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #128]\n\t" + "ldp q1, q3, [%[qinv], #128]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #256]\n\t" + "ldp q1, q3, [%[qinv], #256]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #288]\n\t" + "ldp q1, q3, [%[qinv], #288]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #320]\n\t" + "ldp q1, q3, [%[qinv], #320]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #352]\n\t" + "ldp q1, q3, [%[qinv], #352]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #768]\n\t" + "ldp q1, q3, [%[qinv], #768]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #800]\n\t" + "ldp q1, q3, [%[qinv], #800]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #832]\n\t" + "ldp q1, q3, [%[qinv], #832]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #864]\n\t" + "ldp q1, q3, [%[qinv], #864]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "trn1 v29.4s, v5.4s, v6.4s\n\t" + "trn2 v30.4s, v5.4s, v6.4s\n\t" + "trn1 v5.2d, v29.2d, v30.2d\n\t" + "trn2 v6.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v7.4s, v8.4s\n\t" + "trn2 v30.4s, v7.4s, v8.4s\n\t" + "trn1 v7.2d, v29.2d, v30.2d\n\t" + "trn2 v8.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v9.4s, v10.4s\n\t" + "trn2 v30.4s, v9.4s, v10.4s\n\t" + "trn1 v9.2d, v29.2d, v30.2d\n\t" + "trn2 v10.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v11.4s, v12.4s\n\t" + "trn2 v30.4s, v11.4s, v12.4s\n\t" + "trn1 v11.2d, v29.2d, v30.2d\n\t" + "trn2 v12.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v13.4s, v14.4s\n\t" + "trn2 v30.4s, v13.4s, v14.4s\n\t" + "trn1 v13.2d, v29.2d, v30.2d\n\t" + "trn2 v14.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v15.4s, v16.4s\n\t" + "trn2 v30.4s, v15.4s, v16.4s\n\t" + "trn1 v15.2d, v29.2d, v30.2d\n\t" + "trn2 v16.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v17.4s, v18.4s\n\t" + "trn2 v30.4s, v17.4s, v18.4s\n\t" + "trn1 v17.2d, v29.2d, v30.2d\n\t" + "trn2 v18.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v19.4s, v20.4s\n\t" + "trn2 v30.4s, v19.4s, v20.4s\n\t" + "trn1 v19.2d, v29.2d, v30.2d\n\t" + "trn2 v20.2d, v29.2d, v30.2d\n\t" + "stp q5, q6, [%x[r]]\n\t" + "stp q7, q8, [%x[r], #32]\n\t" + "stp q9, q10, [%x[r], #64]\n\t" + "stp q11, q12, [%x[r], #96]\n\t" + "stp q13, q14, [%x[r], #128]\n\t" + "stp q15, q16, [%x[r], #160]\n\t" + "stp q17, q18, [%x[r], #192]\n\t" + "stp q19, q20, [%x[r], #224]\n\t" + "ldp q5, q6, [%x[r], #256]\n\t" + "ldp q7, q8, [%x[r], #288]\n\t" + "ldp q9, q10, [%x[r], #320]\n\t" + "ldp q11, q12, [%x[r], #352]\n\t" + "ldp q13, q14, [%x[r], #384]\n\t" + "ldp q15, q16, [%x[r], #416]\n\t" + "ldp q17, q18, [%x[r], #448]\n\t" + "ldp q19, q20, [%x[r], #480]\n\t" + "ldr q0, [%[zetas], #80]\n\t" + "ldr q1, [%[qinv], #80]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #160]\n\t" + "ldp q1, q3, [%[qinv], #160]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #384]\n\t" + "ldp q1, q3, [%[qinv], #384]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #416]\n\t" + "ldp q1, q3, [%[qinv], #416]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #448]\n\t" + "ldp q1, q3, [%[qinv], #448]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #480]\n\t" + "ldp q1, q3, [%[qinv], #480]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #896]\n\t" + "ldp q1, q3, [%[qinv], #896]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #928]\n\t" + "ldp q1, q3, [%[qinv], #928]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #960]\n\t" + "ldp q1, q3, [%[qinv], #960]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #992]\n\t" + "ldp q1, q3, [%[qinv], #992]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "trn1 v29.4s, v5.4s, v6.4s\n\t" + "trn2 v30.4s, v5.4s, v6.4s\n\t" + "trn1 v5.2d, v29.2d, v30.2d\n\t" + "trn2 v6.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v7.4s, v8.4s\n\t" + "trn2 v30.4s, v7.4s, v8.4s\n\t" + "trn1 v7.2d, v29.2d, v30.2d\n\t" + "trn2 v8.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v9.4s, v10.4s\n\t" + "trn2 v30.4s, v9.4s, v10.4s\n\t" + "trn1 v9.2d, v29.2d, v30.2d\n\t" + "trn2 v10.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v11.4s, v12.4s\n\t" + "trn2 v30.4s, v11.4s, v12.4s\n\t" + "trn1 v11.2d, v29.2d, v30.2d\n\t" + "trn2 v12.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v13.4s, v14.4s\n\t" + "trn2 v30.4s, v13.4s, v14.4s\n\t" + "trn1 v13.2d, v29.2d, v30.2d\n\t" + "trn2 v14.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v15.4s, v16.4s\n\t" + "trn2 v30.4s, v15.4s, v16.4s\n\t" + "trn1 v15.2d, v29.2d, v30.2d\n\t" + "trn2 v16.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v17.4s, v18.4s\n\t" + "trn2 v30.4s, v17.4s, v18.4s\n\t" + "trn1 v17.2d, v29.2d, v30.2d\n\t" + "trn2 v18.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v19.4s, v20.4s\n\t" + "trn2 v30.4s, v19.4s, v20.4s\n\t" + "trn1 v19.2d, v29.2d, v30.2d\n\t" + "trn2 v20.2d, v29.2d, v30.2d\n\t" + "stp q5, q6, [%x[r], #256]\n\t" + "stp q7, q8, [%x[r], #288]\n\t" + "stp q9, q10, [%x[r], #320]\n\t" + "stp q11, q12, [%x[r], #352]\n\t" + "stp q13, q14, [%x[r], #384]\n\t" + "stp q15, q16, [%x[r], #416]\n\t" + "stp q17, q18, [%x[r], #448]\n\t" + "stp q19, q20, [%x[r], #480]\n\t" + "ldp q5, q6, [%x[r], #512]\n\t" + "ldp q7, q8, [%x[r], #544]\n\t" + "ldp q9, q10, [%x[r], #576]\n\t" + "ldp q11, q12, [%x[r], #608]\n\t" + "ldp q13, q14, [%x[r], #640]\n\t" + "ldp q15, q16, [%x[r], #672]\n\t" + "ldp q17, q18, [%x[r], #704]\n\t" + "ldp q19, q20, [%x[r], #736]\n\t" + "ldr q0, [%[zetas], #96]\n\t" + "ldr q1, [%[qinv], #96]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #192]\n\t" + "ldp q1, q3, [%[qinv], #192]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #512]\n\t" + "ldp q1, q3, [%[qinv], #512]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #544]\n\t" + "ldp q1, q3, [%[qinv], #544]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #576]\n\t" + "ldp q1, q3, [%[qinv], #576]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #608]\n\t" + "ldp q1, q3, [%[qinv], #608]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #1024]\n\t" + "ldr q2, [%[zetas], #1040]\n\t" + "ldr q1, [%[qinv], #1024]\n\t" + "ldr q3, [%[qinv], #1040]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldr q0, [%[zetas], #1056]\n\t" + "ldr q2, [%[zetas], #1072]\n\t" + "ldr q1, [%[qinv], #1056]\n\t" + "ldr q3, [%[qinv], #1072]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldr q0, [%[zetas], #1088]\n\t" + "ldr q2, [%[zetas], #1104]\n\t" + "ldr q1, [%[qinv], #1088]\n\t" + "ldr q3, [%[qinv], #1104]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldr q0, [%[zetas], #1120]\n\t" + "ldr q2, [%[zetas], #1136]\n\t" + "ldr q1, [%[qinv], #1120]\n\t" + "ldr q3, [%[qinv], #1136]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "trn1 v29.4s, v5.4s, v6.4s\n\t" + "trn2 v30.4s, v5.4s, v6.4s\n\t" + "trn1 v5.2d, v29.2d, v30.2d\n\t" + "trn2 v6.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v7.4s, v8.4s\n\t" + "trn2 v30.4s, v7.4s, v8.4s\n\t" + "trn1 v7.2d, v29.2d, v30.2d\n\t" + "trn2 v8.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v9.4s, v10.4s\n\t" + "trn2 v30.4s, v9.4s, v10.4s\n\t" + "trn1 v9.2d, v29.2d, v30.2d\n\t" + "trn2 v10.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v11.4s, v12.4s\n\t" + "trn2 v30.4s, v11.4s, v12.4s\n\t" + "trn1 v11.2d, v29.2d, v30.2d\n\t" + "trn2 v12.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v13.4s, v14.4s\n\t" + "trn2 v30.4s, v13.4s, v14.4s\n\t" + "trn1 v13.2d, v29.2d, v30.2d\n\t" + "trn2 v14.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v15.4s, v16.4s\n\t" + "trn2 v30.4s, v15.4s, v16.4s\n\t" + "trn1 v15.2d, v29.2d, v30.2d\n\t" + "trn2 v16.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v17.4s, v18.4s\n\t" + "trn2 v30.4s, v17.4s, v18.4s\n\t" + "trn1 v17.2d, v29.2d, v30.2d\n\t" + "trn2 v18.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v19.4s, v20.4s\n\t" + "trn2 v30.4s, v19.4s, v20.4s\n\t" + "trn1 v19.2d, v29.2d, v30.2d\n\t" + "trn2 v20.2d, v29.2d, v30.2d\n\t" + "stp q5, q6, [%x[r], #512]\n\t" + "stp q7, q8, [%x[r], #544]\n\t" + "stp q9, q10, [%x[r], #576]\n\t" + "stp q11, q12, [%x[r], #608]\n\t" + "stp q13, q14, [%x[r], #640]\n\t" + "stp q15, q16, [%x[r], #672]\n\t" + "stp q17, q18, [%x[r], #704]\n\t" + "stp q19, q20, [%x[r], #736]\n\t" + "ldp q5, q6, [%x[r], #768]\n\t" + "ldp q7, q8, [%x[r], #800]\n\t" + "ldp q9, q10, [%x[r], #832]\n\t" + "ldp q11, q12, [%x[r], #864]\n\t" + "ldp q13, q14, [%x[r], #896]\n\t" + "ldp q15, q16, [%x[r], #928]\n\t" + "ldp q17, q18, [%x[r], #960]\n\t" + "ldp q19, q20, [%x[r], #992]\n\t" + "ldr q0, [%[zetas], #112]\n\t" + "ldr q1, [%[qinv], #112]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #224]\n\t" + "ldp q1, q3, [%[qinv], #224]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #640]\n\t" + "ldp q1, q3, [%[qinv], #640]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #672]\n\t" + "ldp q1, q3, [%[qinv], #672]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #704]\n\t" + "ldp q1, q3, [%[qinv], #704]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #736]\n\t" + "ldp q1, q3, [%[qinv], #736]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #1152]\n\t" + "ldr q2, [%[zetas], #1168]\n\t" + "ldr q1, [%[qinv], #1152]\n\t" + "ldr q3, [%[qinv], #1168]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v21.4s, v21.4s, v29.4s\n\t" + "sub v22.4s, v22.4s, v30.4s\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldr q0, [%[zetas], #1184]\n\t" + "ldr q2, [%[zetas], #1200]\n\t" + "ldr q1, [%[qinv], #1184]\n\t" + "ldr q3, [%[qinv], #1200]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v23.4s, v23.4s, v29.4s\n\t" + "sub v24.4s, v24.4s, v30.4s\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldr q0, [%[zetas], #1216]\n\t" + "ldr q2, [%[zetas], #1232]\n\t" + "ldr q1, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v25.4s, v25.4s, v29.4s\n\t" + "sub v26.4s, v26.4s, v30.4s\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldr q0, [%[zetas], #1248]\n\t" + "ldr q2, [%[zetas], #1264]\n\t" + "ldr q1, [%[qinv], #1248]\n\t" + "ldr q3, [%[qinv], #1264]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmulh v29.4s, v29.4s, v4.s[0]\n\t" + "sqrdmulh v30.4s, v30.4s, v4.s[0]\n\t" + "sub v27.4s, v27.4s, v29.4s\n\t" + "sub v28.4s, v28.4s, v30.4s\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "trn1 v29.4s, v5.4s, v6.4s\n\t" + "trn2 v30.4s, v5.4s, v6.4s\n\t" + "trn1 v5.2d, v29.2d, v30.2d\n\t" + "trn2 v6.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v7.4s, v8.4s\n\t" + "trn2 v30.4s, v7.4s, v8.4s\n\t" + "trn1 v7.2d, v29.2d, v30.2d\n\t" + "trn2 v8.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v9.4s, v10.4s\n\t" + "trn2 v30.4s, v9.4s, v10.4s\n\t" + "trn1 v9.2d, v29.2d, v30.2d\n\t" + "trn2 v10.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v11.4s, v12.4s\n\t" + "trn2 v30.4s, v11.4s, v12.4s\n\t" + "trn1 v11.2d, v29.2d, v30.2d\n\t" + "trn2 v12.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v13.4s, v14.4s\n\t" + "trn2 v30.4s, v13.4s, v14.4s\n\t" + "trn1 v13.2d, v29.2d, v30.2d\n\t" + "trn2 v14.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v15.4s, v16.4s\n\t" + "trn2 v30.4s, v15.4s, v16.4s\n\t" + "trn1 v15.2d, v29.2d, v30.2d\n\t" + "trn2 v16.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v17.4s, v18.4s\n\t" + "trn2 v30.4s, v17.4s, v18.4s\n\t" + "trn1 v17.2d, v29.2d, v30.2d\n\t" + "trn2 v18.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v19.4s, v20.4s\n\t" + "trn2 v30.4s, v19.4s, v20.4s\n\t" + "trn1 v19.2d, v29.2d, v30.2d\n\t" + "trn2 v20.2d, v29.2d, v30.2d\n\t" + "stp q5, q6, [%x[r], #768]\n\t" + "stp q7, q8, [%x[r], #800]\n\t" + "stp q9, q10, [%x[r], #832]\n\t" + "stp q11, q12, [%x[r], #864]\n\t" + "stp q13, q14, [%x[r], #896]\n\t" + "stp q15, q16, [%x[r], #928]\n\t" + "stp q17, q18, [%x[r], #960]\n\t" + "stp q19, q20, [%x[r], #992]\n\t" + : [r] "+r" (r) + : [zetas] "r" (zetas), [qinv] "r" (qinv), [consts] "r" (consts) + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", + "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", + "v28", "v29", "v30" + ); +} + +void mldsa_invntt_neon(sword32* r) +{ + const word32* inv = L_mldsa_aarch64_zetas_inv; + const word32* qinv = L_mldsa_aarch64_zetas_inv_qinv; + const word32* consts = L_mldsa_aarch64_consts; + __asm__ __volatile__ ( + "ldr q4, [%[consts]]\n\t" + "ldp q5, q6, [%x[r]]\n\t" + "ldp q7, q8, [%x[r], #32]\n\t" + "ldp q9, q10, [%x[r], #64]\n\t" + "ldp q11, q12, [%x[r], #96]\n\t" + "ldp q13, q14, [%x[r], #128]\n\t" + "ldp q15, q16, [%x[r], #160]\n\t" + "ldp q17, q18, [%x[r], #192]\n\t" + "ldp q19, q20, [%x[r], #224]\n\t" + "ldp q0, q1, [%[inv]]\n\t" + "ldp q2, q3, [%[qinv]]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #32]\n\t" + "ldp q2, q3, [%[qinv], #32]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #64]\n\t" + "ldp q2, q3, [%[qinv], #64]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #96]\n\t" + "ldp q2, q3, [%[qinv], #96]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #512]\n\t" + "ldp q2, q3, [%[qinv], #512]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #544]\n\t" + "ldp q2, q3, [%[qinv], #544]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #576]\n\t" + "ldp q2, q3, [%[qinv], #576]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #608]\n\t" + "ldp q2, q3, [%[qinv], #608]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1024]\n\t" + "ldr q1, [%[inv], #1040]\n\t" + "ldr q2, [%[qinv], #1024]\n\t" + "ldr q3, [%[qinv], #1040]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1152]\n\t" + "ldr q2, [%[qinv], #1152]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r]]\n\t" + "stp q7, q8, [%x[r], #32]\n\t" + "stp q9, q10, [%x[r], #64]\n\t" + "stp q11, q12, [%x[r], #96]\n\t" + "stp q13, q14, [%x[r], #128]\n\t" + "stp q15, q16, [%x[r], #160]\n\t" + "stp q17, q18, [%x[r], #192]\n\t" + "stp q19, q20, [%x[r], #224]\n\t" + "ldp q5, q6, [%x[r], #256]\n\t" + "ldp q7, q8, [%x[r], #288]\n\t" + "ldp q9, q10, [%x[r], #320]\n\t" + "ldp q11, q12, [%x[r], #352]\n\t" + "ldp q13, q14, [%x[r], #384]\n\t" + "ldp q15, q16, [%x[r], #416]\n\t" + "ldp q17, q18, [%x[r], #448]\n\t" + "ldp q19, q20, [%x[r], #480]\n\t" + "ldp q0, q1, [%[inv], #128]\n\t" + "ldp q2, q3, [%[qinv], #128]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #160]\n\t" + "ldp q2, q3, [%[qinv], #160]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #192]\n\t" + "ldp q2, q3, [%[qinv], #192]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #224]\n\t" + "ldp q2, q3, [%[qinv], #224]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #640]\n\t" + "ldp q2, q3, [%[qinv], #640]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #672]\n\t" + "ldp q2, q3, [%[qinv], #672]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #704]\n\t" + "ldp q2, q3, [%[qinv], #704]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #736]\n\t" + "ldp q2, q3, [%[qinv], #736]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1056]\n\t" + "ldr q1, [%[inv], #1072]\n\t" + "ldr q2, [%[qinv], #1056]\n\t" + "ldr q3, [%[qinv], #1072]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1168]\n\t" + "ldr q2, [%[qinv], #1168]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r], #256]\n\t" + "stp q7, q8, [%x[r], #288]\n\t" + "stp q9, q10, [%x[r], #320]\n\t" + "stp q11, q12, [%x[r], #352]\n\t" + "stp q13, q14, [%x[r], #384]\n\t" + "stp q15, q16, [%x[r], #416]\n\t" + "stp q17, q18, [%x[r], #448]\n\t" + "stp q19, q20, [%x[r], #480]\n\t" + "ldp q5, q6, [%x[r], #512]\n\t" + "ldp q7, q8, [%x[r], #544]\n\t" + "ldp q9, q10, [%x[r], #576]\n\t" + "ldp q11, q12, [%x[r], #608]\n\t" + "ldp q13, q14, [%x[r], #640]\n\t" + "ldp q15, q16, [%x[r], #672]\n\t" + "ldp q17, q18, [%x[r], #704]\n\t" + "ldp q19, q20, [%x[r], #736]\n\t" + "ldp q0, q1, [%[inv], #256]\n\t" + "ldp q2, q3, [%[qinv], #256]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #288]\n\t" + "ldp q2, q3, [%[qinv], #288]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #320]\n\t" + "ldp q2, q3, [%[qinv], #320]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #352]\n\t" + "ldp q2, q3, [%[qinv], #352]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #768]\n\t" + "ldp q2, q3, [%[qinv], #768]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #800]\n\t" + "ldp q2, q3, [%[qinv], #800]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #832]\n\t" + "ldp q2, q3, [%[qinv], #832]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #864]\n\t" + "ldp q2, q3, [%[qinv], #864]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1088]\n\t" + "ldr q1, [%[inv], #1104]\n\t" + "ldr q2, [%[qinv], #1088]\n\t" + "ldr q3, [%[qinv], #1104]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1184]\n\t" + "ldr q2, [%[qinv], #1184]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r], #512]\n\t" + "stp q7, q8, [%x[r], #544]\n\t" + "stp q9, q10, [%x[r], #576]\n\t" + "stp q11, q12, [%x[r], #608]\n\t" + "stp q13, q14, [%x[r], #640]\n\t" + "stp q15, q16, [%x[r], #672]\n\t" + "stp q17, q18, [%x[r], #704]\n\t" + "stp q19, q20, [%x[r], #736]\n\t" + "ldp q5, q6, [%x[r], #768]\n\t" + "ldp q7, q8, [%x[r], #800]\n\t" + "ldp q9, q10, [%x[r], #832]\n\t" + "ldp q11, q12, [%x[r], #864]\n\t" + "ldp q13, q14, [%x[r], #896]\n\t" + "ldp q15, q16, [%x[r], #928]\n\t" + "ldp q17, q18, [%x[r], #960]\n\t" + "ldp q19, q20, [%x[r], #992]\n\t" + "ldp q0, q1, [%[inv], #384]\n\t" + "ldp q2, q3, [%[qinv], #384]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #416]\n\t" + "ldp q2, q3, [%[qinv], #416]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #448]\n\t" + "ldp q2, q3, [%[qinv], #448]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #480]\n\t" + "ldp q2, q3, [%[qinv], #480]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #896]\n\t" + "ldp q2, q3, [%[qinv], #896]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #928]\n\t" + "ldp q2, q3, [%[qinv], #928]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #960]\n\t" + "ldp q2, q3, [%[qinv], #960]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #992]\n\t" + "ldp q2, q3, [%[qinv], #992]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1120]\n\t" + "ldr q1, [%[inv], #1136]\n\t" + "ldr q2, [%[qinv], #1120]\n\t" + "ldr q3, [%[qinv], #1136]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1200]\n\t" + "ldr q2, [%[qinv], #1200]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r], #768]\n\t" + "stp q7, q8, [%x[r], #800]\n\t" + "stp q9, q10, [%x[r], #832]\n\t" + "stp q11, q12, [%x[r], #864]\n\t" + "stp q13, q14, [%x[r], #896]\n\t" + "stp q15, q16, [%x[r], #928]\n\t" + "stp q17, q18, [%x[r], #960]\n\t" + "stp q19, q20, [%x[r], #992]\n\t" + "ldr q5, [%x[r]]\n\t" + "ldr q6, [%x[r], #64]\n\t" + "ldr q7, [%x[r], #128]\n\t" + "ldr q8, [%x[r], #192]\n\t" + "ldr q9, [%x[r], #256]\n\t" + "ldr q10, [%x[r], #320]\n\t" + "ldr q11, [%x[r], #384]\n\t" + "ldr q12, [%x[r], #448]\n\t" + "ldr q13, [%x[r], #512]\n\t" + "ldr q14, [%x[r], #576]\n\t" + "ldr q15, [%x[r], #640]\n\t" + "ldr q16, [%x[r], #704]\n\t" + "ldr q17, [%x[r], #768]\n\t" + "ldr q18, [%x[r], #832]\n\t" + "ldr q19, [%x[r], #896]\n\t" + "ldr q20, [%x[r], #960]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v23.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v13.4s, v13.4s, v21.4s\n\t" + "sub v14.4s, v14.4s, v23.4s\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v5.4s, v5.4s, v21.4s\n\t" + "sub v6.4s, v6.4s, v22.4s\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v22.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v22.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v22.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r]]\n\t" + "str q6, [%x[r], #64]\n\t" + "str q7, [%x[r], #128]\n\t" + "str q8, [%x[r], #192]\n\t" + "str q9, [%x[r], #256]\n\t" + "str q10, [%x[r], #320]\n\t" + "str q11, [%x[r], #384]\n\t" + "str q12, [%x[r], #448]\n\t" + "str q13, [%x[r], #512]\n\t" + "str q14, [%x[r], #576]\n\t" + "str q15, [%x[r], #640]\n\t" + "str q16, [%x[r], #704]\n\t" + "str q17, [%x[r], #768]\n\t" + "str q18, [%x[r], #832]\n\t" + "str q19, [%x[r], #896]\n\t" + "str q20, [%x[r], #960]\n\t" + "ldr q5, [%x[r], #16]\n\t" + "ldr q6, [%x[r], #80]\n\t" + "ldr q7, [%x[r], #144]\n\t" + "ldr q8, [%x[r], #208]\n\t" + "ldr q9, [%x[r], #272]\n\t" + "ldr q10, [%x[r], #336]\n\t" + "ldr q11, [%x[r], #400]\n\t" + "ldr q12, [%x[r], #464]\n\t" + "ldr q13, [%x[r], #528]\n\t" + "ldr q14, [%x[r], #592]\n\t" + "ldr q15, [%x[r], #656]\n\t" + "ldr q16, [%x[r], #720]\n\t" + "ldr q17, [%x[r], #784]\n\t" + "ldr q18, [%x[r], #848]\n\t" + "ldr q19, [%x[r], #912]\n\t" + "ldr q20, [%x[r], #976]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v23.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v13.4s, v13.4s, v21.4s\n\t" + "sub v14.4s, v14.4s, v23.4s\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v5.4s, v5.4s, v21.4s\n\t" + "sub v6.4s, v6.4s, v22.4s\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v22.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v22.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v22.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r], #16]\n\t" + "str q6, [%x[r], #80]\n\t" + "str q7, [%x[r], #144]\n\t" + "str q8, [%x[r], #208]\n\t" + "str q9, [%x[r], #272]\n\t" + "str q10, [%x[r], #336]\n\t" + "str q11, [%x[r], #400]\n\t" + "str q12, [%x[r], #464]\n\t" + "str q13, [%x[r], #528]\n\t" + "str q14, [%x[r], #592]\n\t" + "str q15, [%x[r], #656]\n\t" + "str q16, [%x[r], #720]\n\t" + "str q17, [%x[r], #784]\n\t" + "str q18, [%x[r], #848]\n\t" + "str q19, [%x[r], #912]\n\t" + "str q20, [%x[r], #976]\n\t" + "ldr q5, [%x[r], #32]\n\t" + "ldr q6, [%x[r], #96]\n\t" + "ldr q7, [%x[r], #160]\n\t" + "ldr q8, [%x[r], #224]\n\t" + "ldr q9, [%x[r], #288]\n\t" + "ldr q10, [%x[r], #352]\n\t" + "ldr q11, [%x[r], #416]\n\t" + "ldr q12, [%x[r], #480]\n\t" + "ldr q13, [%x[r], #544]\n\t" + "ldr q14, [%x[r], #608]\n\t" + "ldr q15, [%x[r], #672]\n\t" + "ldr q16, [%x[r], #736]\n\t" + "ldr q17, [%x[r], #800]\n\t" + "ldr q18, [%x[r], #864]\n\t" + "ldr q19, [%x[r], #928]\n\t" + "ldr q20, [%x[r], #992]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v23.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v13.4s, v13.4s, v21.4s\n\t" + "sub v14.4s, v14.4s, v23.4s\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v5.4s, v5.4s, v21.4s\n\t" + "sub v6.4s, v6.4s, v22.4s\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v22.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v22.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v22.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r], #32]\n\t" + "str q6, [%x[r], #96]\n\t" + "str q7, [%x[r], #160]\n\t" + "str q8, [%x[r], #224]\n\t" + "str q9, [%x[r], #288]\n\t" + "str q10, [%x[r], #352]\n\t" + "str q11, [%x[r], #416]\n\t" + "str q12, [%x[r], #480]\n\t" + "str q13, [%x[r], #544]\n\t" + "str q14, [%x[r], #608]\n\t" + "str q15, [%x[r], #672]\n\t" + "str q16, [%x[r], #736]\n\t" + "str q17, [%x[r], #800]\n\t" + "str q18, [%x[r], #864]\n\t" + "str q19, [%x[r], #928]\n\t" + "str q20, [%x[r], #992]\n\t" + "ldr q5, [%x[r], #48]\n\t" + "ldr q6, [%x[r], #112]\n\t" + "ldr q7, [%x[r], #176]\n\t" + "ldr q8, [%x[r], #240]\n\t" + "ldr q9, [%x[r], #304]\n\t" + "ldr q10, [%x[r], #368]\n\t" + "ldr q11, [%x[r], #432]\n\t" + "ldr q12, [%x[r], #496]\n\t" + "ldr q13, [%x[r], #560]\n\t" + "ldr q14, [%x[r], #624]\n\t" + "ldr q15, [%x[r], #688]\n\t" + "ldr q16, [%x[r], #752]\n\t" + "ldr q17, [%x[r], #816]\n\t" + "ldr q18, [%x[r], #880]\n\t" + "ldr q19, [%x[r], #944]\n\t" + "ldr q20, [%x[r], #1008]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v23.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v13.4s, v13.4s, v21.4s\n\t" + "sub v14.4s, v14.4s, v23.4s\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v5.4s, v5.4s, v21.4s\n\t" + "sub v6.4s, v6.4s, v22.4s\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v22.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v22.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v22.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r], #48]\n\t" + "str q6, [%x[r], #112]\n\t" + "str q7, [%x[r], #176]\n\t" + "str q8, [%x[r], #240]\n\t" + "str q9, [%x[r], #304]\n\t" + "str q10, [%x[r], #368]\n\t" + "str q11, [%x[r], #432]\n\t" + "str q12, [%x[r], #496]\n\t" + "str q13, [%x[r], #560]\n\t" + "str q14, [%x[r], #624]\n\t" + "str q15, [%x[r], #688]\n\t" + "str q16, [%x[r], #752]\n\t" + "str q17, [%x[r], #816]\n\t" + "str q18, [%x[r], #880]\n\t" + "str q19, [%x[r], #944]\n\t" + "str q20, [%x[r], #1008]\n\t" + : [r] "+r" (r) + : [inv] "r" (inv), [qinv] "r" (qinv), [consts] "r" (consts) + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", + "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v23", "v24" + ); +} + +void mldsa_invntt_full_neon(sword32* r) +{ + const word32* inv = L_mldsa_aarch64_zetas_inv; + const word32* qinv = L_mldsa_aarch64_zetas_inv_qinv; + const word32* consts = L_mldsa_aarch64_consts; + __asm__ __volatile__ ( + "ldr q4, [%[consts]]\n\t" + "ldp q5, q6, [%x[r]]\n\t" + "ldp q7, q8, [%x[r], #32]\n\t" + "ldp q9, q10, [%x[r], #64]\n\t" + "ldp q11, q12, [%x[r], #96]\n\t" + "ldp q13, q14, [%x[r], #128]\n\t" + "ldp q15, q16, [%x[r], #160]\n\t" + "ldp q17, q18, [%x[r], #192]\n\t" + "ldp q19, q20, [%x[r], #224]\n\t" + "trn1 v21.2d, v5.2d, v6.2d\n\t" + "trn2 v22.2d, v5.2d, v6.2d\n\t" + "trn1 v5.4s, v21.4s, v22.4s\n\t" + "trn2 v6.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v7.2d, v8.2d\n\t" + "trn2 v22.2d, v7.2d, v8.2d\n\t" + "trn1 v7.4s, v21.4s, v22.4s\n\t" + "trn2 v8.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v9.2d, v10.2d\n\t" + "trn2 v22.2d, v9.2d, v10.2d\n\t" + "trn1 v9.4s, v21.4s, v22.4s\n\t" + "trn2 v10.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v11.2d, v12.2d\n\t" + "trn2 v22.2d, v11.2d, v12.2d\n\t" + "trn1 v11.4s, v21.4s, v22.4s\n\t" + "trn2 v12.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v13.2d, v14.2d\n\t" + "trn2 v22.2d, v13.2d, v14.2d\n\t" + "trn1 v13.4s, v21.4s, v22.4s\n\t" + "trn2 v14.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v15.2d, v16.2d\n\t" + "trn2 v22.2d, v15.2d, v16.2d\n\t" + "trn1 v15.4s, v21.4s, v22.4s\n\t" + "trn2 v16.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v17.2d, v18.2d\n\t" + "trn2 v22.2d, v17.2d, v18.2d\n\t" + "trn1 v17.4s, v21.4s, v22.4s\n\t" + "trn2 v18.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v19.2d, v20.2d\n\t" + "trn2 v22.2d, v19.2d, v20.2d\n\t" + "trn1 v19.4s, v21.4s, v22.4s\n\t" + "trn2 v20.4s, v21.4s, v22.4s\n\t" + "ldp q0, q1, [%[inv]]\n\t" + "ldp q2, q3, [%[qinv]]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #32]\n\t" + "ldp q2, q3, [%[qinv], #32]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #64]\n\t" + "ldp q2, q3, [%[qinv], #64]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #96]\n\t" + "ldp q2, q3, [%[qinv], #96]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #512]\n\t" + "ldp q2, q3, [%[qinv], #512]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #544]\n\t" + "ldp q2, q3, [%[qinv], #544]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #576]\n\t" + "ldp q2, q3, [%[qinv], #576]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #608]\n\t" + "ldp q2, q3, [%[qinv], #608]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1024]\n\t" + "ldr q1, [%[inv], #1040]\n\t" + "ldr q2, [%[qinv], #1024]\n\t" + "ldr q3, [%[qinv], #1040]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1152]\n\t" + "ldr q2, [%[qinv], #1152]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r]]\n\t" + "stp q7, q8, [%x[r], #32]\n\t" + "stp q9, q10, [%x[r], #64]\n\t" + "stp q11, q12, [%x[r], #96]\n\t" + "stp q13, q14, [%x[r], #128]\n\t" + "stp q15, q16, [%x[r], #160]\n\t" + "stp q17, q18, [%x[r], #192]\n\t" + "stp q19, q20, [%x[r], #224]\n\t" + "ldp q5, q6, [%x[r], #256]\n\t" + "ldp q7, q8, [%x[r], #288]\n\t" + "ldp q9, q10, [%x[r], #320]\n\t" + "ldp q11, q12, [%x[r], #352]\n\t" + "ldp q13, q14, [%x[r], #384]\n\t" + "ldp q15, q16, [%x[r], #416]\n\t" + "ldp q17, q18, [%x[r], #448]\n\t" + "ldp q19, q20, [%x[r], #480]\n\t" + "trn1 v21.2d, v5.2d, v6.2d\n\t" + "trn2 v22.2d, v5.2d, v6.2d\n\t" + "trn1 v5.4s, v21.4s, v22.4s\n\t" + "trn2 v6.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v7.2d, v8.2d\n\t" + "trn2 v22.2d, v7.2d, v8.2d\n\t" + "trn1 v7.4s, v21.4s, v22.4s\n\t" + "trn2 v8.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v9.2d, v10.2d\n\t" + "trn2 v22.2d, v9.2d, v10.2d\n\t" + "trn1 v9.4s, v21.4s, v22.4s\n\t" + "trn2 v10.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v11.2d, v12.2d\n\t" + "trn2 v22.2d, v11.2d, v12.2d\n\t" + "trn1 v11.4s, v21.4s, v22.4s\n\t" + "trn2 v12.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v13.2d, v14.2d\n\t" + "trn2 v22.2d, v13.2d, v14.2d\n\t" + "trn1 v13.4s, v21.4s, v22.4s\n\t" + "trn2 v14.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v15.2d, v16.2d\n\t" + "trn2 v22.2d, v15.2d, v16.2d\n\t" + "trn1 v15.4s, v21.4s, v22.4s\n\t" + "trn2 v16.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v17.2d, v18.2d\n\t" + "trn2 v22.2d, v17.2d, v18.2d\n\t" + "trn1 v17.4s, v21.4s, v22.4s\n\t" + "trn2 v18.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v19.2d, v20.2d\n\t" + "trn2 v22.2d, v19.2d, v20.2d\n\t" + "trn1 v19.4s, v21.4s, v22.4s\n\t" + "trn2 v20.4s, v21.4s, v22.4s\n\t" + "ldp q0, q1, [%[inv], #128]\n\t" + "ldp q2, q3, [%[qinv], #128]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #160]\n\t" + "ldp q2, q3, [%[qinv], #160]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #192]\n\t" + "ldp q2, q3, [%[qinv], #192]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #224]\n\t" + "ldp q2, q3, [%[qinv], #224]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #640]\n\t" + "ldp q2, q3, [%[qinv], #640]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #672]\n\t" + "ldp q2, q3, [%[qinv], #672]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #704]\n\t" + "ldp q2, q3, [%[qinv], #704]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #736]\n\t" + "ldp q2, q3, [%[qinv], #736]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1056]\n\t" + "ldr q1, [%[inv], #1072]\n\t" + "ldr q2, [%[qinv], #1056]\n\t" + "ldr q3, [%[qinv], #1072]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1168]\n\t" + "ldr q2, [%[qinv], #1168]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r], #256]\n\t" + "stp q7, q8, [%x[r], #288]\n\t" + "stp q9, q10, [%x[r], #320]\n\t" + "stp q11, q12, [%x[r], #352]\n\t" + "stp q13, q14, [%x[r], #384]\n\t" + "stp q15, q16, [%x[r], #416]\n\t" + "stp q17, q18, [%x[r], #448]\n\t" + "stp q19, q20, [%x[r], #480]\n\t" + "ldp q5, q6, [%x[r], #512]\n\t" + "ldp q7, q8, [%x[r], #544]\n\t" + "ldp q9, q10, [%x[r], #576]\n\t" + "ldp q11, q12, [%x[r], #608]\n\t" + "ldp q13, q14, [%x[r], #640]\n\t" + "ldp q15, q16, [%x[r], #672]\n\t" + "ldp q17, q18, [%x[r], #704]\n\t" + "ldp q19, q20, [%x[r], #736]\n\t" + "trn1 v21.2d, v5.2d, v6.2d\n\t" + "trn2 v22.2d, v5.2d, v6.2d\n\t" + "trn1 v5.4s, v21.4s, v22.4s\n\t" + "trn2 v6.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v7.2d, v8.2d\n\t" + "trn2 v22.2d, v7.2d, v8.2d\n\t" + "trn1 v7.4s, v21.4s, v22.4s\n\t" + "trn2 v8.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v9.2d, v10.2d\n\t" + "trn2 v22.2d, v9.2d, v10.2d\n\t" + "trn1 v9.4s, v21.4s, v22.4s\n\t" + "trn2 v10.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v11.2d, v12.2d\n\t" + "trn2 v22.2d, v11.2d, v12.2d\n\t" + "trn1 v11.4s, v21.4s, v22.4s\n\t" + "trn2 v12.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v13.2d, v14.2d\n\t" + "trn2 v22.2d, v13.2d, v14.2d\n\t" + "trn1 v13.4s, v21.4s, v22.4s\n\t" + "trn2 v14.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v15.2d, v16.2d\n\t" + "trn2 v22.2d, v15.2d, v16.2d\n\t" + "trn1 v15.4s, v21.4s, v22.4s\n\t" + "trn2 v16.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v17.2d, v18.2d\n\t" + "trn2 v22.2d, v17.2d, v18.2d\n\t" + "trn1 v17.4s, v21.4s, v22.4s\n\t" + "trn2 v18.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v19.2d, v20.2d\n\t" + "trn2 v22.2d, v19.2d, v20.2d\n\t" + "trn1 v19.4s, v21.4s, v22.4s\n\t" + "trn2 v20.4s, v21.4s, v22.4s\n\t" + "ldp q0, q1, [%[inv], #256]\n\t" + "ldp q2, q3, [%[qinv], #256]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #288]\n\t" + "ldp q2, q3, [%[qinv], #288]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #320]\n\t" + "ldp q2, q3, [%[qinv], #320]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #352]\n\t" + "ldp q2, q3, [%[qinv], #352]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #768]\n\t" + "ldp q2, q3, [%[qinv], #768]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #800]\n\t" + "ldp q2, q3, [%[qinv], #800]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #832]\n\t" + "ldp q2, q3, [%[qinv], #832]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #864]\n\t" + "ldp q2, q3, [%[qinv], #864]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1088]\n\t" + "ldr q1, [%[inv], #1104]\n\t" + "ldr q2, [%[qinv], #1088]\n\t" + "ldr q3, [%[qinv], #1104]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1184]\n\t" + "ldr q2, [%[qinv], #1184]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r], #512]\n\t" + "stp q7, q8, [%x[r], #544]\n\t" + "stp q9, q10, [%x[r], #576]\n\t" + "stp q11, q12, [%x[r], #608]\n\t" + "stp q13, q14, [%x[r], #640]\n\t" + "stp q15, q16, [%x[r], #672]\n\t" + "stp q17, q18, [%x[r], #704]\n\t" + "stp q19, q20, [%x[r], #736]\n\t" + "ldp q5, q6, [%x[r], #768]\n\t" + "ldp q7, q8, [%x[r], #800]\n\t" + "ldp q9, q10, [%x[r], #832]\n\t" + "ldp q11, q12, [%x[r], #864]\n\t" + "ldp q13, q14, [%x[r], #896]\n\t" + "ldp q15, q16, [%x[r], #928]\n\t" + "ldp q17, q18, [%x[r], #960]\n\t" + "ldp q19, q20, [%x[r], #992]\n\t" + "trn1 v21.2d, v5.2d, v6.2d\n\t" + "trn2 v22.2d, v5.2d, v6.2d\n\t" + "trn1 v5.4s, v21.4s, v22.4s\n\t" + "trn2 v6.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v7.2d, v8.2d\n\t" + "trn2 v22.2d, v7.2d, v8.2d\n\t" + "trn1 v7.4s, v21.4s, v22.4s\n\t" + "trn2 v8.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v9.2d, v10.2d\n\t" + "trn2 v22.2d, v9.2d, v10.2d\n\t" + "trn1 v9.4s, v21.4s, v22.4s\n\t" + "trn2 v10.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v11.2d, v12.2d\n\t" + "trn2 v22.2d, v11.2d, v12.2d\n\t" + "trn1 v11.4s, v21.4s, v22.4s\n\t" + "trn2 v12.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v13.2d, v14.2d\n\t" + "trn2 v22.2d, v13.2d, v14.2d\n\t" + "trn1 v13.4s, v21.4s, v22.4s\n\t" + "trn2 v14.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v15.2d, v16.2d\n\t" + "trn2 v22.2d, v15.2d, v16.2d\n\t" + "trn1 v15.4s, v21.4s, v22.4s\n\t" + "trn2 v16.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v17.2d, v18.2d\n\t" + "trn2 v22.2d, v17.2d, v18.2d\n\t" + "trn1 v17.4s, v21.4s, v22.4s\n\t" + "trn2 v18.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v19.2d, v20.2d\n\t" + "trn2 v22.2d, v19.2d, v20.2d\n\t" + "trn1 v19.4s, v21.4s, v22.4s\n\t" + "trn2 v20.4s, v21.4s, v22.4s\n\t" + "ldp q0, q1, [%[inv], #384]\n\t" + "ldp q2, q3, [%[qinv], #384]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #416]\n\t" + "ldp q2, q3, [%[qinv], #416]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #448]\n\t" + "ldp q2, q3, [%[qinv], #448]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #480]\n\t" + "ldp q2, q3, [%[qinv], #480]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #896]\n\t" + "ldp q2, q3, [%[qinv], #896]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #928]\n\t" + "ldp q2, q3, [%[qinv], #928]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #960]\n\t" + "ldp q2, q3, [%[qinv], #960]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #992]\n\t" + "ldp q2, q3, [%[qinv], #992]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1120]\n\t" + "ldr q1, [%[inv], #1136]\n\t" + "ldr q2, [%[qinv], #1120]\n\t" + "ldr q3, [%[qinv], #1136]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1200]\n\t" + "ldr q2, [%[qinv], #1200]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r], #768]\n\t" + "stp q7, q8, [%x[r], #800]\n\t" + "stp q9, q10, [%x[r], #832]\n\t" + "stp q11, q12, [%x[r], #864]\n\t" + "stp q13, q14, [%x[r], #896]\n\t" + "stp q15, q16, [%x[r], #928]\n\t" + "stp q17, q18, [%x[r], #960]\n\t" + "stp q19, q20, [%x[r], #992]\n\t" + "ldr q5, [%x[r]]\n\t" + "ldr q6, [%x[r], #64]\n\t" + "ldr q7, [%x[r], #128]\n\t" + "ldr q8, [%x[r], #192]\n\t" + "ldr q9, [%x[r], #256]\n\t" + "ldr q10, [%x[r], #320]\n\t" + "ldr q11, [%x[r], #384]\n\t" + "ldr q12, [%x[r], #448]\n\t" + "ldr q13, [%x[r], #512]\n\t" + "ldr q14, [%x[r], #576]\n\t" + "ldr q15, [%x[r], #640]\n\t" + "ldr q16, [%x[r], #704]\n\t" + "ldr q17, [%x[r], #768]\n\t" + "ldr q18, [%x[r], #832]\n\t" + "ldr q19, [%x[r], #896]\n\t" + "ldr q20, [%x[r], #960]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v23.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v13.4s, v13.4s, v21.4s\n\t" + "sub v14.4s, v14.4s, v23.4s\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v5.4s, v5.4s, v21.4s\n\t" + "sub v6.4s, v6.4s, v22.4s\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v22.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v22.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v22.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r]]\n\t" + "str q6, [%x[r], #64]\n\t" + "str q7, [%x[r], #128]\n\t" + "str q8, [%x[r], #192]\n\t" + "str q9, [%x[r], #256]\n\t" + "str q10, [%x[r], #320]\n\t" + "str q11, [%x[r], #384]\n\t" + "str q12, [%x[r], #448]\n\t" + "str q13, [%x[r], #512]\n\t" + "str q14, [%x[r], #576]\n\t" + "str q15, [%x[r], #640]\n\t" + "str q16, [%x[r], #704]\n\t" + "str q17, [%x[r], #768]\n\t" + "str q18, [%x[r], #832]\n\t" + "str q19, [%x[r], #896]\n\t" + "str q20, [%x[r], #960]\n\t" + "ldr q5, [%x[r], #16]\n\t" + "ldr q6, [%x[r], #80]\n\t" + "ldr q7, [%x[r], #144]\n\t" + "ldr q8, [%x[r], #208]\n\t" + "ldr q9, [%x[r], #272]\n\t" + "ldr q10, [%x[r], #336]\n\t" + "ldr q11, [%x[r], #400]\n\t" + "ldr q12, [%x[r], #464]\n\t" + "ldr q13, [%x[r], #528]\n\t" + "ldr q14, [%x[r], #592]\n\t" + "ldr q15, [%x[r], #656]\n\t" + "ldr q16, [%x[r], #720]\n\t" + "ldr q17, [%x[r], #784]\n\t" + "ldr q18, [%x[r], #848]\n\t" + "ldr q19, [%x[r], #912]\n\t" + "ldr q20, [%x[r], #976]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v23.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v13.4s, v13.4s, v21.4s\n\t" + "sub v14.4s, v14.4s, v23.4s\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v5.4s, v5.4s, v21.4s\n\t" + "sub v6.4s, v6.4s, v22.4s\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v22.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v22.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v22.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r], #16]\n\t" + "str q6, [%x[r], #80]\n\t" + "str q7, [%x[r], #144]\n\t" + "str q8, [%x[r], #208]\n\t" + "str q9, [%x[r], #272]\n\t" + "str q10, [%x[r], #336]\n\t" + "str q11, [%x[r], #400]\n\t" + "str q12, [%x[r], #464]\n\t" + "str q13, [%x[r], #528]\n\t" + "str q14, [%x[r], #592]\n\t" + "str q15, [%x[r], #656]\n\t" + "str q16, [%x[r], #720]\n\t" + "str q17, [%x[r], #784]\n\t" + "str q18, [%x[r], #848]\n\t" + "str q19, [%x[r], #912]\n\t" + "str q20, [%x[r], #976]\n\t" + "ldr q5, [%x[r], #32]\n\t" + "ldr q6, [%x[r], #96]\n\t" + "ldr q7, [%x[r], #160]\n\t" + "ldr q8, [%x[r], #224]\n\t" + "ldr q9, [%x[r], #288]\n\t" + "ldr q10, [%x[r], #352]\n\t" + "ldr q11, [%x[r], #416]\n\t" + "ldr q12, [%x[r], #480]\n\t" + "ldr q13, [%x[r], #544]\n\t" + "ldr q14, [%x[r], #608]\n\t" + "ldr q15, [%x[r], #672]\n\t" + "ldr q16, [%x[r], #736]\n\t" + "ldr q17, [%x[r], #800]\n\t" + "ldr q18, [%x[r], #864]\n\t" + "ldr q19, [%x[r], #928]\n\t" + "ldr q20, [%x[r], #992]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v23.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v13.4s, v13.4s, v21.4s\n\t" + "sub v14.4s, v14.4s, v23.4s\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v5.4s, v5.4s, v21.4s\n\t" + "sub v6.4s, v6.4s, v22.4s\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v22.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v22.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v22.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r], #32]\n\t" + "str q6, [%x[r], #96]\n\t" + "str q7, [%x[r], #160]\n\t" + "str q8, [%x[r], #224]\n\t" + "str q9, [%x[r], #288]\n\t" + "str q10, [%x[r], #352]\n\t" + "str q11, [%x[r], #416]\n\t" + "str q12, [%x[r], #480]\n\t" + "str q13, [%x[r], #544]\n\t" + "str q14, [%x[r], #608]\n\t" + "str q15, [%x[r], #672]\n\t" + "str q16, [%x[r], #736]\n\t" + "str q17, [%x[r], #800]\n\t" + "str q18, [%x[r], #864]\n\t" + "str q19, [%x[r], #928]\n\t" + "str q20, [%x[r], #992]\n\t" + "ldr q5, [%x[r], #48]\n\t" + "ldr q6, [%x[r], #112]\n\t" + "ldr q7, [%x[r], #176]\n\t" + "ldr q8, [%x[r], #240]\n\t" + "ldr q9, [%x[r], #304]\n\t" + "ldr q10, [%x[r], #368]\n\t" + "ldr q11, [%x[r], #432]\n\t" + "ldr q12, [%x[r], #496]\n\t" + "ldr q13, [%x[r], #560]\n\t" + "ldr q14, [%x[r], #624]\n\t" + "ldr q15, [%x[r], #688]\n\t" + "ldr q16, [%x[r], #752]\n\t" + "ldr q17, [%x[r], #816]\n\t" + "ldr q18, [%x[r], #880]\n\t" + "ldr q19, [%x[r], #944]\n\t" + "ldr q20, [%x[r], #1008]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v6.4s, v6.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v10.4s, v10.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v14.4s, v14.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v18.4s, v18.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v23.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v23.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v23.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v13.4s, v13.4s, v21.4s\n\t" + "sub v14.4s, v14.4s, v23.4s\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v15.4s, v15.4s, v21.4s\n\t" + "sub v16.4s, v16.4s, v23.4s\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v17.4s, v17.4s, v21.4s\n\t" + "sub v18.4s, v18.4s, v23.4s\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v23.4s, v23.4s, v4.s[0]\n\t" + "sub v19.4s, v19.4s, v21.4s\n\t" + "sub v20.4s, v20.4s, v23.4s\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v5.4s, v5.4s, v21.4s\n\t" + "sub v6.4s, v6.4s, v22.4s\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v7.4s, v7.4s, v21.4s\n\t" + "sub v8.4s, v8.4s, v22.4s\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v9.4s, v9.4s, v21.4s\n\t" + "sub v10.4s, v10.4s, v22.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmulh v21.4s, v21.4s, v4.s[0]\n\t" + "sqrdmulh v22.4s, v22.4s, v4.s[0]\n\t" + "sub v11.4s, v11.4s, v21.4s\n\t" + "sub v12.4s, v12.4s, v22.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r], #48]\n\t" + "str q6, [%x[r], #112]\n\t" + "str q7, [%x[r], #176]\n\t" + "str q8, [%x[r], #240]\n\t" + "str q9, [%x[r], #304]\n\t" + "str q10, [%x[r], #368]\n\t" + "str q11, [%x[r], #432]\n\t" + "str q12, [%x[r], #496]\n\t" + "str q13, [%x[r], #560]\n\t" + "str q14, [%x[r], #624]\n\t" + "str q15, [%x[r], #688]\n\t" + "str q16, [%x[r], #752]\n\t" + "str q17, [%x[r], #816]\n\t" + "str q18, [%x[r], #880]\n\t" + "str q19, [%x[r], #944]\n\t" + "str q20, [%x[r], #1008]\n\t" + : [r] "+r" (r) + : [inv] "r" (inv), [qinv] "r" (qinv), [consts] "r" (consts) + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", + "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v23", "v24" + ); +} + +void mldsa_mul_neon(sword32* r, sword32* a, sword32* b) +{ + const word32* consts = L_mldsa_aarch64_consts; + __asm__ __volatile__ ( + "ldr q0, [%[consts]]\n\t" + "ldr q1, [%x[a]]\n\t" + "ldr q5, [%x[b]]\n\t" + "ldr q2, [%x[a], #16]\n\t" + "ldr q6, [%x[b], #16]\n\t" + "ldr q3, [%x[a], #32]\n\t" + "ldr q7, [%x[b], #32]\n\t" + "ldr q4, [%x[a], #48]\n\t" + "ldr q8, [%x[b], #48]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r]]\n\t" + "str q10, [%x[r], #16]\n\t" + "str q11, [%x[r], #32]\n\t" + "str q12, [%x[r], #48]\n\t" + "ldr q1, [%x[a], #64]\n\t" + "ldr q5, [%x[b], #64]\n\t" + "ldr q2, [%x[a], #80]\n\t" + "ldr q6, [%x[b], #80]\n\t" + "ldr q3, [%x[a], #96]\n\t" + "ldr q7, [%x[b], #96]\n\t" + "ldr q4, [%x[a], #112]\n\t" + "ldr q8, [%x[b], #112]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #64]\n\t" + "str q10, [%x[r], #80]\n\t" + "str q11, [%x[r], #96]\n\t" + "str q12, [%x[r], #112]\n\t" + "ldr q1, [%x[a], #128]\n\t" + "ldr q5, [%x[b], #128]\n\t" + "ldr q2, [%x[a], #144]\n\t" + "ldr q6, [%x[b], #144]\n\t" + "ldr q3, [%x[a], #160]\n\t" + "ldr q7, [%x[b], #160]\n\t" + "ldr q4, [%x[a], #176]\n\t" + "ldr q8, [%x[b], #176]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #128]\n\t" + "str q10, [%x[r], #144]\n\t" + "str q11, [%x[r], #160]\n\t" + "str q12, [%x[r], #176]\n\t" + "ldr q1, [%x[a], #192]\n\t" + "ldr q5, [%x[b], #192]\n\t" + "ldr q2, [%x[a], #208]\n\t" + "ldr q6, [%x[b], #208]\n\t" + "ldr q3, [%x[a], #224]\n\t" + "ldr q7, [%x[b], #224]\n\t" + "ldr q4, [%x[a], #240]\n\t" + "ldr q8, [%x[b], #240]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #192]\n\t" + "str q10, [%x[r], #208]\n\t" + "str q11, [%x[r], #224]\n\t" + "str q12, [%x[r], #240]\n\t" + "ldr q1, [%x[a], #256]\n\t" + "ldr q5, [%x[b], #256]\n\t" + "ldr q2, [%x[a], #272]\n\t" + "ldr q6, [%x[b], #272]\n\t" + "ldr q3, [%x[a], #288]\n\t" + "ldr q7, [%x[b], #288]\n\t" + "ldr q4, [%x[a], #304]\n\t" + "ldr q8, [%x[b], #304]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #256]\n\t" + "str q10, [%x[r], #272]\n\t" + "str q11, [%x[r], #288]\n\t" + "str q12, [%x[r], #304]\n\t" + "ldr q1, [%x[a], #320]\n\t" + "ldr q5, [%x[b], #320]\n\t" + "ldr q2, [%x[a], #336]\n\t" + "ldr q6, [%x[b], #336]\n\t" + "ldr q3, [%x[a], #352]\n\t" + "ldr q7, [%x[b], #352]\n\t" + "ldr q4, [%x[a], #368]\n\t" + "ldr q8, [%x[b], #368]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #320]\n\t" + "str q10, [%x[r], #336]\n\t" + "str q11, [%x[r], #352]\n\t" + "str q12, [%x[r], #368]\n\t" + "ldr q1, [%x[a], #384]\n\t" + "ldr q5, [%x[b], #384]\n\t" + "ldr q2, [%x[a], #400]\n\t" + "ldr q6, [%x[b], #400]\n\t" + "ldr q3, [%x[a], #416]\n\t" + "ldr q7, [%x[b], #416]\n\t" + "ldr q4, [%x[a], #432]\n\t" + "ldr q8, [%x[b], #432]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #384]\n\t" + "str q10, [%x[r], #400]\n\t" + "str q11, [%x[r], #416]\n\t" + "str q12, [%x[r], #432]\n\t" + "ldr q1, [%x[a], #448]\n\t" + "ldr q5, [%x[b], #448]\n\t" + "ldr q2, [%x[a], #464]\n\t" + "ldr q6, [%x[b], #464]\n\t" + "ldr q3, [%x[a], #480]\n\t" + "ldr q7, [%x[b], #480]\n\t" + "ldr q4, [%x[a], #496]\n\t" + "ldr q8, [%x[b], #496]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #448]\n\t" + "str q10, [%x[r], #464]\n\t" + "str q11, [%x[r], #480]\n\t" + "str q12, [%x[r], #496]\n\t" + "ldr q1, [%x[a], #512]\n\t" + "ldr q5, [%x[b], #512]\n\t" + "ldr q2, [%x[a], #528]\n\t" + "ldr q6, [%x[b], #528]\n\t" + "ldr q3, [%x[a], #544]\n\t" + "ldr q7, [%x[b], #544]\n\t" + "ldr q4, [%x[a], #560]\n\t" + "ldr q8, [%x[b], #560]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #512]\n\t" + "str q10, [%x[r], #528]\n\t" + "str q11, [%x[r], #544]\n\t" + "str q12, [%x[r], #560]\n\t" + "ldr q1, [%x[a], #576]\n\t" + "ldr q5, [%x[b], #576]\n\t" + "ldr q2, [%x[a], #592]\n\t" + "ldr q6, [%x[b], #592]\n\t" + "ldr q3, [%x[a], #608]\n\t" + "ldr q7, [%x[b], #608]\n\t" + "ldr q4, [%x[a], #624]\n\t" + "ldr q8, [%x[b], #624]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #576]\n\t" + "str q10, [%x[r], #592]\n\t" + "str q11, [%x[r], #608]\n\t" + "str q12, [%x[r], #624]\n\t" + "ldr q1, [%x[a], #640]\n\t" + "ldr q5, [%x[b], #640]\n\t" + "ldr q2, [%x[a], #656]\n\t" + "ldr q6, [%x[b], #656]\n\t" + "ldr q3, [%x[a], #672]\n\t" + "ldr q7, [%x[b], #672]\n\t" + "ldr q4, [%x[a], #688]\n\t" + "ldr q8, [%x[b], #688]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #640]\n\t" + "str q10, [%x[r], #656]\n\t" + "str q11, [%x[r], #672]\n\t" + "str q12, [%x[r], #688]\n\t" + "ldr q1, [%x[a], #704]\n\t" + "ldr q5, [%x[b], #704]\n\t" + "ldr q2, [%x[a], #720]\n\t" + "ldr q6, [%x[b], #720]\n\t" + "ldr q3, [%x[a], #736]\n\t" + "ldr q7, [%x[b], #736]\n\t" + "ldr q4, [%x[a], #752]\n\t" + "ldr q8, [%x[b], #752]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #704]\n\t" + "str q10, [%x[r], #720]\n\t" + "str q11, [%x[r], #736]\n\t" + "str q12, [%x[r], #752]\n\t" + "ldr q1, [%x[a], #768]\n\t" + "ldr q5, [%x[b], #768]\n\t" + "ldr q2, [%x[a], #784]\n\t" + "ldr q6, [%x[b], #784]\n\t" + "ldr q3, [%x[a], #800]\n\t" + "ldr q7, [%x[b], #800]\n\t" + "ldr q4, [%x[a], #816]\n\t" + "ldr q8, [%x[b], #816]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #768]\n\t" + "str q10, [%x[r], #784]\n\t" + "str q11, [%x[r], #800]\n\t" + "str q12, [%x[r], #816]\n\t" + "ldr q1, [%x[a], #832]\n\t" + "ldr q5, [%x[b], #832]\n\t" + "ldr q2, [%x[a], #848]\n\t" + "ldr q6, [%x[b], #848]\n\t" + "ldr q3, [%x[a], #864]\n\t" + "ldr q7, [%x[b], #864]\n\t" + "ldr q4, [%x[a], #880]\n\t" + "ldr q8, [%x[b], #880]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #832]\n\t" + "str q10, [%x[r], #848]\n\t" + "str q11, [%x[r], #864]\n\t" + "str q12, [%x[r], #880]\n\t" + "ldr q1, [%x[a], #896]\n\t" + "ldr q5, [%x[b], #896]\n\t" + "ldr q2, [%x[a], #912]\n\t" + "ldr q6, [%x[b], #912]\n\t" + "ldr q3, [%x[a], #928]\n\t" + "ldr q7, [%x[b], #928]\n\t" + "ldr q4, [%x[a], #944]\n\t" + "ldr q8, [%x[b], #944]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #896]\n\t" + "str q10, [%x[r], #912]\n\t" + "str q11, [%x[r], #928]\n\t" + "str q12, [%x[r], #944]\n\t" + "ldr q1, [%x[a], #960]\n\t" + "ldr q5, [%x[b], #960]\n\t" + "ldr q2, [%x[a], #976]\n\t" + "ldr q6, [%x[b], #976]\n\t" + "ldr q3, [%x[a], #992]\n\t" + "ldr q7, [%x[b], #992]\n\t" + "ldr q4, [%x[a], #1008]\n\t" + "ldr q8, [%x[b], #1008]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmulh v17.4s, v17.4s, v0.s[0]\n\t" + "sqrdmulh v18.4s, v18.4s, v0.s[0]\n\t" + "sub v11.4s, v11.4s, v17.4s\n\t" + "sub v12.4s, v12.4s, v18.4s\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #960]\n\t" + "str q10, [%x[r], #976]\n\t" + "str q11, [%x[r], #992]\n\t" + "str q12, [%x[r], #1008]\n\t" + : [r] "+r" (r), [a] "+r" (a), [b] "+r" (b) + : [consts] "r" (consts) + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", + "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18" + ); +} + +#ifndef WOLFSSL_AARCH64_NO_SQRDMLSH +void mldsa_ntt_sqrdmlsh_neon(sword32* r) +{ + const word32* zetas = L_mldsa_aarch64_zetas; + const word32* qinv = L_mldsa_aarch64_zetas_qinv; + const word32* consts = L_mldsa_aarch64_consts; + __asm__ __volatile__ ( + "ldr q4, [%[consts]]\n\t" + "ldr q5, [%x[r]]\n\t" + "ldr q6, [%x[r], #64]\n\t" + "ldr q7, [%x[r], #128]\n\t" + "ldr q8, [%x[r], #192]\n\t" + "ldr q9, [%x[r], #256]\n\t" + "ldr q10, [%x[r], #320]\n\t" + "ldr q11, [%x[r], #384]\n\t" + "ldr q12, [%x[r], #448]\n\t" + "ldr q13, [%x[r], #512]\n\t" + "ldr q14, [%x[r], #576]\n\t" + "ldr q15, [%x[r], #640]\n\t" + "ldr q16, [%x[r], #704]\n\t" + "ldr q17, [%x[r], #768]\n\t" + "ldr q18, [%x[r], #832]\n\t" + "ldr q19, [%x[r], #896]\n\t" + "ldr q20, [%x[r], #960]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r]]\n\t" + "str q6, [%x[r], #64]\n\t" + "str q7, [%x[r], #128]\n\t" + "str q8, [%x[r], #192]\n\t" + "str q9, [%x[r], #256]\n\t" + "str q10, [%x[r], #320]\n\t" + "str q11, [%x[r], #384]\n\t" + "str q12, [%x[r], #448]\n\t" + "str q13, [%x[r], #512]\n\t" + "str q14, [%x[r], #576]\n\t" + "str q15, [%x[r], #640]\n\t" + "str q16, [%x[r], #704]\n\t" + "str q17, [%x[r], #768]\n\t" + "str q18, [%x[r], #832]\n\t" + "str q19, [%x[r], #896]\n\t" + "str q20, [%x[r], #960]\n\t" + "ldr q5, [%x[r], #16]\n\t" + "ldr q6, [%x[r], #80]\n\t" + "ldr q7, [%x[r], #144]\n\t" + "ldr q8, [%x[r], #208]\n\t" + "ldr q9, [%x[r], #272]\n\t" + "ldr q10, [%x[r], #336]\n\t" + "ldr q11, [%x[r], #400]\n\t" + "ldr q12, [%x[r], #464]\n\t" + "ldr q13, [%x[r], #528]\n\t" + "ldr q14, [%x[r], #592]\n\t" + "ldr q15, [%x[r], #656]\n\t" + "ldr q16, [%x[r], #720]\n\t" + "ldr q17, [%x[r], #784]\n\t" + "ldr q18, [%x[r], #848]\n\t" + "ldr q19, [%x[r], #912]\n\t" + "ldr q20, [%x[r], #976]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r], #16]\n\t" + "str q6, [%x[r], #80]\n\t" + "str q7, [%x[r], #144]\n\t" + "str q8, [%x[r], #208]\n\t" + "str q9, [%x[r], #272]\n\t" + "str q10, [%x[r], #336]\n\t" + "str q11, [%x[r], #400]\n\t" + "str q12, [%x[r], #464]\n\t" + "str q13, [%x[r], #528]\n\t" + "str q14, [%x[r], #592]\n\t" + "str q15, [%x[r], #656]\n\t" + "str q16, [%x[r], #720]\n\t" + "str q17, [%x[r], #784]\n\t" + "str q18, [%x[r], #848]\n\t" + "str q19, [%x[r], #912]\n\t" + "str q20, [%x[r], #976]\n\t" + "ldr q5, [%x[r], #32]\n\t" + "ldr q6, [%x[r], #96]\n\t" + "ldr q7, [%x[r], #160]\n\t" + "ldr q8, [%x[r], #224]\n\t" + "ldr q9, [%x[r], #288]\n\t" + "ldr q10, [%x[r], #352]\n\t" + "ldr q11, [%x[r], #416]\n\t" + "ldr q12, [%x[r], #480]\n\t" + "ldr q13, [%x[r], #544]\n\t" + "ldr q14, [%x[r], #608]\n\t" + "ldr q15, [%x[r], #672]\n\t" + "ldr q16, [%x[r], #736]\n\t" + "ldr q17, [%x[r], #800]\n\t" + "ldr q18, [%x[r], #864]\n\t" + "ldr q19, [%x[r], #928]\n\t" + "ldr q20, [%x[r], #992]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r], #32]\n\t" + "str q6, [%x[r], #96]\n\t" + "str q7, [%x[r], #160]\n\t" + "str q8, [%x[r], #224]\n\t" + "str q9, [%x[r], #288]\n\t" + "str q10, [%x[r], #352]\n\t" + "str q11, [%x[r], #416]\n\t" + "str q12, [%x[r], #480]\n\t" + "str q13, [%x[r], #544]\n\t" + "str q14, [%x[r], #608]\n\t" + "str q15, [%x[r], #672]\n\t" + "str q16, [%x[r], #736]\n\t" + "str q17, [%x[r], #800]\n\t" + "str q18, [%x[r], #864]\n\t" + "str q19, [%x[r], #928]\n\t" + "str q20, [%x[r], #992]\n\t" + "ldr q5, [%x[r], #48]\n\t" + "ldr q6, [%x[r], #112]\n\t" + "ldr q7, [%x[r], #176]\n\t" + "ldr q8, [%x[r], #240]\n\t" + "ldr q9, [%x[r], #304]\n\t" + "ldr q10, [%x[r], #368]\n\t" + "ldr q11, [%x[r], #432]\n\t" + "ldr q12, [%x[r], #496]\n\t" + "ldr q13, [%x[r], #560]\n\t" + "ldr q14, [%x[r], #624]\n\t" + "ldr q15, [%x[r], #688]\n\t" + "ldr q16, [%x[r], #752]\n\t" + "ldr q17, [%x[r], #816]\n\t" + "ldr q18, [%x[r], #880]\n\t" + "ldr q19, [%x[r], #944]\n\t" + "ldr q20, [%x[r], #1008]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r], #48]\n\t" + "str q6, [%x[r], #112]\n\t" + "str q7, [%x[r], #176]\n\t" + "str q8, [%x[r], #240]\n\t" + "str q9, [%x[r], #304]\n\t" + "str q10, [%x[r], #368]\n\t" + "str q11, [%x[r], #432]\n\t" + "str q12, [%x[r], #496]\n\t" + "str q13, [%x[r], #560]\n\t" + "str q14, [%x[r], #624]\n\t" + "str q15, [%x[r], #688]\n\t" + "str q16, [%x[r], #752]\n\t" + "str q17, [%x[r], #816]\n\t" + "str q18, [%x[r], #880]\n\t" + "str q19, [%x[r], #944]\n\t" + "str q20, [%x[r], #1008]\n\t" + "ldp q5, q6, [%x[r]]\n\t" + "ldp q7, q8, [%x[r], #32]\n\t" + "ldp q9, q10, [%x[r], #64]\n\t" + "ldp q11, q12, [%x[r], #96]\n\t" + "ldp q13, q14, [%x[r], #128]\n\t" + "ldp q15, q16, [%x[r], #160]\n\t" + "ldp q17, q18, [%x[r], #192]\n\t" + "ldp q19, q20, [%x[r], #224]\n\t" + "ldr q0, [%[zetas], #64]\n\t" + "ldr q1, [%[qinv], #64]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #128]\n\t" + "ldp q1, q3, [%[qinv], #128]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #256]\n\t" + "ldp q1, q3, [%[qinv], #256]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #288]\n\t" + "ldp q1, q3, [%[qinv], #288]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #320]\n\t" + "ldp q1, q3, [%[qinv], #320]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #352]\n\t" + "ldp q1, q3, [%[qinv], #352]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #768]\n\t" + "ldp q1, q3, [%[qinv], #768]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #800]\n\t" + "ldp q1, q3, [%[qinv], #800]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #832]\n\t" + "ldp q1, q3, [%[qinv], #832]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #864]\n\t" + "ldp q1, q3, [%[qinv], #864]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "stp q5, q6, [%x[r]]\n\t" + "stp q7, q8, [%x[r], #32]\n\t" + "stp q9, q10, [%x[r], #64]\n\t" + "stp q11, q12, [%x[r], #96]\n\t" + "stp q13, q14, [%x[r], #128]\n\t" + "stp q15, q16, [%x[r], #160]\n\t" + "stp q17, q18, [%x[r], #192]\n\t" + "stp q19, q20, [%x[r], #224]\n\t" + "ldp q5, q6, [%x[r], #256]\n\t" + "ldp q7, q8, [%x[r], #288]\n\t" + "ldp q9, q10, [%x[r], #320]\n\t" + "ldp q11, q12, [%x[r], #352]\n\t" + "ldp q13, q14, [%x[r], #384]\n\t" + "ldp q15, q16, [%x[r], #416]\n\t" + "ldp q17, q18, [%x[r], #448]\n\t" + "ldp q19, q20, [%x[r], #480]\n\t" + "ldr q0, [%[zetas], #80]\n\t" + "ldr q1, [%[qinv], #80]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #160]\n\t" + "ldp q1, q3, [%[qinv], #160]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #384]\n\t" + "ldp q1, q3, [%[qinv], #384]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #416]\n\t" + "ldp q1, q3, [%[qinv], #416]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #448]\n\t" + "ldp q1, q3, [%[qinv], #448]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #480]\n\t" + "ldp q1, q3, [%[qinv], #480]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #896]\n\t" + "ldp q1, q3, [%[qinv], #896]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #928]\n\t" + "ldp q1, q3, [%[qinv], #928]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #960]\n\t" + "ldp q1, q3, [%[qinv], #960]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #992]\n\t" + "ldp q1, q3, [%[qinv], #992]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "stp q5, q6, [%x[r], #256]\n\t" + "stp q7, q8, [%x[r], #288]\n\t" + "stp q9, q10, [%x[r], #320]\n\t" + "stp q11, q12, [%x[r], #352]\n\t" + "stp q13, q14, [%x[r], #384]\n\t" + "stp q15, q16, [%x[r], #416]\n\t" + "stp q17, q18, [%x[r], #448]\n\t" + "stp q19, q20, [%x[r], #480]\n\t" + "ldp q5, q6, [%x[r], #512]\n\t" + "ldp q7, q8, [%x[r], #544]\n\t" + "ldp q9, q10, [%x[r], #576]\n\t" + "ldp q11, q12, [%x[r], #608]\n\t" + "ldp q13, q14, [%x[r], #640]\n\t" + "ldp q15, q16, [%x[r], #672]\n\t" + "ldp q17, q18, [%x[r], #704]\n\t" + "ldp q19, q20, [%x[r], #736]\n\t" + "ldr q0, [%[zetas], #96]\n\t" + "ldr q1, [%[qinv], #96]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #192]\n\t" + "ldp q1, q3, [%[qinv], #192]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #512]\n\t" + "ldp q1, q3, [%[qinv], #512]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #544]\n\t" + "ldp q1, q3, [%[qinv], #544]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #576]\n\t" + "ldp q1, q3, [%[qinv], #576]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #608]\n\t" + "ldp q1, q3, [%[qinv], #608]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #1024]\n\t" + "ldr q2, [%[zetas], #1040]\n\t" + "ldr q1, [%[qinv], #1024]\n\t" + "ldr q3, [%[qinv], #1040]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldr q0, [%[zetas], #1056]\n\t" + "ldr q2, [%[zetas], #1072]\n\t" + "ldr q1, [%[qinv], #1056]\n\t" + "ldr q3, [%[qinv], #1072]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldr q0, [%[zetas], #1088]\n\t" + "ldr q2, [%[zetas], #1104]\n\t" + "ldr q1, [%[qinv], #1088]\n\t" + "ldr q3, [%[qinv], #1104]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldr q0, [%[zetas], #1120]\n\t" + "ldr q2, [%[zetas], #1136]\n\t" + "ldr q1, [%[qinv], #1120]\n\t" + "ldr q3, [%[qinv], #1136]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "stp q5, q6, [%x[r], #512]\n\t" + "stp q7, q8, [%x[r], #544]\n\t" + "stp q9, q10, [%x[r], #576]\n\t" + "stp q11, q12, [%x[r], #608]\n\t" + "stp q13, q14, [%x[r], #640]\n\t" + "stp q15, q16, [%x[r], #672]\n\t" + "stp q17, q18, [%x[r], #704]\n\t" + "stp q19, q20, [%x[r], #736]\n\t" + "ldp q5, q6, [%x[r], #768]\n\t" + "ldp q7, q8, [%x[r], #800]\n\t" + "ldp q9, q10, [%x[r], #832]\n\t" + "ldp q11, q12, [%x[r], #864]\n\t" + "ldp q13, q14, [%x[r], #896]\n\t" + "ldp q15, q16, [%x[r], #928]\n\t" + "ldp q17, q18, [%x[r], #960]\n\t" + "ldp q19, q20, [%x[r], #992]\n\t" + "ldr q0, [%[zetas], #112]\n\t" + "ldr q1, [%[qinv], #112]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #224]\n\t" + "ldp q1, q3, [%[qinv], #224]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #640]\n\t" + "ldp q1, q3, [%[qinv], #640]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #672]\n\t" + "ldp q1, q3, [%[qinv], #672]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #704]\n\t" + "ldp q1, q3, [%[qinv], #704]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #736]\n\t" + "ldp q1, q3, [%[qinv], #736]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #1152]\n\t" + "ldr q2, [%[zetas], #1168]\n\t" + "ldr q1, [%[qinv], #1152]\n\t" + "ldr q3, [%[qinv], #1168]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldr q0, [%[zetas], #1184]\n\t" + "ldr q2, [%[zetas], #1200]\n\t" + "ldr q1, [%[qinv], #1184]\n\t" + "ldr q3, [%[qinv], #1200]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldr q0, [%[zetas], #1216]\n\t" + "ldr q2, [%[zetas], #1232]\n\t" + "ldr q1, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldr q0, [%[zetas], #1248]\n\t" + "ldr q2, [%[zetas], #1264]\n\t" + "ldr q1, [%[qinv], #1248]\n\t" + "ldr q3, [%[qinv], #1264]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "stp q5, q6, [%x[r], #768]\n\t" + "stp q7, q8, [%x[r], #800]\n\t" + "stp q9, q10, [%x[r], #832]\n\t" + "stp q11, q12, [%x[r], #864]\n\t" + "stp q13, q14, [%x[r], #896]\n\t" + "stp q15, q16, [%x[r], #928]\n\t" + "stp q17, q18, [%x[r], #960]\n\t" + "stp q19, q20, [%x[r], #992]\n\t" + : [r] "+r" (r) + : [zetas] "r" (zetas), [qinv] "r" (qinv), [consts] "r" (consts) + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", + "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", + "v28", "v29", "v30" + ); +} + +void mldsa_ntt_full_sqrdmlsh_neon(sword32* r) +{ + const word32* zetas = L_mldsa_aarch64_zetas; + const word32* qinv = L_mldsa_aarch64_zetas_qinv; + const word32* consts = L_mldsa_aarch64_consts; + __asm__ __volatile__ ( + "ldr q4, [%[consts]]\n\t" + "ldr q5, [%x[r]]\n\t" + "ldr q6, [%x[r], #64]\n\t" + "ldr q7, [%x[r], #128]\n\t" + "ldr q8, [%x[r], #192]\n\t" + "ldr q9, [%x[r], #256]\n\t" + "ldr q10, [%x[r], #320]\n\t" + "ldr q11, [%x[r], #384]\n\t" + "ldr q12, [%x[r], #448]\n\t" + "ldr q13, [%x[r], #512]\n\t" + "ldr q14, [%x[r], #576]\n\t" + "ldr q15, [%x[r], #640]\n\t" + "ldr q16, [%x[r], #704]\n\t" + "ldr q17, [%x[r], #768]\n\t" + "ldr q18, [%x[r], #832]\n\t" + "ldr q19, [%x[r], #896]\n\t" + "ldr q20, [%x[r], #960]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r]]\n\t" + "str q6, [%x[r], #64]\n\t" + "str q7, [%x[r], #128]\n\t" + "str q8, [%x[r], #192]\n\t" + "str q9, [%x[r], #256]\n\t" + "str q10, [%x[r], #320]\n\t" + "str q11, [%x[r], #384]\n\t" + "str q12, [%x[r], #448]\n\t" + "str q13, [%x[r], #512]\n\t" + "str q14, [%x[r], #576]\n\t" + "str q15, [%x[r], #640]\n\t" + "str q16, [%x[r], #704]\n\t" + "str q17, [%x[r], #768]\n\t" + "str q18, [%x[r], #832]\n\t" + "str q19, [%x[r], #896]\n\t" + "str q20, [%x[r], #960]\n\t" + "ldr q5, [%x[r], #16]\n\t" + "ldr q6, [%x[r], #80]\n\t" + "ldr q7, [%x[r], #144]\n\t" + "ldr q8, [%x[r], #208]\n\t" + "ldr q9, [%x[r], #272]\n\t" + "ldr q10, [%x[r], #336]\n\t" + "ldr q11, [%x[r], #400]\n\t" + "ldr q12, [%x[r], #464]\n\t" + "ldr q13, [%x[r], #528]\n\t" + "ldr q14, [%x[r], #592]\n\t" + "ldr q15, [%x[r], #656]\n\t" + "ldr q16, [%x[r], #720]\n\t" + "ldr q17, [%x[r], #784]\n\t" + "ldr q18, [%x[r], #848]\n\t" + "ldr q19, [%x[r], #912]\n\t" + "ldr q20, [%x[r], #976]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r], #16]\n\t" + "str q6, [%x[r], #80]\n\t" + "str q7, [%x[r], #144]\n\t" + "str q8, [%x[r], #208]\n\t" + "str q9, [%x[r], #272]\n\t" + "str q10, [%x[r], #336]\n\t" + "str q11, [%x[r], #400]\n\t" + "str q12, [%x[r], #464]\n\t" + "str q13, [%x[r], #528]\n\t" + "str q14, [%x[r], #592]\n\t" + "str q15, [%x[r], #656]\n\t" + "str q16, [%x[r], #720]\n\t" + "str q17, [%x[r], #784]\n\t" + "str q18, [%x[r], #848]\n\t" + "str q19, [%x[r], #912]\n\t" + "str q20, [%x[r], #976]\n\t" + "ldr q5, [%x[r], #32]\n\t" + "ldr q6, [%x[r], #96]\n\t" + "ldr q7, [%x[r], #160]\n\t" + "ldr q8, [%x[r], #224]\n\t" + "ldr q9, [%x[r], #288]\n\t" + "ldr q10, [%x[r], #352]\n\t" + "ldr q11, [%x[r], #416]\n\t" + "ldr q12, [%x[r], #480]\n\t" + "ldr q13, [%x[r], #544]\n\t" + "ldr q14, [%x[r], #608]\n\t" + "ldr q15, [%x[r], #672]\n\t" + "ldr q16, [%x[r], #736]\n\t" + "ldr q17, [%x[r], #800]\n\t" + "ldr q18, [%x[r], #864]\n\t" + "ldr q19, [%x[r], #928]\n\t" + "ldr q20, [%x[r], #992]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r], #32]\n\t" + "str q6, [%x[r], #96]\n\t" + "str q7, [%x[r], #160]\n\t" + "str q8, [%x[r], #224]\n\t" + "str q9, [%x[r], #288]\n\t" + "str q10, [%x[r], #352]\n\t" + "str q11, [%x[r], #416]\n\t" + "str q12, [%x[r], #480]\n\t" + "str q13, [%x[r], #544]\n\t" + "str q14, [%x[r], #608]\n\t" + "str q15, [%x[r], #672]\n\t" + "str q16, [%x[r], #736]\n\t" + "str q17, [%x[r], #800]\n\t" + "str q18, [%x[r], #864]\n\t" + "str q19, [%x[r], #928]\n\t" + "str q20, [%x[r], #992]\n\t" + "ldr q5, [%x[r], #48]\n\t" + "ldr q6, [%x[r], #112]\n\t" + "ldr q7, [%x[r], #176]\n\t" + "ldr q8, [%x[r], #240]\n\t" + "ldr q9, [%x[r], #304]\n\t" + "ldr q10, [%x[r], #368]\n\t" + "ldr q11, [%x[r], #432]\n\t" + "ldr q12, [%x[r], #496]\n\t" + "ldr q13, [%x[r], #560]\n\t" + "ldr q14, [%x[r], #624]\n\t" + "ldr q15, [%x[r], #688]\n\t" + "ldr q16, [%x[r], #752]\n\t" + "ldr q17, [%x[r], #816]\n\t" + "ldr q18, [%x[r], #880]\n\t" + "ldr q19, [%x[r], #944]\n\t" + "ldr q20, [%x[r], #1008]\n\t" + "ldr q0, [%[zetas]]\n\t" + "ldr q1, [%[qinv]]\n\t" + "mul v29.4s, v13.4s, v1.s[1]\n\t" + "mul v30.4s, v14.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v13.4s, v0.s[1]\n\t" + "sqrdmulh v22.4s, v14.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[1]\n\t" + "mul v30.4s, v16.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v15.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v16.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[1]\n\t" + "mul v30.4s, v18.4s, v1.s[1]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[1]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[1]\n\t" + "mul v30.4s, v20.4s, v1.s[1]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[1]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[1]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v13.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v14.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v15.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v16.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v9.4s, v25.4s\n\t" + "add v9.4s, v9.4s, v25.4s\n\t" + "sub v18.4s, v10.4s, v26.4s\n\t" + "add v10.4s, v10.4s, v26.4s\n\t" + "sub v19.4s, v11.4s, v27.4s\n\t" + "add v11.4s, v11.4s, v27.4s\n\t" + "sub v20.4s, v12.4s, v28.4s\n\t" + "add v12.4s, v12.4s, v28.4s\n\t" + "mul v29.4s, v9.4s, v1.s[2]\n\t" + "mul v30.4s, v10.4s, v1.s[2]\n\t" + "sqrdmulh v21.4s, v9.4s, v0.s[2]\n\t" + "sqrdmulh v22.4s, v10.4s, v0.s[2]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[2]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[2]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v17.4s, v1.s[3]\n\t" + "mul v30.4s, v18.4s, v1.s[3]\n\t" + "sqrdmulh v25.4s, v17.4s, v0.s[3]\n\t" + "sqrdmulh v26.4s, v18.4s, v0.s[3]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v9.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v10.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v7.4s, v23.4s\n\t" + "add v7.4s, v7.4s, v23.4s\n\t" + "sub v12.4s, v8.4s, v24.4s\n\t" + "add v8.4s, v8.4s, v24.4s\n\t" + "sub v17.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v18.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v15.4s, v27.4s\n\t" + "add v15.4s, v15.4s, v27.4s\n\t" + "sub v20.4s, v16.4s, v28.4s\n\t" + "add v16.4s, v16.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #16]\n\t" + "ldr q1, [%[qinv], #16]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #32]\n\t" + "ldp q1, q3, [%[qinv], #32]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "str q5, [%x[r], #48]\n\t" + "str q6, [%x[r], #112]\n\t" + "str q7, [%x[r], #176]\n\t" + "str q8, [%x[r], #240]\n\t" + "str q9, [%x[r], #304]\n\t" + "str q10, [%x[r], #368]\n\t" + "str q11, [%x[r], #432]\n\t" + "str q12, [%x[r], #496]\n\t" + "str q13, [%x[r], #560]\n\t" + "str q14, [%x[r], #624]\n\t" + "str q15, [%x[r], #688]\n\t" + "str q16, [%x[r], #752]\n\t" + "str q17, [%x[r], #816]\n\t" + "str q18, [%x[r], #880]\n\t" + "str q19, [%x[r], #944]\n\t" + "str q20, [%x[r], #1008]\n\t" + "ldp q5, q6, [%x[r]]\n\t" + "ldp q7, q8, [%x[r], #32]\n\t" + "ldp q9, q10, [%x[r], #64]\n\t" + "ldp q11, q12, [%x[r], #96]\n\t" + "ldp q13, q14, [%x[r], #128]\n\t" + "ldp q15, q16, [%x[r], #160]\n\t" + "ldp q17, q18, [%x[r], #192]\n\t" + "ldp q19, q20, [%x[r], #224]\n\t" + "ldr q0, [%[zetas], #64]\n\t" + "ldr q1, [%[qinv], #64]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #128]\n\t" + "ldp q1, q3, [%[qinv], #128]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #256]\n\t" + "ldp q1, q3, [%[qinv], #256]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #288]\n\t" + "ldp q1, q3, [%[qinv], #288]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #320]\n\t" + "ldp q1, q3, [%[qinv], #320]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #352]\n\t" + "ldp q1, q3, [%[qinv], #352]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #768]\n\t" + "ldp q1, q3, [%[qinv], #768]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #800]\n\t" + "ldp q1, q3, [%[qinv], #800]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #832]\n\t" + "ldp q1, q3, [%[qinv], #832]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #864]\n\t" + "ldp q1, q3, [%[qinv], #864]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "trn1 v29.4s, v5.4s, v6.4s\n\t" + "trn2 v30.4s, v5.4s, v6.4s\n\t" + "trn1 v5.2d, v29.2d, v30.2d\n\t" + "trn2 v6.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v7.4s, v8.4s\n\t" + "trn2 v30.4s, v7.4s, v8.4s\n\t" + "trn1 v7.2d, v29.2d, v30.2d\n\t" + "trn2 v8.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v9.4s, v10.4s\n\t" + "trn2 v30.4s, v9.4s, v10.4s\n\t" + "trn1 v9.2d, v29.2d, v30.2d\n\t" + "trn2 v10.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v11.4s, v12.4s\n\t" + "trn2 v30.4s, v11.4s, v12.4s\n\t" + "trn1 v11.2d, v29.2d, v30.2d\n\t" + "trn2 v12.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v13.4s, v14.4s\n\t" + "trn2 v30.4s, v13.4s, v14.4s\n\t" + "trn1 v13.2d, v29.2d, v30.2d\n\t" + "trn2 v14.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v15.4s, v16.4s\n\t" + "trn2 v30.4s, v15.4s, v16.4s\n\t" + "trn1 v15.2d, v29.2d, v30.2d\n\t" + "trn2 v16.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v17.4s, v18.4s\n\t" + "trn2 v30.4s, v17.4s, v18.4s\n\t" + "trn1 v17.2d, v29.2d, v30.2d\n\t" + "trn2 v18.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v19.4s, v20.4s\n\t" + "trn2 v30.4s, v19.4s, v20.4s\n\t" + "trn1 v19.2d, v29.2d, v30.2d\n\t" + "trn2 v20.2d, v29.2d, v30.2d\n\t" + "stp q5, q6, [%x[r]]\n\t" + "stp q7, q8, [%x[r], #32]\n\t" + "stp q9, q10, [%x[r], #64]\n\t" + "stp q11, q12, [%x[r], #96]\n\t" + "stp q13, q14, [%x[r], #128]\n\t" + "stp q15, q16, [%x[r], #160]\n\t" + "stp q17, q18, [%x[r], #192]\n\t" + "stp q19, q20, [%x[r], #224]\n\t" + "ldp q5, q6, [%x[r], #256]\n\t" + "ldp q7, q8, [%x[r], #288]\n\t" + "ldp q9, q10, [%x[r], #320]\n\t" + "ldp q11, q12, [%x[r], #352]\n\t" + "ldp q13, q14, [%x[r], #384]\n\t" + "ldp q15, q16, [%x[r], #416]\n\t" + "ldp q17, q18, [%x[r], #448]\n\t" + "ldp q19, q20, [%x[r], #480]\n\t" + "ldr q0, [%[zetas], #80]\n\t" + "ldr q1, [%[qinv], #80]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #160]\n\t" + "ldp q1, q3, [%[qinv], #160]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #384]\n\t" + "ldp q1, q3, [%[qinv], #384]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #416]\n\t" + "ldp q1, q3, [%[qinv], #416]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #448]\n\t" + "ldp q1, q3, [%[qinv], #448]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #480]\n\t" + "ldp q1, q3, [%[qinv], #480]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #896]\n\t" + "ldp q1, q3, [%[qinv], #896]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #928]\n\t" + "ldp q1, q3, [%[qinv], #928]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #960]\n\t" + "ldp q1, q3, [%[qinv], #960]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #992]\n\t" + "ldp q1, q3, [%[qinv], #992]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "trn1 v29.4s, v5.4s, v6.4s\n\t" + "trn2 v30.4s, v5.4s, v6.4s\n\t" + "trn1 v5.2d, v29.2d, v30.2d\n\t" + "trn2 v6.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v7.4s, v8.4s\n\t" + "trn2 v30.4s, v7.4s, v8.4s\n\t" + "trn1 v7.2d, v29.2d, v30.2d\n\t" + "trn2 v8.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v9.4s, v10.4s\n\t" + "trn2 v30.4s, v9.4s, v10.4s\n\t" + "trn1 v9.2d, v29.2d, v30.2d\n\t" + "trn2 v10.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v11.4s, v12.4s\n\t" + "trn2 v30.4s, v11.4s, v12.4s\n\t" + "trn1 v11.2d, v29.2d, v30.2d\n\t" + "trn2 v12.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v13.4s, v14.4s\n\t" + "trn2 v30.4s, v13.4s, v14.4s\n\t" + "trn1 v13.2d, v29.2d, v30.2d\n\t" + "trn2 v14.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v15.4s, v16.4s\n\t" + "trn2 v30.4s, v15.4s, v16.4s\n\t" + "trn1 v15.2d, v29.2d, v30.2d\n\t" + "trn2 v16.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v17.4s, v18.4s\n\t" + "trn2 v30.4s, v17.4s, v18.4s\n\t" + "trn1 v17.2d, v29.2d, v30.2d\n\t" + "trn2 v18.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v19.4s, v20.4s\n\t" + "trn2 v30.4s, v19.4s, v20.4s\n\t" + "trn1 v19.2d, v29.2d, v30.2d\n\t" + "trn2 v20.2d, v29.2d, v30.2d\n\t" + "stp q5, q6, [%x[r], #256]\n\t" + "stp q7, q8, [%x[r], #288]\n\t" + "stp q9, q10, [%x[r], #320]\n\t" + "stp q11, q12, [%x[r], #352]\n\t" + "stp q13, q14, [%x[r], #384]\n\t" + "stp q15, q16, [%x[r], #416]\n\t" + "stp q17, q18, [%x[r], #448]\n\t" + "stp q19, q20, [%x[r], #480]\n\t" + "ldp q5, q6, [%x[r], #512]\n\t" + "ldp q7, q8, [%x[r], #544]\n\t" + "ldp q9, q10, [%x[r], #576]\n\t" + "ldp q11, q12, [%x[r], #608]\n\t" + "ldp q13, q14, [%x[r], #640]\n\t" + "ldp q15, q16, [%x[r], #672]\n\t" + "ldp q17, q18, [%x[r], #704]\n\t" + "ldp q19, q20, [%x[r], #736]\n\t" + "ldr q0, [%[zetas], #96]\n\t" + "ldr q1, [%[qinv], #96]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #192]\n\t" + "ldp q1, q3, [%[qinv], #192]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #512]\n\t" + "ldp q1, q3, [%[qinv], #512]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #544]\n\t" + "ldp q1, q3, [%[qinv], #544]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #576]\n\t" + "ldp q1, q3, [%[qinv], #576]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #608]\n\t" + "ldp q1, q3, [%[qinv], #608]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #1024]\n\t" + "ldr q2, [%[zetas], #1040]\n\t" + "ldr q1, [%[qinv], #1024]\n\t" + "ldr q3, [%[qinv], #1040]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldr q0, [%[zetas], #1056]\n\t" + "ldr q2, [%[zetas], #1072]\n\t" + "ldr q1, [%[qinv], #1056]\n\t" + "ldr q3, [%[qinv], #1072]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldr q0, [%[zetas], #1088]\n\t" + "ldr q2, [%[zetas], #1104]\n\t" + "ldr q1, [%[qinv], #1088]\n\t" + "ldr q3, [%[qinv], #1104]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldr q0, [%[zetas], #1120]\n\t" + "ldr q2, [%[zetas], #1136]\n\t" + "ldr q1, [%[qinv], #1120]\n\t" + "ldr q3, [%[qinv], #1136]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "trn1 v29.4s, v5.4s, v6.4s\n\t" + "trn2 v30.4s, v5.4s, v6.4s\n\t" + "trn1 v5.2d, v29.2d, v30.2d\n\t" + "trn2 v6.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v7.4s, v8.4s\n\t" + "trn2 v30.4s, v7.4s, v8.4s\n\t" + "trn1 v7.2d, v29.2d, v30.2d\n\t" + "trn2 v8.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v9.4s, v10.4s\n\t" + "trn2 v30.4s, v9.4s, v10.4s\n\t" + "trn1 v9.2d, v29.2d, v30.2d\n\t" + "trn2 v10.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v11.4s, v12.4s\n\t" + "trn2 v30.4s, v11.4s, v12.4s\n\t" + "trn1 v11.2d, v29.2d, v30.2d\n\t" + "trn2 v12.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v13.4s, v14.4s\n\t" + "trn2 v30.4s, v13.4s, v14.4s\n\t" + "trn1 v13.2d, v29.2d, v30.2d\n\t" + "trn2 v14.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v15.4s, v16.4s\n\t" + "trn2 v30.4s, v15.4s, v16.4s\n\t" + "trn1 v15.2d, v29.2d, v30.2d\n\t" + "trn2 v16.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v17.4s, v18.4s\n\t" + "trn2 v30.4s, v17.4s, v18.4s\n\t" + "trn1 v17.2d, v29.2d, v30.2d\n\t" + "trn2 v18.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v19.4s, v20.4s\n\t" + "trn2 v30.4s, v19.4s, v20.4s\n\t" + "trn1 v19.2d, v29.2d, v30.2d\n\t" + "trn2 v20.2d, v29.2d, v30.2d\n\t" + "stp q5, q6, [%x[r], #512]\n\t" + "stp q7, q8, [%x[r], #544]\n\t" + "stp q9, q10, [%x[r], #576]\n\t" + "stp q11, q12, [%x[r], #608]\n\t" + "stp q13, q14, [%x[r], #640]\n\t" + "stp q15, q16, [%x[r], #672]\n\t" + "stp q17, q18, [%x[r], #704]\n\t" + "stp q19, q20, [%x[r], #736]\n\t" + "ldp q5, q6, [%x[r], #768]\n\t" + "ldp q7, q8, [%x[r], #800]\n\t" + "ldp q9, q10, [%x[r], #832]\n\t" + "ldp q11, q12, [%x[r], #864]\n\t" + "ldp q13, q14, [%x[r], #896]\n\t" + "ldp q15, q16, [%x[r], #928]\n\t" + "ldp q17, q18, [%x[r], #960]\n\t" + "ldp q19, q20, [%x[r], #992]\n\t" + "ldr q0, [%[zetas], #112]\n\t" + "ldr q1, [%[qinv], #112]\n\t" + "mul v29.4s, v7.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[0]\n\t" + "sqrdmulh v21.4s, v7.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[0]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v11.4s, v1.s[1]\n\t" + "mul v30.4s, v12.4s, v1.s[1]\n\t" + "sqrdmulh v23.4s, v11.4s, v0.s[1]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[1]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v15.4s, v1.s[2]\n\t" + "mul v30.4s, v16.4s, v1.s[2]\n\t" + "sqrdmulh v25.4s, v15.4s, v0.s[2]\n\t" + "sqrdmulh v26.4s, v16.4s, v0.s[2]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v19.4s, v1.s[3]\n\t" + "mul v30.4s, v20.4s, v1.s[3]\n\t" + "sqrdmulh v27.4s, v19.4s, v0.s[3]\n\t" + "sqrdmulh v28.4s, v20.4s, v0.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v7.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v6.4s, v22.4s\n\t" + "add v6.4s, v6.4s, v22.4s\n\t" + "sub v11.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v10.4s, v24.4s\n\t" + "add v10.4s, v10.4s, v24.4s\n\t" + "sub v15.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v14.4s, v26.4s\n\t" + "add v14.4s, v14.4s, v26.4s\n\t" + "sub v19.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v18.4s, v28.4s\n\t" + "add v18.4s, v18.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #224]\n\t" + "ldp q1, q3, [%[qinv], #224]\n\t" + "mul v29.4s, v6.4s, v1.s[0]\n\t" + "mul v30.4s, v8.4s, v1.s[1]\n\t" + "sqrdmulh v21.4s, v6.4s, v0.s[0]\n\t" + "sqrdmulh v22.4s, v8.4s, v0.s[1]\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "mul v29.4s, v10.4s, v1.s[2]\n\t" + "mul v30.4s, v12.4s, v1.s[3]\n\t" + "sqrdmulh v23.4s, v10.4s, v0.s[2]\n\t" + "sqrdmulh v24.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "mul v29.4s, v14.4s, v3.s[0]\n\t" + "mul v30.4s, v16.4s, v3.s[1]\n\t" + "sqrdmulh v25.4s, v14.4s, v2.s[0]\n\t" + "sqrdmulh v26.4s, v16.4s, v2.s[1]\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "mul v29.4s, v18.4s, v3.s[2]\n\t" + "mul v30.4s, v20.4s, v3.s[3]\n\t" + "sqrdmulh v27.4s, v18.4s, v2.s[2]\n\t" + "sqrdmulh v28.4s, v20.4s, v2.s[3]\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldp q0, q2, [%[zetas], #640]\n\t" + "ldp q1, q3, [%[qinv], #640]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v29.2d, v6.2d\n\t" + "trn2 v8.2d, v30.2d, v8.2d\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #672]\n\t" + "ldp q1, q3, [%[qinv], #672]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v29.2d, v10.2d\n\t" + "trn2 v12.2d, v30.2d, v12.2d\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #704]\n\t" + "ldp q1, q3, [%[qinv], #704]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v29.2d, v14.2d\n\t" + "trn2 v16.2d, v30.2d, v16.2d\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldp q0, q2, [%[zetas], #736]\n\t" + "ldp q1, q3, [%[qinv], #736]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v29.2d, v18.2d\n\t" + "trn2 v20.2d, v30.2d, v20.2d\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "ldr q0, [%[zetas], #1152]\n\t" + "ldr q2, [%[zetas], #1168]\n\t" + "ldr q1, [%[qinv], #1152]\n\t" + "ldr q3, [%[qinv], #1168]\n\t" + "mov v29.16b, v5.16b\n\t" + "mov v30.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v29.4s, v6.4s\n\t" + "trn2 v8.4s, v30.4s, v8.4s\n\t" + "mul v29.4s, v6.4s, v1.4s\n\t" + "mul v30.4s, v8.4s, v3.4s\n\t" + "sqrdmulh v21.4s, v6.4s, v0.4s\n\t" + "sqrdmulh v22.4s, v8.4s, v2.4s\n\t" + "sqrdmlsh v21.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v22.4s, v30.4s, v4.s[0]\n\t" + "sshr v21.4s, v21.4s, #1\n\t" + "sshr v22.4s, v22.4s, #1\n\t" + "ldr q0, [%[zetas], #1184]\n\t" + "ldr q2, [%[zetas], #1200]\n\t" + "ldr q1, [%[qinv], #1184]\n\t" + "ldr q3, [%[qinv], #1200]\n\t" + "mov v29.16b, v9.16b\n\t" + "mov v30.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v29.4s, v10.4s\n\t" + "trn2 v12.4s, v30.4s, v12.4s\n\t" + "mul v29.4s, v10.4s, v1.4s\n\t" + "mul v30.4s, v12.4s, v3.4s\n\t" + "sqrdmulh v23.4s, v10.4s, v0.4s\n\t" + "sqrdmulh v24.4s, v12.4s, v2.4s\n\t" + "sqrdmlsh v23.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v24.4s, v30.4s, v4.s[0]\n\t" + "sshr v23.4s, v23.4s, #1\n\t" + "sshr v24.4s, v24.4s, #1\n\t" + "ldr q0, [%[zetas], #1216]\n\t" + "ldr q2, [%[zetas], #1232]\n\t" + "ldr q1, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "mov v29.16b, v13.16b\n\t" + "mov v30.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v29.4s, v14.4s\n\t" + "trn2 v16.4s, v30.4s, v16.4s\n\t" + "mul v29.4s, v14.4s, v1.4s\n\t" + "mul v30.4s, v16.4s, v3.4s\n\t" + "sqrdmulh v25.4s, v14.4s, v0.4s\n\t" + "sqrdmulh v26.4s, v16.4s, v2.4s\n\t" + "sqrdmlsh v25.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v26.4s, v30.4s, v4.s[0]\n\t" + "sshr v25.4s, v25.4s, #1\n\t" + "sshr v26.4s, v26.4s, #1\n\t" + "ldr q0, [%[zetas], #1248]\n\t" + "ldr q2, [%[zetas], #1264]\n\t" + "ldr q1, [%[qinv], #1248]\n\t" + "ldr q3, [%[qinv], #1264]\n\t" + "mov v29.16b, v17.16b\n\t" + "mov v30.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v29.4s, v18.4s\n\t" + "trn2 v20.4s, v30.4s, v20.4s\n\t" + "mul v29.4s, v18.4s, v1.4s\n\t" + "mul v30.4s, v20.4s, v3.4s\n\t" + "sqrdmulh v27.4s, v18.4s, v0.4s\n\t" + "sqrdmulh v28.4s, v20.4s, v2.4s\n\t" + "sqrdmlsh v27.4s, v29.4s, v4.s[0]\n\t" + "sqrdmlsh v28.4s, v30.4s, v4.s[0]\n\t" + "sshr v27.4s, v27.4s, #1\n\t" + "sshr v28.4s, v28.4s, #1\n\t" + "sub v6.4s, v5.4s, v21.4s\n\t" + "add v5.4s, v5.4s, v21.4s\n\t" + "sub v8.4s, v7.4s, v22.4s\n\t" + "add v7.4s, v7.4s, v22.4s\n\t" + "sub v10.4s, v9.4s, v23.4s\n\t" + "add v9.4s, v9.4s, v23.4s\n\t" + "sub v12.4s, v11.4s, v24.4s\n\t" + "add v11.4s, v11.4s, v24.4s\n\t" + "sub v14.4s, v13.4s, v25.4s\n\t" + "add v13.4s, v13.4s, v25.4s\n\t" + "sub v16.4s, v15.4s, v26.4s\n\t" + "add v15.4s, v15.4s, v26.4s\n\t" + "sub v18.4s, v17.4s, v27.4s\n\t" + "add v17.4s, v17.4s, v27.4s\n\t" + "sub v20.4s, v19.4s, v28.4s\n\t" + "add v19.4s, v19.4s, v28.4s\n\t" + "trn1 v29.4s, v5.4s, v6.4s\n\t" + "trn2 v30.4s, v5.4s, v6.4s\n\t" + "trn1 v5.2d, v29.2d, v30.2d\n\t" + "trn2 v6.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v7.4s, v8.4s\n\t" + "trn2 v30.4s, v7.4s, v8.4s\n\t" + "trn1 v7.2d, v29.2d, v30.2d\n\t" + "trn2 v8.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v9.4s, v10.4s\n\t" + "trn2 v30.4s, v9.4s, v10.4s\n\t" + "trn1 v9.2d, v29.2d, v30.2d\n\t" + "trn2 v10.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v11.4s, v12.4s\n\t" + "trn2 v30.4s, v11.4s, v12.4s\n\t" + "trn1 v11.2d, v29.2d, v30.2d\n\t" + "trn2 v12.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v13.4s, v14.4s\n\t" + "trn2 v30.4s, v13.4s, v14.4s\n\t" + "trn1 v13.2d, v29.2d, v30.2d\n\t" + "trn2 v14.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v15.4s, v16.4s\n\t" + "trn2 v30.4s, v15.4s, v16.4s\n\t" + "trn1 v15.2d, v29.2d, v30.2d\n\t" + "trn2 v16.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v17.4s, v18.4s\n\t" + "trn2 v30.4s, v17.4s, v18.4s\n\t" + "trn1 v17.2d, v29.2d, v30.2d\n\t" + "trn2 v18.2d, v29.2d, v30.2d\n\t" + "trn1 v29.4s, v19.4s, v20.4s\n\t" + "trn2 v30.4s, v19.4s, v20.4s\n\t" + "trn1 v19.2d, v29.2d, v30.2d\n\t" + "trn2 v20.2d, v29.2d, v30.2d\n\t" + "stp q5, q6, [%x[r], #768]\n\t" + "stp q7, q8, [%x[r], #800]\n\t" + "stp q9, q10, [%x[r], #832]\n\t" + "stp q11, q12, [%x[r], #864]\n\t" + "stp q13, q14, [%x[r], #896]\n\t" + "stp q15, q16, [%x[r], #928]\n\t" + "stp q17, q18, [%x[r], #960]\n\t" + "stp q19, q20, [%x[r], #992]\n\t" + : [r] "+r" (r) + : [zetas] "r" (zetas), [qinv] "r" (qinv), [consts] "r" (consts) + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", + "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v23", "v24", "v25", "v26", "v27", + "v28", "v29", "v30" + ); +} + +void mldsa_invntt_sqrdmlsh_neon(sword32* r) +{ + const word32* inv = L_mldsa_aarch64_zetas_inv; + const word32* qinv = L_mldsa_aarch64_zetas_inv_qinv; + const word32* consts = L_mldsa_aarch64_consts; + __asm__ __volatile__ ( + "ldr q4, [%[consts]]\n\t" + "ldp q5, q6, [%x[r]]\n\t" + "ldp q7, q8, [%x[r], #32]\n\t" + "ldp q9, q10, [%x[r], #64]\n\t" + "ldp q11, q12, [%x[r], #96]\n\t" + "ldp q13, q14, [%x[r], #128]\n\t" + "ldp q15, q16, [%x[r], #160]\n\t" + "ldp q17, q18, [%x[r], #192]\n\t" + "ldp q19, q20, [%x[r], #224]\n\t" + "ldp q0, q1, [%[inv]]\n\t" + "ldp q2, q3, [%[qinv]]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #32]\n\t" + "ldp q2, q3, [%[qinv], #32]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #64]\n\t" + "ldp q2, q3, [%[qinv], #64]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #96]\n\t" + "ldp q2, q3, [%[qinv], #96]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #512]\n\t" + "ldp q2, q3, [%[qinv], #512]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #544]\n\t" + "ldp q2, q3, [%[qinv], #544]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #576]\n\t" + "ldp q2, q3, [%[qinv], #576]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #608]\n\t" + "ldp q2, q3, [%[qinv], #608]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1024]\n\t" + "ldr q1, [%[inv], #1040]\n\t" + "ldr q2, [%[qinv], #1024]\n\t" + "ldr q3, [%[qinv], #1040]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1152]\n\t" + "ldr q2, [%[qinv], #1152]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r]]\n\t" + "stp q7, q8, [%x[r], #32]\n\t" + "stp q9, q10, [%x[r], #64]\n\t" + "stp q11, q12, [%x[r], #96]\n\t" + "stp q13, q14, [%x[r], #128]\n\t" + "stp q15, q16, [%x[r], #160]\n\t" + "stp q17, q18, [%x[r], #192]\n\t" + "stp q19, q20, [%x[r], #224]\n\t" + "ldp q5, q6, [%x[r], #256]\n\t" + "ldp q7, q8, [%x[r], #288]\n\t" + "ldp q9, q10, [%x[r], #320]\n\t" + "ldp q11, q12, [%x[r], #352]\n\t" + "ldp q13, q14, [%x[r], #384]\n\t" + "ldp q15, q16, [%x[r], #416]\n\t" + "ldp q17, q18, [%x[r], #448]\n\t" + "ldp q19, q20, [%x[r], #480]\n\t" + "ldp q0, q1, [%[inv], #128]\n\t" + "ldp q2, q3, [%[qinv], #128]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #160]\n\t" + "ldp q2, q3, [%[qinv], #160]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #192]\n\t" + "ldp q2, q3, [%[qinv], #192]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #224]\n\t" + "ldp q2, q3, [%[qinv], #224]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #640]\n\t" + "ldp q2, q3, [%[qinv], #640]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #672]\n\t" + "ldp q2, q3, [%[qinv], #672]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #704]\n\t" + "ldp q2, q3, [%[qinv], #704]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #736]\n\t" + "ldp q2, q3, [%[qinv], #736]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1056]\n\t" + "ldr q1, [%[inv], #1072]\n\t" + "ldr q2, [%[qinv], #1056]\n\t" + "ldr q3, [%[qinv], #1072]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1168]\n\t" + "ldr q2, [%[qinv], #1168]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r], #256]\n\t" + "stp q7, q8, [%x[r], #288]\n\t" + "stp q9, q10, [%x[r], #320]\n\t" + "stp q11, q12, [%x[r], #352]\n\t" + "stp q13, q14, [%x[r], #384]\n\t" + "stp q15, q16, [%x[r], #416]\n\t" + "stp q17, q18, [%x[r], #448]\n\t" + "stp q19, q20, [%x[r], #480]\n\t" + "ldp q5, q6, [%x[r], #512]\n\t" + "ldp q7, q8, [%x[r], #544]\n\t" + "ldp q9, q10, [%x[r], #576]\n\t" + "ldp q11, q12, [%x[r], #608]\n\t" + "ldp q13, q14, [%x[r], #640]\n\t" + "ldp q15, q16, [%x[r], #672]\n\t" + "ldp q17, q18, [%x[r], #704]\n\t" + "ldp q19, q20, [%x[r], #736]\n\t" + "ldp q0, q1, [%[inv], #256]\n\t" + "ldp q2, q3, [%[qinv], #256]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #288]\n\t" + "ldp q2, q3, [%[qinv], #288]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #320]\n\t" + "ldp q2, q3, [%[qinv], #320]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #352]\n\t" + "ldp q2, q3, [%[qinv], #352]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #768]\n\t" + "ldp q2, q3, [%[qinv], #768]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #800]\n\t" + "ldp q2, q3, [%[qinv], #800]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #832]\n\t" + "ldp q2, q3, [%[qinv], #832]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #864]\n\t" + "ldp q2, q3, [%[qinv], #864]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1088]\n\t" + "ldr q1, [%[inv], #1104]\n\t" + "ldr q2, [%[qinv], #1088]\n\t" + "ldr q3, [%[qinv], #1104]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1184]\n\t" + "ldr q2, [%[qinv], #1184]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r], #512]\n\t" + "stp q7, q8, [%x[r], #544]\n\t" + "stp q9, q10, [%x[r], #576]\n\t" + "stp q11, q12, [%x[r], #608]\n\t" + "stp q13, q14, [%x[r], #640]\n\t" + "stp q15, q16, [%x[r], #672]\n\t" + "stp q17, q18, [%x[r], #704]\n\t" + "stp q19, q20, [%x[r], #736]\n\t" + "ldp q5, q6, [%x[r], #768]\n\t" + "ldp q7, q8, [%x[r], #800]\n\t" + "ldp q9, q10, [%x[r], #832]\n\t" + "ldp q11, q12, [%x[r], #864]\n\t" + "ldp q13, q14, [%x[r], #896]\n\t" + "ldp q15, q16, [%x[r], #928]\n\t" + "ldp q17, q18, [%x[r], #960]\n\t" + "ldp q19, q20, [%x[r], #992]\n\t" + "ldp q0, q1, [%[inv], #384]\n\t" + "ldp q2, q3, [%[qinv], #384]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #416]\n\t" + "ldp q2, q3, [%[qinv], #416]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #448]\n\t" + "ldp q2, q3, [%[qinv], #448]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #480]\n\t" + "ldp q2, q3, [%[qinv], #480]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #896]\n\t" + "ldp q2, q3, [%[qinv], #896]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #928]\n\t" + "ldp q2, q3, [%[qinv], #928]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #960]\n\t" + "ldp q2, q3, [%[qinv], #960]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #992]\n\t" + "ldp q2, q3, [%[qinv], #992]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1120]\n\t" + "ldr q1, [%[inv], #1136]\n\t" + "ldr q2, [%[qinv], #1120]\n\t" + "ldr q3, [%[qinv], #1136]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1200]\n\t" + "ldr q2, [%[qinv], #1200]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r], #768]\n\t" + "stp q7, q8, [%x[r], #800]\n\t" + "stp q9, q10, [%x[r], #832]\n\t" + "stp q11, q12, [%x[r], #864]\n\t" + "stp q13, q14, [%x[r], #896]\n\t" + "stp q15, q16, [%x[r], #928]\n\t" + "stp q17, q18, [%x[r], #960]\n\t" + "stp q19, q20, [%x[r], #992]\n\t" + "ldr q5, [%x[r]]\n\t" + "ldr q6, [%x[r], #64]\n\t" + "ldr q7, [%x[r], #128]\n\t" + "ldr q8, [%x[r], #192]\n\t" + "ldr q9, [%x[r], #256]\n\t" + "ldr q10, [%x[r], #320]\n\t" + "ldr q11, [%x[r], #384]\n\t" + "ldr q12, [%x[r], #448]\n\t" + "ldr q13, [%x[r], #512]\n\t" + "ldr q14, [%x[r], #576]\n\t" + "ldr q15, [%x[r], #640]\n\t" + "ldr q16, [%x[r], #704]\n\t" + "ldr q17, [%x[r], #768]\n\t" + "ldr q18, [%x[r], #832]\n\t" + "ldr q19, [%x[r], #896]\n\t" + "ldr q20, [%x[r], #960]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v23.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v13.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v14.4s, v23.4s, v4.s[0]\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmlsh v5.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v6.4s, v22.4s, v4.s[0]\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v22.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v22.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v22.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r]]\n\t" + "str q6, [%x[r], #64]\n\t" + "str q7, [%x[r], #128]\n\t" + "str q8, [%x[r], #192]\n\t" + "str q9, [%x[r], #256]\n\t" + "str q10, [%x[r], #320]\n\t" + "str q11, [%x[r], #384]\n\t" + "str q12, [%x[r], #448]\n\t" + "str q13, [%x[r], #512]\n\t" + "str q14, [%x[r], #576]\n\t" + "str q15, [%x[r], #640]\n\t" + "str q16, [%x[r], #704]\n\t" + "str q17, [%x[r], #768]\n\t" + "str q18, [%x[r], #832]\n\t" + "str q19, [%x[r], #896]\n\t" + "str q20, [%x[r], #960]\n\t" + "ldr q5, [%x[r], #16]\n\t" + "ldr q6, [%x[r], #80]\n\t" + "ldr q7, [%x[r], #144]\n\t" + "ldr q8, [%x[r], #208]\n\t" + "ldr q9, [%x[r], #272]\n\t" + "ldr q10, [%x[r], #336]\n\t" + "ldr q11, [%x[r], #400]\n\t" + "ldr q12, [%x[r], #464]\n\t" + "ldr q13, [%x[r], #528]\n\t" + "ldr q14, [%x[r], #592]\n\t" + "ldr q15, [%x[r], #656]\n\t" + "ldr q16, [%x[r], #720]\n\t" + "ldr q17, [%x[r], #784]\n\t" + "ldr q18, [%x[r], #848]\n\t" + "ldr q19, [%x[r], #912]\n\t" + "ldr q20, [%x[r], #976]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v23.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v13.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v14.4s, v23.4s, v4.s[0]\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmlsh v5.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v6.4s, v22.4s, v4.s[0]\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v22.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v22.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v22.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r], #16]\n\t" + "str q6, [%x[r], #80]\n\t" + "str q7, [%x[r], #144]\n\t" + "str q8, [%x[r], #208]\n\t" + "str q9, [%x[r], #272]\n\t" + "str q10, [%x[r], #336]\n\t" + "str q11, [%x[r], #400]\n\t" + "str q12, [%x[r], #464]\n\t" + "str q13, [%x[r], #528]\n\t" + "str q14, [%x[r], #592]\n\t" + "str q15, [%x[r], #656]\n\t" + "str q16, [%x[r], #720]\n\t" + "str q17, [%x[r], #784]\n\t" + "str q18, [%x[r], #848]\n\t" + "str q19, [%x[r], #912]\n\t" + "str q20, [%x[r], #976]\n\t" + "ldr q5, [%x[r], #32]\n\t" + "ldr q6, [%x[r], #96]\n\t" + "ldr q7, [%x[r], #160]\n\t" + "ldr q8, [%x[r], #224]\n\t" + "ldr q9, [%x[r], #288]\n\t" + "ldr q10, [%x[r], #352]\n\t" + "ldr q11, [%x[r], #416]\n\t" + "ldr q12, [%x[r], #480]\n\t" + "ldr q13, [%x[r], #544]\n\t" + "ldr q14, [%x[r], #608]\n\t" + "ldr q15, [%x[r], #672]\n\t" + "ldr q16, [%x[r], #736]\n\t" + "ldr q17, [%x[r], #800]\n\t" + "ldr q18, [%x[r], #864]\n\t" + "ldr q19, [%x[r], #928]\n\t" + "ldr q20, [%x[r], #992]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v23.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v13.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v14.4s, v23.4s, v4.s[0]\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmlsh v5.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v6.4s, v22.4s, v4.s[0]\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v22.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v22.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v22.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r], #32]\n\t" + "str q6, [%x[r], #96]\n\t" + "str q7, [%x[r], #160]\n\t" + "str q8, [%x[r], #224]\n\t" + "str q9, [%x[r], #288]\n\t" + "str q10, [%x[r], #352]\n\t" + "str q11, [%x[r], #416]\n\t" + "str q12, [%x[r], #480]\n\t" + "str q13, [%x[r], #544]\n\t" + "str q14, [%x[r], #608]\n\t" + "str q15, [%x[r], #672]\n\t" + "str q16, [%x[r], #736]\n\t" + "str q17, [%x[r], #800]\n\t" + "str q18, [%x[r], #864]\n\t" + "str q19, [%x[r], #928]\n\t" + "str q20, [%x[r], #992]\n\t" + "ldr q5, [%x[r], #48]\n\t" + "ldr q6, [%x[r], #112]\n\t" + "ldr q7, [%x[r], #176]\n\t" + "ldr q8, [%x[r], #240]\n\t" + "ldr q9, [%x[r], #304]\n\t" + "ldr q10, [%x[r], #368]\n\t" + "ldr q11, [%x[r], #432]\n\t" + "ldr q12, [%x[r], #496]\n\t" + "ldr q13, [%x[r], #560]\n\t" + "ldr q14, [%x[r], #624]\n\t" + "ldr q15, [%x[r], #688]\n\t" + "ldr q16, [%x[r], #752]\n\t" + "ldr q17, [%x[r], #816]\n\t" + "ldr q18, [%x[r], #880]\n\t" + "ldr q19, [%x[r], #944]\n\t" + "ldr q20, [%x[r], #1008]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v23.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v13.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v14.4s, v23.4s, v4.s[0]\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmlsh v5.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v6.4s, v22.4s, v4.s[0]\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v22.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v22.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v22.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r], #48]\n\t" + "str q6, [%x[r], #112]\n\t" + "str q7, [%x[r], #176]\n\t" + "str q8, [%x[r], #240]\n\t" + "str q9, [%x[r], #304]\n\t" + "str q10, [%x[r], #368]\n\t" + "str q11, [%x[r], #432]\n\t" + "str q12, [%x[r], #496]\n\t" + "str q13, [%x[r], #560]\n\t" + "str q14, [%x[r], #624]\n\t" + "str q15, [%x[r], #688]\n\t" + "str q16, [%x[r], #752]\n\t" + "str q17, [%x[r], #816]\n\t" + "str q18, [%x[r], #880]\n\t" + "str q19, [%x[r], #944]\n\t" + "str q20, [%x[r], #1008]\n\t" + : [r] "+r" (r) + : [inv] "r" (inv), [qinv] "r" (qinv), [consts] "r" (consts) + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", + "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v23", "v24" + ); +} + +void mldsa_invntt_full_sqrdmlsh_neon(sword32* r) +{ + const word32* inv = L_mldsa_aarch64_zetas_inv; + const word32* qinv = L_mldsa_aarch64_zetas_inv_qinv; + const word32* consts = L_mldsa_aarch64_consts; + __asm__ __volatile__ ( + "ldr q4, [%[consts]]\n\t" + "ldp q5, q6, [%x[r]]\n\t" + "ldp q7, q8, [%x[r], #32]\n\t" + "ldp q9, q10, [%x[r], #64]\n\t" + "ldp q11, q12, [%x[r], #96]\n\t" + "ldp q13, q14, [%x[r], #128]\n\t" + "ldp q15, q16, [%x[r], #160]\n\t" + "ldp q17, q18, [%x[r], #192]\n\t" + "ldp q19, q20, [%x[r], #224]\n\t" + "trn1 v21.2d, v5.2d, v6.2d\n\t" + "trn2 v22.2d, v5.2d, v6.2d\n\t" + "trn1 v5.4s, v21.4s, v22.4s\n\t" + "trn2 v6.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v7.2d, v8.2d\n\t" + "trn2 v22.2d, v7.2d, v8.2d\n\t" + "trn1 v7.4s, v21.4s, v22.4s\n\t" + "trn2 v8.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v9.2d, v10.2d\n\t" + "trn2 v22.2d, v9.2d, v10.2d\n\t" + "trn1 v9.4s, v21.4s, v22.4s\n\t" + "trn2 v10.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v11.2d, v12.2d\n\t" + "trn2 v22.2d, v11.2d, v12.2d\n\t" + "trn1 v11.4s, v21.4s, v22.4s\n\t" + "trn2 v12.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v13.2d, v14.2d\n\t" + "trn2 v22.2d, v13.2d, v14.2d\n\t" + "trn1 v13.4s, v21.4s, v22.4s\n\t" + "trn2 v14.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v15.2d, v16.2d\n\t" + "trn2 v22.2d, v15.2d, v16.2d\n\t" + "trn1 v15.4s, v21.4s, v22.4s\n\t" + "trn2 v16.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v17.2d, v18.2d\n\t" + "trn2 v22.2d, v17.2d, v18.2d\n\t" + "trn1 v17.4s, v21.4s, v22.4s\n\t" + "trn2 v18.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v19.2d, v20.2d\n\t" + "trn2 v22.2d, v19.2d, v20.2d\n\t" + "trn1 v19.4s, v21.4s, v22.4s\n\t" + "trn2 v20.4s, v21.4s, v22.4s\n\t" + "ldp q0, q1, [%[inv]]\n\t" + "ldp q2, q3, [%[qinv]]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #32]\n\t" + "ldp q2, q3, [%[qinv], #32]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #64]\n\t" + "ldp q2, q3, [%[qinv], #64]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #96]\n\t" + "ldp q2, q3, [%[qinv], #96]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #512]\n\t" + "ldp q2, q3, [%[qinv], #512]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #544]\n\t" + "ldp q2, q3, [%[qinv], #544]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #576]\n\t" + "ldp q2, q3, [%[qinv], #576]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #608]\n\t" + "ldp q2, q3, [%[qinv], #608]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1024]\n\t" + "ldr q1, [%[inv], #1040]\n\t" + "ldr q2, [%[qinv], #1024]\n\t" + "ldr q3, [%[qinv], #1040]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1152]\n\t" + "ldr q2, [%[qinv], #1152]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r]]\n\t" + "stp q7, q8, [%x[r], #32]\n\t" + "stp q9, q10, [%x[r], #64]\n\t" + "stp q11, q12, [%x[r], #96]\n\t" + "stp q13, q14, [%x[r], #128]\n\t" + "stp q15, q16, [%x[r], #160]\n\t" + "stp q17, q18, [%x[r], #192]\n\t" + "stp q19, q20, [%x[r], #224]\n\t" + "ldp q5, q6, [%x[r], #256]\n\t" + "ldp q7, q8, [%x[r], #288]\n\t" + "ldp q9, q10, [%x[r], #320]\n\t" + "ldp q11, q12, [%x[r], #352]\n\t" + "ldp q13, q14, [%x[r], #384]\n\t" + "ldp q15, q16, [%x[r], #416]\n\t" + "ldp q17, q18, [%x[r], #448]\n\t" + "ldp q19, q20, [%x[r], #480]\n\t" + "trn1 v21.2d, v5.2d, v6.2d\n\t" + "trn2 v22.2d, v5.2d, v6.2d\n\t" + "trn1 v5.4s, v21.4s, v22.4s\n\t" + "trn2 v6.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v7.2d, v8.2d\n\t" + "trn2 v22.2d, v7.2d, v8.2d\n\t" + "trn1 v7.4s, v21.4s, v22.4s\n\t" + "trn2 v8.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v9.2d, v10.2d\n\t" + "trn2 v22.2d, v9.2d, v10.2d\n\t" + "trn1 v9.4s, v21.4s, v22.4s\n\t" + "trn2 v10.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v11.2d, v12.2d\n\t" + "trn2 v22.2d, v11.2d, v12.2d\n\t" + "trn1 v11.4s, v21.4s, v22.4s\n\t" + "trn2 v12.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v13.2d, v14.2d\n\t" + "trn2 v22.2d, v13.2d, v14.2d\n\t" + "trn1 v13.4s, v21.4s, v22.4s\n\t" + "trn2 v14.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v15.2d, v16.2d\n\t" + "trn2 v22.2d, v15.2d, v16.2d\n\t" + "trn1 v15.4s, v21.4s, v22.4s\n\t" + "trn2 v16.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v17.2d, v18.2d\n\t" + "trn2 v22.2d, v17.2d, v18.2d\n\t" + "trn1 v17.4s, v21.4s, v22.4s\n\t" + "trn2 v18.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v19.2d, v20.2d\n\t" + "trn2 v22.2d, v19.2d, v20.2d\n\t" + "trn1 v19.4s, v21.4s, v22.4s\n\t" + "trn2 v20.4s, v21.4s, v22.4s\n\t" + "ldp q0, q1, [%[inv], #128]\n\t" + "ldp q2, q3, [%[qinv], #128]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #160]\n\t" + "ldp q2, q3, [%[qinv], #160]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #192]\n\t" + "ldp q2, q3, [%[qinv], #192]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #224]\n\t" + "ldp q2, q3, [%[qinv], #224]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #640]\n\t" + "ldp q2, q3, [%[qinv], #640]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #672]\n\t" + "ldp q2, q3, [%[qinv], #672]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #704]\n\t" + "ldp q2, q3, [%[qinv], #704]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #736]\n\t" + "ldp q2, q3, [%[qinv], #736]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1056]\n\t" + "ldr q1, [%[inv], #1072]\n\t" + "ldr q2, [%[qinv], #1056]\n\t" + "ldr q3, [%[qinv], #1072]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1168]\n\t" + "ldr q2, [%[qinv], #1168]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r], #256]\n\t" + "stp q7, q8, [%x[r], #288]\n\t" + "stp q9, q10, [%x[r], #320]\n\t" + "stp q11, q12, [%x[r], #352]\n\t" + "stp q13, q14, [%x[r], #384]\n\t" + "stp q15, q16, [%x[r], #416]\n\t" + "stp q17, q18, [%x[r], #448]\n\t" + "stp q19, q20, [%x[r], #480]\n\t" + "ldp q5, q6, [%x[r], #512]\n\t" + "ldp q7, q8, [%x[r], #544]\n\t" + "ldp q9, q10, [%x[r], #576]\n\t" + "ldp q11, q12, [%x[r], #608]\n\t" + "ldp q13, q14, [%x[r], #640]\n\t" + "ldp q15, q16, [%x[r], #672]\n\t" + "ldp q17, q18, [%x[r], #704]\n\t" + "ldp q19, q20, [%x[r], #736]\n\t" + "trn1 v21.2d, v5.2d, v6.2d\n\t" + "trn2 v22.2d, v5.2d, v6.2d\n\t" + "trn1 v5.4s, v21.4s, v22.4s\n\t" + "trn2 v6.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v7.2d, v8.2d\n\t" + "trn2 v22.2d, v7.2d, v8.2d\n\t" + "trn1 v7.4s, v21.4s, v22.4s\n\t" + "trn2 v8.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v9.2d, v10.2d\n\t" + "trn2 v22.2d, v9.2d, v10.2d\n\t" + "trn1 v9.4s, v21.4s, v22.4s\n\t" + "trn2 v10.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v11.2d, v12.2d\n\t" + "trn2 v22.2d, v11.2d, v12.2d\n\t" + "trn1 v11.4s, v21.4s, v22.4s\n\t" + "trn2 v12.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v13.2d, v14.2d\n\t" + "trn2 v22.2d, v13.2d, v14.2d\n\t" + "trn1 v13.4s, v21.4s, v22.4s\n\t" + "trn2 v14.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v15.2d, v16.2d\n\t" + "trn2 v22.2d, v15.2d, v16.2d\n\t" + "trn1 v15.4s, v21.4s, v22.4s\n\t" + "trn2 v16.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v17.2d, v18.2d\n\t" + "trn2 v22.2d, v17.2d, v18.2d\n\t" + "trn1 v17.4s, v21.4s, v22.4s\n\t" + "trn2 v18.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v19.2d, v20.2d\n\t" + "trn2 v22.2d, v19.2d, v20.2d\n\t" + "trn1 v19.4s, v21.4s, v22.4s\n\t" + "trn2 v20.4s, v21.4s, v22.4s\n\t" + "ldp q0, q1, [%[inv], #256]\n\t" + "ldp q2, q3, [%[qinv], #256]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #288]\n\t" + "ldp q2, q3, [%[qinv], #288]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #320]\n\t" + "ldp q2, q3, [%[qinv], #320]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #352]\n\t" + "ldp q2, q3, [%[qinv], #352]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #768]\n\t" + "ldp q2, q3, [%[qinv], #768]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #800]\n\t" + "ldp q2, q3, [%[qinv], #800]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #832]\n\t" + "ldp q2, q3, [%[qinv], #832]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #864]\n\t" + "ldp q2, q3, [%[qinv], #864]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1088]\n\t" + "ldr q1, [%[inv], #1104]\n\t" + "ldr q2, [%[qinv], #1088]\n\t" + "ldr q3, [%[qinv], #1104]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1184]\n\t" + "ldr q2, [%[qinv], #1184]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r], #512]\n\t" + "stp q7, q8, [%x[r], #544]\n\t" + "stp q9, q10, [%x[r], #576]\n\t" + "stp q11, q12, [%x[r], #608]\n\t" + "stp q13, q14, [%x[r], #640]\n\t" + "stp q15, q16, [%x[r], #672]\n\t" + "stp q17, q18, [%x[r], #704]\n\t" + "stp q19, q20, [%x[r], #736]\n\t" + "ldp q5, q6, [%x[r], #768]\n\t" + "ldp q7, q8, [%x[r], #800]\n\t" + "ldp q9, q10, [%x[r], #832]\n\t" + "ldp q11, q12, [%x[r], #864]\n\t" + "ldp q13, q14, [%x[r], #896]\n\t" + "ldp q15, q16, [%x[r], #928]\n\t" + "ldp q17, q18, [%x[r], #960]\n\t" + "ldp q19, q20, [%x[r], #992]\n\t" + "trn1 v21.2d, v5.2d, v6.2d\n\t" + "trn2 v22.2d, v5.2d, v6.2d\n\t" + "trn1 v5.4s, v21.4s, v22.4s\n\t" + "trn2 v6.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v7.2d, v8.2d\n\t" + "trn2 v22.2d, v7.2d, v8.2d\n\t" + "trn1 v7.4s, v21.4s, v22.4s\n\t" + "trn2 v8.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v9.2d, v10.2d\n\t" + "trn2 v22.2d, v9.2d, v10.2d\n\t" + "trn1 v9.4s, v21.4s, v22.4s\n\t" + "trn2 v10.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v11.2d, v12.2d\n\t" + "trn2 v22.2d, v11.2d, v12.2d\n\t" + "trn1 v11.4s, v21.4s, v22.4s\n\t" + "trn2 v12.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v13.2d, v14.2d\n\t" + "trn2 v22.2d, v13.2d, v14.2d\n\t" + "trn1 v13.4s, v21.4s, v22.4s\n\t" + "trn2 v14.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v15.2d, v16.2d\n\t" + "trn2 v22.2d, v15.2d, v16.2d\n\t" + "trn1 v15.4s, v21.4s, v22.4s\n\t" + "trn2 v16.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v17.2d, v18.2d\n\t" + "trn2 v22.2d, v17.2d, v18.2d\n\t" + "trn1 v17.4s, v21.4s, v22.4s\n\t" + "trn2 v18.4s, v21.4s, v22.4s\n\t" + "trn1 v21.2d, v19.2d, v20.2d\n\t" + "trn2 v22.2d, v19.2d, v20.2d\n\t" + "trn1 v19.4s, v21.4s, v22.4s\n\t" + "trn2 v20.4s, v21.4s, v22.4s\n\t" + "ldp q0, q1, [%[inv], #384]\n\t" + "ldp q2, q3, [%[qinv], #384]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #416]\n\t" + "ldp q2, q3, [%[qinv], #416]\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #448]\n\t" + "ldp q2, q3, [%[qinv], #448]\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #480]\n\t" + "ldp q2, q3, [%[qinv], #480]\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldp q0, q1, [%[inv], #896]\n\t" + "ldp q2, q3, [%[qinv], #896]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.4s, v5.4s, v6.4s\n\t" + "trn1 v7.4s, v7.4s, v8.4s\n\t" + "trn2 v6.4s, v21.4s, v6.4s\n\t" + "trn2 v8.4s, v22.4s, v8.4s\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v6.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v8.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "ldp q0, q1, [%[inv], #928]\n\t" + "ldp q2, q3, [%[qinv], #928]\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.4s, v9.4s, v10.4s\n\t" + "trn1 v11.4s, v11.4s, v12.4s\n\t" + "trn2 v10.4s, v21.4s, v10.4s\n\t" + "trn2 v12.4s, v22.4s, v12.4s\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v10.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v12.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "ldp q0, q1, [%[inv], #960]\n\t" + "ldp q2, q3, [%[qinv], #960]\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.4s, v13.4s, v14.4s\n\t" + "trn1 v15.4s, v15.4s, v16.4s\n\t" + "trn2 v14.4s, v21.4s, v14.4s\n\t" + "trn2 v16.4s, v22.4s, v16.4s\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v14.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v16.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "ldp q0, q1, [%[inv], #992]\n\t" + "ldp q2, q3, [%[qinv], #992]\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.4s, v17.4s, v18.4s\n\t" + "trn1 v19.4s, v19.4s, v20.4s\n\t" + "trn2 v18.4s, v21.4s, v18.4s\n\t" + "trn2 v20.4s, v22.4s, v20.4s\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.4s\n\t" + "mul v23.4s, v24.4s, v3.4s\n\t" + "sqrdmulh v18.4s, v22.4s, v0.4s\n\t" + "sqrdmulh v20.4s, v24.4s, v1.4s\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1120]\n\t" + "ldr q1, [%[inv], #1136]\n\t" + "ldr q2, [%[qinv], #1120]\n\t" + "ldr q3, [%[qinv], #1136]\n\t" + "mov v21.16b, v5.16b\n\t" + "mov v22.16b, v7.16b\n\t" + "trn1 v5.2d, v5.2d, v6.2d\n\t" + "trn1 v7.2d, v7.2d, v8.2d\n\t" + "trn2 v6.2d, v21.2d, v6.2d\n\t" + "trn2 v8.2d, v22.2d, v8.2d\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mov v21.16b, v9.16b\n\t" + "mov v22.16b, v11.16b\n\t" + "trn1 v9.2d, v9.2d, v10.2d\n\t" + "trn1 v11.2d, v11.2d, v12.2d\n\t" + "trn2 v10.2d, v21.2d, v10.2d\n\t" + "trn2 v12.2d, v22.2d, v12.2d\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "mov v21.16b, v13.16b\n\t" + "mov v22.16b, v15.16b\n\t" + "trn1 v13.2d, v13.2d, v14.2d\n\t" + "trn1 v15.2d, v15.2d, v16.2d\n\t" + "trn2 v14.2d, v21.2d, v14.2d\n\t" + "trn2 v16.2d, v22.2d, v16.2d\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "mov v21.16b, v17.16b\n\t" + "mov v22.16b, v19.16b\n\t" + "trn1 v17.2d, v17.2d, v18.2d\n\t" + "trn1 v19.2d, v19.2d, v20.2d\n\t" + "trn2 v18.2d, v21.2d, v18.2d\n\t" + "trn2 v20.2d, v22.2d, v20.2d\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1200]\n\t" + "ldr q2, [%[qinv], #1200]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "stp q5, q6, [%x[r], #768]\n\t" + "stp q7, q8, [%x[r], #800]\n\t" + "stp q9, q10, [%x[r], #832]\n\t" + "stp q11, q12, [%x[r], #864]\n\t" + "stp q13, q14, [%x[r], #896]\n\t" + "stp q15, q16, [%x[r], #928]\n\t" + "stp q17, q18, [%x[r], #960]\n\t" + "stp q19, q20, [%x[r], #992]\n\t" + "ldr q5, [%x[r]]\n\t" + "ldr q6, [%x[r], #64]\n\t" + "ldr q7, [%x[r], #128]\n\t" + "ldr q8, [%x[r], #192]\n\t" + "ldr q9, [%x[r], #256]\n\t" + "ldr q10, [%x[r], #320]\n\t" + "ldr q11, [%x[r], #384]\n\t" + "ldr q12, [%x[r], #448]\n\t" + "ldr q13, [%x[r], #512]\n\t" + "ldr q14, [%x[r], #576]\n\t" + "ldr q15, [%x[r], #640]\n\t" + "ldr q16, [%x[r], #704]\n\t" + "ldr q17, [%x[r], #768]\n\t" + "ldr q18, [%x[r], #832]\n\t" + "ldr q19, [%x[r], #896]\n\t" + "ldr q20, [%x[r], #960]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v23.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v13.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v14.4s, v23.4s, v4.s[0]\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmlsh v5.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v6.4s, v22.4s, v4.s[0]\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v22.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v22.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v22.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r]]\n\t" + "str q6, [%x[r], #64]\n\t" + "str q7, [%x[r], #128]\n\t" + "str q8, [%x[r], #192]\n\t" + "str q9, [%x[r], #256]\n\t" + "str q10, [%x[r], #320]\n\t" + "str q11, [%x[r], #384]\n\t" + "str q12, [%x[r], #448]\n\t" + "str q13, [%x[r], #512]\n\t" + "str q14, [%x[r], #576]\n\t" + "str q15, [%x[r], #640]\n\t" + "str q16, [%x[r], #704]\n\t" + "str q17, [%x[r], #768]\n\t" + "str q18, [%x[r], #832]\n\t" + "str q19, [%x[r], #896]\n\t" + "str q20, [%x[r], #960]\n\t" + "ldr q5, [%x[r], #16]\n\t" + "ldr q6, [%x[r], #80]\n\t" + "ldr q7, [%x[r], #144]\n\t" + "ldr q8, [%x[r], #208]\n\t" + "ldr q9, [%x[r], #272]\n\t" + "ldr q10, [%x[r], #336]\n\t" + "ldr q11, [%x[r], #400]\n\t" + "ldr q12, [%x[r], #464]\n\t" + "ldr q13, [%x[r], #528]\n\t" + "ldr q14, [%x[r], #592]\n\t" + "ldr q15, [%x[r], #656]\n\t" + "ldr q16, [%x[r], #720]\n\t" + "ldr q17, [%x[r], #784]\n\t" + "ldr q18, [%x[r], #848]\n\t" + "ldr q19, [%x[r], #912]\n\t" + "ldr q20, [%x[r], #976]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v23.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v13.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v14.4s, v23.4s, v4.s[0]\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmlsh v5.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v6.4s, v22.4s, v4.s[0]\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v22.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v22.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v22.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r], #16]\n\t" + "str q6, [%x[r], #80]\n\t" + "str q7, [%x[r], #144]\n\t" + "str q8, [%x[r], #208]\n\t" + "str q9, [%x[r], #272]\n\t" + "str q10, [%x[r], #336]\n\t" + "str q11, [%x[r], #400]\n\t" + "str q12, [%x[r], #464]\n\t" + "str q13, [%x[r], #528]\n\t" + "str q14, [%x[r], #592]\n\t" + "str q15, [%x[r], #656]\n\t" + "str q16, [%x[r], #720]\n\t" + "str q17, [%x[r], #784]\n\t" + "str q18, [%x[r], #848]\n\t" + "str q19, [%x[r], #912]\n\t" + "str q20, [%x[r], #976]\n\t" + "ldr q5, [%x[r], #32]\n\t" + "ldr q6, [%x[r], #96]\n\t" + "ldr q7, [%x[r], #160]\n\t" + "ldr q8, [%x[r], #224]\n\t" + "ldr q9, [%x[r], #288]\n\t" + "ldr q10, [%x[r], #352]\n\t" + "ldr q11, [%x[r], #416]\n\t" + "ldr q12, [%x[r], #480]\n\t" + "ldr q13, [%x[r], #544]\n\t" + "ldr q14, [%x[r], #608]\n\t" + "ldr q15, [%x[r], #672]\n\t" + "ldr q16, [%x[r], #736]\n\t" + "ldr q17, [%x[r], #800]\n\t" + "ldr q18, [%x[r], #864]\n\t" + "ldr q19, [%x[r], #928]\n\t" + "ldr q20, [%x[r], #992]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v23.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v13.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v14.4s, v23.4s, v4.s[0]\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmlsh v5.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v6.4s, v22.4s, v4.s[0]\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v22.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v22.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v22.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r], #32]\n\t" + "str q6, [%x[r], #96]\n\t" + "str q7, [%x[r], #160]\n\t" + "str q8, [%x[r], #224]\n\t" + "str q9, [%x[r], #288]\n\t" + "str q10, [%x[r], #352]\n\t" + "str q11, [%x[r], #416]\n\t" + "str q12, [%x[r], #480]\n\t" + "str q13, [%x[r], #544]\n\t" + "str q14, [%x[r], #608]\n\t" + "str q15, [%x[r], #672]\n\t" + "str q16, [%x[r], #736]\n\t" + "str q17, [%x[r], #800]\n\t" + "str q18, [%x[r], #864]\n\t" + "str q19, [%x[r], #928]\n\t" + "str q20, [%x[r], #992]\n\t" + "ldr q5, [%x[r], #48]\n\t" + "ldr q6, [%x[r], #112]\n\t" + "ldr q7, [%x[r], #176]\n\t" + "ldr q8, [%x[r], #240]\n\t" + "ldr q9, [%x[r], #304]\n\t" + "ldr q10, [%x[r], #368]\n\t" + "ldr q11, [%x[r], #432]\n\t" + "ldr q12, [%x[r], #496]\n\t" + "ldr q13, [%x[r], #560]\n\t" + "ldr q14, [%x[r], #624]\n\t" + "ldr q15, [%x[r], #688]\n\t" + "ldr q16, [%x[r], #752]\n\t" + "ldr q17, [%x[r], #816]\n\t" + "ldr q18, [%x[r], #880]\n\t" + "ldr q19, [%x[r], #944]\n\t" + "ldr q20, [%x[r], #1008]\n\t" + "ldr q0, [%[inv], #1216]\n\t" + "ldr q1, [%[inv], #1232]\n\t" + "ldr q2, [%[qinv], #1216]\n\t" + "ldr q3, [%[qinv], #1232]\n\t" + "sub v22.4s, v5.4s, v6.4s\n\t" + "sub v24.4s, v7.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v6.4s\n\t" + "add v7.4s, v7.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v6.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v6.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v10.4s\n\t" + "sub v24.4s, v11.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v10.4s\n\t" + "add v11.4s, v11.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v10.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v10.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v14.4s\n\t" + "sub v24.4s, v15.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v14.4s\n\t" + "add v15.4s, v15.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v3.s[0]\n\t" + "mul v23.4s, v24.4s, v3.s[1]\n\t" + "sqrdmulh v14.4s, v22.4s, v1.s[0]\n\t" + "sqrdmulh v16.4s, v24.4s, v1.s[1]\n\t" + "sqrdmlsh v14.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v18.4s\n\t" + "sub v24.4s, v19.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v18.4s\n\t" + "add v19.4s, v19.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v3.s[2]\n\t" + "mul v23.4s, v24.4s, v3.s[3]\n\t" + "sqrdmulh v18.4s, v22.4s, v1.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v1.s[3]\n\t" + "sqrdmlsh v18.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1248]\n\t" + "ldr q2, [%[qinv], #1248]\n\t" + "sub v22.4s, v5.4s, v7.4s\n\t" + "sub v24.4s, v6.4s, v8.4s\n\t" + "add v5.4s, v5.4s, v7.4s\n\t" + "add v6.4s, v6.4s, v8.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v7.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v8.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v23.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "sub v22.4s, v9.4s, v11.4s\n\t" + "sub v24.4s, v10.4s, v12.4s\n\t" + "add v9.4s, v9.4s, v11.4s\n\t" + "add v10.4s, v10.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v15.4s\n\t" + "sub v24.4s, v14.4s, v16.4s\n\t" + "add v13.4s, v13.4s, v15.4s\n\t" + "add v14.4s, v14.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v17.4s, v19.4s\n\t" + "sub v24.4s, v18.4s, v20.4s\n\t" + "add v17.4s, v17.4s, v19.4s\n\t" + "add v18.4s, v18.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[3]\n\t" + "mul v23.4s, v24.4s, v2.s[3]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[3]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[3]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "ldr q0, [%[inv], #1264]\n\t" + "ldr q2, [%[qinv], #1264]\n\t" + "sub v22.4s, v5.4s, v9.4s\n\t" + "sub v24.4s, v6.4s, v10.4s\n\t" + "add v5.4s, v5.4s, v9.4s\n\t" + "add v6.4s, v6.4s, v10.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v9.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v10.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v23.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "sub v22.4s, v7.4s, v11.4s\n\t" + "sub v24.4s, v8.4s, v12.4s\n\t" + "add v7.4s, v7.4s, v11.4s\n\t" + "add v8.4s, v8.4s, v12.4s\n\t" + "mul v21.4s, v22.4s, v2.s[0]\n\t" + "mul v23.4s, v24.4s, v2.s[0]\n\t" + "sqrdmulh v11.4s, v22.4s, v0.s[0]\n\t" + "sqrdmulh v12.4s, v24.4s, v0.s[0]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v23.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "sub v22.4s, v13.4s, v17.4s\n\t" + "sub v24.4s, v14.4s, v18.4s\n\t" + "add v13.4s, v13.4s, v17.4s\n\t" + "add v14.4s, v14.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v15.4s, v19.4s\n\t" + "sub v24.4s, v16.4s, v20.4s\n\t" + "add v15.4s, v15.4s, v19.4s\n\t" + "add v16.4s, v16.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[1]\n\t" + "mul v23.4s, v24.4s, v2.s[1]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[1]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[1]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "sub v22.4s, v5.4s, v13.4s\n\t" + "sub v24.4s, v6.4s, v14.4s\n\t" + "add v5.4s, v5.4s, v13.4s\n\t" + "add v6.4s, v6.4s, v14.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v13.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v14.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v13.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v14.4s, v23.4s, v4.s[0]\n\t" + "sshr v13.4s, v13.4s, #1\n\t" + "sshr v14.4s, v14.4s, #1\n\t" + "sub v22.4s, v7.4s, v15.4s\n\t" + "sub v24.4s, v8.4s, v16.4s\n\t" + "add v7.4s, v7.4s, v15.4s\n\t" + "add v8.4s, v8.4s, v16.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v15.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v16.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v15.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v16.4s, v23.4s, v4.s[0]\n\t" + "sshr v15.4s, v15.4s, #1\n\t" + "sshr v16.4s, v16.4s, #1\n\t" + "sub v22.4s, v9.4s, v17.4s\n\t" + "sub v24.4s, v10.4s, v18.4s\n\t" + "add v9.4s, v9.4s, v17.4s\n\t" + "add v10.4s, v10.4s, v18.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v17.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v18.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v17.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v18.4s, v23.4s, v4.s[0]\n\t" + "sshr v17.4s, v17.4s, #1\n\t" + "sshr v18.4s, v18.4s, #1\n\t" + "sub v22.4s, v11.4s, v19.4s\n\t" + "sub v24.4s, v12.4s, v20.4s\n\t" + "add v11.4s, v11.4s, v19.4s\n\t" + "add v12.4s, v12.4s, v20.4s\n\t" + "mul v21.4s, v22.4s, v2.s[2]\n\t" + "mul v23.4s, v24.4s, v2.s[2]\n\t" + "sqrdmulh v19.4s, v22.4s, v0.s[2]\n\t" + "sqrdmulh v20.4s, v24.4s, v0.s[2]\n\t" + "sqrdmlsh v19.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v20.4s, v23.4s, v4.s[0]\n\t" + "sshr v19.4s, v19.4s, #1\n\t" + "sshr v20.4s, v20.4s, #1\n\t" + "mul v21.4s, v5.4s, v2.s[3]\n\t" + "mul v22.4s, v6.4s, v2.s[3]\n\t" + "sqrdmulh v5.4s, v5.4s, v0.s[3]\n\t" + "sqrdmulh v6.4s, v6.4s, v0.s[3]\n\t" + "sqrdmlsh v5.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v6.4s, v22.4s, v4.s[0]\n\t" + "sshr v5.4s, v5.4s, #1\n\t" + "sshr v6.4s, v6.4s, #1\n\t" + "mul v21.4s, v7.4s, v2.s[3]\n\t" + "mul v22.4s, v8.4s, v2.s[3]\n\t" + "sqrdmulh v7.4s, v7.4s, v0.s[3]\n\t" + "sqrdmulh v8.4s, v8.4s, v0.s[3]\n\t" + "sqrdmlsh v7.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v8.4s, v22.4s, v4.s[0]\n\t" + "sshr v7.4s, v7.4s, #1\n\t" + "sshr v8.4s, v8.4s, #1\n\t" + "mul v21.4s, v9.4s, v2.s[3]\n\t" + "mul v22.4s, v10.4s, v2.s[3]\n\t" + "sqrdmulh v9.4s, v9.4s, v0.s[3]\n\t" + "sqrdmulh v10.4s, v10.4s, v0.s[3]\n\t" + "sqrdmlsh v9.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v10.4s, v22.4s, v4.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v21.4s, v11.4s, v2.s[3]\n\t" + "mul v22.4s, v12.4s, v2.s[3]\n\t" + "sqrdmulh v11.4s, v11.4s, v0.s[3]\n\t" + "sqrdmulh v12.4s, v12.4s, v0.s[3]\n\t" + "sqrdmlsh v11.4s, v21.4s, v4.s[0]\n\t" + "sqrdmlsh v12.4s, v22.4s, v4.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q5, [%x[r], #48]\n\t" + "str q6, [%x[r], #112]\n\t" + "str q7, [%x[r], #176]\n\t" + "str q8, [%x[r], #240]\n\t" + "str q9, [%x[r], #304]\n\t" + "str q10, [%x[r], #368]\n\t" + "str q11, [%x[r], #432]\n\t" + "str q12, [%x[r], #496]\n\t" + "str q13, [%x[r], #560]\n\t" + "str q14, [%x[r], #624]\n\t" + "str q15, [%x[r], #688]\n\t" + "str q16, [%x[r], #752]\n\t" + "str q17, [%x[r], #816]\n\t" + "str q18, [%x[r], #880]\n\t" + "str q19, [%x[r], #944]\n\t" + "str q20, [%x[r], #1008]\n\t" + : [r] "+r" (r) + : [inv] "r" (inv), [qinv] "r" (qinv), [consts] "r" (consts) + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", + "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18", + "v19", "v20", "v21", "v22", "v23", "v24" + ); +} + +void mldsa_mul_sqrdmlsh_neon(sword32* r, sword32* a, sword32* b) +{ + const word32* consts = L_mldsa_aarch64_consts; + __asm__ __volatile__ ( + "ldr q0, [%[consts]]\n\t" + "ldr q1, [%x[a]]\n\t" + "ldr q5, [%x[b]]\n\t" + "ldr q2, [%x[a], #16]\n\t" + "ldr q6, [%x[b], #16]\n\t" + "ldr q3, [%x[a], #32]\n\t" + "ldr q7, [%x[b], #32]\n\t" + "ldr q4, [%x[a], #48]\n\t" + "ldr q8, [%x[b], #48]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r]]\n\t" + "str q10, [%x[r], #16]\n\t" + "str q11, [%x[r], #32]\n\t" + "str q12, [%x[r], #48]\n\t" + "ldr q1, [%x[a], #64]\n\t" + "ldr q5, [%x[b], #64]\n\t" + "ldr q2, [%x[a], #80]\n\t" + "ldr q6, [%x[b], #80]\n\t" + "ldr q3, [%x[a], #96]\n\t" + "ldr q7, [%x[b], #96]\n\t" + "ldr q4, [%x[a], #112]\n\t" + "ldr q8, [%x[b], #112]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #64]\n\t" + "str q10, [%x[r], #80]\n\t" + "str q11, [%x[r], #96]\n\t" + "str q12, [%x[r], #112]\n\t" + "ldr q1, [%x[a], #128]\n\t" + "ldr q5, [%x[b], #128]\n\t" + "ldr q2, [%x[a], #144]\n\t" + "ldr q6, [%x[b], #144]\n\t" + "ldr q3, [%x[a], #160]\n\t" + "ldr q7, [%x[b], #160]\n\t" + "ldr q4, [%x[a], #176]\n\t" + "ldr q8, [%x[b], #176]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #128]\n\t" + "str q10, [%x[r], #144]\n\t" + "str q11, [%x[r], #160]\n\t" + "str q12, [%x[r], #176]\n\t" + "ldr q1, [%x[a], #192]\n\t" + "ldr q5, [%x[b], #192]\n\t" + "ldr q2, [%x[a], #208]\n\t" + "ldr q6, [%x[b], #208]\n\t" + "ldr q3, [%x[a], #224]\n\t" + "ldr q7, [%x[b], #224]\n\t" + "ldr q4, [%x[a], #240]\n\t" + "ldr q8, [%x[b], #240]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #192]\n\t" + "str q10, [%x[r], #208]\n\t" + "str q11, [%x[r], #224]\n\t" + "str q12, [%x[r], #240]\n\t" + "ldr q1, [%x[a], #256]\n\t" + "ldr q5, [%x[b], #256]\n\t" + "ldr q2, [%x[a], #272]\n\t" + "ldr q6, [%x[b], #272]\n\t" + "ldr q3, [%x[a], #288]\n\t" + "ldr q7, [%x[b], #288]\n\t" + "ldr q4, [%x[a], #304]\n\t" + "ldr q8, [%x[b], #304]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #256]\n\t" + "str q10, [%x[r], #272]\n\t" + "str q11, [%x[r], #288]\n\t" + "str q12, [%x[r], #304]\n\t" + "ldr q1, [%x[a], #320]\n\t" + "ldr q5, [%x[b], #320]\n\t" + "ldr q2, [%x[a], #336]\n\t" + "ldr q6, [%x[b], #336]\n\t" + "ldr q3, [%x[a], #352]\n\t" + "ldr q7, [%x[b], #352]\n\t" + "ldr q4, [%x[a], #368]\n\t" + "ldr q8, [%x[b], #368]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #320]\n\t" + "str q10, [%x[r], #336]\n\t" + "str q11, [%x[r], #352]\n\t" + "str q12, [%x[r], #368]\n\t" + "ldr q1, [%x[a], #384]\n\t" + "ldr q5, [%x[b], #384]\n\t" + "ldr q2, [%x[a], #400]\n\t" + "ldr q6, [%x[b], #400]\n\t" + "ldr q3, [%x[a], #416]\n\t" + "ldr q7, [%x[b], #416]\n\t" + "ldr q4, [%x[a], #432]\n\t" + "ldr q8, [%x[b], #432]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #384]\n\t" + "str q10, [%x[r], #400]\n\t" + "str q11, [%x[r], #416]\n\t" + "str q12, [%x[r], #432]\n\t" + "ldr q1, [%x[a], #448]\n\t" + "ldr q5, [%x[b], #448]\n\t" + "ldr q2, [%x[a], #464]\n\t" + "ldr q6, [%x[b], #464]\n\t" + "ldr q3, [%x[a], #480]\n\t" + "ldr q7, [%x[b], #480]\n\t" + "ldr q4, [%x[a], #496]\n\t" + "ldr q8, [%x[b], #496]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #448]\n\t" + "str q10, [%x[r], #464]\n\t" + "str q11, [%x[r], #480]\n\t" + "str q12, [%x[r], #496]\n\t" + "ldr q1, [%x[a], #512]\n\t" + "ldr q5, [%x[b], #512]\n\t" + "ldr q2, [%x[a], #528]\n\t" + "ldr q6, [%x[b], #528]\n\t" + "ldr q3, [%x[a], #544]\n\t" + "ldr q7, [%x[b], #544]\n\t" + "ldr q4, [%x[a], #560]\n\t" + "ldr q8, [%x[b], #560]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #512]\n\t" + "str q10, [%x[r], #528]\n\t" + "str q11, [%x[r], #544]\n\t" + "str q12, [%x[r], #560]\n\t" + "ldr q1, [%x[a], #576]\n\t" + "ldr q5, [%x[b], #576]\n\t" + "ldr q2, [%x[a], #592]\n\t" + "ldr q6, [%x[b], #592]\n\t" + "ldr q3, [%x[a], #608]\n\t" + "ldr q7, [%x[b], #608]\n\t" + "ldr q4, [%x[a], #624]\n\t" + "ldr q8, [%x[b], #624]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #576]\n\t" + "str q10, [%x[r], #592]\n\t" + "str q11, [%x[r], #608]\n\t" + "str q12, [%x[r], #624]\n\t" + "ldr q1, [%x[a], #640]\n\t" + "ldr q5, [%x[b], #640]\n\t" + "ldr q2, [%x[a], #656]\n\t" + "ldr q6, [%x[b], #656]\n\t" + "ldr q3, [%x[a], #672]\n\t" + "ldr q7, [%x[b], #672]\n\t" + "ldr q4, [%x[a], #688]\n\t" + "ldr q8, [%x[b], #688]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #640]\n\t" + "str q10, [%x[r], #656]\n\t" + "str q11, [%x[r], #672]\n\t" + "str q12, [%x[r], #688]\n\t" + "ldr q1, [%x[a], #704]\n\t" + "ldr q5, [%x[b], #704]\n\t" + "ldr q2, [%x[a], #720]\n\t" + "ldr q6, [%x[b], #720]\n\t" + "ldr q3, [%x[a], #736]\n\t" + "ldr q7, [%x[b], #736]\n\t" + "ldr q4, [%x[a], #752]\n\t" + "ldr q8, [%x[b], #752]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #704]\n\t" + "str q10, [%x[r], #720]\n\t" + "str q11, [%x[r], #736]\n\t" + "str q12, [%x[r], #752]\n\t" + "ldr q1, [%x[a], #768]\n\t" + "ldr q5, [%x[b], #768]\n\t" + "ldr q2, [%x[a], #784]\n\t" + "ldr q6, [%x[b], #784]\n\t" + "ldr q3, [%x[a], #800]\n\t" + "ldr q7, [%x[b], #800]\n\t" + "ldr q4, [%x[a], #816]\n\t" + "ldr q8, [%x[b], #816]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #768]\n\t" + "str q10, [%x[r], #784]\n\t" + "str q11, [%x[r], #800]\n\t" + "str q12, [%x[r], #816]\n\t" + "ldr q1, [%x[a], #832]\n\t" + "ldr q5, [%x[b], #832]\n\t" + "ldr q2, [%x[a], #848]\n\t" + "ldr q6, [%x[b], #848]\n\t" + "ldr q3, [%x[a], #864]\n\t" + "ldr q7, [%x[b], #864]\n\t" + "ldr q4, [%x[a], #880]\n\t" + "ldr q8, [%x[b], #880]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #832]\n\t" + "str q10, [%x[r], #848]\n\t" + "str q11, [%x[r], #864]\n\t" + "str q12, [%x[r], #880]\n\t" + "ldr q1, [%x[a], #896]\n\t" + "ldr q5, [%x[b], #896]\n\t" + "ldr q2, [%x[a], #912]\n\t" + "ldr q6, [%x[b], #912]\n\t" + "ldr q3, [%x[a], #928]\n\t" + "ldr q7, [%x[b], #928]\n\t" + "ldr q4, [%x[a], #944]\n\t" + "ldr q8, [%x[b], #944]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #896]\n\t" + "str q10, [%x[r], #912]\n\t" + "str q11, [%x[r], #928]\n\t" + "str q12, [%x[r], #944]\n\t" + "ldr q1, [%x[a], #960]\n\t" + "ldr q5, [%x[b], #960]\n\t" + "ldr q2, [%x[a], #976]\n\t" + "ldr q6, [%x[b], #976]\n\t" + "ldr q3, [%x[a], #992]\n\t" + "ldr q7, [%x[b], #992]\n\t" + "ldr q4, [%x[a], #1008]\n\t" + "ldr q8, [%x[b], #1008]\n\t" + "mul v13.4s, v5.4s, v0.s[1]\n\t" + "mul v14.4s, v6.4s, v0.s[1]\n\t" + "mul v17.4s, v1.4s, v13.4s\n\t" + "mul v18.4s, v2.4s, v14.4s\n\t" + "sqrdmulh v9.4s, v1.4s, v5.4s\n\t" + "sqrdmulh v10.4s, v2.4s, v6.4s\n\t" + "sqrdmlsh v9.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v10.4s, v18.4s, v0.s[0]\n\t" + "sshr v9.4s, v9.4s, #1\n\t" + "sshr v10.4s, v10.4s, #1\n\t" + "mul v15.4s, v7.4s, v0.s[1]\n\t" + "mul v16.4s, v8.4s, v0.s[1]\n\t" + "mul v17.4s, v3.4s, v15.4s\n\t" + "mul v18.4s, v4.4s, v16.4s\n\t" + "sqrdmulh v11.4s, v3.4s, v7.4s\n\t" + "sqrdmulh v12.4s, v4.4s, v8.4s\n\t" + "sqrdmlsh v11.4s, v17.4s, v0.s[0]\n\t" + "sqrdmlsh v12.4s, v18.4s, v0.s[0]\n\t" + "sshr v11.4s, v11.4s, #1\n\t" + "sshr v12.4s, v12.4s, #1\n\t" + "str q9, [%x[r], #960]\n\t" + "str q10, [%x[r], #976]\n\t" + "str q11, [%x[r], #992]\n\t" + "str q12, [%x[r], #1008]\n\t" + : [r] "+r" (r), [a] "+r" (a), [b] "+r" (b) + : [consts] "r" (consts) + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", + "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", "v18" + ); +} + +#endif /* WOLFSSL_AARCH64_NO_SQRDMLSH */ +XALIGNED(8) static const word32 L_mldsa_decompose_q88_consts[] = { + 0x0000002c, 0x0000002c, 0x0000002c, 0x0000002c, + 0x003fefd4, 0x003fefd4, 0x003fefd4, 0x003fefd4, + 0x0002e800, 0x0002e800, 0x0002e800, 0x0002e800, + 0x00017400, 0x00017400, 0x00017400, 0x00017400, + 0x0000002b, 0x0000002b, 0x0000002b, 0x0000002b, + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +void mldsa_decompose_q88_neon(const sword32* r, sword32* r0, sword32* r1) +{ + const word32* consts = L_mldsa_decompose_q88_consts; + __asm__ __volatile__ ( + "ldr q0, [%[consts]]\n\t" + "ldr q1, [%[consts], #16]\n\t" + "ldr q2, [%[consts], #32]\n\t" + "ldr q3, [%[consts], #48]\n\t" + "ldr q4, [%[consts], #64]\n\t" + "mov x4, #16\n\t" + "\n" + "L_mldsa_decompose_q88_loop_%=:\n\t" + "ldr q5, [%x[r]]\n\t" + "ldr q6, [%x[r], #16]\n\t" + "ldr q7, [%x[r], #32]\n\t" + "ldr q8, [%x[r], #48]\n\t" + "mov v13.16b, v1.16b\n\t" + "mla v13.4s, v5.4s, v0.4s\n\t" + "sshr v13.4s, v13.4s, #23\n\t" + "mov v9.16b, v5.16b\n\t" + "mls v9.4s, v13.4s, v2.4s\n\t" + "sub v17.4s, v3.4s, v9.4s\n\t" + "usra v13.4s, v17.4s, #31\n\t" + "mov v9.16b, v5.16b\n\t" + "mls v9.4s, v13.4s, v2.4s\n\t" + "sub v17.4s, v4.4s, v13.4s\n\t" + "ushr v17.4s, v17.4s, #31\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "sub v17.4s, v13.4s, v0.4s\n\t" + "ushr v17.4s, v17.4s, #31\n\t" + "neg v17.4s, v17.4s\n\t" + "and v13.16b, v13.16b, v17.16b\n\t" + "mov v14.16b, v1.16b\n\t" + "mla v14.4s, v6.4s, v0.4s\n\t" + "sshr v14.4s, v14.4s, #23\n\t" + "mov v10.16b, v6.16b\n\t" + "mls v10.4s, v14.4s, v2.4s\n\t" + "sub v18.4s, v3.4s, v10.4s\n\t" + "usra v14.4s, v18.4s, #31\n\t" + "mov v10.16b, v6.16b\n\t" + "mls v10.4s, v14.4s, v2.4s\n\t" + "sub v18.4s, v4.4s, v14.4s\n\t" + "ushr v18.4s, v18.4s, #31\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sub v18.4s, v14.4s, v0.4s\n\t" + "ushr v18.4s, v18.4s, #31\n\t" + "neg v18.4s, v18.4s\n\t" + "and v14.16b, v14.16b, v18.16b\n\t" + "mov v15.16b, v1.16b\n\t" + "mla v15.4s, v7.4s, v0.4s\n\t" + "sshr v15.4s, v15.4s, #23\n\t" + "mov v11.16b, v7.16b\n\t" + "mls v11.4s, v15.4s, v2.4s\n\t" + "sub v19.4s, v3.4s, v11.4s\n\t" + "usra v15.4s, v19.4s, #31\n\t" + "mov v11.16b, v7.16b\n\t" + "mls v11.4s, v15.4s, v2.4s\n\t" + "sub v19.4s, v4.4s, v15.4s\n\t" + "ushr v19.4s, v19.4s, #31\n\t" + "sub v11.4s, v11.4s, v19.4s\n\t" + "sub v19.4s, v15.4s, v0.4s\n\t" + "ushr v19.4s, v19.4s, #31\n\t" + "neg v19.4s, v19.4s\n\t" + "and v15.16b, v15.16b, v19.16b\n\t" + "mov v16.16b, v1.16b\n\t" + "mla v16.4s, v8.4s, v0.4s\n\t" + "sshr v16.4s, v16.4s, #23\n\t" + "mov v12.16b, v8.16b\n\t" + "mls v12.4s, v16.4s, v2.4s\n\t" + "sub v20.4s, v3.4s, v12.4s\n\t" + "usra v16.4s, v20.4s, #31\n\t" + "mov v12.16b, v8.16b\n\t" + "mls v12.4s, v16.4s, v2.4s\n\t" + "sub v20.4s, v4.4s, v16.4s\n\t" + "ushr v20.4s, v20.4s, #31\n\t" + "sub v12.4s, v12.4s, v20.4s\n\t" + "sub v20.4s, v16.4s, v0.4s\n\t" + "ushr v20.4s, v20.4s, #31\n\t" + "neg v20.4s, v20.4s\n\t" + "and v16.16b, v16.16b, v20.16b\n\t" + "str q9, [%x[r0]]\n\t" + "str q13, [%x[r1]]\n\t" + "str q10, [%x[r0], #16]\n\t" + "str q14, [%x[r1], #16]\n\t" + "str q11, [%x[r0], #32]\n\t" + "str q15, [%x[r1], #32]\n\t" + "str q12, [%x[r0], #48]\n\t" + "str q16, [%x[r1], #48]\n\t" + "add %x[r], %x[r], #0x40\n\t" + "add %x[r0], %x[r0], #0x40\n\t" + "add %x[r1], %x[r1], #0x40\n\t" + "subs x4, x4, #1\n\t" + "b.ne L_mldsa_decompose_q88_loop_%=\n\t" + : [r0] "+r" (r0), [r1] "+r" (r1) + : [r] "r" (r), [consts] "r" (consts) + : "memory", "cc", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", + "v18", "v19", "v20" + ); +} + +XALIGNED(8) static const word32 L_mldsa_decompose_q32_consts[] = { + 0x0003feff, 0x0003feff, 0x0003feff, 0x0003feff, + 0x0003ff00, 0x0003ff00, 0x0003ff00, 0x0003ff00, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0007fe00, 0x0007fe00, 0x0007fe00, 0x0007fe00, + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +void mldsa_decompose_q32_neon(const sword32* r, sword32* r0, sword32* r1) +{ + const word32* consts = L_mldsa_decompose_q32_consts; + __asm__ __volatile__ ( + "ldr q0, [%[consts]]\n\t" + "ldr q1, [%[consts], #16]\n\t" + "ldr q2, [%[consts], #32]\n\t" + "ldr q3, [%[consts], #48]\n\t" + "mov x4, #16\n\t" + "\n" + "L_mldsa_decompose_q32_loop_%=:\n\t" + "ldr q4, [%x[r]]\n\t" + "ldr q5, [%x[r], #16]\n\t" + "ldr q6, [%x[r], #32]\n\t" + "ldr q7, [%x[r], #48]\n\t" + "add v12.4s, v4.4s, v0.4s\n\t" + "sshr v12.4s, v12.4s, #19\n\t" + "mov v8.16b, v4.16b\n\t" + "mls v8.4s, v12.4s, v3.4s\n\t" + "sub v16.4s, v1.4s, v8.4s\n\t" + "usra v12.4s, v16.4s, #31\n\t" + "mov v8.16b, v4.16b\n\t" + "mls v8.4s, v12.4s, v3.4s\n\t" + "sshr v16.4s, v12.4s, #4\n\t" + "sub v8.4s, v8.4s, v16.4s\n\t" + "and v12.16b, v12.16b, v2.16b\n\t" + "add v13.4s, v5.4s, v0.4s\n\t" + "sshr v13.4s, v13.4s, #19\n\t" + "mov v9.16b, v5.16b\n\t" + "mls v9.4s, v13.4s, v3.4s\n\t" + "sub v17.4s, v1.4s, v9.4s\n\t" + "usra v13.4s, v17.4s, #31\n\t" + "mov v9.16b, v5.16b\n\t" + "mls v9.4s, v13.4s, v3.4s\n\t" + "sshr v17.4s, v13.4s, #4\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "and v13.16b, v13.16b, v2.16b\n\t" + "add v14.4s, v6.4s, v0.4s\n\t" + "sshr v14.4s, v14.4s, #19\n\t" + "mov v10.16b, v6.16b\n\t" + "mls v10.4s, v14.4s, v3.4s\n\t" + "sub v18.4s, v1.4s, v10.4s\n\t" + "usra v14.4s, v18.4s, #31\n\t" + "mov v10.16b, v6.16b\n\t" + "mls v10.4s, v14.4s, v3.4s\n\t" + "sshr v18.4s, v14.4s, #4\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "and v14.16b, v14.16b, v2.16b\n\t" + "add v15.4s, v7.4s, v0.4s\n\t" + "sshr v15.4s, v15.4s, #19\n\t" + "mov v11.16b, v7.16b\n\t" + "mls v11.4s, v15.4s, v3.4s\n\t" + "sub v19.4s, v1.4s, v11.4s\n\t" + "usra v15.4s, v19.4s, #31\n\t" + "mov v11.16b, v7.16b\n\t" + "mls v11.4s, v15.4s, v3.4s\n\t" + "sshr v19.4s, v15.4s, #4\n\t" + "sub v11.4s, v11.4s, v19.4s\n\t" + "and v15.16b, v15.16b, v2.16b\n\t" + "str q8, [%x[r0]]\n\t" + "str q12, [%x[r1]]\n\t" + "str q9, [%x[r0], #16]\n\t" + "str q13, [%x[r1], #16]\n\t" + "str q10, [%x[r0], #32]\n\t" + "str q14, [%x[r1], #32]\n\t" + "str q11, [%x[r0], #48]\n\t" + "str q15, [%x[r1], #48]\n\t" + "add %x[r], %x[r], #0x40\n\t" + "add %x[r0], %x[r0], #0x40\n\t" + "add %x[r1], %x[r1], #0x40\n\t" + "subs x4, x4, #1\n\t" + "b.ne L_mldsa_decompose_q32_loop_%=\n\t" + : [r0] "+r" (r0), [r1] "+r" (r1) + : [r] "r" (r), [consts] "r" (consts) + : "memory", "cc", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", + "v18", "v19" + ); +} + +XALIGNED(8) static const word32 L_mldsa_use_hint_q88_consts[] = { + 0x0000002c, 0x0000002c, 0x0000002c, 0x0000002c, + 0x003fefd4, 0x003fefd4, 0x003fefd4, 0x003fefd4, + 0x0002e800, 0x0002e800, 0x0002e800, 0x0002e800, + 0x00017400, 0x00017400, 0x00017400, 0x00017400, + 0x0000002b, 0x0000002b, 0x0000002b, 0x0000002b, + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +void mldsa_use_hint_q88_neon(sword32* w1) +{ + const word32* consts = L_mldsa_use_hint_q88_consts; + __asm__ __volatile__ ( + "ldr q0, [%[consts]]\n\t" + "ldr q1, [%[consts], #16]\n\t" + "ldr q2, [%[consts], #32]\n\t" + "ldr q3, [%[consts], #48]\n\t" + "ldr q4, [%[consts], #64]\n\t" + "ldr q5, [%[consts], #80]\n\t" + "mov x2, #16\n\t" + "\n" + "L_mldsa_use_hint_q88_loop_%=:\n\t" + "ldr q6, [%x[w1]]\n\t" + "ldr q7, [%x[w1], #16]\n\t" + "ldr q8, [%x[w1], #32]\n\t" + "ldr q9, [%x[w1], #48]\n\t" + "sshr v18.4s, v6.4s, #31\n\t" + "and v18.16b, v18.16b, v5.16b\n\t" + "add v6.4s, v6.4s, v18.4s\n\t" + "sshr v19.4s, v7.4s, #31\n\t" + "and v19.16b, v19.16b, v5.16b\n\t" + "add v7.4s, v7.4s, v19.4s\n\t" + "sshr v20.4s, v8.4s, #31\n\t" + "and v20.16b, v20.16b, v5.16b\n\t" + "add v8.4s, v8.4s, v20.4s\n\t" + "sshr v21.4s, v9.4s, #31\n\t" + "and v21.16b, v21.16b, v5.16b\n\t" + "add v9.4s, v9.4s, v21.4s\n\t" + "mov v14.16b, v1.16b\n\t" + "mla v14.4s, v6.4s, v0.4s\n\t" + "sshr v14.4s, v14.4s, #23\n\t" + "mov v10.16b, v6.16b\n\t" + "mls v10.4s, v14.4s, v2.4s\n\t" + "sub v18.4s, v3.4s, v10.4s\n\t" + "usra v14.4s, v18.4s, #31\n\t" + "mov v10.16b, v6.16b\n\t" + "mls v10.4s, v14.4s, v2.4s\n\t" + "sub v18.4s, v4.4s, v14.4s\n\t" + "ushr v18.4s, v18.4s, #31\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "sub v18.4s, v14.4s, v0.4s\n\t" + "ushr v18.4s, v18.4s, #31\n\t" + "neg v18.4s, v18.4s\n\t" + "and v14.16b, v14.16b, v18.16b\n\t" + "mov v15.16b, v1.16b\n\t" + "mla v15.4s, v7.4s, v0.4s\n\t" + "sshr v15.4s, v15.4s, #23\n\t" + "mov v11.16b, v7.16b\n\t" + "mls v11.4s, v15.4s, v2.4s\n\t" + "sub v19.4s, v3.4s, v11.4s\n\t" + "usra v15.4s, v19.4s, #31\n\t" + "mov v11.16b, v7.16b\n\t" + "mls v11.4s, v15.4s, v2.4s\n\t" + "sub v19.4s, v4.4s, v15.4s\n\t" + "ushr v19.4s, v19.4s, #31\n\t" + "sub v11.4s, v11.4s, v19.4s\n\t" + "sub v19.4s, v15.4s, v0.4s\n\t" + "ushr v19.4s, v19.4s, #31\n\t" + "neg v19.4s, v19.4s\n\t" + "and v15.16b, v15.16b, v19.16b\n\t" + "mov v16.16b, v1.16b\n\t" + "mla v16.4s, v8.4s, v0.4s\n\t" + "sshr v16.4s, v16.4s, #23\n\t" + "mov v12.16b, v8.16b\n\t" + "mls v12.4s, v16.4s, v2.4s\n\t" + "sub v20.4s, v3.4s, v12.4s\n\t" + "usra v16.4s, v20.4s, #31\n\t" + "mov v12.16b, v8.16b\n\t" + "mls v12.4s, v16.4s, v2.4s\n\t" + "sub v20.4s, v4.4s, v16.4s\n\t" + "ushr v20.4s, v20.4s, #31\n\t" + "sub v12.4s, v12.4s, v20.4s\n\t" + "sub v20.4s, v16.4s, v0.4s\n\t" + "ushr v20.4s, v20.4s, #31\n\t" + "neg v20.4s, v20.4s\n\t" + "and v16.16b, v16.16b, v20.16b\n\t" + "mov v17.16b, v1.16b\n\t" + "mla v17.4s, v9.4s, v0.4s\n\t" + "sshr v17.4s, v17.4s, #23\n\t" + "mov v13.16b, v9.16b\n\t" + "mls v13.4s, v17.4s, v2.4s\n\t" + "sub v21.4s, v3.4s, v13.4s\n\t" + "usra v17.4s, v21.4s, #31\n\t" + "mov v13.16b, v9.16b\n\t" + "mls v13.4s, v17.4s, v2.4s\n\t" + "sub v21.4s, v4.4s, v17.4s\n\t" + "ushr v21.4s, v21.4s, #31\n\t" + "sub v13.4s, v13.4s, v21.4s\n\t" + "sub v21.4s, v17.4s, v0.4s\n\t" + "ushr v21.4s, v21.4s, #31\n\t" + "neg v21.4s, v21.4s\n\t" + "and v17.16b, v17.16b, v21.16b\n\t" + "str q14, [%x[w1]]\n\t" + "str q15, [%x[w1], #16]\n\t" + "str q16, [%x[w1], #32]\n\t" + "str q17, [%x[w1], #48]\n\t" + "add %x[w1], %x[w1], #0x40\n\t" + "subs x2, x2, #1\n\t" + "b.ne L_mldsa_use_hint_q88_loop_%=\n\t" + : [w1] "+r" (w1) + : [consts] "r" (consts) + : "memory", "cc", "x2", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", + "v18", "v19", "v20", "v21" + ); +} + +XALIGNED(8) static const word32 L_mldsa_use_hint_q32_consts[] = { + 0x0003feff, 0x0003feff, 0x0003feff, 0x0003feff, + 0x0003ff00, 0x0003ff00, 0x0003ff00, 0x0003ff00, + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, + 0x0007fe00, 0x0007fe00, 0x0007fe00, 0x0007fe00, + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +void mldsa_use_hint_q32_neon(sword32* w1) +{ + const word32* consts = L_mldsa_use_hint_q32_consts; + __asm__ __volatile__ ( + "ldr q0, [%[consts]]\n\t" + "ldr q1, [%[consts], #16]\n\t" + "ldr q2, [%[consts], #32]\n\t" + "ldr q3, [%[consts], #48]\n\t" + "ldr q4, [%[consts], #64]\n\t" + "mov x2, #16\n\t" + "\n" + "L_mldsa_use_hint_q32_loop_%=:\n\t" + "ldr q5, [%x[w1]]\n\t" + "ldr q6, [%x[w1], #16]\n\t" + "ldr q7, [%x[w1], #32]\n\t" + "ldr q8, [%x[w1], #48]\n\t" + "sshr v17.4s, v5.4s, #31\n\t" + "and v17.16b, v17.16b, v4.16b\n\t" + "add v5.4s, v5.4s, v17.4s\n\t" + "sshr v18.4s, v6.4s, #31\n\t" + "and v18.16b, v18.16b, v4.16b\n\t" + "add v6.4s, v6.4s, v18.4s\n\t" + "sshr v19.4s, v7.4s, #31\n\t" + "and v19.16b, v19.16b, v4.16b\n\t" + "add v7.4s, v7.4s, v19.4s\n\t" + "sshr v20.4s, v8.4s, #31\n\t" + "and v20.16b, v20.16b, v4.16b\n\t" + "add v8.4s, v8.4s, v20.4s\n\t" + "add v13.4s, v5.4s, v0.4s\n\t" + "sshr v13.4s, v13.4s, #19\n\t" + "mov v9.16b, v5.16b\n\t" + "mls v9.4s, v13.4s, v3.4s\n\t" + "sub v17.4s, v1.4s, v9.4s\n\t" + "usra v13.4s, v17.4s, #31\n\t" + "mov v9.16b, v5.16b\n\t" + "mls v9.4s, v13.4s, v3.4s\n\t" + "sshr v17.4s, v13.4s, #4\n\t" + "sub v9.4s, v9.4s, v17.4s\n\t" + "and v13.16b, v13.16b, v2.16b\n\t" + "add v14.4s, v6.4s, v0.4s\n\t" + "sshr v14.4s, v14.4s, #19\n\t" + "mov v10.16b, v6.16b\n\t" + "mls v10.4s, v14.4s, v3.4s\n\t" + "sub v18.4s, v1.4s, v10.4s\n\t" + "usra v14.4s, v18.4s, #31\n\t" + "mov v10.16b, v6.16b\n\t" + "mls v10.4s, v14.4s, v3.4s\n\t" + "sshr v18.4s, v14.4s, #4\n\t" + "sub v10.4s, v10.4s, v18.4s\n\t" + "and v14.16b, v14.16b, v2.16b\n\t" + "add v15.4s, v7.4s, v0.4s\n\t" + "sshr v15.4s, v15.4s, #19\n\t" + "mov v11.16b, v7.16b\n\t" + "mls v11.4s, v15.4s, v3.4s\n\t" + "sub v19.4s, v1.4s, v11.4s\n\t" + "usra v15.4s, v19.4s, #31\n\t" + "mov v11.16b, v7.16b\n\t" + "mls v11.4s, v15.4s, v3.4s\n\t" + "sshr v19.4s, v15.4s, #4\n\t" + "sub v11.4s, v11.4s, v19.4s\n\t" + "and v15.16b, v15.16b, v2.16b\n\t" + "add v16.4s, v8.4s, v0.4s\n\t" + "sshr v16.4s, v16.4s, #19\n\t" + "mov v12.16b, v8.16b\n\t" + "mls v12.4s, v16.4s, v3.4s\n\t" + "sub v20.4s, v1.4s, v12.4s\n\t" + "usra v16.4s, v20.4s, #31\n\t" + "mov v12.16b, v8.16b\n\t" + "mls v12.4s, v16.4s, v3.4s\n\t" + "sshr v20.4s, v16.4s, #4\n\t" + "sub v12.4s, v12.4s, v20.4s\n\t" + "and v16.16b, v16.16b, v2.16b\n\t" + "str q13, [%x[w1]]\n\t" + "str q14, [%x[w1], #16]\n\t" + "str q15, [%x[w1], #32]\n\t" + "str q16, [%x[w1], #48]\n\t" + "add %x[w1], %x[w1], #0x40\n\t" + "subs x2, x2, #1\n\t" + "b.ne L_mldsa_use_hint_q32_loop_%=\n\t" + : [w1] "+r" (w1) + : [consts] "r" (consts) + : "memory", "cc", "x2", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", + "v18", "v19", "v20" + ); +} + +XALIGNED(8) static const word32 L_mldsa_make_hint_bits[] = { + 0x00000001, 0x00000002, 0x00000004, 0x00000008, + 0x00000010, 0x00000020, 0x00000040, 0x00000080, + 0x00000100, 0x00000200, 0x00000400, 0x00000800, + 0x00001000, 0x00002000, 0x00004000, 0x00008000, +}; + +int mldsa_make_hint_neon(const sword32* s, const sword32* w1, byte* h, int idx, + int omega, sword32 gamma2) +{ + const word32* bits = L_mldsa_make_hint_bits; + __asm__ __volatile__ ( + "dup v0.4s, %w[gamma2]\n\t" + "neg v1.4s, v0.4s\n\t" + "ldr q2, [%[bits]]\n\t" + "ldr q3, [%[bits], #16]\n\t" + "ldr q4, [%[bits], #32]\n\t" + "ldr q5, [%[bits], #48]\n\t" + "eor x7, x7, x7\n\t" + "\n" + "L_mldsa_make_hint_loop_%=:\n\t" + "ldr q6, [%x[s]]\n\t" + "ldr q10, [%x[w1]]\n\t" + "ldr q7, [%x[s], #16]\n\t" + "ldr q11, [%x[w1], #16]\n\t" + "ldr q8, [%x[s], #32]\n\t" + "ldr q12, [%x[w1], #32]\n\t" + "ldr q9, [%x[s], #48]\n\t" + "ldr q13, [%x[w1], #48]\n\t" + "cmgt v14.4s, v6.4s, v0.4s\n\t" + "cmgt v18.4s, v1.4s, v6.4s\n\t" + "cmeq v22.4s, v6.4s, v1.4s\n\t" + "cmeq v10.4s, v10.4s, #0\n\t" + "bic v22.16b, v22.16b, v10.16b\n\t" + "orr v14.16b, v14.16b, v18.16b\n\t" + "orr v14.16b, v14.16b, v22.16b\n\t" + "and v14.16b, v14.16b, v2.16b\n\t" + "cmgt v15.4s, v7.4s, v0.4s\n\t" + "cmgt v19.4s, v1.4s, v7.4s\n\t" + "cmeq v23.4s, v7.4s, v1.4s\n\t" + "cmeq v11.4s, v11.4s, #0\n\t" + "bic v23.16b, v23.16b, v11.16b\n\t" + "orr v15.16b, v15.16b, v19.16b\n\t" + "orr v15.16b, v15.16b, v23.16b\n\t" + "and v15.16b, v15.16b, v3.16b\n\t" + "cmgt v16.4s, v8.4s, v0.4s\n\t" + "cmgt v20.4s, v1.4s, v8.4s\n\t" + "cmeq v24.4s, v8.4s, v1.4s\n\t" + "cmeq v12.4s, v12.4s, #0\n\t" + "bic v24.16b, v24.16b, v12.16b\n\t" + "orr v16.16b, v16.16b, v20.16b\n\t" + "orr v16.16b, v16.16b, v24.16b\n\t" + "and v16.16b, v16.16b, v4.16b\n\t" + "cmgt v17.4s, v9.4s, v0.4s\n\t" + "cmgt v21.4s, v1.4s, v9.4s\n\t" + "cmeq v25.4s, v9.4s, v1.4s\n\t" + "cmeq v13.4s, v13.4s, #0\n\t" + "bic v25.16b, v25.16b, v13.16b\n\t" + "orr v17.16b, v17.16b, v21.16b\n\t" + "orr v17.16b, v17.16b, v25.16b\n\t" + "and v17.16b, v17.16b, v5.16b\n\t" + "orr v14.16b, v14.16b, v15.16b\n\t" + "orr v16.16b, v16.16b, v17.16b\n\t" + "orr v14.16b, v14.16b, v16.16b\n\t" + "addv s14, v14.4s\n\t" + "mov w8, v14.s[0]\n\t" + "cbz w8, L_mldsa_make_hint_next_%=\n\t" + "\n" + "L_mldsa_make_hint_bit_%=:\n\t" + "rbit w9, w8\n\t" + "clz w9, w9\n\t" + "add w9, w9, w7\n\t" + "strb w9, [%x[h]]\n\t" + "add %x[h], %x[h], #1\n\t" + "add %w[idx], %w[idx], #1\n\t" + "subs wzr, %w[idx], %w[omega]\n\t" + "b.gt L_mldsa_make_hint_fail_%=\n\t" + "sub w10, w8, #1\n\t" + "and w8, w8, w10\n\t" + "cbnz w8, L_mldsa_make_hint_bit_%=\n\t" + "\n" + "L_mldsa_make_hint_next_%=:\n\t" + "add %x[s], %x[s], #0x40\n\t" + "add %x[w1], %x[w1], #0x40\n\t" + "add x7, x7, #16\n\t" + "subs wzr, w7, #0x100\n\t" + "b.lt L_mldsa_make_hint_loop_%=\n\t" + "mov %x[s], %x[idx]\n\t" + "b L_mldsa_make_hint_done_%=\n\t" + "\n" + "L_mldsa_make_hint_fail_%=:\n\t" + "movn %x[s], #0\n\t" + "\n" + "L_mldsa_make_hint_done_%=:\n\t" + : [s] "+r" (s), [h] "+r" (h), [idx] "+r" (idx), [omega] "+r" (omega), + [gamma2] "+r" (gamma2) + : [w1] "r" (w1), [bits] "r" (bits) + : "memory", "cc", "x7", "x8", "x9", "x10", "v0", "v1", "v2", "v3", "v4", + "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13", "v14", + "v15", "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", + "v24", "v25" + ); + return (word32)(size_t)s; +} + +void mldsa_mul_vec_4_neon(sword32* r, const sword32* m, const sword32* v) +{ + const word32* consts = L_mldsa_aarch64_consts; + __asm__ __volatile__ ( + "ldr q0, [%[consts]]\n\t" + "ldr q1, [%x[m]]\n\t" + "ldr q2, [%x[v]]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1024]\n\t" + "ldr q2, [%x[v], #1024]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2048]\n\t" + "ldr q2, [%x[v], #2048]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3072]\n\t" + "ldr q2, [%x[v], #3072]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r]]\n\t" + "ldr q1, [%x[m], #16]\n\t" + "ldr q2, [%x[v], #16]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1040]\n\t" + "ldr q2, [%x[v], #1040]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2064]\n\t" + "ldr q2, [%x[v], #2064]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3088]\n\t" + "ldr q2, [%x[v], #3088]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #16]\n\t" + "ldr q1, [%x[m], #32]\n\t" + "ldr q2, [%x[v], #32]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1056]\n\t" + "ldr q2, [%x[v], #1056]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2080]\n\t" + "ldr q2, [%x[v], #2080]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3104]\n\t" + "ldr q2, [%x[v], #3104]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #32]\n\t" + "ldr q1, [%x[m], #48]\n\t" + "ldr q2, [%x[v], #48]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1072]\n\t" + "ldr q2, [%x[v], #1072]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2096]\n\t" + "ldr q2, [%x[v], #2096]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3120]\n\t" + "ldr q2, [%x[v], #3120]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #48]\n\t" + "ldr q1, [%x[m], #64]\n\t" + "ldr q2, [%x[v], #64]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1088]\n\t" + "ldr q2, [%x[v], #1088]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2112]\n\t" + "ldr q2, [%x[v], #2112]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3136]\n\t" + "ldr q2, [%x[v], #3136]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #64]\n\t" + "ldr q1, [%x[m], #80]\n\t" + "ldr q2, [%x[v], #80]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1104]\n\t" + "ldr q2, [%x[v], #1104]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2128]\n\t" + "ldr q2, [%x[v], #2128]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3152]\n\t" + "ldr q2, [%x[v], #3152]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #80]\n\t" + "ldr q1, [%x[m], #96]\n\t" + "ldr q2, [%x[v], #96]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1120]\n\t" + "ldr q2, [%x[v], #1120]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2144]\n\t" + "ldr q2, [%x[v], #2144]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3168]\n\t" + "ldr q2, [%x[v], #3168]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #96]\n\t" + "ldr q1, [%x[m], #112]\n\t" + "ldr q2, [%x[v], #112]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1136]\n\t" + "ldr q2, [%x[v], #1136]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2160]\n\t" + "ldr q2, [%x[v], #2160]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3184]\n\t" + "ldr q2, [%x[v], #3184]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #112]\n\t" + "ldr q1, [%x[m], #128]\n\t" + "ldr q2, [%x[v], #128]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1152]\n\t" + "ldr q2, [%x[v], #1152]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2176]\n\t" + "ldr q2, [%x[v], #2176]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3200]\n\t" + "ldr q2, [%x[v], #3200]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #128]\n\t" + "ldr q1, [%x[m], #144]\n\t" + "ldr q2, [%x[v], #144]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1168]\n\t" + "ldr q2, [%x[v], #1168]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2192]\n\t" + "ldr q2, [%x[v], #2192]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3216]\n\t" + "ldr q2, [%x[v], #3216]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #144]\n\t" + "ldr q1, [%x[m], #160]\n\t" + "ldr q2, [%x[v], #160]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1184]\n\t" + "ldr q2, [%x[v], #1184]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2208]\n\t" + "ldr q2, [%x[v], #2208]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3232]\n\t" + "ldr q2, [%x[v], #3232]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #160]\n\t" + "ldr q1, [%x[m], #176]\n\t" + "ldr q2, [%x[v], #176]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1200]\n\t" + "ldr q2, [%x[v], #1200]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2224]\n\t" + "ldr q2, [%x[v], #2224]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3248]\n\t" + "ldr q2, [%x[v], #3248]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #176]\n\t" + "ldr q1, [%x[m], #192]\n\t" + "ldr q2, [%x[v], #192]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1216]\n\t" + "ldr q2, [%x[v], #1216]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2240]\n\t" + "ldr q2, [%x[v], #2240]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3264]\n\t" + "ldr q2, [%x[v], #3264]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #192]\n\t" + "ldr q1, [%x[m], #208]\n\t" + "ldr q2, [%x[v], #208]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1232]\n\t" + "ldr q2, [%x[v], #1232]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2256]\n\t" + "ldr q2, [%x[v], #2256]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3280]\n\t" + "ldr q2, [%x[v], #3280]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #208]\n\t" + "ldr q1, [%x[m], #224]\n\t" + "ldr q2, [%x[v], #224]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1248]\n\t" + "ldr q2, [%x[v], #1248]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2272]\n\t" + "ldr q2, [%x[v], #2272]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3296]\n\t" + "ldr q2, [%x[v], #3296]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #224]\n\t" + "ldr q1, [%x[m], #240]\n\t" + "ldr q2, [%x[v], #240]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1264]\n\t" + "ldr q2, [%x[v], #1264]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2288]\n\t" + "ldr q2, [%x[v], #2288]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3312]\n\t" + "ldr q2, [%x[v], #3312]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #240]\n\t" + "ldr q1, [%x[m], #256]\n\t" + "ldr q2, [%x[v], #256]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1280]\n\t" + "ldr q2, [%x[v], #1280]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2304]\n\t" + "ldr q2, [%x[v], #2304]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3328]\n\t" + "ldr q2, [%x[v], #3328]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #256]\n\t" + "ldr q1, [%x[m], #272]\n\t" + "ldr q2, [%x[v], #272]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1296]\n\t" + "ldr q2, [%x[v], #1296]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2320]\n\t" + "ldr q2, [%x[v], #2320]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3344]\n\t" + "ldr q2, [%x[v], #3344]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #272]\n\t" + "ldr q1, [%x[m], #288]\n\t" + "ldr q2, [%x[v], #288]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1312]\n\t" + "ldr q2, [%x[v], #1312]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2336]\n\t" + "ldr q2, [%x[v], #2336]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3360]\n\t" + "ldr q2, [%x[v], #3360]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #288]\n\t" + "ldr q1, [%x[m], #304]\n\t" + "ldr q2, [%x[v], #304]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1328]\n\t" + "ldr q2, [%x[v], #1328]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2352]\n\t" + "ldr q2, [%x[v], #2352]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3376]\n\t" + "ldr q2, [%x[v], #3376]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #304]\n\t" + "ldr q1, [%x[m], #320]\n\t" + "ldr q2, [%x[v], #320]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1344]\n\t" + "ldr q2, [%x[v], #1344]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2368]\n\t" + "ldr q2, [%x[v], #2368]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3392]\n\t" + "ldr q2, [%x[v], #3392]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #320]\n\t" + "ldr q1, [%x[m], #336]\n\t" + "ldr q2, [%x[v], #336]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1360]\n\t" + "ldr q2, [%x[v], #1360]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2384]\n\t" + "ldr q2, [%x[v], #2384]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3408]\n\t" + "ldr q2, [%x[v], #3408]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #336]\n\t" + "ldr q1, [%x[m], #352]\n\t" + "ldr q2, [%x[v], #352]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1376]\n\t" + "ldr q2, [%x[v], #1376]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2400]\n\t" + "ldr q2, [%x[v], #2400]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3424]\n\t" + "ldr q2, [%x[v], #3424]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #352]\n\t" + "ldr q1, [%x[m], #368]\n\t" + "ldr q2, [%x[v], #368]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1392]\n\t" + "ldr q2, [%x[v], #1392]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2416]\n\t" + "ldr q2, [%x[v], #2416]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3440]\n\t" + "ldr q2, [%x[v], #3440]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #368]\n\t" + "ldr q1, [%x[m], #384]\n\t" + "ldr q2, [%x[v], #384]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1408]\n\t" + "ldr q2, [%x[v], #1408]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2432]\n\t" + "ldr q2, [%x[v], #2432]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3456]\n\t" + "ldr q2, [%x[v], #3456]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #384]\n\t" + "ldr q1, [%x[m], #400]\n\t" + "ldr q2, [%x[v], #400]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1424]\n\t" + "ldr q2, [%x[v], #1424]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2448]\n\t" + "ldr q2, [%x[v], #2448]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3472]\n\t" + "ldr q2, [%x[v], #3472]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #400]\n\t" + "ldr q1, [%x[m], #416]\n\t" + "ldr q2, [%x[v], #416]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1440]\n\t" + "ldr q2, [%x[v], #1440]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2464]\n\t" + "ldr q2, [%x[v], #2464]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3488]\n\t" + "ldr q2, [%x[v], #3488]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #416]\n\t" + "ldr q1, [%x[m], #432]\n\t" + "ldr q2, [%x[v], #432]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1456]\n\t" + "ldr q2, [%x[v], #1456]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2480]\n\t" + "ldr q2, [%x[v], #2480]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3504]\n\t" + "ldr q2, [%x[v], #3504]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #432]\n\t" + "ldr q1, [%x[m], #448]\n\t" + "ldr q2, [%x[v], #448]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1472]\n\t" + "ldr q2, [%x[v], #1472]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2496]\n\t" + "ldr q2, [%x[v], #2496]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3520]\n\t" + "ldr q2, [%x[v], #3520]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #448]\n\t" + "ldr q1, [%x[m], #464]\n\t" + "ldr q2, [%x[v], #464]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1488]\n\t" + "ldr q2, [%x[v], #1488]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2512]\n\t" + "ldr q2, [%x[v], #2512]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3536]\n\t" + "ldr q2, [%x[v], #3536]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #464]\n\t" + "ldr q1, [%x[m], #480]\n\t" + "ldr q2, [%x[v], #480]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1504]\n\t" + "ldr q2, [%x[v], #1504]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2528]\n\t" + "ldr q2, [%x[v], #2528]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3552]\n\t" + "ldr q2, [%x[v], #3552]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #480]\n\t" + "ldr q1, [%x[m], #496]\n\t" + "ldr q2, [%x[v], #496]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1520]\n\t" + "ldr q2, [%x[v], #1520]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2544]\n\t" + "ldr q2, [%x[v], #2544]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3568]\n\t" + "ldr q2, [%x[v], #3568]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #496]\n\t" + "ldr q1, [%x[m], #512]\n\t" + "ldr q2, [%x[v], #512]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1536]\n\t" + "ldr q2, [%x[v], #1536]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2560]\n\t" + "ldr q2, [%x[v], #2560]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3584]\n\t" + "ldr q2, [%x[v], #3584]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #512]\n\t" + "ldr q1, [%x[m], #528]\n\t" + "ldr q2, [%x[v], #528]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1552]\n\t" + "ldr q2, [%x[v], #1552]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2576]\n\t" + "ldr q2, [%x[v], #2576]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3600]\n\t" + "ldr q2, [%x[v], #3600]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #528]\n\t" + "ldr q1, [%x[m], #544]\n\t" + "ldr q2, [%x[v], #544]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1568]\n\t" + "ldr q2, [%x[v], #1568]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2592]\n\t" + "ldr q2, [%x[v], #2592]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3616]\n\t" + "ldr q2, [%x[v], #3616]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #544]\n\t" + "ldr q1, [%x[m], #560]\n\t" + "ldr q2, [%x[v], #560]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1584]\n\t" + "ldr q2, [%x[v], #1584]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2608]\n\t" + "ldr q2, [%x[v], #2608]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3632]\n\t" + "ldr q2, [%x[v], #3632]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #560]\n\t" + "ldr q1, [%x[m], #576]\n\t" + "ldr q2, [%x[v], #576]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1600]\n\t" + "ldr q2, [%x[v], #1600]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2624]\n\t" + "ldr q2, [%x[v], #2624]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3648]\n\t" + "ldr q2, [%x[v], #3648]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #576]\n\t" + "ldr q1, [%x[m], #592]\n\t" + "ldr q2, [%x[v], #592]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1616]\n\t" + "ldr q2, [%x[v], #1616]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2640]\n\t" + "ldr q2, [%x[v], #2640]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3664]\n\t" + "ldr q2, [%x[v], #3664]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #592]\n\t" + "ldr q1, [%x[m], #608]\n\t" + "ldr q2, [%x[v], #608]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1632]\n\t" + "ldr q2, [%x[v], #1632]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2656]\n\t" + "ldr q2, [%x[v], #2656]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3680]\n\t" + "ldr q2, [%x[v], #3680]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #608]\n\t" + "ldr q1, [%x[m], #624]\n\t" + "ldr q2, [%x[v], #624]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1648]\n\t" + "ldr q2, [%x[v], #1648]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2672]\n\t" + "ldr q2, [%x[v], #2672]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3696]\n\t" + "ldr q2, [%x[v], #3696]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #624]\n\t" + "ldr q1, [%x[m], #640]\n\t" + "ldr q2, [%x[v], #640]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1664]\n\t" + "ldr q2, [%x[v], #1664]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2688]\n\t" + "ldr q2, [%x[v], #2688]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3712]\n\t" + "ldr q2, [%x[v], #3712]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #640]\n\t" + "ldr q1, [%x[m], #656]\n\t" + "ldr q2, [%x[v], #656]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1680]\n\t" + "ldr q2, [%x[v], #1680]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2704]\n\t" + "ldr q2, [%x[v], #2704]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3728]\n\t" + "ldr q2, [%x[v], #3728]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #656]\n\t" + "ldr q1, [%x[m], #672]\n\t" + "ldr q2, [%x[v], #672]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1696]\n\t" + "ldr q2, [%x[v], #1696]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2720]\n\t" + "ldr q2, [%x[v], #2720]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3744]\n\t" + "ldr q2, [%x[v], #3744]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #672]\n\t" + "ldr q1, [%x[m], #688]\n\t" + "ldr q2, [%x[v], #688]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1712]\n\t" + "ldr q2, [%x[v], #1712]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2736]\n\t" + "ldr q2, [%x[v], #2736]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3760]\n\t" + "ldr q2, [%x[v], #3760]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #688]\n\t" + "ldr q1, [%x[m], #704]\n\t" + "ldr q2, [%x[v], #704]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1728]\n\t" + "ldr q2, [%x[v], #1728]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2752]\n\t" + "ldr q2, [%x[v], #2752]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3776]\n\t" + "ldr q2, [%x[v], #3776]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #704]\n\t" + "ldr q1, [%x[m], #720]\n\t" + "ldr q2, [%x[v], #720]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1744]\n\t" + "ldr q2, [%x[v], #1744]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2768]\n\t" + "ldr q2, [%x[v], #2768]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3792]\n\t" + "ldr q2, [%x[v], #3792]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #720]\n\t" + "ldr q1, [%x[m], #736]\n\t" + "ldr q2, [%x[v], #736]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1760]\n\t" + "ldr q2, [%x[v], #1760]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2784]\n\t" + "ldr q2, [%x[v], #2784]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3808]\n\t" + "ldr q2, [%x[v], #3808]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #736]\n\t" + "ldr q1, [%x[m], #752]\n\t" + "ldr q2, [%x[v], #752]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1776]\n\t" + "ldr q2, [%x[v], #1776]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2800]\n\t" + "ldr q2, [%x[v], #2800]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3824]\n\t" + "ldr q2, [%x[v], #3824]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #752]\n\t" + "ldr q1, [%x[m], #768]\n\t" + "ldr q2, [%x[v], #768]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1792]\n\t" + "ldr q2, [%x[v], #1792]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2816]\n\t" + "ldr q2, [%x[v], #2816]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3840]\n\t" + "ldr q2, [%x[v], #3840]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #768]\n\t" + "ldr q1, [%x[m], #784]\n\t" + "ldr q2, [%x[v], #784]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1808]\n\t" + "ldr q2, [%x[v], #1808]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2832]\n\t" + "ldr q2, [%x[v], #2832]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3856]\n\t" + "ldr q2, [%x[v], #3856]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #784]\n\t" + "ldr q1, [%x[m], #800]\n\t" + "ldr q2, [%x[v], #800]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1824]\n\t" + "ldr q2, [%x[v], #1824]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2848]\n\t" + "ldr q2, [%x[v], #2848]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3872]\n\t" + "ldr q2, [%x[v], #3872]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #800]\n\t" + "ldr q1, [%x[m], #816]\n\t" + "ldr q2, [%x[v], #816]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1840]\n\t" + "ldr q2, [%x[v], #1840]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2864]\n\t" + "ldr q2, [%x[v], #2864]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3888]\n\t" + "ldr q2, [%x[v], #3888]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #816]\n\t" + "ldr q1, [%x[m], #832]\n\t" + "ldr q2, [%x[v], #832]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1856]\n\t" + "ldr q2, [%x[v], #1856]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2880]\n\t" + "ldr q2, [%x[v], #2880]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3904]\n\t" + "ldr q2, [%x[v], #3904]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #832]\n\t" + "ldr q1, [%x[m], #848]\n\t" + "ldr q2, [%x[v], #848]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1872]\n\t" + "ldr q2, [%x[v], #1872]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2896]\n\t" + "ldr q2, [%x[v], #2896]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3920]\n\t" + "ldr q2, [%x[v], #3920]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #848]\n\t" + "ldr q1, [%x[m], #864]\n\t" + "ldr q2, [%x[v], #864]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1888]\n\t" + "ldr q2, [%x[v], #1888]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2912]\n\t" + "ldr q2, [%x[v], #2912]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3936]\n\t" + "ldr q2, [%x[v], #3936]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #864]\n\t" + "ldr q1, [%x[m], #880]\n\t" + "ldr q2, [%x[v], #880]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1904]\n\t" + "ldr q2, [%x[v], #1904]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2928]\n\t" + "ldr q2, [%x[v], #2928]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3952]\n\t" + "ldr q2, [%x[v], #3952]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #880]\n\t" + "ldr q1, [%x[m], #896]\n\t" + "ldr q2, [%x[v], #896]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1920]\n\t" + "ldr q2, [%x[v], #1920]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2944]\n\t" + "ldr q2, [%x[v], #2944]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3968]\n\t" + "ldr q2, [%x[v], #3968]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #896]\n\t" + "ldr q1, [%x[m], #912]\n\t" + "ldr q2, [%x[v], #912]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1936]\n\t" + "ldr q2, [%x[v], #1936]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2960]\n\t" + "ldr q2, [%x[v], #2960]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3984]\n\t" + "ldr q2, [%x[v], #3984]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #912]\n\t" + "ldr q1, [%x[m], #928]\n\t" + "ldr q2, [%x[v], #928]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1952]\n\t" + "ldr q2, [%x[v], #1952]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2976]\n\t" + "ldr q2, [%x[v], #2976]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4000]\n\t" + "ldr q2, [%x[v], #4000]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #928]\n\t" + "ldr q1, [%x[m], #944]\n\t" + "ldr q2, [%x[v], #944]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1968]\n\t" + "ldr q2, [%x[v], #1968]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2992]\n\t" + "ldr q2, [%x[v], #2992]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4016]\n\t" + "ldr q2, [%x[v], #4016]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #944]\n\t" + "ldr q1, [%x[m], #960]\n\t" + "ldr q2, [%x[v], #960]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1984]\n\t" + "ldr q2, [%x[v], #1984]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3008]\n\t" + "ldr q2, [%x[v], #3008]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4032]\n\t" + "ldr q2, [%x[v], #4032]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #960]\n\t" + "ldr q1, [%x[m], #976]\n\t" + "ldr q2, [%x[v], #976]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2000]\n\t" + "ldr q2, [%x[v], #2000]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3024]\n\t" + "ldr q2, [%x[v], #3024]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4048]\n\t" + "ldr q2, [%x[v], #4048]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #976]\n\t" + "ldr q1, [%x[m], #992]\n\t" + "ldr q2, [%x[v], #992]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2016]\n\t" + "ldr q2, [%x[v], #2016]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3040]\n\t" + "ldr q2, [%x[v], #3040]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4064]\n\t" + "ldr q2, [%x[v], #4064]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #992]\n\t" + "ldr q1, [%x[m], #1008]\n\t" + "ldr q2, [%x[v], #1008]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2032]\n\t" + "ldr q2, [%x[v], #2032]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3056]\n\t" + "ldr q2, [%x[v], #3056]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4080]\n\t" + "ldr q2, [%x[v], #4080]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #1008]\n\t" + : [r] "+r" (r) + : [m] "r" (m), [v] "r" (v), [consts] "r" (consts) + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + ); +} + +void mldsa_mul_vec_5_neon(sword32* r, const sword32* m, const sword32* v) +{ + const word32* consts = L_mldsa_aarch64_consts; + __asm__ __volatile__ ( + "ldr q0, [%[consts]]\n\t" + "ldr q1, [%x[m]]\n\t" + "ldr q2, [%x[v]]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1024]\n\t" + "ldr q2, [%x[v], #1024]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2048]\n\t" + "ldr q2, [%x[v], #2048]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3072]\n\t" + "ldr q2, [%x[v], #3072]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4096]\n\t" + "ldr q2, [%x[v], #4096]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r]]\n\t" + "ldr q1, [%x[m], #16]\n\t" + "ldr q2, [%x[v], #16]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1040]\n\t" + "ldr q2, [%x[v], #1040]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2064]\n\t" + "ldr q2, [%x[v], #2064]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3088]\n\t" + "ldr q2, [%x[v], #3088]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4112]\n\t" + "ldr q2, [%x[v], #4112]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #16]\n\t" + "ldr q1, [%x[m], #32]\n\t" + "ldr q2, [%x[v], #32]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1056]\n\t" + "ldr q2, [%x[v], #1056]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2080]\n\t" + "ldr q2, [%x[v], #2080]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3104]\n\t" + "ldr q2, [%x[v], #3104]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4128]\n\t" + "ldr q2, [%x[v], #4128]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #32]\n\t" + "ldr q1, [%x[m], #48]\n\t" + "ldr q2, [%x[v], #48]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1072]\n\t" + "ldr q2, [%x[v], #1072]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2096]\n\t" + "ldr q2, [%x[v], #2096]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3120]\n\t" + "ldr q2, [%x[v], #3120]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4144]\n\t" + "ldr q2, [%x[v], #4144]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #48]\n\t" + "ldr q1, [%x[m], #64]\n\t" + "ldr q2, [%x[v], #64]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1088]\n\t" + "ldr q2, [%x[v], #1088]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2112]\n\t" + "ldr q2, [%x[v], #2112]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3136]\n\t" + "ldr q2, [%x[v], #3136]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4160]\n\t" + "ldr q2, [%x[v], #4160]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #64]\n\t" + "ldr q1, [%x[m], #80]\n\t" + "ldr q2, [%x[v], #80]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1104]\n\t" + "ldr q2, [%x[v], #1104]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2128]\n\t" + "ldr q2, [%x[v], #2128]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3152]\n\t" + "ldr q2, [%x[v], #3152]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4176]\n\t" + "ldr q2, [%x[v], #4176]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #80]\n\t" + "ldr q1, [%x[m], #96]\n\t" + "ldr q2, [%x[v], #96]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1120]\n\t" + "ldr q2, [%x[v], #1120]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2144]\n\t" + "ldr q2, [%x[v], #2144]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3168]\n\t" + "ldr q2, [%x[v], #3168]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4192]\n\t" + "ldr q2, [%x[v], #4192]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #96]\n\t" + "ldr q1, [%x[m], #112]\n\t" + "ldr q2, [%x[v], #112]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1136]\n\t" + "ldr q2, [%x[v], #1136]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2160]\n\t" + "ldr q2, [%x[v], #2160]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3184]\n\t" + "ldr q2, [%x[v], #3184]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4208]\n\t" + "ldr q2, [%x[v], #4208]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #112]\n\t" + "ldr q1, [%x[m], #128]\n\t" + "ldr q2, [%x[v], #128]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1152]\n\t" + "ldr q2, [%x[v], #1152]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2176]\n\t" + "ldr q2, [%x[v], #2176]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3200]\n\t" + "ldr q2, [%x[v], #3200]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4224]\n\t" + "ldr q2, [%x[v], #4224]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #128]\n\t" + "ldr q1, [%x[m], #144]\n\t" + "ldr q2, [%x[v], #144]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1168]\n\t" + "ldr q2, [%x[v], #1168]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2192]\n\t" + "ldr q2, [%x[v], #2192]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3216]\n\t" + "ldr q2, [%x[v], #3216]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4240]\n\t" + "ldr q2, [%x[v], #4240]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #144]\n\t" + "ldr q1, [%x[m], #160]\n\t" + "ldr q2, [%x[v], #160]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1184]\n\t" + "ldr q2, [%x[v], #1184]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2208]\n\t" + "ldr q2, [%x[v], #2208]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3232]\n\t" + "ldr q2, [%x[v], #3232]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4256]\n\t" + "ldr q2, [%x[v], #4256]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #160]\n\t" + "ldr q1, [%x[m], #176]\n\t" + "ldr q2, [%x[v], #176]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1200]\n\t" + "ldr q2, [%x[v], #1200]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2224]\n\t" + "ldr q2, [%x[v], #2224]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3248]\n\t" + "ldr q2, [%x[v], #3248]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4272]\n\t" + "ldr q2, [%x[v], #4272]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #176]\n\t" + "ldr q1, [%x[m], #192]\n\t" + "ldr q2, [%x[v], #192]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1216]\n\t" + "ldr q2, [%x[v], #1216]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2240]\n\t" + "ldr q2, [%x[v], #2240]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3264]\n\t" + "ldr q2, [%x[v], #3264]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4288]\n\t" + "ldr q2, [%x[v], #4288]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #192]\n\t" + "ldr q1, [%x[m], #208]\n\t" + "ldr q2, [%x[v], #208]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1232]\n\t" + "ldr q2, [%x[v], #1232]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2256]\n\t" + "ldr q2, [%x[v], #2256]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3280]\n\t" + "ldr q2, [%x[v], #3280]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4304]\n\t" + "ldr q2, [%x[v], #4304]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #208]\n\t" + "ldr q1, [%x[m], #224]\n\t" + "ldr q2, [%x[v], #224]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1248]\n\t" + "ldr q2, [%x[v], #1248]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2272]\n\t" + "ldr q2, [%x[v], #2272]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3296]\n\t" + "ldr q2, [%x[v], #3296]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4320]\n\t" + "ldr q2, [%x[v], #4320]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #224]\n\t" + "ldr q1, [%x[m], #240]\n\t" + "ldr q2, [%x[v], #240]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1264]\n\t" + "ldr q2, [%x[v], #1264]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2288]\n\t" + "ldr q2, [%x[v], #2288]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3312]\n\t" + "ldr q2, [%x[v], #3312]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4336]\n\t" + "ldr q2, [%x[v], #4336]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #240]\n\t" + "ldr q1, [%x[m], #256]\n\t" + "ldr q2, [%x[v], #256]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1280]\n\t" + "ldr q2, [%x[v], #1280]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2304]\n\t" + "ldr q2, [%x[v], #2304]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3328]\n\t" + "ldr q2, [%x[v], #3328]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4352]\n\t" + "ldr q2, [%x[v], #4352]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #256]\n\t" + "ldr q1, [%x[m], #272]\n\t" + "ldr q2, [%x[v], #272]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1296]\n\t" + "ldr q2, [%x[v], #1296]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2320]\n\t" + "ldr q2, [%x[v], #2320]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3344]\n\t" + "ldr q2, [%x[v], #3344]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4368]\n\t" + "ldr q2, [%x[v], #4368]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #272]\n\t" + "ldr q1, [%x[m], #288]\n\t" + "ldr q2, [%x[v], #288]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1312]\n\t" + "ldr q2, [%x[v], #1312]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2336]\n\t" + "ldr q2, [%x[v], #2336]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3360]\n\t" + "ldr q2, [%x[v], #3360]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4384]\n\t" + "ldr q2, [%x[v], #4384]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #288]\n\t" + "ldr q1, [%x[m], #304]\n\t" + "ldr q2, [%x[v], #304]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1328]\n\t" + "ldr q2, [%x[v], #1328]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2352]\n\t" + "ldr q2, [%x[v], #2352]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3376]\n\t" + "ldr q2, [%x[v], #3376]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4400]\n\t" + "ldr q2, [%x[v], #4400]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #304]\n\t" + "ldr q1, [%x[m], #320]\n\t" + "ldr q2, [%x[v], #320]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1344]\n\t" + "ldr q2, [%x[v], #1344]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2368]\n\t" + "ldr q2, [%x[v], #2368]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3392]\n\t" + "ldr q2, [%x[v], #3392]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4416]\n\t" + "ldr q2, [%x[v], #4416]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #320]\n\t" + "ldr q1, [%x[m], #336]\n\t" + "ldr q2, [%x[v], #336]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1360]\n\t" + "ldr q2, [%x[v], #1360]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2384]\n\t" + "ldr q2, [%x[v], #2384]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3408]\n\t" + "ldr q2, [%x[v], #3408]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4432]\n\t" + "ldr q2, [%x[v], #4432]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #336]\n\t" + "ldr q1, [%x[m], #352]\n\t" + "ldr q2, [%x[v], #352]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1376]\n\t" + "ldr q2, [%x[v], #1376]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2400]\n\t" + "ldr q2, [%x[v], #2400]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3424]\n\t" + "ldr q2, [%x[v], #3424]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4448]\n\t" + "ldr q2, [%x[v], #4448]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #352]\n\t" + "ldr q1, [%x[m], #368]\n\t" + "ldr q2, [%x[v], #368]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1392]\n\t" + "ldr q2, [%x[v], #1392]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2416]\n\t" + "ldr q2, [%x[v], #2416]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3440]\n\t" + "ldr q2, [%x[v], #3440]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4464]\n\t" + "ldr q2, [%x[v], #4464]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #368]\n\t" + "ldr q1, [%x[m], #384]\n\t" + "ldr q2, [%x[v], #384]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1408]\n\t" + "ldr q2, [%x[v], #1408]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2432]\n\t" + "ldr q2, [%x[v], #2432]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3456]\n\t" + "ldr q2, [%x[v], #3456]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4480]\n\t" + "ldr q2, [%x[v], #4480]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #384]\n\t" + "ldr q1, [%x[m], #400]\n\t" + "ldr q2, [%x[v], #400]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1424]\n\t" + "ldr q2, [%x[v], #1424]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2448]\n\t" + "ldr q2, [%x[v], #2448]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3472]\n\t" + "ldr q2, [%x[v], #3472]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4496]\n\t" + "ldr q2, [%x[v], #4496]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #400]\n\t" + "ldr q1, [%x[m], #416]\n\t" + "ldr q2, [%x[v], #416]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1440]\n\t" + "ldr q2, [%x[v], #1440]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2464]\n\t" + "ldr q2, [%x[v], #2464]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3488]\n\t" + "ldr q2, [%x[v], #3488]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4512]\n\t" + "ldr q2, [%x[v], #4512]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #416]\n\t" + "ldr q1, [%x[m], #432]\n\t" + "ldr q2, [%x[v], #432]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1456]\n\t" + "ldr q2, [%x[v], #1456]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2480]\n\t" + "ldr q2, [%x[v], #2480]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3504]\n\t" + "ldr q2, [%x[v], #3504]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4528]\n\t" + "ldr q2, [%x[v], #4528]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #432]\n\t" + "ldr q1, [%x[m], #448]\n\t" + "ldr q2, [%x[v], #448]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1472]\n\t" + "ldr q2, [%x[v], #1472]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2496]\n\t" + "ldr q2, [%x[v], #2496]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3520]\n\t" + "ldr q2, [%x[v], #3520]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4544]\n\t" + "ldr q2, [%x[v], #4544]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #448]\n\t" + "ldr q1, [%x[m], #464]\n\t" + "ldr q2, [%x[v], #464]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1488]\n\t" + "ldr q2, [%x[v], #1488]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2512]\n\t" + "ldr q2, [%x[v], #2512]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3536]\n\t" + "ldr q2, [%x[v], #3536]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4560]\n\t" + "ldr q2, [%x[v], #4560]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #464]\n\t" + "ldr q1, [%x[m], #480]\n\t" + "ldr q2, [%x[v], #480]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1504]\n\t" + "ldr q2, [%x[v], #1504]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2528]\n\t" + "ldr q2, [%x[v], #2528]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3552]\n\t" + "ldr q2, [%x[v], #3552]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4576]\n\t" + "ldr q2, [%x[v], #4576]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #480]\n\t" + "ldr q1, [%x[m], #496]\n\t" + "ldr q2, [%x[v], #496]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1520]\n\t" + "ldr q2, [%x[v], #1520]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2544]\n\t" + "ldr q2, [%x[v], #2544]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3568]\n\t" + "ldr q2, [%x[v], #3568]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4592]\n\t" + "ldr q2, [%x[v], #4592]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #496]\n\t" + "ldr q1, [%x[m], #512]\n\t" + "ldr q2, [%x[v], #512]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1536]\n\t" + "ldr q2, [%x[v], #1536]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2560]\n\t" + "ldr q2, [%x[v], #2560]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3584]\n\t" + "ldr q2, [%x[v], #3584]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4608]\n\t" + "ldr q2, [%x[v], #4608]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #512]\n\t" + "ldr q1, [%x[m], #528]\n\t" + "ldr q2, [%x[v], #528]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1552]\n\t" + "ldr q2, [%x[v], #1552]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2576]\n\t" + "ldr q2, [%x[v], #2576]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3600]\n\t" + "ldr q2, [%x[v], #3600]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4624]\n\t" + "ldr q2, [%x[v], #4624]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #528]\n\t" + "ldr q1, [%x[m], #544]\n\t" + "ldr q2, [%x[v], #544]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1568]\n\t" + "ldr q2, [%x[v], #1568]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2592]\n\t" + "ldr q2, [%x[v], #2592]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3616]\n\t" + "ldr q2, [%x[v], #3616]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4640]\n\t" + "ldr q2, [%x[v], #4640]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #544]\n\t" + "ldr q1, [%x[m], #560]\n\t" + "ldr q2, [%x[v], #560]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1584]\n\t" + "ldr q2, [%x[v], #1584]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2608]\n\t" + "ldr q2, [%x[v], #2608]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3632]\n\t" + "ldr q2, [%x[v], #3632]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4656]\n\t" + "ldr q2, [%x[v], #4656]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #560]\n\t" + "ldr q1, [%x[m], #576]\n\t" + "ldr q2, [%x[v], #576]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1600]\n\t" + "ldr q2, [%x[v], #1600]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2624]\n\t" + "ldr q2, [%x[v], #2624]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3648]\n\t" + "ldr q2, [%x[v], #3648]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4672]\n\t" + "ldr q2, [%x[v], #4672]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #576]\n\t" + "ldr q1, [%x[m], #592]\n\t" + "ldr q2, [%x[v], #592]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1616]\n\t" + "ldr q2, [%x[v], #1616]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2640]\n\t" + "ldr q2, [%x[v], #2640]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3664]\n\t" + "ldr q2, [%x[v], #3664]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4688]\n\t" + "ldr q2, [%x[v], #4688]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #592]\n\t" + "ldr q1, [%x[m], #608]\n\t" + "ldr q2, [%x[v], #608]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1632]\n\t" + "ldr q2, [%x[v], #1632]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2656]\n\t" + "ldr q2, [%x[v], #2656]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3680]\n\t" + "ldr q2, [%x[v], #3680]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4704]\n\t" + "ldr q2, [%x[v], #4704]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #608]\n\t" + "ldr q1, [%x[m], #624]\n\t" + "ldr q2, [%x[v], #624]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1648]\n\t" + "ldr q2, [%x[v], #1648]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2672]\n\t" + "ldr q2, [%x[v], #2672]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3696]\n\t" + "ldr q2, [%x[v], #3696]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4720]\n\t" + "ldr q2, [%x[v], #4720]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #624]\n\t" + "ldr q1, [%x[m], #640]\n\t" + "ldr q2, [%x[v], #640]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1664]\n\t" + "ldr q2, [%x[v], #1664]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2688]\n\t" + "ldr q2, [%x[v], #2688]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3712]\n\t" + "ldr q2, [%x[v], #3712]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4736]\n\t" + "ldr q2, [%x[v], #4736]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #640]\n\t" + "ldr q1, [%x[m], #656]\n\t" + "ldr q2, [%x[v], #656]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1680]\n\t" + "ldr q2, [%x[v], #1680]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2704]\n\t" + "ldr q2, [%x[v], #2704]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3728]\n\t" + "ldr q2, [%x[v], #3728]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4752]\n\t" + "ldr q2, [%x[v], #4752]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #656]\n\t" + "ldr q1, [%x[m], #672]\n\t" + "ldr q2, [%x[v], #672]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1696]\n\t" + "ldr q2, [%x[v], #1696]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2720]\n\t" + "ldr q2, [%x[v], #2720]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3744]\n\t" + "ldr q2, [%x[v], #3744]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4768]\n\t" + "ldr q2, [%x[v], #4768]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #672]\n\t" + "ldr q1, [%x[m], #688]\n\t" + "ldr q2, [%x[v], #688]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1712]\n\t" + "ldr q2, [%x[v], #1712]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2736]\n\t" + "ldr q2, [%x[v], #2736]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3760]\n\t" + "ldr q2, [%x[v], #3760]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4784]\n\t" + "ldr q2, [%x[v], #4784]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #688]\n\t" + "ldr q1, [%x[m], #704]\n\t" + "ldr q2, [%x[v], #704]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1728]\n\t" + "ldr q2, [%x[v], #1728]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2752]\n\t" + "ldr q2, [%x[v], #2752]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3776]\n\t" + "ldr q2, [%x[v], #3776]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4800]\n\t" + "ldr q2, [%x[v], #4800]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #704]\n\t" + "ldr q1, [%x[m], #720]\n\t" + "ldr q2, [%x[v], #720]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1744]\n\t" + "ldr q2, [%x[v], #1744]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2768]\n\t" + "ldr q2, [%x[v], #2768]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3792]\n\t" + "ldr q2, [%x[v], #3792]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4816]\n\t" + "ldr q2, [%x[v], #4816]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #720]\n\t" + "ldr q1, [%x[m], #736]\n\t" + "ldr q2, [%x[v], #736]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1760]\n\t" + "ldr q2, [%x[v], #1760]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2784]\n\t" + "ldr q2, [%x[v], #2784]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3808]\n\t" + "ldr q2, [%x[v], #3808]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4832]\n\t" + "ldr q2, [%x[v], #4832]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #736]\n\t" + "ldr q1, [%x[m], #752]\n\t" + "ldr q2, [%x[v], #752]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1776]\n\t" + "ldr q2, [%x[v], #1776]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2800]\n\t" + "ldr q2, [%x[v], #2800]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3824]\n\t" + "ldr q2, [%x[v], #3824]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4848]\n\t" + "ldr q2, [%x[v], #4848]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #752]\n\t" + "ldr q1, [%x[m], #768]\n\t" + "ldr q2, [%x[v], #768]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1792]\n\t" + "ldr q2, [%x[v], #1792]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2816]\n\t" + "ldr q2, [%x[v], #2816]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3840]\n\t" + "ldr q2, [%x[v], #3840]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4864]\n\t" + "ldr q2, [%x[v], #4864]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #768]\n\t" + "ldr q1, [%x[m], #784]\n\t" + "ldr q2, [%x[v], #784]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1808]\n\t" + "ldr q2, [%x[v], #1808]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2832]\n\t" + "ldr q2, [%x[v], #2832]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3856]\n\t" + "ldr q2, [%x[v], #3856]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4880]\n\t" + "ldr q2, [%x[v], #4880]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #784]\n\t" + "ldr q1, [%x[m], #800]\n\t" + "ldr q2, [%x[v], #800]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1824]\n\t" + "ldr q2, [%x[v], #1824]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2848]\n\t" + "ldr q2, [%x[v], #2848]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3872]\n\t" + "ldr q2, [%x[v], #3872]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4896]\n\t" + "ldr q2, [%x[v], #4896]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #800]\n\t" + "ldr q1, [%x[m], #816]\n\t" + "ldr q2, [%x[v], #816]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1840]\n\t" + "ldr q2, [%x[v], #1840]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2864]\n\t" + "ldr q2, [%x[v], #2864]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3888]\n\t" + "ldr q2, [%x[v], #3888]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4912]\n\t" + "ldr q2, [%x[v], #4912]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #816]\n\t" + "ldr q1, [%x[m], #832]\n\t" + "ldr q2, [%x[v], #832]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1856]\n\t" + "ldr q2, [%x[v], #1856]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2880]\n\t" + "ldr q2, [%x[v], #2880]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3904]\n\t" + "ldr q2, [%x[v], #3904]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4928]\n\t" + "ldr q2, [%x[v], #4928]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #832]\n\t" + "ldr q1, [%x[m], #848]\n\t" + "ldr q2, [%x[v], #848]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1872]\n\t" + "ldr q2, [%x[v], #1872]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2896]\n\t" + "ldr q2, [%x[v], #2896]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3920]\n\t" + "ldr q2, [%x[v], #3920]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4944]\n\t" + "ldr q2, [%x[v], #4944]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #848]\n\t" + "ldr q1, [%x[m], #864]\n\t" + "ldr q2, [%x[v], #864]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1888]\n\t" + "ldr q2, [%x[v], #1888]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2912]\n\t" + "ldr q2, [%x[v], #2912]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3936]\n\t" + "ldr q2, [%x[v], #3936]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4960]\n\t" + "ldr q2, [%x[v], #4960]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #864]\n\t" + "ldr q1, [%x[m], #880]\n\t" + "ldr q2, [%x[v], #880]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1904]\n\t" + "ldr q2, [%x[v], #1904]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2928]\n\t" + "ldr q2, [%x[v], #2928]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3952]\n\t" + "ldr q2, [%x[v], #3952]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4976]\n\t" + "ldr q2, [%x[v], #4976]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #880]\n\t" + "ldr q1, [%x[m], #896]\n\t" + "ldr q2, [%x[v], #896]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1920]\n\t" + "ldr q2, [%x[v], #1920]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2944]\n\t" + "ldr q2, [%x[v], #2944]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3968]\n\t" + "ldr q2, [%x[v], #3968]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4992]\n\t" + "ldr q2, [%x[v], #4992]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #896]\n\t" + "ldr q1, [%x[m], #912]\n\t" + "ldr q2, [%x[v], #912]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1936]\n\t" + "ldr q2, [%x[v], #1936]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2960]\n\t" + "ldr q2, [%x[v], #2960]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3984]\n\t" + "ldr q2, [%x[v], #3984]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5008]\n\t" + "ldr q2, [%x[v], #5008]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #912]\n\t" + "ldr q1, [%x[m], #928]\n\t" + "ldr q2, [%x[v], #928]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1952]\n\t" + "ldr q2, [%x[v], #1952]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2976]\n\t" + "ldr q2, [%x[v], #2976]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4000]\n\t" + "ldr q2, [%x[v], #4000]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5024]\n\t" + "ldr q2, [%x[v], #5024]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #928]\n\t" + "ldr q1, [%x[m], #944]\n\t" + "ldr q2, [%x[v], #944]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1968]\n\t" + "ldr q2, [%x[v], #1968]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2992]\n\t" + "ldr q2, [%x[v], #2992]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4016]\n\t" + "ldr q2, [%x[v], #4016]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5040]\n\t" + "ldr q2, [%x[v], #5040]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #944]\n\t" + "ldr q1, [%x[m], #960]\n\t" + "ldr q2, [%x[v], #960]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1984]\n\t" + "ldr q2, [%x[v], #1984]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3008]\n\t" + "ldr q2, [%x[v], #3008]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4032]\n\t" + "ldr q2, [%x[v], #4032]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5056]\n\t" + "ldr q2, [%x[v], #5056]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #960]\n\t" + "ldr q1, [%x[m], #976]\n\t" + "ldr q2, [%x[v], #976]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2000]\n\t" + "ldr q2, [%x[v], #2000]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3024]\n\t" + "ldr q2, [%x[v], #3024]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4048]\n\t" + "ldr q2, [%x[v], #4048]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5072]\n\t" + "ldr q2, [%x[v], #5072]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #976]\n\t" + "ldr q1, [%x[m], #992]\n\t" + "ldr q2, [%x[v], #992]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2016]\n\t" + "ldr q2, [%x[v], #2016]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3040]\n\t" + "ldr q2, [%x[v], #3040]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4064]\n\t" + "ldr q2, [%x[v], #4064]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5088]\n\t" + "ldr q2, [%x[v], #5088]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #992]\n\t" + "ldr q1, [%x[m], #1008]\n\t" + "ldr q2, [%x[v], #1008]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2032]\n\t" + "ldr q2, [%x[v], #2032]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3056]\n\t" + "ldr q2, [%x[v], #3056]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4080]\n\t" + "ldr q2, [%x[v], #4080]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5104]\n\t" + "ldr q2, [%x[v], #5104]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #1008]\n\t" + : [r] "+r" (r) + : [m] "r" (m), [v] "r" (v), [consts] "r" (consts) + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + ); +} + +void mldsa_mul_vec_7_neon(sword32* r, const sword32* m, const sword32* v) +{ + const word32* consts = L_mldsa_aarch64_consts; + __asm__ __volatile__ ( + "ldr q0, [%[consts]]\n\t" + "ldr q1, [%x[m]]\n\t" + "ldr q2, [%x[v]]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1024]\n\t" + "ldr q2, [%x[v], #1024]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2048]\n\t" + "ldr q2, [%x[v], #2048]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3072]\n\t" + "ldr q2, [%x[v], #3072]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4096]\n\t" + "ldr q2, [%x[v], #4096]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5120]\n\t" + "ldr q2, [%x[v], #5120]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6144]\n\t" + "ldr q2, [%x[v], #6144]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r]]\n\t" + "ldr q1, [%x[m], #16]\n\t" + "ldr q2, [%x[v], #16]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1040]\n\t" + "ldr q2, [%x[v], #1040]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2064]\n\t" + "ldr q2, [%x[v], #2064]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3088]\n\t" + "ldr q2, [%x[v], #3088]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4112]\n\t" + "ldr q2, [%x[v], #4112]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5136]\n\t" + "ldr q2, [%x[v], #5136]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6160]\n\t" + "ldr q2, [%x[v], #6160]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #16]\n\t" + "ldr q1, [%x[m], #32]\n\t" + "ldr q2, [%x[v], #32]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1056]\n\t" + "ldr q2, [%x[v], #1056]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2080]\n\t" + "ldr q2, [%x[v], #2080]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3104]\n\t" + "ldr q2, [%x[v], #3104]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4128]\n\t" + "ldr q2, [%x[v], #4128]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5152]\n\t" + "ldr q2, [%x[v], #5152]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6176]\n\t" + "ldr q2, [%x[v], #6176]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #32]\n\t" + "ldr q1, [%x[m], #48]\n\t" + "ldr q2, [%x[v], #48]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1072]\n\t" + "ldr q2, [%x[v], #1072]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2096]\n\t" + "ldr q2, [%x[v], #2096]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3120]\n\t" + "ldr q2, [%x[v], #3120]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4144]\n\t" + "ldr q2, [%x[v], #4144]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5168]\n\t" + "ldr q2, [%x[v], #5168]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6192]\n\t" + "ldr q2, [%x[v], #6192]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #48]\n\t" + "ldr q1, [%x[m], #64]\n\t" + "ldr q2, [%x[v], #64]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1088]\n\t" + "ldr q2, [%x[v], #1088]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2112]\n\t" + "ldr q2, [%x[v], #2112]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3136]\n\t" + "ldr q2, [%x[v], #3136]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4160]\n\t" + "ldr q2, [%x[v], #4160]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5184]\n\t" + "ldr q2, [%x[v], #5184]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6208]\n\t" + "ldr q2, [%x[v], #6208]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #64]\n\t" + "ldr q1, [%x[m], #80]\n\t" + "ldr q2, [%x[v], #80]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1104]\n\t" + "ldr q2, [%x[v], #1104]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2128]\n\t" + "ldr q2, [%x[v], #2128]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3152]\n\t" + "ldr q2, [%x[v], #3152]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4176]\n\t" + "ldr q2, [%x[v], #4176]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5200]\n\t" + "ldr q2, [%x[v], #5200]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6224]\n\t" + "ldr q2, [%x[v], #6224]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #80]\n\t" + "ldr q1, [%x[m], #96]\n\t" + "ldr q2, [%x[v], #96]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1120]\n\t" + "ldr q2, [%x[v], #1120]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2144]\n\t" + "ldr q2, [%x[v], #2144]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3168]\n\t" + "ldr q2, [%x[v], #3168]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4192]\n\t" + "ldr q2, [%x[v], #4192]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5216]\n\t" + "ldr q2, [%x[v], #5216]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6240]\n\t" + "ldr q2, [%x[v], #6240]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #96]\n\t" + "ldr q1, [%x[m], #112]\n\t" + "ldr q2, [%x[v], #112]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1136]\n\t" + "ldr q2, [%x[v], #1136]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2160]\n\t" + "ldr q2, [%x[v], #2160]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3184]\n\t" + "ldr q2, [%x[v], #3184]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4208]\n\t" + "ldr q2, [%x[v], #4208]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5232]\n\t" + "ldr q2, [%x[v], #5232]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6256]\n\t" + "ldr q2, [%x[v], #6256]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #112]\n\t" + "ldr q1, [%x[m], #128]\n\t" + "ldr q2, [%x[v], #128]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1152]\n\t" + "ldr q2, [%x[v], #1152]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2176]\n\t" + "ldr q2, [%x[v], #2176]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3200]\n\t" + "ldr q2, [%x[v], #3200]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4224]\n\t" + "ldr q2, [%x[v], #4224]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5248]\n\t" + "ldr q2, [%x[v], #5248]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6272]\n\t" + "ldr q2, [%x[v], #6272]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #128]\n\t" + "ldr q1, [%x[m], #144]\n\t" + "ldr q2, [%x[v], #144]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1168]\n\t" + "ldr q2, [%x[v], #1168]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2192]\n\t" + "ldr q2, [%x[v], #2192]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3216]\n\t" + "ldr q2, [%x[v], #3216]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4240]\n\t" + "ldr q2, [%x[v], #4240]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5264]\n\t" + "ldr q2, [%x[v], #5264]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6288]\n\t" + "ldr q2, [%x[v], #6288]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #144]\n\t" + "ldr q1, [%x[m], #160]\n\t" + "ldr q2, [%x[v], #160]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1184]\n\t" + "ldr q2, [%x[v], #1184]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2208]\n\t" + "ldr q2, [%x[v], #2208]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3232]\n\t" + "ldr q2, [%x[v], #3232]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4256]\n\t" + "ldr q2, [%x[v], #4256]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5280]\n\t" + "ldr q2, [%x[v], #5280]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6304]\n\t" + "ldr q2, [%x[v], #6304]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #160]\n\t" + "ldr q1, [%x[m], #176]\n\t" + "ldr q2, [%x[v], #176]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1200]\n\t" + "ldr q2, [%x[v], #1200]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2224]\n\t" + "ldr q2, [%x[v], #2224]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3248]\n\t" + "ldr q2, [%x[v], #3248]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4272]\n\t" + "ldr q2, [%x[v], #4272]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5296]\n\t" + "ldr q2, [%x[v], #5296]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6320]\n\t" + "ldr q2, [%x[v], #6320]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #176]\n\t" + "ldr q1, [%x[m], #192]\n\t" + "ldr q2, [%x[v], #192]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1216]\n\t" + "ldr q2, [%x[v], #1216]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2240]\n\t" + "ldr q2, [%x[v], #2240]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3264]\n\t" + "ldr q2, [%x[v], #3264]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4288]\n\t" + "ldr q2, [%x[v], #4288]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5312]\n\t" + "ldr q2, [%x[v], #5312]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6336]\n\t" + "ldr q2, [%x[v], #6336]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #192]\n\t" + "ldr q1, [%x[m], #208]\n\t" + "ldr q2, [%x[v], #208]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1232]\n\t" + "ldr q2, [%x[v], #1232]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2256]\n\t" + "ldr q2, [%x[v], #2256]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3280]\n\t" + "ldr q2, [%x[v], #3280]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4304]\n\t" + "ldr q2, [%x[v], #4304]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5328]\n\t" + "ldr q2, [%x[v], #5328]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6352]\n\t" + "ldr q2, [%x[v], #6352]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #208]\n\t" + "ldr q1, [%x[m], #224]\n\t" + "ldr q2, [%x[v], #224]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1248]\n\t" + "ldr q2, [%x[v], #1248]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2272]\n\t" + "ldr q2, [%x[v], #2272]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3296]\n\t" + "ldr q2, [%x[v], #3296]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4320]\n\t" + "ldr q2, [%x[v], #4320]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5344]\n\t" + "ldr q2, [%x[v], #5344]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6368]\n\t" + "ldr q2, [%x[v], #6368]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #224]\n\t" + "ldr q1, [%x[m], #240]\n\t" + "ldr q2, [%x[v], #240]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1264]\n\t" + "ldr q2, [%x[v], #1264]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2288]\n\t" + "ldr q2, [%x[v], #2288]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3312]\n\t" + "ldr q2, [%x[v], #3312]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4336]\n\t" + "ldr q2, [%x[v], #4336]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5360]\n\t" + "ldr q2, [%x[v], #5360]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6384]\n\t" + "ldr q2, [%x[v], #6384]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #240]\n\t" + "ldr q1, [%x[m], #256]\n\t" + "ldr q2, [%x[v], #256]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1280]\n\t" + "ldr q2, [%x[v], #1280]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2304]\n\t" + "ldr q2, [%x[v], #2304]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3328]\n\t" + "ldr q2, [%x[v], #3328]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4352]\n\t" + "ldr q2, [%x[v], #4352]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5376]\n\t" + "ldr q2, [%x[v], #5376]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6400]\n\t" + "ldr q2, [%x[v], #6400]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #256]\n\t" + "ldr q1, [%x[m], #272]\n\t" + "ldr q2, [%x[v], #272]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1296]\n\t" + "ldr q2, [%x[v], #1296]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2320]\n\t" + "ldr q2, [%x[v], #2320]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3344]\n\t" + "ldr q2, [%x[v], #3344]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4368]\n\t" + "ldr q2, [%x[v], #4368]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5392]\n\t" + "ldr q2, [%x[v], #5392]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6416]\n\t" + "ldr q2, [%x[v], #6416]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #272]\n\t" + "ldr q1, [%x[m], #288]\n\t" + "ldr q2, [%x[v], #288]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1312]\n\t" + "ldr q2, [%x[v], #1312]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2336]\n\t" + "ldr q2, [%x[v], #2336]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3360]\n\t" + "ldr q2, [%x[v], #3360]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4384]\n\t" + "ldr q2, [%x[v], #4384]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5408]\n\t" + "ldr q2, [%x[v], #5408]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6432]\n\t" + "ldr q2, [%x[v], #6432]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #288]\n\t" + "ldr q1, [%x[m], #304]\n\t" + "ldr q2, [%x[v], #304]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1328]\n\t" + "ldr q2, [%x[v], #1328]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2352]\n\t" + "ldr q2, [%x[v], #2352]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3376]\n\t" + "ldr q2, [%x[v], #3376]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4400]\n\t" + "ldr q2, [%x[v], #4400]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5424]\n\t" + "ldr q2, [%x[v], #5424]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6448]\n\t" + "ldr q2, [%x[v], #6448]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #304]\n\t" + "ldr q1, [%x[m], #320]\n\t" + "ldr q2, [%x[v], #320]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1344]\n\t" + "ldr q2, [%x[v], #1344]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2368]\n\t" + "ldr q2, [%x[v], #2368]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3392]\n\t" + "ldr q2, [%x[v], #3392]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4416]\n\t" + "ldr q2, [%x[v], #4416]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5440]\n\t" + "ldr q2, [%x[v], #5440]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6464]\n\t" + "ldr q2, [%x[v], #6464]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #320]\n\t" + "ldr q1, [%x[m], #336]\n\t" + "ldr q2, [%x[v], #336]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1360]\n\t" + "ldr q2, [%x[v], #1360]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2384]\n\t" + "ldr q2, [%x[v], #2384]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3408]\n\t" + "ldr q2, [%x[v], #3408]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4432]\n\t" + "ldr q2, [%x[v], #4432]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5456]\n\t" + "ldr q2, [%x[v], #5456]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6480]\n\t" + "ldr q2, [%x[v], #6480]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #336]\n\t" + "ldr q1, [%x[m], #352]\n\t" + "ldr q2, [%x[v], #352]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1376]\n\t" + "ldr q2, [%x[v], #1376]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2400]\n\t" + "ldr q2, [%x[v], #2400]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3424]\n\t" + "ldr q2, [%x[v], #3424]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4448]\n\t" + "ldr q2, [%x[v], #4448]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5472]\n\t" + "ldr q2, [%x[v], #5472]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6496]\n\t" + "ldr q2, [%x[v], #6496]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #352]\n\t" + "ldr q1, [%x[m], #368]\n\t" + "ldr q2, [%x[v], #368]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1392]\n\t" + "ldr q2, [%x[v], #1392]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2416]\n\t" + "ldr q2, [%x[v], #2416]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3440]\n\t" + "ldr q2, [%x[v], #3440]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4464]\n\t" + "ldr q2, [%x[v], #4464]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5488]\n\t" + "ldr q2, [%x[v], #5488]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6512]\n\t" + "ldr q2, [%x[v], #6512]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #368]\n\t" + "ldr q1, [%x[m], #384]\n\t" + "ldr q2, [%x[v], #384]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1408]\n\t" + "ldr q2, [%x[v], #1408]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2432]\n\t" + "ldr q2, [%x[v], #2432]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3456]\n\t" + "ldr q2, [%x[v], #3456]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4480]\n\t" + "ldr q2, [%x[v], #4480]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5504]\n\t" + "ldr q2, [%x[v], #5504]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6528]\n\t" + "ldr q2, [%x[v], #6528]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #384]\n\t" + "ldr q1, [%x[m], #400]\n\t" + "ldr q2, [%x[v], #400]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1424]\n\t" + "ldr q2, [%x[v], #1424]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2448]\n\t" + "ldr q2, [%x[v], #2448]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3472]\n\t" + "ldr q2, [%x[v], #3472]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4496]\n\t" + "ldr q2, [%x[v], #4496]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5520]\n\t" + "ldr q2, [%x[v], #5520]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6544]\n\t" + "ldr q2, [%x[v], #6544]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #400]\n\t" + "ldr q1, [%x[m], #416]\n\t" + "ldr q2, [%x[v], #416]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1440]\n\t" + "ldr q2, [%x[v], #1440]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2464]\n\t" + "ldr q2, [%x[v], #2464]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3488]\n\t" + "ldr q2, [%x[v], #3488]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4512]\n\t" + "ldr q2, [%x[v], #4512]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5536]\n\t" + "ldr q2, [%x[v], #5536]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6560]\n\t" + "ldr q2, [%x[v], #6560]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #416]\n\t" + "ldr q1, [%x[m], #432]\n\t" + "ldr q2, [%x[v], #432]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1456]\n\t" + "ldr q2, [%x[v], #1456]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2480]\n\t" + "ldr q2, [%x[v], #2480]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3504]\n\t" + "ldr q2, [%x[v], #3504]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4528]\n\t" + "ldr q2, [%x[v], #4528]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5552]\n\t" + "ldr q2, [%x[v], #5552]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6576]\n\t" + "ldr q2, [%x[v], #6576]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #432]\n\t" + "ldr q1, [%x[m], #448]\n\t" + "ldr q2, [%x[v], #448]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1472]\n\t" + "ldr q2, [%x[v], #1472]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2496]\n\t" + "ldr q2, [%x[v], #2496]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3520]\n\t" + "ldr q2, [%x[v], #3520]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4544]\n\t" + "ldr q2, [%x[v], #4544]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5568]\n\t" + "ldr q2, [%x[v], #5568]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6592]\n\t" + "ldr q2, [%x[v], #6592]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #448]\n\t" + "ldr q1, [%x[m], #464]\n\t" + "ldr q2, [%x[v], #464]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1488]\n\t" + "ldr q2, [%x[v], #1488]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2512]\n\t" + "ldr q2, [%x[v], #2512]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3536]\n\t" + "ldr q2, [%x[v], #3536]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4560]\n\t" + "ldr q2, [%x[v], #4560]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5584]\n\t" + "ldr q2, [%x[v], #5584]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6608]\n\t" + "ldr q2, [%x[v], #6608]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #464]\n\t" + "ldr q1, [%x[m], #480]\n\t" + "ldr q2, [%x[v], #480]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1504]\n\t" + "ldr q2, [%x[v], #1504]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2528]\n\t" + "ldr q2, [%x[v], #2528]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3552]\n\t" + "ldr q2, [%x[v], #3552]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4576]\n\t" + "ldr q2, [%x[v], #4576]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5600]\n\t" + "ldr q2, [%x[v], #5600]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6624]\n\t" + "ldr q2, [%x[v], #6624]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #480]\n\t" + "ldr q1, [%x[m], #496]\n\t" + "ldr q2, [%x[v], #496]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1520]\n\t" + "ldr q2, [%x[v], #1520]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2544]\n\t" + "ldr q2, [%x[v], #2544]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3568]\n\t" + "ldr q2, [%x[v], #3568]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4592]\n\t" + "ldr q2, [%x[v], #4592]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5616]\n\t" + "ldr q2, [%x[v], #5616]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6640]\n\t" + "ldr q2, [%x[v], #6640]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #496]\n\t" + "ldr q1, [%x[m], #512]\n\t" + "ldr q2, [%x[v], #512]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1536]\n\t" + "ldr q2, [%x[v], #1536]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2560]\n\t" + "ldr q2, [%x[v], #2560]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3584]\n\t" + "ldr q2, [%x[v], #3584]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4608]\n\t" + "ldr q2, [%x[v], #4608]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5632]\n\t" + "ldr q2, [%x[v], #5632]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6656]\n\t" + "ldr q2, [%x[v], #6656]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #512]\n\t" + "ldr q1, [%x[m], #528]\n\t" + "ldr q2, [%x[v], #528]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1552]\n\t" + "ldr q2, [%x[v], #1552]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2576]\n\t" + "ldr q2, [%x[v], #2576]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3600]\n\t" + "ldr q2, [%x[v], #3600]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4624]\n\t" + "ldr q2, [%x[v], #4624]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5648]\n\t" + "ldr q2, [%x[v], #5648]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6672]\n\t" + "ldr q2, [%x[v], #6672]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #528]\n\t" + "ldr q1, [%x[m], #544]\n\t" + "ldr q2, [%x[v], #544]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1568]\n\t" + "ldr q2, [%x[v], #1568]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2592]\n\t" + "ldr q2, [%x[v], #2592]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3616]\n\t" + "ldr q2, [%x[v], #3616]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4640]\n\t" + "ldr q2, [%x[v], #4640]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5664]\n\t" + "ldr q2, [%x[v], #5664]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6688]\n\t" + "ldr q2, [%x[v], #6688]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #544]\n\t" + "ldr q1, [%x[m], #560]\n\t" + "ldr q2, [%x[v], #560]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1584]\n\t" + "ldr q2, [%x[v], #1584]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2608]\n\t" + "ldr q2, [%x[v], #2608]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3632]\n\t" + "ldr q2, [%x[v], #3632]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4656]\n\t" + "ldr q2, [%x[v], #4656]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5680]\n\t" + "ldr q2, [%x[v], #5680]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6704]\n\t" + "ldr q2, [%x[v], #6704]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #560]\n\t" + "ldr q1, [%x[m], #576]\n\t" + "ldr q2, [%x[v], #576]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1600]\n\t" + "ldr q2, [%x[v], #1600]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2624]\n\t" + "ldr q2, [%x[v], #2624]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3648]\n\t" + "ldr q2, [%x[v], #3648]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4672]\n\t" + "ldr q2, [%x[v], #4672]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5696]\n\t" + "ldr q2, [%x[v], #5696]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6720]\n\t" + "ldr q2, [%x[v], #6720]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #576]\n\t" + "ldr q1, [%x[m], #592]\n\t" + "ldr q2, [%x[v], #592]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1616]\n\t" + "ldr q2, [%x[v], #1616]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2640]\n\t" + "ldr q2, [%x[v], #2640]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3664]\n\t" + "ldr q2, [%x[v], #3664]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4688]\n\t" + "ldr q2, [%x[v], #4688]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5712]\n\t" + "ldr q2, [%x[v], #5712]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6736]\n\t" + "ldr q2, [%x[v], #6736]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #592]\n\t" + "ldr q1, [%x[m], #608]\n\t" + "ldr q2, [%x[v], #608]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1632]\n\t" + "ldr q2, [%x[v], #1632]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2656]\n\t" + "ldr q2, [%x[v], #2656]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3680]\n\t" + "ldr q2, [%x[v], #3680]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4704]\n\t" + "ldr q2, [%x[v], #4704]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5728]\n\t" + "ldr q2, [%x[v], #5728]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6752]\n\t" + "ldr q2, [%x[v], #6752]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #608]\n\t" + "ldr q1, [%x[m], #624]\n\t" + "ldr q2, [%x[v], #624]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1648]\n\t" + "ldr q2, [%x[v], #1648]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2672]\n\t" + "ldr q2, [%x[v], #2672]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3696]\n\t" + "ldr q2, [%x[v], #3696]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4720]\n\t" + "ldr q2, [%x[v], #4720]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5744]\n\t" + "ldr q2, [%x[v], #5744]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6768]\n\t" + "ldr q2, [%x[v], #6768]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #624]\n\t" + "ldr q1, [%x[m], #640]\n\t" + "ldr q2, [%x[v], #640]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1664]\n\t" + "ldr q2, [%x[v], #1664]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2688]\n\t" + "ldr q2, [%x[v], #2688]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3712]\n\t" + "ldr q2, [%x[v], #3712]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4736]\n\t" + "ldr q2, [%x[v], #4736]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5760]\n\t" + "ldr q2, [%x[v], #5760]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6784]\n\t" + "ldr q2, [%x[v], #6784]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #640]\n\t" + "ldr q1, [%x[m], #656]\n\t" + "ldr q2, [%x[v], #656]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1680]\n\t" + "ldr q2, [%x[v], #1680]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2704]\n\t" + "ldr q2, [%x[v], #2704]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3728]\n\t" + "ldr q2, [%x[v], #3728]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4752]\n\t" + "ldr q2, [%x[v], #4752]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5776]\n\t" + "ldr q2, [%x[v], #5776]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6800]\n\t" + "ldr q2, [%x[v], #6800]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #656]\n\t" + "ldr q1, [%x[m], #672]\n\t" + "ldr q2, [%x[v], #672]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1696]\n\t" + "ldr q2, [%x[v], #1696]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2720]\n\t" + "ldr q2, [%x[v], #2720]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3744]\n\t" + "ldr q2, [%x[v], #3744]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4768]\n\t" + "ldr q2, [%x[v], #4768]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5792]\n\t" + "ldr q2, [%x[v], #5792]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6816]\n\t" + "ldr q2, [%x[v], #6816]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #672]\n\t" + "ldr q1, [%x[m], #688]\n\t" + "ldr q2, [%x[v], #688]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1712]\n\t" + "ldr q2, [%x[v], #1712]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2736]\n\t" + "ldr q2, [%x[v], #2736]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3760]\n\t" + "ldr q2, [%x[v], #3760]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4784]\n\t" + "ldr q2, [%x[v], #4784]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5808]\n\t" + "ldr q2, [%x[v], #5808]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6832]\n\t" + "ldr q2, [%x[v], #6832]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #688]\n\t" + "ldr q1, [%x[m], #704]\n\t" + "ldr q2, [%x[v], #704]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1728]\n\t" + "ldr q2, [%x[v], #1728]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2752]\n\t" + "ldr q2, [%x[v], #2752]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3776]\n\t" + "ldr q2, [%x[v], #3776]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4800]\n\t" + "ldr q2, [%x[v], #4800]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5824]\n\t" + "ldr q2, [%x[v], #5824]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6848]\n\t" + "ldr q2, [%x[v], #6848]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #704]\n\t" + "ldr q1, [%x[m], #720]\n\t" + "ldr q2, [%x[v], #720]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1744]\n\t" + "ldr q2, [%x[v], #1744]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2768]\n\t" + "ldr q2, [%x[v], #2768]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3792]\n\t" + "ldr q2, [%x[v], #3792]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4816]\n\t" + "ldr q2, [%x[v], #4816]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5840]\n\t" + "ldr q2, [%x[v], #5840]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6864]\n\t" + "ldr q2, [%x[v], #6864]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #720]\n\t" + "ldr q1, [%x[m], #736]\n\t" + "ldr q2, [%x[v], #736]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1760]\n\t" + "ldr q2, [%x[v], #1760]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2784]\n\t" + "ldr q2, [%x[v], #2784]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3808]\n\t" + "ldr q2, [%x[v], #3808]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4832]\n\t" + "ldr q2, [%x[v], #4832]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5856]\n\t" + "ldr q2, [%x[v], #5856]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6880]\n\t" + "ldr q2, [%x[v], #6880]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #736]\n\t" + "ldr q1, [%x[m], #752]\n\t" + "ldr q2, [%x[v], #752]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1776]\n\t" + "ldr q2, [%x[v], #1776]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2800]\n\t" + "ldr q2, [%x[v], #2800]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3824]\n\t" + "ldr q2, [%x[v], #3824]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4848]\n\t" + "ldr q2, [%x[v], #4848]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5872]\n\t" + "ldr q2, [%x[v], #5872]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6896]\n\t" + "ldr q2, [%x[v], #6896]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #752]\n\t" + "ldr q1, [%x[m], #768]\n\t" + "ldr q2, [%x[v], #768]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1792]\n\t" + "ldr q2, [%x[v], #1792]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2816]\n\t" + "ldr q2, [%x[v], #2816]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3840]\n\t" + "ldr q2, [%x[v], #3840]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4864]\n\t" + "ldr q2, [%x[v], #4864]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5888]\n\t" + "ldr q2, [%x[v], #5888]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6912]\n\t" + "ldr q2, [%x[v], #6912]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #768]\n\t" + "ldr q1, [%x[m], #784]\n\t" + "ldr q2, [%x[v], #784]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1808]\n\t" + "ldr q2, [%x[v], #1808]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2832]\n\t" + "ldr q2, [%x[v], #2832]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3856]\n\t" + "ldr q2, [%x[v], #3856]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4880]\n\t" + "ldr q2, [%x[v], #4880]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5904]\n\t" + "ldr q2, [%x[v], #5904]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6928]\n\t" + "ldr q2, [%x[v], #6928]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #784]\n\t" + "ldr q1, [%x[m], #800]\n\t" + "ldr q2, [%x[v], #800]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1824]\n\t" + "ldr q2, [%x[v], #1824]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2848]\n\t" + "ldr q2, [%x[v], #2848]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3872]\n\t" + "ldr q2, [%x[v], #3872]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4896]\n\t" + "ldr q2, [%x[v], #4896]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5920]\n\t" + "ldr q2, [%x[v], #5920]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6944]\n\t" + "ldr q2, [%x[v], #6944]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #800]\n\t" + "ldr q1, [%x[m], #816]\n\t" + "ldr q2, [%x[v], #816]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1840]\n\t" + "ldr q2, [%x[v], #1840]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2864]\n\t" + "ldr q2, [%x[v], #2864]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3888]\n\t" + "ldr q2, [%x[v], #3888]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4912]\n\t" + "ldr q2, [%x[v], #4912]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5936]\n\t" + "ldr q2, [%x[v], #5936]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6960]\n\t" + "ldr q2, [%x[v], #6960]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #816]\n\t" + "ldr q1, [%x[m], #832]\n\t" + "ldr q2, [%x[v], #832]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1856]\n\t" + "ldr q2, [%x[v], #1856]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2880]\n\t" + "ldr q2, [%x[v], #2880]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3904]\n\t" + "ldr q2, [%x[v], #3904]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4928]\n\t" + "ldr q2, [%x[v], #4928]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5952]\n\t" + "ldr q2, [%x[v], #5952]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6976]\n\t" + "ldr q2, [%x[v], #6976]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #832]\n\t" + "ldr q1, [%x[m], #848]\n\t" + "ldr q2, [%x[v], #848]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1872]\n\t" + "ldr q2, [%x[v], #1872]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2896]\n\t" + "ldr q2, [%x[v], #2896]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3920]\n\t" + "ldr q2, [%x[v], #3920]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4944]\n\t" + "ldr q2, [%x[v], #4944]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5968]\n\t" + "ldr q2, [%x[v], #5968]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6992]\n\t" + "ldr q2, [%x[v], #6992]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #848]\n\t" + "ldr q1, [%x[m], #864]\n\t" + "ldr q2, [%x[v], #864]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1888]\n\t" + "ldr q2, [%x[v], #1888]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2912]\n\t" + "ldr q2, [%x[v], #2912]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3936]\n\t" + "ldr q2, [%x[v], #3936]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4960]\n\t" + "ldr q2, [%x[v], #4960]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5984]\n\t" + "ldr q2, [%x[v], #5984]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #7008]\n\t" + "ldr q2, [%x[v], #7008]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #864]\n\t" + "ldr q1, [%x[m], #880]\n\t" + "ldr q2, [%x[v], #880]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1904]\n\t" + "ldr q2, [%x[v], #1904]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2928]\n\t" + "ldr q2, [%x[v], #2928]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3952]\n\t" + "ldr q2, [%x[v], #3952]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4976]\n\t" + "ldr q2, [%x[v], #4976]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6000]\n\t" + "ldr q2, [%x[v], #6000]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #7024]\n\t" + "ldr q2, [%x[v], #7024]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #880]\n\t" + "ldr q1, [%x[m], #896]\n\t" + "ldr q2, [%x[v], #896]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1920]\n\t" + "ldr q2, [%x[v], #1920]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2944]\n\t" + "ldr q2, [%x[v], #2944]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3968]\n\t" + "ldr q2, [%x[v], #3968]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4992]\n\t" + "ldr q2, [%x[v], #4992]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6016]\n\t" + "ldr q2, [%x[v], #6016]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #7040]\n\t" + "ldr q2, [%x[v], #7040]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #896]\n\t" + "ldr q1, [%x[m], #912]\n\t" + "ldr q2, [%x[v], #912]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1936]\n\t" + "ldr q2, [%x[v], #1936]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2960]\n\t" + "ldr q2, [%x[v], #2960]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3984]\n\t" + "ldr q2, [%x[v], #3984]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5008]\n\t" + "ldr q2, [%x[v], #5008]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6032]\n\t" + "ldr q2, [%x[v], #6032]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #7056]\n\t" + "ldr q2, [%x[v], #7056]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #912]\n\t" + "ldr q1, [%x[m], #928]\n\t" + "ldr q2, [%x[v], #928]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1952]\n\t" + "ldr q2, [%x[v], #1952]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2976]\n\t" + "ldr q2, [%x[v], #2976]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4000]\n\t" + "ldr q2, [%x[v], #4000]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5024]\n\t" + "ldr q2, [%x[v], #5024]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6048]\n\t" + "ldr q2, [%x[v], #6048]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #7072]\n\t" + "ldr q2, [%x[v], #7072]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #928]\n\t" + "ldr q1, [%x[m], #944]\n\t" + "ldr q2, [%x[v], #944]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1968]\n\t" + "ldr q2, [%x[v], #1968]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2992]\n\t" + "ldr q2, [%x[v], #2992]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4016]\n\t" + "ldr q2, [%x[v], #4016]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5040]\n\t" + "ldr q2, [%x[v], #5040]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6064]\n\t" + "ldr q2, [%x[v], #6064]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #7088]\n\t" + "ldr q2, [%x[v], #7088]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #944]\n\t" + "ldr q1, [%x[m], #960]\n\t" + "ldr q2, [%x[v], #960]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #1984]\n\t" + "ldr q2, [%x[v], #1984]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3008]\n\t" + "ldr q2, [%x[v], #3008]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4032]\n\t" + "ldr q2, [%x[v], #4032]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5056]\n\t" + "ldr q2, [%x[v], #5056]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6080]\n\t" + "ldr q2, [%x[v], #6080]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #7104]\n\t" + "ldr q2, [%x[v], #7104]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #960]\n\t" + "ldr q1, [%x[m], #976]\n\t" + "ldr q2, [%x[v], #976]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2000]\n\t" + "ldr q2, [%x[v], #2000]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3024]\n\t" + "ldr q2, [%x[v], #3024]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4048]\n\t" + "ldr q2, [%x[v], #4048]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5072]\n\t" + "ldr q2, [%x[v], #5072]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6096]\n\t" + "ldr q2, [%x[v], #6096]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #7120]\n\t" + "ldr q2, [%x[v], #7120]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #976]\n\t" + "ldr q1, [%x[m], #992]\n\t" + "ldr q2, [%x[v], #992]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2016]\n\t" + "ldr q2, [%x[v], #2016]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3040]\n\t" + "ldr q2, [%x[v], #3040]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4064]\n\t" + "ldr q2, [%x[v], #4064]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5088]\n\t" + "ldr q2, [%x[v], #5088]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6112]\n\t" + "ldr q2, [%x[v], #6112]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #7136]\n\t" + "ldr q2, [%x[v], #7136]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #992]\n\t" + "ldr q1, [%x[m], #1008]\n\t" + "ldr q2, [%x[v], #1008]\n\t" + "smull v3.2d, v1.2s, v2.2s\n\t" + "smull2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #2032]\n\t" + "ldr q2, [%x[v], #2032]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #3056]\n\t" + "ldr q2, [%x[v], #3056]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #4080]\n\t" + "ldr q2, [%x[v], #4080]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #5104]\n\t" + "ldr q2, [%x[v], #5104]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #6128]\n\t" + "ldr q2, [%x[v], #6128]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "ldr q1, [%x[m], #7152]\n\t" + "ldr q2, [%x[v], #7152]\n\t" + "smlal v3.2d, v1.2s, v2.2s\n\t" + "smlal2 v4.2d, v1.4s, v2.4s\n\t" + "uzp1 v5.4s, v3.4s, v4.4s\n\t" + "mul v5.4s, v5.4s, v0.s[1]\n\t" + "smull v6.2d, v5.2s, v0.s[0]\n\t" + "smull2 v7.2d, v5.4s, v0.s[0]\n\t" + "sub v3.2d, v3.2d, v6.2d\n\t" + "sub v4.2d, v4.2d, v7.2d\n\t" + "uzp2 v5.4s, v3.4s, v4.4s\n\t" + "str q5, [%x[r], #1008]\n\t" + : [r] "+r" (r) + : [m] "r" (m), [v] "r" (v), [consts] "r" (consts) + : "memory", "cc", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + ); +} + +XALIGNED(8) static const word32 L_mldsa_aarch64_q[] = { + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +XALIGNED(4) static const word8 L_mldsa_decode_eta_2_neon_idx[] = { + 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, + 0x00, 0x01, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff, + 0x01, 0xff, 0xff, 0xff, 0x01, 0x02, 0xff, 0xff, + 0x02, 0xff, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, + 0x01, 0xff, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff, + 0x01, 0x02, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, + 0x02, 0xff, 0xff, 0xff, 0x02, 0x03, 0xff, 0xff, + 0x03, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, + 0x04, 0xff, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff, + 0x04, 0x05, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff, + 0x05, 0xff, 0xff, 0xff, 0x05, 0x06, 0xff, 0xff, + 0x06, 0xff, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff, + 0x07, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, + 0x07, 0x08, 0xff, 0xff, 0x08, 0xff, 0xff, 0xff, + 0x08, 0xff, 0xff, 0xff, 0x08, 0x09, 0xff, 0xff, + 0x09, 0xff, 0xff, 0xff, 0x09, 0xff, 0xff, 0xff, + 0x0a, 0xff, 0xff, 0xff, 0x0a, 0xff, 0xff, 0xff, + 0x0a, 0x0b, 0xff, 0xff, 0x0b, 0xff, 0xff, 0xff, + 0x0b, 0xff, 0xff, 0xff, 0x0b, 0x0c, 0xff, 0xff, + 0x0c, 0xff, 0xff, 0xff, 0x0c, 0xff, 0xff, 0xff, + 0x0d, 0xff, 0xff, 0xff, 0x0d, 0xff, 0xff, 0xff, + 0x0d, 0x0e, 0xff, 0xff, 0x0e, 0xff, 0xff, 0xff, + 0x0e, 0xff, 0xff, 0xff, 0x0e, 0x0f, 0xff, 0xff, + 0x0f, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_eta_2_neon_shift[] = { + 0x00000000, 0xfffffffd, 0xfffffffa, 0xffffffff, + 0xfffffffc, 0xfffffff9, 0xfffffffe, 0xfffffffb, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_eta_2_neon_mask[] = { + 0x00000007, 0x00000007, 0x00000007, 0x00000007, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_eta_2_neon_konst[] = { + 0x00000002, 0x00000002, 0x00000002, 0x00000002, +}; + +void mldsa_decode_eta_2_neon(const byte* p, sword32* z) +{ + const word8* idx = L_mldsa_decode_eta_2_neon_idx; + const word32* shift = L_mldsa_decode_eta_2_neon_shift; + const word32* mask = L_mldsa_decode_eta_2_neon_mask; + const word32* konst = L_mldsa_decode_eta_2_neon_konst; + __asm__ __volatile__ ( + "ldr q1, [%[idx]]\n\t" + "ldr q3, [%[shift]]\n\t" + "ldr q2, [%[idx], #16]\n\t" + "ldr q4, [%[shift], #16]\n\t" + "ldr q5, [%[mask]]\n\t" + "ldr q6, [%[konst]]\n\t" + "mov x6, #27\n\t" + "\n" + "L_mldsa_decode_eta_2_neon_loop_%=:\n\t" + "ldr q0, [%x[p]]\n\t" + "tbl v7.16b, {v0.16b}, v1.16b\n\t" + "ushl v7.4s, v7.4s, v3.4s\n\t" + "and v7.16b, v7.16b, v5.16b\n\t" + "sub v7.4s, v6.4s, v7.4s\n\t" + "str q7, [%x[z]]\n\t" + "tbl v8.16b, {v0.16b}, v2.16b\n\t" + "ushl v8.4s, v8.4s, v4.4s\n\t" + "and v8.16b, v8.16b, v5.16b\n\t" + "sub v8.4s, v6.4s, v8.4s\n\t" + "str q8, [%x[z], #16]\n\t" + "add %x[p], %x[p], #3\n\t" + "add %x[z], %x[z], #32\n\t" + "subs x6, x6, #1\n\t" + "b.ne L_mldsa_decode_eta_2_neon_loop_%=\n\t" + "sub %x[p], %x[p], #1\n\t" + "ldr q0, [%x[p]]\n\t" + "ldr q1, [%[idx], #32]\n\t" + "ldr q2, [%[idx], #48]\n\t" + "tbl v7.16b, {v0.16b}, v1.16b\n\t" + "ushl v7.4s, v7.4s, v3.4s\n\t" + "and v7.16b, v7.16b, v5.16b\n\t" + "sub v7.4s, v6.4s, v7.4s\n\t" + "str q7, [%x[z]]\n\t" + "tbl v8.16b, {v0.16b}, v2.16b\n\t" + "ushl v8.4s, v8.4s, v4.4s\n\t" + "and v8.16b, v8.16b, v5.16b\n\t" + "sub v8.4s, v6.4s, v8.4s\n\t" + "str q8, [%x[z], #16]\n\t" + "add %x[z], %x[z], #32\n\t" + "ldr q1, [%[idx], #64]\n\t" + "ldr q2, [%[idx], #80]\n\t" + "tbl v7.16b, {v0.16b}, v1.16b\n\t" + "ushl v7.4s, v7.4s, v3.4s\n\t" + "and v7.16b, v7.16b, v5.16b\n\t" + "sub v7.4s, v6.4s, v7.4s\n\t" + "str q7, [%x[z]]\n\t" + "tbl v8.16b, {v0.16b}, v2.16b\n\t" + "ushl v8.4s, v8.4s, v4.4s\n\t" + "and v8.16b, v8.16b, v5.16b\n\t" + "sub v8.4s, v6.4s, v8.4s\n\t" + "str q8, [%x[z], #16]\n\t" + "add %x[z], %x[z], #32\n\t" + "ldr q1, [%[idx], #96]\n\t" + "ldr q2, [%[idx], #112]\n\t" + "tbl v7.16b, {v0.16b}, v1.16b\n\t" + "ushl v7.4s, v7.4s, v3.4s\n\t" + "and v7.16b, v7.16b, v5.16b\n\t" + "sub v7.4s, v6.4s, v7.4s\n\t" + "str q7, [%x[z]]\n\t" + "tbl v8.16b, {v0.16b}, v2.16b\n\t" + "ushl v8.4s, v8.4s, v4.4s\n\t" + "and v8.16b, v8.16b, v5.16b\n\t" + "sub v8.4s, v6.4s, v8.4s\n\t" + "str q8, [%x[z], #16]\n\t" + "add %x[z], %x[z], #32\n\t" + "ldr q1, [%[idx], #128]\n\t" + "ldr q2, [%[idx], #144]\n\t" + "tbl v7.16b, {v0.16b}, v1.16b\n\t" + "ushl v7.4s, v7.4s, v3.4s\n\t" + "and v7.16b, v7.16b, v5.16b\n\t" + "sub v7.4s, v6.4s, v7.4s\n\t" + "str q7, [%x[z]]\n\t" + "tbl v8.16b, {v0.16b}, v2.16b\n\t" + "ushl v8.4s, v8.4s, v4.4s\n\t" + "and v8.16b, v8.16b, v5.16b\n\t" + "sub v8.4s, v6.4s, v8.4s\n\t" + "str q8, [%x[z], #16]\n\t" + "add %x[z], %x[z], #32\n\t" + "ldr q1, [%[idx], #160]\n\t" + "ldr q2, [%[idx], #176]\n\t" + "tbl v7.16b, {v0.16b}, v1.16b\n\t" + "ushl v7.4s, v7.4s, v3.4s\n\t" + "and v7.16b, v7.16b, v5.16b\n\t" + "sub v7.4s, v6.4s, v7.4s\n\t" + "str q7, [%x[z]]\n\t" + "tbl v8.16b, {v0.16b}, v2.16b\n\t" + "ushl v8.4s, v8.4s, v4.4s\n\t" + "and v8.16b, v8.16b, v5.16b\n\t" + "sub v8.4s, v6.4s, v8.4s\n\t" + "str q8, [%x[z], #16]\n\t" + : [z] "+r" (z) + : [p] "r" (p), [idx] "r" (idx), [shift] "r" (shift), [mask] "r" (mask), + [konst] "r" (konst) + : "memory", "cc", "x6", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8" + ); +} + +XALIGNED(4) static const word8 L_mldsa_decode_eta_4_neon_idx[] = { + 0x00, 0xff, 0xff, 0xff, 0x00, 0xff, 0xff, 0xff, + 0x01, 0xff, 0xff, 0xff, 0x01, 0xff, 0xff, 0xff, + 0x02, 0xff, 0xff, 0xff, 0x02, 0xff, 0xff, 0xff, + 0x03, 0xff, 0xff, 0xff, 0x03, 0xff, 0xff, 0xff, + 0x04, 0xff, 0xff, 0xff, 0x04, 0xff, 0xff, 0xff, + 0x05, 0xff, 0xff, 0xff, 0x05, 0xff, 0xff, 0xff, + 0x06, 0xff, 0xff, 0xff, 0x06, 0xff, 0xff, 0xff, + 0x07, 0xff, 0xff, 0xff, 0x07, 0xff, 0xff, 0xff, + 0x08, 0xff, 0xff, 0xff, 0x08, 0xff, 0xff, 0xff, + 0x09, 0xff, 0xff, 0xff, 0x09, 0xff, 0xff, 0xff, + 0x0a, 0xff, 0xff, 0xff, 0x0a, 0xff, 0xff, 0xff, + 0x0b, 0xff, 0xff, 0xff, 0x0b, 0xff, 0xff, 0xff, + 0x0c, 0xff, 0xff, 0xff, 0x0c, 0xff, 0xff, 0xff, + 0x0d, 0xff, 0xff, 0xff, 0x0d, 0xff, 0xff, 0xff, + 0x0e, 0xff, 0xff, 0xff, 0x0e, 0xff, 0xff, 0xff, + 0x0f, 0xff, 0xff, 0xff, 0x0f, 0xff, 0xff, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_eta_4_neon_shift[] = { + 0x00000000, 0xfffffffc, 0x00000000, 0xfffffffc, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_eta_4_neon_mask[] = { + 0x0000000f, 0x0000000f, 0x0000000f, 0x0000000f, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_eta_4_neon_konst[] = { + 0x00000004, 0x00000004, 0x00000004, 0x00000004, +}; + +void mldsa_decode_eta_4_neon(const byte* p, sword32* z) +{ + const word8* idx = L_mldsa_decode_eta_4_neon_idx; + const word32* shift = L_mldsa_decode_eta_4_neon_shift; + const word32* mask = L_mldsa_decode_eta_4_neon_mask; + const word32* konst = L_mldsa_decode_eta_4_neon_konst; + __asm__ __volatile__ ( + "ldr q1, [%[idx]]\n\t" + "ldr q2, [%[shift]]\n\t" + "ldr q3, [%[mask]]\n\t" + "ldr q4, [%[konst]]\n\t" + "mov x6, #57\n\t" + "\n" + "L_mldsa_decode_eta_4_neon_loop_%=:\n\t" + "ldr q0, [%x[p]]\n\t" + "tbl v5.16b, {v0.16b}, v1.16b\n\t" + "ushl v5.4s, v5.4s, v2.4s\n\t" + "and v5.16b, v5.16b, v3.16b\n\t" + "sub v5.4s, v4.4s, v5.4s\n\t" + "str q5, [%x[z]]\n\t" + "add %x[p], %x[p], #2\n\t" + "add %x[z], %x[z], #16\n\t" + "subs x6, x6, #1\n\t" + "b.ne L_mldsa_decode_eta_4_neon_loop_%=\n\t" + "sub %x[p], %x[p], #2\n\t" + "ldr q0, [%x[p]]\n\t" + "ldr q1, [%[idx], #16]\n\t" + "tbl v5.16b, {v0.16b}, v1.16b\n\t" + "ushl v5.4s, v5.4s, v2.4s\n\t" + "and v5.16b, v5.16b, v3.16b\n\t" + "sub v5.4s, v4.4s, v5.4s\n\t" + "str q5, [%x[z]]\n\t" + "add %x[z], %x[z], #16\n\t" + "ldr q1, [%[idx], #32]\n\t" + "tbl v5.16b, {v0.16b}, v1.16b\n\t" + "ushl v5.4s, v5.4s, v2.4s\n\t" + "and v5.16b, v5.16b, v3.16b\n\t" + "sub v5.4s, v4.4s, v5.4s\n\t" + "str q5, [%x[z]]\n\t" + "add %x[z], %x[z], #16\n\t" + "ldr q1, [%[idx], #48]\n\t" + "tbl v5.16b, {v0.16b}, v1.16b\n\t" + "ushl v5.4s, v5.4s, v2.4s\n\t" + "and v5.16b, v5.16b, v3.16b\n\t" + "sub v5.4s, v4.4s, v5.4s\n\t" + "str q5, [%x[z]]\n\t" + "add %x[z], %x[z], #16\n\t" + "ldr q1, [%[idx], #64]\n\t" + "tbl v5.16b, {v0.16b}, v1.16b\n\t" + "ushl v5.4s, v5.4s, v2.4s\n\t" + "and v5.16b, v5.16b, v3.16b\n\t" + "sub v5.4s, v4.4s, v5.4s\n\t" + "str q5, [%x[z]]\n\t" + "add %x[z], %x[z], #16\n\t" + "ldr q1, [%[idx], #80]\n\t" + "tbl v5.16b, {v0.16b}, v1.16b\n\t" + "ushl v5.4s, v5.4s, v2.4s\n\t" + "and v5.16b, v5.16b, v3.16b\n\t" + "sub v5.4s, v4.4s, v5.4s\n\t" + "str q5, [%x[z]]\n\t" + "add %x[z], %x[z], #16\n\t" + "ldr q1, [%[idx], #96]\n\t" + "tbl v5.16b, {v0.16b}, v1.16b\n\t" + "ushl v5.4s, v5.4s, v2.4s\n\t" + "and v5.16b, v5.16b, v3.16b\n\t" + "sub v5.4s, v4.4s, v5.4s\n\t" + "str q5, [%x[z]]\n\t" + "add %x[z], %x[z], #16\n\t" + "ldr q1, [%[idx], #112]\n\t" + "tbl v5.16b, {v0.16b}, v1.16b\n\t" + "ushl v5.4s, v5.4s, v2.4s\n\t" + "and v5.16b, v5.16b, v3.16b\n\t" + "sub v5.4s, v4.4s, v5.4s\n\t" + "str q5, [%x[z]]\n\t" + : [z] "+r" (z) + : [p] "r" (p), [idx] "r" (idx), [shift] "r" (shift), [mask] "r" (mask), + [konst] "r" (konst) + : "memory", "cc", "x6", "v0", "v1", "v2", "v3", "v4", "v5" + ); +} + +XALIGNED(4) static const word8 L_mldsa_decode_t1_neon_idx[] = { + 0x00, 0x01, 0xff, 0xff, 0x01, 0x02, 0xff, 0xff, + 0x02, 0x03, 0xff, 0xff, 0x03, 0x04, 0xff, 0xff, + 0x01, 0x02, 0xff, 0xff, 0x02, 0x03, 0xff, 0xff, + 0x03, 0x04, 0xff, 0xff, 0x04, 0x05, 0xff, 0xff, + 0x06, 0x07, 0xff, 0xff, 0x07, 0x08, 0xff, 0xff, + 0x08, 0x09, 0xff, 0xff, 0x09, 0x0a, 0xff, 0xff, + 0x0b, 0x0c, 0xff, 0xff, 0x0c, 0x0d, 0xff, 0xff, + 0x0d, 0x0e, 0xff, 0xff, 0x0e, 0x0f, 0xff, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_t1_neon_shift[] = { + 0x00000000, 0xfffffffe, 0xfffffffc, 0xfffffffa, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_t1_neon_mask[] = { + 0x000003ff, 0x000003ff, 0x000003ff, 0x000003ff, +}; + +void mldsa_decode_t1_neon(const byte* p, sword32* z) +{ + const word8* idx = L_mldsa_decode_t1_neon_idx; + const word32* shift = L_mldsa_decode_t1_neon_shift; + const word32* mask = L_mldsa_decode_t1_neon_mask; + __asm__ __volatile__ ( + "ldr q1, [%[idx]]\n\t" + "ldr q2, [%[shift]]\n\t" + "ldr q3, [%[mask]]\n\t" + "mov x5, #61\n\t" + "\n" + "L_mldsa_decode_t1_neon_loop_%=:\n\t" + "ldr q0, [%x[p]]\n\t" + "tbl v4.16b, {v0.16b}, v1.16b\n\t" + "ushl v4.4s, v4.4s, v2.4s\n\t" + "and v4.16b, v4.16b, v3.16b\n\t" + "shl v4.4s, v4.4s, #13\n\t" + "str q4, [%x[z]]\n\t" + "add %x[p], %x[p], #5\n\t" + "add %x[z], %x[z], #16\n\t" + "subs x5, x5, #1\n\t" + "b.ne L_mldsa_decode_t1_neon_loop_%=\n\t" + "sub %x[p], %x[p], #1\n\t" + "ldr q0, [%x[p]]\n\t" + "ldr q1, [%[idx], #16]\n\t" + "tbl v4.16b, {v0.16b}, v1.16b\n\t" + "ushl v4.4s, v4.4s, v2.4s\n\t" + "and v4.16b, v4.16b, v3.16b\n\t" + "shl v4.4s, v4.4s, #13\n\t" + "str q4, [%x[z]]\n\t" + "add %x[z], %x[z], #16\n\t" + "ldr q1, [%[idx], #32]\n\t" + "tbl v4.16b, {v0.16b}, v1.16b\n\t" + "ushl v4.4s, v4.4s, v2.4s\n\t" + "and v4.16b, v4.16b, v3.16b\n\t" + "shl v4.4s, v4.4s, #13\n\t" + "str q4, [%x[z]]\n\t" + "add %x[z], %x[z], #16\n\t" + "ldr q1, [%[idx], #48]\n\t" + "tbl v4.16b, {v0.16b}, v1.16b\n\t" + "ushl v4.4s, v4.4s, v2.4s\n\t" + "and v4.16b, v4.16b, v3.16b\n\t" + "shl v4.4s, v4.4s, #13\n\t" + "str q4, [%x[z]]\n\t" + : [z] "+r" (z) + : [p] "r" (p), [idx] "r" (idx), [shift] "r" (shift), [mask] "r" (mask) + : "memory", "cc", "x5", "v0", "v1", "v2", "v3", "v4" + ); +} + +XALIGNED(4) static const word8 L_mldsa_decode_t0_neon_idx[] = { + 0x00, 0x01, 0xff, 0xff, 0x01, 0x02, 0x03, 0xff, + 0x03, 0x04, 0xff, 0xff, 0x04, 0x05, 0x06, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x08, 0x09, 0xff, 0xff, + 0x09, 0x0a, 0x0b, 0xff, 0x0b, 0x0c, 0xff, 0xff, + 0x03, 0x04, 0xff, 0xff, 0x04, 0x05, 0x06, 0xff, + 0x06, 0x07, 0xff, 0xff, 0x07, 0x08, 0x09, 0xff, + 0x09, 0x0a, 0x0b, 0xff, 0x0b, 0x0c, 0xff, 0xff, + 0x0c, 0x0d, 0x0e, 0xff, 0x0e, 0x0f, 0xff, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_t0_neon_shift[] = { + 0x00000000, 0xfffffffb, 0xfffffffe, 0xfffffff9, + 0xfffffffc, 0xffffffff, 0xfffffffa, 0xfffffffd, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_t0_neon_mask[] = { + 0x00001fff, 0x00001fff, 0x00001fff, 0x00001fff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_t0_neon_konst[] = { + 0x00001000, 0x00001000, 0x00001000, 0x00001000, +}; + +void mldsa_decode_t0_neon(const byte* p, sword32* z) +{ + const word8* idx = L_mldsa_decode_t0_neon_idx; + const word32* shift = L_mldsa_decode_t0_neon_shift; + const word32* mask = L_mldsa_decode_t0_neon_mask; + const word32* konst = L_mldsa_decode_t0_neon_konst; + __asm__ __volatile__ ( + "ldr q1, [%[idx]]\n\t" + "ldr q3, [%[shift]]\n\t" + "ldr q2, [%[idx], #16]\n\t" + "ldr q4, [%[shift], #16]\n\t" + "ldr q5, [%[mask]]\n\t" + "ldr q6, [%[konst]]\n\t" + "mov x6, #31\n\t" + "\n" + "L_mldsa_decode_t0_neon_loop_%=:\n\t" + "ldr q0, [%x[p]]\n\t" + "tbl v7.16b, {v0.16b}, v1.16b\n\t" + "ushl v7.4s, v7.4s, v3.4s\n\t" + "and v7.16b, v7.16b, v5.16b\n\t" + "sub v7.4s, v6.4s, v7.4s\n\t" + "str q7, [%x[z]]\n\t" + "tbl v8.16b, {v0.16b}, v2.16b\n\t" + "ushl v8.4s, v8.4s, v4.4s\n\t" + "and v8.16b, v8.16b, v5.16b\n\t" + "sub v8.4s, v6.4s, v8.4s\n\t" + "str q8, [%x[z], #16]\n\t" + "add %x[p], %x[p], #13\n\t" + "add %x[z], %x[z], #32\n\t" + "subs x6, x6, #1\n\t" + "b.ne L_mldsa_decode_t0_neon_loop_%=\n\t" + "sub %x[p], %x[p], #3\n\t" + "ldr q0, [%x[p]]\n\t" + "ldr q1, [%[idx], #32]\n\t" + "ldr q2, [%[idx], #48]\n\t" + "tbl v7.16b, {v0.16b}, v1.16b\n\t" + "ushl v7.4s, v7.4s, v3.4s\n\t" + "and v7.16b, v7.16b, v5.16b\n\t" + "sub v7.4s, v6.4s, v7.4s\n\t" + "str q7, [%x[z]]\n\t" + "tbl v8.16b, {v0.16b}, v2.16b\n\t" + "ushl v8.4s, v8.4s, v4.4s\n\t" + "and v8.16b, v8.16b, v5.16b\n\t" + "sub v8.4s, v6.4s, v8.4s\n\t" + "str q8, [%x[z], #16]\n\t" + : [z] "+r" (z) + : [p] "r" (p), [idx] "r" (idx), [shift] "r" (shift), [mask] "r" (mask), + [konst] "r" (konst) + : "memory", "cc", "x6", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8" + ); +} + +XALIGNED(4) static const word8 L_mldsa_decode_gamma1_17_neon_idx[] = { + 0x00, 0x01, 0x02, 0xff, 0x02, 0x03, 0x04, 0xff, + 0x04, 0x05, 0x06, 0xff, 0x06, 0x07, 0x08, 0xff, + 0x07, 0x08, 0x09, 0xff, 0x09, 0x0a, 0x0b, 0xff, + 0x0b, 0x0c, 0x0d, 0xff, 0x0d, 0x0e, 0x0f, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_gamma1_17_neon_shift[] = { + 0x00000000, 0xfffffffe, 0xfffffffc, 0xfffffffa, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_gamma1_17_neon_mask[] = { + 0x0003ffff, 0x0003ffff, 0x0003ffff, 0x0003ffff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_gamma1_17_neon_konst[] = { + 0x00020000, 0x00020000, 0x00020000, 0x00020000, +}; + +void mldsa_decode_gamma1_17_neon(const byte* p, sword32* z) +{ + const word8* idx = L_mldsa_decode_gamma1_17_neon_idx; + const word32* shift = L_mldsa_decode_gamma1_17_neon_shift; + const word32* mask = L_mldsa_decode_gamma1_17_neon_mask; + const word32* konst = L_mldsa_decode_gamma1_17_neon_konst; + __asm__ __volatile__ ( + "ldr q1, [%[idx]]\n\t" + "ldr q2, [%[shift]]\n\t" + "ldr q3, [%[mask]]\n\t" + "ldr q4, [%[konst]]\n\t" + "mov x6, #63\n\t" + "\n" + "L_mldsa_decode_gamma1_17_neon_loop_%=:\n\t" + "ldr q0, [%x[p]]\n\t" + "tbl v5.16b, {v0.16b}, v1.16b\n\t" + "ushl v5.4s, v5.4s, v2.4s\n\t" + "and v5.16b, v5.16b, v3.16b\n\t" + "sub v5.4s, v4.4s, v5.4s\n\t" + "str q5, [%x[z]]\n\t" + "add %x[p], %x[p], #9\n\t" + "add %x[z], %x[z], #16\n\t" + "subs x6, x6, #1\n\t" + "b.ne L_mldsa_decode_gamma1_17_neon_loop_%=\n\t" + "sub %x[p], %x[p], #7\n\t" + "ldr q0, [%x[p]]\n\t" + "ldr q1, [%[idx], #16]\n\t" + "tbl v5.16b, {v0.16b}, v1.16b\n\t" + "ushl v5.4s, v5.4s, v2.4s\n\t" + "and v5.16b, v5.16b, v3.16b\n\t" + "sub v5.4s, v4.4s, v5.4s\n\t" + "str q5, [%x[z]]\n\t" + : [z] "+r" (z) + : [p] "r" (p), [idx] "r" (idx), [shift] "r" (shift), [mask] "r" (mask), + [konst] "r" (konst) + : "memory", "cc", "x6", "v0", "v1", "v2", "v3", "v4", "v5" + ); +} + +XALIGNED(4) static const word8 L_mldsa_decode_gamma1_19_neon_idx[] = { + 0x00, 0x01, 0x02, 0xff, 0x02, 0x03, 0x04, 0xff, + 0x05, 0x06, 0x07, 0xff, 0x07, 0x08, 0x09, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x08, 0x09, 0x0a, 0xff, + 0x0b, 0x0c, 0x0d, 0xff, 0x0d, 0x0e, 0x0f, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_gamma1_19_neon_shift[] = { + 0x00000000, 0xfffffffc, 0x00000000, 0xfffffffc, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_gamma1_19_neon_mask[] = { + 0x000fffff, 0x000fffff, 0x000fffff, 0x000fffff, +}; + +XALIGNED(8) static const word32 L_mldsa_decode_gamma1_19_neon_konst[] = { + 0x00080000, 0x00080000, 0x00080000, 0x00080000, +}; + +void mldsa_decode_gamma1_19_neon(const byte* p, sword32* z) +{ + const word8* idx = L_mldsa_decode_gamma1_19_neon_idx; + const word32* shift = L_mldsa_decode_gamma1_19_neon_shift; + const word32* mask = L_mldsa_decode_gamma1_19_neon_mask; + const word32* konst = L_mldsa_decode_gamma1_19_neon_konst; + __asm__ __volatile__ ( + "ldr q1, [%[idx]]\n\t" + "ldr q2, [%[shift]]\n\t" + "ldr q3, [%[mask]]\n\t" + "ldr q4, [%[konst]]\n\t" + "mov x6, #63\n\t" + "\n" + "L_mldsa_decode_gamma1_19_neon_loop_%=:\n\t" + "ldr q0, [%x[p]]\n\t" + "tbl v5.16b, {v0.16b}, v1.16b\n\t" + "ushl v5.4s, v5.4s, v2.4s\n\t" + "and v5.16b, v5.16b, v3.16b\n\t" + "sub v5.4s, v4.4s, v5.4s\n\t" + "str q5, [%x[z]]\n\t" + "add %x[p], %x[p], #10\n\t" + "add %x[z], %x[z], #16\n\t" + "subs x6, x6, #1\n\t" + "b.ne L_mldsa_decode_gamma1_19_neon_loop_%=\n\t" + "sub %x[p], %x[p], #6\n\t" + "ldr q0, [%x[p]]\n\t" + "ldr q1, [%[idx], #16]\n\t" + "tbl v5.16b, {v0.16b}, v1.16b\n\t" + "ushl v5.4s, v5.4s, v2.4s\n\t" + "and v5.16b, v5.16b, v3.16b\n\t" + "sub v5.4s, v4.4s, v5.4s\n\t" + "str q5, [%x[z]]\n\t" + : [z] "+r" (z) + : [p] "r" (p), [idx] "r" (idx), [shift] "r" (shift), [mask] "r" (mask), + [konst] "r" (konst) + : "memory", "cc", "x6", "v0", "v1", "v2", "v3", "v4", "v5" + ); +} + +XALIGNED(4) static const word8 L_mldsa_encode_w1_88_neon_idx[] = { + 0x00, 0x01, 0x02, 0x04, 0x05, 0x06, 0x08, 0x09, + 0x0a, 0x0c, 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff, +}; + +void mldsa_encode_w1_88_neon(const sword32* w1, byte* w1e) +{ + const word8* idx = L_mldsa_encode_w1_88_neon_idx; + __asm__ __volatile__ ( + "ldr q8, [%[idx]]\n\t" + "mov x3, #16\n\t" + "\n" + "L_mldsa_encode_w1_88_neon_loop_%=:\n\t" + "ldr q0, [%x[w1]]\n\t" + "ldr q1, [%x[w1], #16]\n\t" + "ldr q2, [%x[w1], #32]\n\t" + "ldr q3, [%x[w1], #48]\n\t" + "uzp1 v4.4s, v0.4s, v1.4s\n\t" + "uzp2 v7.4s, v0.4s, v1.4s\n\t" + "sli v4.4s, v7.4s, #6\n\t" + "uzp1 v5.4s, v2.4s, v3.4s\n\t" + "uzp2 v7.4s, v2.4s, v3.4s\n\t" + "sli v5.4s, v7.4s, #6\n\t" + "uzp1 v6.4s, v4.4s, v5.4s\n\t" + "uzp2 v7.4s, v4.4s, v5.4s\n\t" + "sli v6.4s, v7.4s, #12\n\t" + "tbl v9.16b, {v6.16b}, v8.16b\n\t" + "str d9, [%x[w1e]]\n\t" + "ext v9.16b, v9.16b, v9.16b, #8\n\t" + "str s9, [%x[w1e], #8]\n\t" + "add %x[w1], %x[w1], #0x40\n\t" + "add %x[w1e], %x[w1e], #12\n\t" + "subs x3, x3, #1\n\t" + "b.ne L_mldsa_encode_w1_88_neon_loop_%=\n\t" + : [w1e] "+r" (w1e) + : [w1] "r" (w1), [idx] "r" (idx) + : "memory", "cc", "x3", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9" + ); +} + +XALIGNED(4) static const word8 L_mldsa_encode_w1_32_neon_idx[] = { + 0x00, 0x04, 0x08, 0x0c, 0x10, 0x14, 0x18, 0x1c, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +void mldsa_encode_w1_32_neon(const sword32* w1, byte* w1e) +{ + const word8* idx = L_mldsa_encode_w1_32_neon_idx; + __asm__ __volatile__ ( + "ldr q8, [%[idx]]\n\t" + "mov x3, #16\n\t" + "\n" + "L_mldsa_encode_w1_32_neon_loop_%=:\n\t" + "ldr q0, [%x[w1]]\n\t" + "ldr q1, [%x[w1], #16]\n\t" + "ldr q2, [%x[w1], #32]\n\t" + "ldr q3, [%x[w1], #48]\n\t" + "uzp1 v4.4s, v0.4s, v1.4s\n\t" + "uzp2 v7.4s, v0.4s, v1.4s\n\t" + "sli v4.4s, v7.4s, #4\n\t" + "uzp1 v5.4s, v2.4s, v3.4s\n\t" + "uzp2 v7.4s, v2.4s, v3.4s\n\t" + "sli v5.4s, v7.4s, #4\n\t" + "tbl v9.16b, {v4.16b, v5.16b}, v8.16b\n\t" + "str d9, [%x[w1e]]\n\t" + "add %x[w1], %x[w1], #0x40\n\t" + "add %x[w1e], %x[w1e], #8\n\t" + "subs x3, x3, #1\n\t" + "b.ne L_mldsa_encode_w1_32_neon_loop_%=\n\t" + : [w1e] "+r" (w1e) + : [w1] "r" (w1), [idx] "r" (idx) + : "memory", "cc", "x3", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9" + ); +} + +XALIGNED(4) static const word8 L_mldsa_encode_gamma1_17_neon_idx[] = { + 0x00, 0x01, 0x02, 0xff, 0x08, 0x09, 0x0a, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x04, 0x05, 0x06, 0xff, 0x0c, 0x0d, + 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_encode_gamma1_17_neon_shift[] = { + 0x00000000, 0x00000002, 0x00000004, 0x00000006, +}; + +XALIGNED(8) static const word32 L_mldsa_encode_gamma1_17_neon_gam[] = { + 0x00020000, 0x00020000, 0x00020000, 0x00020000, +}; + +void mldsa_encode_gamma1_17_neon(const sword32* z, byte* s) +{ + const word8* idx = L_mldsa_encode_gamma1_17_neon_idx; + const word32* shift = L_mldsa_encode_gamma1_17_neon_shift; + const word32* gam = L_mldsa_encode_gamma1_17_neon_gam; + __asm__ __volatile__ ( + "ldr q3, [%[idx]]\n\t" + "ldr q4, [%[idx], #16]\n\t" + "ldr q1, [%[shift]]\n\t" + "ldr q2, [%[gam]]\n\t" + "mov x5, #0x40\n\t" + "\n" + "L_mldsa_encode_gamma1_17_neon_loop_%=:\n\t" + "ldr q0, [%x[z]]\n\t" + "sub v0.4s, v2.4s, v0.4s\n\t" + "ushl v0.4s, v0.4s, v1.4s\n\t" + "tbl v5.16b, {v0.16b}, v3.16b\n\t" + "tbl v6.16b, {v0.16b}, v4.16b\n\t" + "orr v5.16b, v5.16b, v6.16b\n\t" + "str d5, [%x[s]]\n\t" + "ext v5.16b, v5.16b, v5.16b, #8\n\t" + "str b5, [%x[s], #8]\n\t" + "add %x[z], %x[z], #16\n\t" + "add %x[s], %x[s], #9\n\t" + "subs x5, x5, #1\n\t" + "b.ne L_mldsa_encode_gamma1_17_neon_loop_%=\n\t" + : [s] "+r" (s) + : [z] "r" (z), [idx] "r" (idx), [shift] "r" (shift), [gam] "r" (gam) + : "memory", "cc", "x5", "v0", "v1", "v2", "v3", "v4", "v5", "v6" + ); +} + +XALIGNED(4) static const word8 L_mldsa_encode_gamma1_19_neon_idx[] = { + 0x00, 0x01, 0x02, 0xff, 0xff, 0x08, 0x09, 0x0a, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x04, 0x05, 0x06, 0xff, 0xff, 0x0c, + 0x0d, 0x0e, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_encode_gamma1_19_neon_shift[] = { + 0x00000000, 0x00000004, 0x00000000, 0x00000004, +}; + +XALIGNED(8) static const word32 L_mldsa_encode_gamma1_19_neon_gam[] = { + 0x00080000, 0x00080000, 0x00080000, 0x00080000, +}; + +void mldsa_encode_gamma1_19_neon(const sword32* z, byte* s) +{ + const word8* idx = L_mldsa_encode_gamma1_19_neon_idx; + const word32* shift = L_mldsa_encode_gamma1_19_neon_shift; + const word32* gam = L_mldsa_encode_gamma1_19_neon_gam; + __asm__ __volatile__ ( + "ldr q3, [%[idx]]\n\t" + "ldr q4, [%[idx], #16]\n\t" + "ldr q1, [%[shift]]\n\t" + "ldr q2, [%[gam]]\n\t" + "mov x5, #0x40\n\t" + "\n" + "L_mldsa_encode_gamma1_19_neon_loop_%=:\n\t" + "ldr q0, [%x[z]]\n\t" + "sub v0.4s, v2.4s, v0.4s\n\t" + "ushl v0.4s, v0.4s, v1.4s\n\t" + "tbl v5.16b, {v0.16b}, v3.16b\n\t" + "tbl v6.16b, {v0.16b}, v4.16b\n\t" + "orr v5.16b, v5.16b, v6.16b\n\t" + "str d5, [%x[s]]\n\t" + "ext v5.16b, v5.16b, v5.16b, #8\n\t" + "str h5, [%x[s], #8]\n\t" + "add %x[z], %x[z], #16\n\t" + "add %x[s], %x[s], #10\n\t" + "subs x5, x5, #1\n\t" + "b.ne L_mldsa_encode_gamma1_19_neon_loop_%=\n\t" + : [s] "+r" (s) + : [z] "r" (z), [idx] "r" (idx), [shift] "r" (shift), [gam] "r" (gam) + : "memory", "cc", "x5", "v0", "v1", "v2", "v3", "v4", "v5", "v6" + ); +} + +int mldsa_vec_check_low_neon(const sword32* a, byte l, sword32 hi) +{ + __asm__ __volatile__ ( + "dup v0.4s, %w[hi]\n\t" + "eor v1.16b, v1.16b, v1.16b\n\t" + "eor v2.16b, v2.16b, v2.16b\n\t" + "eor v3.16b, v3.16b, v3.16b\n\t" + "eor v4.16b, v4.16b, v4.16b\n\t" + "\n" + "L_mldsa_vec_check_low_outer_%=:\n\t" + "mov x3, #16\n\t" + "\n" + "L_mldsa_vec_check_low_inner_%=:\n\t" + "ldr q5, [%x[a]]\n\t" + "ldr q6, [%x[a], #16]\n\t" + "ldr q7, [%x[a], #32]\n\t" + "ldr q8, [%x[a], #48]\n\t" + "sqabs v9.4s, v5.4s\n\t" + "cmge v9.4s, v9.4s, v0.4s\n\t" + "orr v1.16b, v1.16b, v9.16b\n\t" + "sqabs v10.4s, v6.4s\n\t" + "cmge v10.4s, v10.4s, v0.4s\n\t" + "orr v2.16b, v2.16b, v10.16b\n\t" + "sqabs v11.4s, v7.4s\n\t" + "cmge v11.4s, v11.4s, v0.4s\n\t" + "orr v3.16b, v3.16b, v11.16b\n\t" + "sqabs v12.4s, v8.4s\n\t" + "cmge v12.4s, v12.4s, v0.4s\n\t" + "orr v4.16b, v4.16b, v12.16b\n\t" + "add %x[a], %x[a], #0x40\n\t" + "subs x3, x3, #1\n\t" + "b.ne L_mldsa_vec_check_low_inner_%=\n\t" + "subs %w[l], %w[l], #1\n\t" + "b.ne L_mldsa_vec_check_low_outer_%=\n\t" + "orr v1.16b, v1.16b, v2.16b\n\t" + "orr v3.16b, v3.16b, v4.16b\n\t" + "orr v1.16b, v1.16b, v3.16b\n\t" + "umaxv s1, v1.4s\n\t" + "mov w4, v1.s[0]\n\t" + "cmp w4, #0\n\t" + "cset w4, eq\n\t" + "mov %x[a], x4\n\t" + : [a] "+r" (a), [l] "+r" (l), [hi] "+r" (hi) + : + : "memory", "cc", "x3", "x4", "v0", "v1", "v2", "v3", "v4", "v5", "v6", + "v7", "v8", "v9", "v10", "v11", "v12" + ); + return (word32)(size_t)a; +} + +void mldsa_poly_add_neon(sword32* r, const sword32* a) +{ + __asm__ __volatile__ ( + "mov x2, #16\n\t" + "\n" + "L_mldsa_poly_add_neon_loop_%=:\n\t" + "ldr q0, [%x[r]]\n\t" + "ldr q4, [%x[a]]\n\t" + "ldr q1, [%x[r], #16]\n\t" + "ldr q5, [%x[a], #16]\n\t" + "ldr q2, [%x[r], #32]\n\t" + "ldr q6, [%x[a], #32]\n\t" + "ldr q3, [%x[r], #48]\n\t" + "ldr q7, [%x[a], #48]\n\t" + "add v0.4s, v0.4s, v4.4s\n\t" + "add v1.4s, v1.4s, v5.4s\n\t" + "add v2.4s, v2.4s, v6.4s\n\t" + "add v3.4s, v3.4s, v7.4s\n\t" + "str q0, [%x[r]]\n\t" + "str q1, [%x[r], #16]\n\t" + "str q2, [%x[r], #32]\n\t" + "str q3, [%x[r], #48]\n\t" + "add %x[r], %x[r], #0x40\n\t" + "add %x[a], %x[a], #0x40\n\t" + "subs x2, x2, #1\n\t" + "b.ne L_mldsa_poly_add_neon_loop_%=\n\t" + : [r] "+r" (r) + : [a] "r" (a) + : "memory", "cc", "x2", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + ); +} + +void mldsa_poly_sub_neon(sword32* r, const sword32* a) +{ + __asm__ __volatile__ ( + "mov x2, #16\n\t" + "\n" + "L_mldsa_poly_sub_neon_loop_%=:\n\t" + "ldr q0, [%x[r]]\n\t" + "ldr q4, [%x[a]]\n\t" + "ldr q1, [%x[r], #16]\n\t" + "ldr q5, [%x[a], #16]\n\t" + "ldr q2, [%x[r], #32]\n\t" + "ldr q6, [%x[a], #32]\n\t" + "ldr q3, [%x[r], #48]\n\t" + "ldr q7, [%x[a], #48]\n\t" + "sub v0.4s, v0.4s, v4.4s\n\t" + "sub v1.4s, v1.4s, v5.4s\n\t" + "sub v2.4s, v2.4s, v6.4s\n\t" + "sub v3.4s, v3.4s, v7.4s\n\t" + "str q0, [%x[r]]\n\t" + "str q1, [%x[r], #16]\n\t" + "str q2, [%x[r], #32]\n\t" + "str q3, [%x[r], #48]\n\t" + "add %x[r], %x[r], #0x40\n\t" + "add %x[a], %x[a], #0x40\n\t" + "subs x2, x2, #1\n\t" + "b.ne L_mldsa_poly_sub_neon_loop_%=\n\t" + : [r] "+r" (r) + : [a] "r" (a) + : "memory", "cc", "x2", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7" + ); +} + +XALIGNED(8) static const word32 L_mldsa_poly_make_pos_neon_q[] = { + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, +}; + +void mldsa_poly_make_pos_neon(sword32* a) +{ + const word32* q = L_mldsa_poly_make_pos_neon_q; + __asm__ __volatile__ ( + "ldr q0, [%[q]]\n\t" + "mov x2, #16\n\t" + "\n" + "L_mldsa_poly_make_pos_neon_loop_%=:\n\t" + "ldr q1, [%x[a]]\n\t" + "ldr q2, [%x[a], #16]\n\t" + "ldr q3, [%x[a], #32]\n\t" + "ldr q4, [%x[a], #48]\n\t" + "sshr v5.4s, v1.4s, #31\n\t" + "mls v1.4s, v5.4s, v0.4s\n\t" + "sshr v6.4s, v2.4s, #31\n\t" + "mls v2.4s, v6.4s, v0.4s\n\t" + "sshr v7.4s, v3.4s, #31\n\t" + "mls v3.4s, v7.4s, v0.4s\n\t" + "sshr v8.4s, v4.4s, #31\n\t" + "mls v4.4s, v8.4s, v0.4s\n\t" + "str q1, [%x[a]]\n\t" + "str q2, [%x[a], #16]\n\t" + "str q3, [%x[a], #32]\n\t" + "str q4, [%x[a], #48]\n\t" + "add %x[a], %x[a], #0x40\n\t" + "subs x2, x2, #1\n\t" + "b.ne L_mldsa_poly_make_pos_neon_loop_%=\n\t" + : [a] "+r" (a) + : [q] "r" (q) + : "memory", "cc", "x2", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8" + ); +} + +XALIGNED(8) static const word32 L_mldsa_poly_red_neon_q[] = { + 0x007fe001, 0x007fe001, 0x007fe001, 0x007fe001, + 0x00400000, 0x00400000, 0x00400000, 0x00400000, +}; + +void mldsa_poly_red_neon(sword32* a) +{ + const word32* q = L_mldsa_poly_red_neon_q; + __asm__ __volatile__ ( + "ldr q0, [%[q]]\n\t" + "ldr q1, [%[q], #16]\n\t" + "mov x2, #16\n\t" + "\n" + "L_mldsa_poly_red_neon_loop_%=:\n\t" + "ldr q2, [%x[a]]\n\t" + "ldr q3, [%x[a], #16]\n\t" + "ldr q4, [%x[a], #32]\n\t" + "ldr q5, [%x[a], #48]\n\t" + "add v6.4s, v2.4s, v1.4s\n\t" + "sshr v6.4s, v6.4s, #23\n\t" + "mls v2.4s, v6.4s, v0.4s\n\t" + "add v7.4s, v3.4s, v1.4s\n\t" + "sshr v7.4s, v7.4s, #23\n\t" + "mls v3.4s, v7.4s, v0.4s\n\t" + "add v8.4s, v4.4s, v1.4s\n\t" + "sshr v8.4s, v8.4s, #23\n\t" + "mls v4.4s, v8.4s, v0.4s\n\t" + "add v9.4s, v5.4s, v1.4s\n\t" + "sshr v9.4s, v9.4s, #23\n\t" + "mls v5.4s, v9.4s, v0.4s\n\t" + "str q2, [%x[a]]\n\t" + "str q3, [%x[a], #16]\n\t" + "str q4, [%x[a], #32]\n\t" + "str q5, [%x[a], #48]\n\t" + "add %x[a], %x[a], #0x40\n\t" + "subs x2, x2, #1\n\t" + "b.ne L_mldsa_poly_red_neon_loop_%=\n\t" + : [a] "+r" (a) + : [q] "r" (q) + : "memory", "cc", "x2", "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", + "v8", "v9" + ); +} + +XALIGNED(4) static const word8 L_mldsa_extract_coeffs_eta2_neon_map[] = { + 0x02, 0x01, 0x00, 0xff, 0xfe, 0x02, 0x01, 0x00, + 0xff, 0xfe, 0x02, 0x01, 0x00, 0xff, 0xfe, 0x00, +}; + +XALIGNED(4) static const word8 L_mldsa_extract_coeffs_eta2_neon_lim[] = { + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, +}; + +XALIGNED(8) static const word32 L_mldsa_extract_coeffs_eta2_neon_bits[] = { + 0x00000001, 0x00000002, 0x00000004, 0x00000008, +}; + +XALIGNED(4) static const word8 L_mldsa_extract_coeffs_eta2_neon_idx[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, +}; + +void mldsa_extract_coeffs_eta2_neon(const byte* z, unsigned int zLen, + sword32* s, unsigned int* cnt) +{ + const word8* map = L_mldsa_extract_coeffs_eta2_neon_map; + const word8* lim = L_mldsa_extract_coeffs_eta2_neon_lim; + const word32* bits = L_mldsa_extract_coeffs_eta2_neon_bits; + const word8* idx = L_mldsa_extract_coeffs_eta2_neon_idx; + __asm__ __volatile__ ( + "ldr q2, [%[map]]\n\t" + "ldr q0, [%[lim]]\n\t" + "ldr q1, [%[bits]]\n\t" + "ldr w8, [%x[cnt]]\n\t" + "add %x[s], %x[s], x8, lsl 2\n\t" + "\n" + "L_mldsa_extract_coeffs_eta2_neon_wide_%=:\n\t" + "subs wzr, %w[zLen], #8\n\t" + "b.lt L_mldsa_extract_coeffs_eta2_neon_tail_%=\n\t" + "subs wzr, w8, #0xf0\n\t" + "b.gt L_mldsa_extract_coeffs_eta2_neon_tail_%=\n\t" + "ldr d3, [%x[z]]\n\t" + "movi v18.16b, #15\n\t" + "ushr v5.16b, v3.16b, #4\n\t" + "and v4.16b, v3.16b, v18.16b\n\t" + "zip1 v3.16b, v4.16b, v5.16b\n\t" + "tbl v6.16b, {v2.16b}, v3.16b\n\t" + "cmhi v8.16b, v0.16b, v3.16b\n\t" + "sxtl v7.8h, v6.8b\n\t" + "sxtl2 v18.8h, v6.16b\n\t" + "sxtl v10.4s, v7.4h\n\t" + "sxtl2 v11.4s, v7.8h\n\t" + "sxtl v12.4s, v18.4h\n\t" + "sxtl2 v13.4s, v18.8h\n\t" + "sxtl v9.8h, v8.8b\n\t" + "sxtl2 v18.8h, v8.16b\n\t" + "sxtl v14.4s, v9.4h\n\t" + "sxtl2 v15.4s, v9.8h\n\t" + "sxtl v16.4s, v18.4h\n\t" + "sxtl2 v17.4s, v18.8h\n\t" + "ushr v18.4s, v14.4s, #31\n\t" + "addv s18, v18.4s\n\t" + "mov w10, v18.s[0]\n\t" + "and v18.16b, v14.16b, v1.16b\n\t" + "addv s18, v18.4s\n\t" + "mov w9, v18.s[0]\n\t" + "lsl x9, x9, #4\n\t" + "ldr q18, [%[idx], x9]\n\t" + "tbl v10.16b, {v10.16b}, v18.16b\n\t" + "str q10, [%x[s]]\n\t" + "add %x[s], %x[s], x10, lsl 2\n\t" + "add x8, x8, x10\n\t" + "ushr v18.4s, v15.4s, #31\n\t" + "addv s18, v18.4s\n\t" + "mov w10, v18.s[0]\n\t" + "and v18.16b, v15.16b, v1.16b\n\t" + "addv s18, v18.4s\n\t" + "mov w9, v18.s[0]\n\t" + "lsl x9, x9, #4\n\t" + "ldr q18, [%[idx], x9]\n\t" + "tbl v11.16b, {v11.16b}, v18.16b\n\t" + "str q11, [%x[s]]\n\t" + "add %x[s], %x[s], x10, lsl 2\n\t" + "add x8, x8, x10\n\t" + "ushr v18.4s, v16.4s, #31\n\t" + "addv s18, v18.4s\n\t" + "mov w10, v18.s[0]\n\t" + "and v18.16b, v16.16b, v1.16b\n\t" + "addv s18, v18.4s\n\t" + "mov w9, v18.s[0]\n\t" + "lsl x9, x9, #4\n\t" + "ldr q18, [%[idx], x9]\n\t" + "tbl v12.16b, {v12.16b}, v18.16b\n\t" + "str q12, [%x[s]]\n\t" + "add %x[s], %x[s], x10, lsl 2\n\t" + "add x8, x8, x10\n\t" + "ushr v18.4s, v17.4s, #31\n\t" + "addv s18, v18.4s\n\t" + "mov w10, v18.s[0]\n\t" + "and v18.16b, v17.16b, v1.16b\n\t" + "addv s18, v18.4s\n\t" + "mov w9, v18.s[0]\n\t" + "lsl x9, x9, #4\n\t" + "ldr q18, [%[idx], x9]\n\t" + "tbl v13.16b, {v13.16b}, v18.16b\n\t" + "str q13, [%x[s]]\n\t" + "add %x[s], %x[s], x10, lsl 2\n\t" + "add x8, x8, x10\n\t" + "add %x[z], %x[z], #8\n\t" + "sub %w[zLen], %w[zLen], #8\n\t" + "b L_mldsa_extract_coeffs_eta2_neon_wide_%=\n\t" + "\n" + "L_mldsa_extract_coeffs_eta2_neon_tail_%=:\n\t" + "subs wzr, %w[zLen], #0\n\t" + "b.eq L_mldsa_extract_coeffs_eta2_neon_done_%=\n\t" + "subs wzr, w8, #0x100\n\t" + "b.eq L_mldsa_extract_coeffs_eta2_neon_done_%=\n\t" + "ldrb w11, [%x[z]]\n\t" + "add %x[z], %x[z], #1\n\t" + "sub %w[zLen], %w[zLen], #1\n\t" + "and w12, w11, #15\n\t" + "subs wzr, w12, #15\n\t" + "bcs L_mldsa_extract_coeffs_eta2_neon_skip_lo_%=\n\t" + "ldrsb w12, [%[map], x12]\n\t" + "str w12, [%x[s]]\n\t" + "add %x[s], %x[s], #4\n\t" + "add x8, x8, #1\n\t" + "subs wzr, w8, #0x100\n\t" + "b.eq L_mldsa_extract_coeffs_eta2_neon_done_%=\n\t" + "\n" + "L_mldsa_extract_coeffs_eta2_neon_skip_lo_%=:\n\t" + "lsr w12, w11, #4\n\t" + "subs wzr, w12, #15\n\t" + "bcs L_mldsa_extract_coeffs_eta2_neon_skip_hi_%=\n\t" + "ldrsb w12, [%[map], x12]\n\t" + "str w12, [%x[s]]\n\t" + "add %x[s], %x[s], #4\n\t" + "add x8, x8, #1\n\t" + "\n" + "L_mldsa_extract_coeffs_eta2_neon_skip_hi_%=:\n\t" + "b L_mldsa_extract_coeffs_eta2_neon_tail_%=\n\t" + "\n" + "L_mldsa_extract_coeffs_eta2_neon_done_%=:\n\t" + "str w8, [%x[cnt]]\n\t" + : [zLen] "+r" (zLen), [s] "+r" (s), [cnt] "+r" (cnt) + : [z] "r" (z), [map] "r" (map), [lim] "r" (lim), [bits] "r" (bits), + [idx] "r" (idx) + : "memory", "cc", "x8", "x9", "x10", "x11", "x12", "v0", "v1", "v2", + "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18" + ); +} + +XALIGNED(4) static const word8 L_mldsa_extract_coeffs_eta4_neon_map[] = { + 0x04, 0x03, 0x02, 0x01, 0x00, 0xff, 0xfe, 0xfd, + 0xfc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +XALIGNED(4) static const word8 L_mldsa_extract_coeffs_eta4_neon_lim[] = { + 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, + 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, 0x09, +}; + +XALIGNED(8) static const word32 L_mldsa_extract_coeffs_eta4_neon_bits[] = { + 0x00000001, 0x00000002, 0x00000004, 0x00000008, +}; + +XALIGNED(4) static const word8 L_mldsa_extract_coeffs_eta4_neon_idx[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, +}; + +void mldsa_extract_coeffs_eta4_neon(const byte* z, unsigned int zLen, + sword32* s, unsigned int* cnt) +{ + const word8* map = L_mldsa_extract_coeffs_eta4_neon_map; + const word8* lim = L_mldsa_extract_coeffs_eta4_neon_lim; + const word32* bits = L_mldsa_extract_coeffs_eta4_neon_bits; + const word8* idx = L_mldsa_extract_coeffs_eta4_neon_idx; + __asm__ __volatile__ ( + "ldr q2, [%[map]]\n\t" + "ldr q0, [%[lim]]\n\t" + "ldr q1, [%[bits]]\n\t" + "ldr w8, [%x[cnt]]\n\t" + "add %x[s], %x[s], x8, lsl 2\n\t" + "\n" + "L_mldsa_extract_coeffs_eta4_neon_wide_%=:\n\t" + "subs wzr, %w[zLen], #8\n\t" + "b.lt L_mldsa_extract_coeffs_eta4_neon_tail_%=\n\t" + "subs wzr, w8, #0xf0\n\t" + "b.gt L_mldsa_extract_coeffs_eta4_neon_tail_%=\n\t" + "ldr d3, [%x[z]]\n\t" + "movi v18.16b, #15\n\t" + "ushr v5.16b, v3.16b, #4\n\t" + "and v4.16b, v3.16b, v18.16b\n\t" + "zip1 v3.16b, v4.16b, v5.16b\n\t" + "tbl v6.16b, {v2.16b}, v3.16b\n\t" + "cmhi v8.16b, v0.16b, v3.16b\n\t" + "sxtl v7.8h, v6.8b\n\t" + "sxtl2 v18.8h, v6.16b\n\t" + "sxtl v10.4s, v7.4h\n\t" + "sxtl2 v11.4s, v7.8h\n\t" + "sxtl v12.4s, v18.4h\n\t" + "sxtl2 v13.4s, v18.8h\n\t" + "sxtl v9.8h, v8.8b\n\t" + "sxtl2 v18.8h, v8.16b\n\t" + "sxtl v14.4s, v9.4h\n\t" + "sxtl2 v15.4s, v9.8h\n\t" + "sxtl v16.4s, v18.4h\n\t" + "sxtl2 v17.4s, v18.8h\n\t" + "ushr v18.4s, v14.4s, #31\n\t" + "addv s18, v18.4s\n\t" + "mov w10, v18.s[0]\n\t" + "and v18.16b, v14.16b, v1.16b\n\t" + "addv s18, v18.4s\n\t" + "mov w9, v18.s[0]\n\t" + "lsl x9, x9, #4\n\t" + "ldr q18, [%[idx], x9]\n\t" + "tbl v10.16b, {v10.16b}, v18.16b\n\t" + "str q10, [%x[s]]\n\t" + "add %x[s], %x[s], x10, lsl 2\n\t" + "add x8, x8, x10\n\t" + "ushr v18.4s, v15.4s, #31\n\t" + "addv s18, v18.4s\n\t" + "mov w10, v18.s[0]\n\t" + "and v18.16b, v15.16b, v1.16b\n\t" + "addv s18, v18.4s\n\t" + "mov w9, v18.s[0]\n\t" + "lsl x9, x9, #4\n\t" + "ldr q18, [%[idx], x9]\n\t" + "tbl v11.16b, {v11.16b}, v18.16b\n\t" + "str q11, [%x[s]]\n\t" + "add %x[s], %x[s], x10, lsl 2\n\t" + "add x8, x8, x10\n\t" + "ushr v18.4s, v16.4s, #31\n\t" + "addv s18, v18.4s\n\t" + "mov w10, v18.s[0]\n\t" + "and v18.16b, v16.16b, v1.16b\n\t" + "addv s18, v18.4s\n\t" + "mov w9, v18.s[0]\n\t" + "lsl x9, x9, #4\n\t" + "ldr q18, [%[idx], x9]\n\t" + "tbl v12.16b, {v12.16b}, v18.16b\n\t" + "str q12, [%x[s]]\n\t" + "add %x[s], %x[s], x10, lsl 2\n\t" + "add x8, x8, x10\n\t" + "ushr v18.4s, v17.4s, #31\n\t" + "addv s18, v18.4s\n\t" + "mov w10, v18.s[0]\n\t" + "and v18.16b, v17.16b, v1.16b\n\t" + "addv s18, v18.4s\n\t" + "mov w9, v18.s[0]\n\t" + "lsl x9, x9, #4\n\t" + "ldr q18, [%[idx], x9]\n\t" + "tbl v13.16b, {v13.16b}, v18.16b\n\t" + "str q13, [%x[s]]\n\t" + "add %x[s], %x[s], x10, lsl 2\n\t" + "add x8, x8, x10\n\t" + "add %x[z], %x[z], #8\n\t" + "sub %w[zLen], %w[zLen], #8\n\t" + "b L_mldsa_extract_coeffs_eta4_neon_wide_%=\n\t" + "\n" + "L_mldsa_extract_coeffs_eta4_neon_tail_%=:\n\t" + "subs wzr, %w[zLen], #0\n\t" + "b.eq L_mldsa_extract_coeffs_eta4_neon_done_%=\n\t" + "subs wzr, w8, #0x100\n\t" + "b.eq L_mldsa_extract_coeffs_eta4_neon_done_%=\n\t" + "ldrb w11, [%x[z]]\n\t" + "add %x[z], %x[z], #1\n\t" + "sub %w[zLen], %w[zLen], #1\n\t" + "and w12, w11, #15\n\t" + "subs wzr, w12, #9\n\t" + "bcs L_mldsa_extract_coeffs_eta4_neon_skip_lo_%=\n\t" + "ldrsb w12, [%[map], x12]\n\t" + "str w12, [%x[s]]\n\t" + "add %x[s], %x[s], #4\n\t" + "add x8, x8, #1\n\t" + "subs wzr, w8, #0x100\n\t" + "b.eq L_mldsa_extract_coeffs_eta4_neon_done_%=\n\t" + "\n" + "L_mldsa_extract_coeffs_eta4_neon_skip_lo_%=:\n\t" + "lsr w12, w11, #4\n\t" + "subs wzr, w12, #9\n\t" + "bcs L_mldsa_extract_coeffs_eta4_neon_skip_hi_%=\n\t" + "ldrsb w12, [%[map], x12]\n\t" + "str w12, [%x[s]]\n\t" + "add %x[s], %x[s], #4\n\t" + "add x8, x8, #1\n\t" + "\n" + "L_mldsa_extract_coeffs_eta4_neon_skip_hi_%=:\n\t" + "b L_mldsa_extract_coeffs_eta4_neon_tail_%=\n\t" + "\n" + "L_mldsa_extract_coeffs_eta4_neon_done_%=:\n\t" + "str w8, [%x[cnt]]\n\t" + : [zLen] "+r" (zLen), [s] "+r" (s), [cnt] "+r" (cnt) + : [z] "r" (z), [map] "r" (map), [lim] "r" (lim), [bits] "r" (bits), + [idx] "r" (idx) + : "memory", "cc", "x8", "x9", "x10", "x11", "x12", "v0", "v1", "v2", + "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18" + ); +} + +XALIGNED(8) static const word32 L_mldsa_rej_uniform_mask[] = { + 0x007fffff, 0x007fffff, 0x007fffff, 0x007fffff, +}; + +XALIGNED(4) static const word8 L_mldsa_rej_uniform_shuffle0[] = { + 0x00, 0x01, 0x02, 0xff, 0x03, 0x04, 0x05, 0xff, + 0x06, 0x07, 0x08, 0xff, 0x09, 0x0a, 0x0b, 0xff, +}; + +XALIGNED(4) static const word8 L_mldsa_rej_uniform_shuffle1[] = { + 0x0c, 0x0d, 0x0e, 0xff, 0x0f, 0x10, 0x11, 0xff, + 0x12, 0x13, 0x14, 0xff, 0x15, 0x16, 0x17, 0xff, +}; + +XALIGNED(8) static const word32 L_mldsa_rej_uniform_bits[] = { + 0x00000001, 0x00000002, 0x00000004, 0x00000008, +}; + +XALIGNED(4) static const word8 L_mldsa_rej_uniform_indices[] = { + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0xff, 0xff, 0xff, 0xff, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, + 0x0c, 0x0d, 0x0e, 0x0f, 0xff, 0xff, 0xff, 0xff, + 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, + 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, +}; + +unsigned int mldsa_rej_uniform_neon(sword32* p, unsigned int len, const byte* r, + unsigned int rLen) +{ + const word32* mask = L_mldsa_rej_uniform_mask; + const word32* q = L_mldsa_aarch64_q; + const word32* bits = L_mldsa_rej_uniform_bits; + const word8* indices = L_mldsa_rej_uniform_indices; + const word8* shuffle0 = L_mldsa_rej_uniform_shuffle0; + const word8* shuffle1 = L_mldsa_rej_uniform_shuffle1; + __asm__ __volatile__ ( + "movz x15, #0xe001\n\t" + "eor v1.16b, v1.16b, v1.16b\n\t" + "eor v12.16b, v12.16b, v12.16b\n\t" + "eor v13.16b, v13.16b, v13.16b\n\t" + "movk x15, #0x7f, lsl 16\n\t" + "eor x14, x14, x14\n\t" + "eor v10.16b, v10.16b, v10.16b\n\t" + "eor v11.16b, v11.16b, v11.16b\n\t" + "ldr q0, [%[mask]]\n\t" + "ldr q3, [%[q]]\n\t" + "ldr q2, [%[bits]]\n\t" + "ldr q4, [%[shuffle0]]\n\t" + "ldr q5, [%[shuffle1]]\n\t" + "subs wzr, %w[len], #0\n\t" + "b.eq L_mldsa_rej_uniform_done_%=\n\t" + "subs wzr, %w[len], #16\n\t" + "b.lt L_mldsa_rej_uniform_loop_2_%=\n\t" + "\n" + "L_mldsa_rej_uniform_loop_8_%=:\n\t" + "ldr q6, [%x[r]]\n\t" + "ldr d7, [%x[r], #16]\n\t" + "add %x[r], %x[r], #24\n\t" + "tbl v8.16b, {v6.16b, v7.16b}, v4.16b\n\t" + "tbl v9.16b, {v6.16b, v7.16b}, v5.16b\n\t" + "and v6.16b, v8.16b, v0.16b\n\t" + "and v7.16b, v9.16b, v0.16b\n\t" + "cmgt v8.4s, v3.4s, v6.4s\n\t" + "cmgt v9.4s, v3.4s, v7.4s\n\t" + "ushr v12.4s, v8.4s, #31\n\t" + "ushr v13.4s, v9.4s, #31\n\t" + "addv s12, v12.4s\n\t" + "addv s13, v13.4s\n\t" + "mov x12, v12.d[0]\n\t" + "mov x13, v13.d[0]\n\t" + "and v10.16b, v8.16b, v2.16b\n\t" + "and v11.16b, v9.16b, v2.16b\n\t" + "addv s10, v10.4s\n\t" + "addv s11, v11.4s\n\t" + "mov w10, v10.s[0]\n\t" + "mov w11, v11.s[0]\n\t" + "lsl w10, w10, #4\n\t" + "lsl w11, w11, #4\n\t" + "ldr q10, [%[indices], x10]\n\t" + "ldr q11, [%[indices], x11]\n\t" + "tbl v8.16b, {v6.16b}, v10.16b\n\t" + "tbl v9.16b, {v7.16b}, v11.16b\n\t" + "str q8, [%x[p]]\n\t" + "add %x[p], %x[p], x12, lsl 2\n\t" + "add x14, x14, x12\n\t" + "str q9, [%x[p]]\n\t" + "add %x[p], %x[p], x13, lsl 2\n\t" + "add x14, x14, x13\n\t" + "subs %w[rLen], %w[rLen], #24\n\t" + "b.eq L_mldsa_rej_uniform_done_%=\n\t" + "sub w12, %w[len], w14\n\t" + "subs x12, x12, #8\n\t" + "b.lt L_mldsa_rej_uniform_loop_2_%=\n\t" + "b L_mldsa_rej_uniform_loop_8_%=\n\t" + "\n" + "L_mldsa_rej_uniform_loop_2_%=:\n\t" + "subs w12, %w[len], w14\n\t" + "b.eq L_mldsa_rej_uniform_done_%=\n\t" + "subs x12, x12, #2\n\t" + "b.lt L_mldsa_rej_uniform_loop_lt_2_%=\n\t" + "ldr %[mask], [%x[r]], #6\n\t" + "lsr %[q], %[mask], #24\n\t" + "and %[mask], %[mask], #0x7fffff\n\t" + "and %[q], %[q], #0x7fffff\n\t" + "str %w[mask], [%x[p]]\n\t" + "add %[bits], %x[p], #4\n\t" + "subs xzr, %[mask], x15\n\t" + "csel %x[p], %[bits], %x[p], lt\n\t" + "cinc x14, x14, lt\n\t" + "str %w[q], [%x[p]]\n\t" + "add %[bits], %x[p], #4\n\t" + "subs xzr, %[q], x15\n\t" + "csel %x[p], %[bits], %x[p], lt\n\t" + "cinc x14, x14, lt\n\t" + "subs %w[rLen], %w[rLen], #6\n\t" + "b.eq L_mldsa_rej_uniform_done_%=\n\t" + "b L_mldsa_rej_uniform_loop_2_%=\n\t" + "\n" + "L_mldsa_rej_uniform_loop_lt_2_%=:\n\t" + "ldr %[mask], [%x[r]], #6\n\t" + "lsr %[q], %[mask], #24\n\t" + "and %[mask], %[mask], #0x7fffff\n\t" + "and %[q], %[q], #0x7fffff\n\t" + "str %w[mask], [%x[p]]\n\t" + "add %[bits], %x[p], #4\n\t" + "subs xzr, %[mask], x15\n\t" + "csel %x[p], %[bits], %x[p], lt\n\t" + "cinc x14, x14, lt\n\t" + "subs wzr, %w[len], w14\n\t" + "b.eq L_mldsa_rej_uniform_done_%=\n\t" + "str %w[q], [%x[p]]\n\t" + "add %[bits], %x[p], #4\n\t" + "subs xzr, %[q], x15\n\t" + "csel %x[p], %[bits], %x[p], lt\n\t" + "cinc x14, x14, lt\n\t" + "subs wzr, %w[len], w14\n\t" + "b.eq L_mldsa_rej_uniform_done_%=\n\t" + "subs %w[rLen], %w[rLen], #6\n\t" + "b.eq L_mldsa_rej_uniform_done_%=\n\t" + "b L_mldsa_rej_uniform_loop_lt_2_%=\n\t" + "\n" + "L_mldsa_rej_uniform_done_%=:\n\t" + "mov %x[p], x14\n\t" + : [p] "+r" (p), [len] "+r" (len), [rLen] "+r" (rLen) + : [r] "r" (r), [mask] "r" (mask), [q] "r" (q), [bits] "r" (bits), + [indices] "r" (indices), [shuffle0] "r" (shuffle0), + [shuffle1] "r" (shuffle1) + : "memory", "cc", "x10", "x11", "x12", "x13", "x14", "x15", "v0", "v1", + "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13" + ); + return (word32)(size_t)p; +} + +#endif /* WOLFSSL_HAVE_MLDSA */ +#endif /* __aarch64__ */ +#endif /* WOLFSSL_ARMASM */ +#endif /* WOLFSSL_ARMASM_INLINE */ diff --git a/wolfcrypt/src/port/arm/armv8-mlkem-asm.S b/wolfcrypt/src/port/arm/armv8-mlkem-asm.S index 14010de9035..ceaab6d4ed8 100644 --- a/wolfcrypt/src/port/arm/armv8-mlkem-asm.S +++ b/wolfcrypt/src/port/arm/armv8-mlkem-asm.S @@ -9737,2340 +9737,6 @@ L_mlkem_rej_uniform_done: #ifndef __APPLE__ .size mlkem_rej_uniform_neon,.-mlkem_rej_uniform_neon #endif /* __APPLE__ */ -#ifndef __APPLE__ - .text - .section .rodata - .type L_sha3_aarch64_r, %object - .size L_sha3_aarch64_r, 192 -#else - .section __DATA,__data -#endif /* __APPLE__ */ - # 16-byte aligned, 128-bit aligned -#ifndef __APPLE__ - .align 4 -#else - .p2align 4 -#endif /* __APPLE__ */ -L_sha3_aarch64_r: - .quad 0x0000000000000001,0x0000000000008082 - .quad 0x800000000000808a,0x8000000080008000 - .quad 0x000000000000808b,0x0000000080000001 - .quad 0x8000000080008081,0x8000000000008009 - .quad 0x000000000000008a,0x0000000000000088 - .quad 0x0000000080008009,0x000000008000000a - .quad 0x000000008000808b,0x800000000000008b - .quad 0x8000000000008089,0x8000000000008003 - .quad 0x8000000000008002,0x8000000000000080 - .quad 0x000000000000800a,0x800000008000000a - .quad 0x8000000080008081,0x8000000000008080 - .quad 0x0000000080000001,0x8000000080008008 -#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 -#ifndef __APPLE__ -.text -.globl mlkem_sha3_blocksx3_crypto -WC_ASM_ATT_HIDDEN(mlkem_sha3_blocksx3_crypto) -.type mlkem_sha3_blocksx3_crypto,@function -.align 2 -mlkem_sha3_blocksx3_crypto: -#else -.section __TEXT,__text -.globl _mlkem_sha3_blocksx3_crypto -WC_ASM_ATT_HIDDEN(_mlkem_sha3_blocksx3_crypto) -.p2align 2 -_mlkem_sha3_blocksx3_crypto: -#endif /* __APPLE__ */ - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] -#ifndef __APPLE__ - adrp x27, L_sha3_aarch64_r - add x27, x27, :lo12:L_sha3_aarch64_r -#else - adrp x27, L_sha3_aarch64_r@PAGE - add x27, x27, L_sha3_aarch64_r@PAGEOFF -#endif /* __APPLE__ */ - str x0, [x29, #40] - ld4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - ld4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - ld4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - ld4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - ld4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - ld4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - ld1 {v24.d}[0], [x0] - add x0, x0, #8 - ld4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - ld4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - ld4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - ld4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - ld4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - ld4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - ld1 {v24.d}[1], [x0] - add x0, x0, #8 - ldp x1, x2, [x0] - ldp x3, x4, [x0, #16] - ldp x5, x6, [x0, #32] - ldp x7, x8, [x0, #48] - ldp x9, x10, [x0, #64] - ldp x11, x12, [x0, #80] - ldp x13, x14, [x0, #96] - ldp x15, x16, [x0, #112] - ldp x17, x19, [x0, #128] - ldp x20, x21, [x0, #144] - ldp x22, x23, [x0, #160] - ldp x24, x25, [x0, #176] - ldr x26, [x0, #192] - mov x28, #24 - # Start of 24 rounds -L_SHA3_transform_blocksx3_crypto_begin: - stp x27, x28, [x29, #48] - # Col Mix - eor3 v31.16b, v0.16b, v5.16b, v10.16b - eor x0, x5, x10 - eor3 v27.16b, v1.16b, v6.16b, v11.16b - eor x30, x1, x6 - eor3 v28.16b, v2.16b, v7.16b, v12.16b - eor x28, x3, x8 - eor3 v29.16b, v3.16b, v8.16b, v13.16b - eor x0, x0, x15 - eor3 v30.16b, v4.16b, v9.16b, v14.16b - eor x30, x30, x11 - eor3 v31.16b, v31.16b, v15.16b, v20.16b - eor x28, x28, x13 - eor3 v27.16b, v27.16b, v16.16b, v21.16b - eor x0, x0, x21 - eor3 v28.16b, v28.16b, v17.16b, v22.16b - eor x30, x30, x16 - eor3 v29.16b, v29.16b, v18.16b, v23.16b - eor x28, x28, x19 - eor3 v30.16b, v30.16b, v19.16b, v24.16b - eor x0, x0, x26 - rax1 v25.2d, v30.2d, v27.2d - eor x30, x30, x22 - rax1 v26.2d, v31.2d, v28.2d - eor x28, x28, x24 - rax1 v27.2d, v27.2d, v29.2d - str x0, [x29, #32] - rax1 v28.2d, v28.2d, v30.2d - str x28, [x29, #24] - rax1 v29.2d, v29.2d, v31.2d - eor x27, x2, x7 - eor v0.16b, v0.16b, v25.16b - xar v30.2d, v1.2d, v26.2d, #63 - eor x28, x4, x9 - xar v1.2d, v6.2d, v26.2d, #20 - eor x27, x27, x12 - xar v6.2d, v9.2d, v29.2d, #44 - eor x28, x28, x14 - xar v9.2d, v22.2d, v27.2d, #3 - eor x27, x27, x17 - xar v22.2d, v14.2d, v29.2d, #25 - eor x28, x28, x20 - xar v14.2d, v20.2d, v25.2d, #46 - eor x27, x27, x23 - xar v20.2d, v2.2d, v27.2d, #2 - eor x28, x28, x25 - xar v2.2d, v12.2d, v27.2d, #21 - eor x0, x0, x27, ror 63 - xar v12.2d, v13.2d, v28.2d, #39 - eor x27, x27, x28, ror 63 - xar v13.2d, v19.2d, v29.2d, #56 - eor x1, x1, x0 - xar v19.2d, v23.2d, v28.2d, #8 - eor x6, x6, x0 - xar v23.2d, v15.2d, v25.2d, #23 - eor x11, x11, x0 - xar v15.2d, v4.2d, v29.2d, #37 - eor x16, x16, x0 - xar v4.2d, v24.2d, v29.2d, #50 - eor x22, x22, x0 - xar v24.2d, v21.2d, v26.2d, #62 - eor x3, x3, x27 - xar v21.2d, v8.2d, v28.2d, #9 - eor x8, x8, x27 - xar v8.2d, v16.2d, v26.2d, #19 - eor x13, x13, x27 - xar v16.2d, v5.2d, v25.2d, #28 - eor x19, x19, x27 - xar v5.2d, v3.2d, v28.2d, #36 - eor x24, x24, x27 - xar v3.2d, v18.2d, v28.2d, #43 - ldr x0, [x29, #32] - xar v18.2d, v17.2d, v27.2d, #49 - ldr x27, [x29, #24] - xar v17.2d, v11.2d, v26.2d, #54 - eor x28, x28, x30, ror 63 - xar v11.2d, v7.2d, v27.2d, #58 - eor x30, x30, x27, ror 63 - xar v7.2d, v10.2d, v25.2d, #61 - eor x27, x27, x0, ror 63 - # Row Mix - mov v25.16b, v0.16b - eor x5, x5, x28 - mov v26.16b, v1.16b - eor x10, x10, x28 - bcax v0.16b, v25.16b, v2.16b, v26.16b - eor x15, x15, x28 - bcax v1.16b, v26.16b, v3.16b, v2.16b - eor x21, x21, x28 - bcax v2.16b, v2.16b, v4.16b, v3.16b - eor x26, x26, x28 - bcax v3.16b, v3.16b, v25.16b, v4.16b - eor x2, x2, x30 - bcax v4.16b, v4.16b, v26.16b, v25.16b - eor x7, x7, x30 - mov v25.16b, v5.16b - eor x12, x12, x30 - mov v26.16b, v6.16b - eor x17, x17, x30 - bcax v5.16b, v25.16b, v7.16b, v26.16b - eor x23, x23, x30 - bcax v6.16b, v26.16b, v8.16b, v7.16b - eor x4, x4, x27 - bcax v7.16b, v7.16b, v9.16b, v8.16b - eor x9, x9, x27 - bcax v8.16b, v8.16b, v25.16b, v9.16b - eor x14, x14, x27 - bcax v9.16b, v9.16b, v26.16b, v25.16b - eor x20, x20, x27 - mov v26.16b, v11.16b - eor x25, x25, x27 - # Swap Rotate Base - bcax v10.16b, v30.16b, v12.16b, v26.16b - ror x0, x2, #63 - bcax v11.16b, v26.16b, v13.16b, v12.16b - ror x2, x7, #20 - bcax v12.16b, v12.16b, v14.16b, v13.16b - ror x7, x10, #44 - bcax v13.16b, v13.16b, v30.16b, v14.16b - ror x10, x24, #3 - bcax v14.16b, v14.16b, v26.16b, v30.16b - ror x24, x15, #25 - mov v25.16b, v15.16b - ror x15, x22, #46 - mov v26.16b, v16.16b - ror x22, x3, #2 - bcax v15.16b, v25.16b, v17.16b, v26.16b - ror x3, x13, #21 - bcax v16.16b, v26.16b, v18.16b, v17.16b - ror x13, x14, #39 - bcax v17.16b, v17.16b, v19.16b, v18.16b - ror x14, x21, #56 - bcax v18.16b, v18.16b, v25.16b, v19.16b - ror x21, x25, #8 - bcax v19.16b, v19.16b, v26.16b, v25.16b - ror x25, x16, #23 - mov v25.16b, v20.16b - ror x16, x5, #37 - mov v26.16b, v21.16b - ror x5, x26, #50 - bcax v20.16b, v25.16b, v22.16b, v26.16b - ror x26, x23, #62 - bcax v21.16b, v26.16b, v23.16b, v22.16b - ror x23, x9, #9 - bcax v22.16b, v22.16b, v24.16b, v23.16b - ror x9, x17, #19 - bcax v23.16b, v23.16b, v25.16b, v24.16b - ror x17, x6, #28 - bcax v24.16b, v24.16b, v26.16b, v25.16b - ror x6, x4, #36 - ror x4, x20, #43 - ror x20, x19, #49 - ror x19, x12, #54 - ror x12, x8, #58 - ror x8, x11, #61 - # Row Mix Base - bic x11, x3, x2 - bic x27, x4, x3 - bic x28, x1, x5 - bic x30, x2, x1 - eor x1, x1, x11 - eor x2, x2, x27 - bic x11, x5, x4 - eor x4, x4, x28 - eor x3, x3, x11 - eor x5, x5, x30 - bic x11, x8, x7 - bic x27, x9, x8 - bic x28, x6, x10 - bic x30, x7, x6 - eor x6, x6, x11 - eor x7, x7, x27 - bic x11, x10, x9 - eor x9, x9, x28 - eor x8, x8, x11 - eor x10, x10, x30 - bic x11, x13, x12 - bic x27, x14, x13 - bic x28, x0, x15 - bic x30, x12, x0 - eor x11, x0, x11 - eor x12, x12, x27 - bic x0, x15, x14 - eor x14, x14, x28 - eor x13, x13, x0 - eor x15, x15, x30 - bic x0, x19, x17 - bic x27, x20, x19 - bic x28, x16, x21 - bic x30, x17, x16 - eor x16, x16, x0 - eor x17, x17, x27 - bic x0, x21, x20 - eor x20, x20, x28 - eor x19, x19, x0 - eor x21, x21, x30 - bic x0, x24, x23 - bic x27, x25, x24 - bic x28, x22, x26 - bic x30, x23, x22 - eor x22, x22, x0 - eor x23, x23, x27 - bic x0, x26, x25 - eor x25, x25, x28 - eor x24, x24, x0 - eor x26, x26, x30 - # Done transforming - ldp x27, x28, [x29, #48] - ldr x0, [x27], #8 - subs x28, x28, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 - eor x1, x1, x0 - eor v0.16b, v0.16b, v30.16b - bne L_SHA3_transform_blocksx3_crypto_begin - ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] - add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] - add x0, x0, #8 - stp x1, x2, [x0] - stp x3, x4, [x0, #16] - stp x5, x6, [x0, #32] - stp x7, x8, [x0, #48] - stp x9, x10, [x0, #64] - stp x11, x12, [x0, #80] - stp x13, x14, [x0, #96] - stp x15, x16, [x0, #112] - stp x17, x19, [x0, #128] - stp x20, x21, [x0, #144] - stp x22, x23, [x0, #160] - stp x24, x25, [x0, #176] - str x26, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret -#ifndef __APPLE__ - .size mlkem_sha3_blocksx3_crypto,.-mlkem_sha3_blocksx3_crypto -#endif /* __APPLE__ */ -#ifndef __APPLE__ -.text -.globl mlkem_shake128_blocksx3_seed_crypto -WC_ASM_ATT_HIDDEN(mlkem_shake128_blocksx3_seed_crypto) -.type mlkem_shake128_blocksx3_seed_crypto,@function -.align 2 -mlkem_shake128_blocksx3_seed_crypto: -#else -.section __TEXT,__text -.globl _mlkem_shake128_blocksx3_seed_crypto -WC_ASM_ATT_HIDDEN(_mlkem_shake128_blocksx3_seed_crypto) -.p2align 2 -_mlkem_shake128_blocksx3_seed_crypto: -#endif /* __APPLE__ */ - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] -#ifndef __APPLE__ - adrp x28, L_sha3_aarch64_r - add x28, x28, :lo12:L_sha3_aarch64_r -#else - adrp x28, L_sha3_aarch64_r@PAGE - add x28, x28, L_sha3_aarch64_r@PAGEOFF -#endif /* __APPLE__ */ - str x0, [x29, #40] - add x0, x0, #32 - ld1 {v4.d}[0], [x0] - ldp x2, x3, [x1], #16 - add x0, x0, #0xc8 - ld1 {v4.d}[1], [x0] - ldp x4, x5, [x1], #16 - ldr x6, [x0, #200] - eor v5.16b, v5.16b, v5.16b - eor x7, x7, x7 - eor v6.16b, v6.16b, v6.16b - eor x8, x8, x8 - eor v7.16b, v7.16b, v7.16b - eor x9, x9, x9 - eor v8.16b, v8.16b, v8.16b - eor x10, x10, x10 - eor v9.16b, v9.16b, v9.16b - eor x11, x11, x11 - eor v10.16b, v10.16b, v10.16b - eor x12, x12, x12 - eor v11.16b, v11.16b, v11.16b - eor x13, x13, x13 - eor v12.16b, v12.16b, v12.16b - eor x14, x14, x14 - eor v13.16b, v13.16b, v13.16b - eor x15, x15, x15 - eor v14.16b, v14.16b, v14.16b - eor x16, x16, x16 - eor v15.16b, v15.16b, v15.16b - eor x17, x17, x17 - eor v16.16b, v16.16b, v16.16b - eor x19, x19, x19 - eor v17.16b, v17.16b, v17.16b - eor x20, x20, x20 - eor v18.16b, v18.16b, v18.16b - eor x21, x21, x21 - eor v19.16b, v19.16b, v19.16b - eor x22, x22, x22 - movz x23, #0x8000, lsl 48 - eor v21.16b, v21.16b, v21.16b - eor x24, x24, x24 - eor v22.16b, v22.16b, v22.16b - eor x25, x25, x25 - eor v23.16b, v23.16b, v23.16b - eor x26, x26, x26 - eor v24.16b, v24.16b, v24.16b - eor x27, x27, x27 - dup v0.2d, x2 - dup v1.2d, x3 - dup v2.2d, x4 - dup v3.2d, x5 - dup v20.2d, x23 - mov x1, #24 - # Start of 24 rounds -L_SHA3_shake128_blocksx3_seed_crypto_begin: - stp x28, x1, [x29, #48] - # Col Mix - eor3 v31.16b, v0.16b, v5.16b, v10.16b - eor x0, x6, x11 - eor3 v27.16b, v1.16b, v6.16b, v11.16b - eor x30, x2, x7 - eor3 v28.16b, v2.16b, v7.16b, v12.16b - eor x28, x4, x9 - eor3 v29.16b, v3.16b, v8.16b, v13.16b - eor x0, x0, x16 - eor3 v30.16b, v4.16b, v9.16b, v14.16b - eor x30, x30, x12 - eor3 v31.16b, v31.16b, v15.16b, v20.16b - eor x28, x28, x14 - eor3 v27.16b, v27.16b, v16.16b, v21.16b - eor x0, x0, x22 - eor3 v28.16b, v28.16b, v17.16b, v22.16b - eor x30, x30, x17 - eor3 v29.16b, v29.16b, v18.16b, v23.16b - eor x28, x28, x20 - eor3 v30.16b, v30.16b, v19.16b, v24.16b - eor x0, x0, x27 - rax1 v25.2d, v30.2d, v27.2d - eor x30, x30, x23 - rax1 v26.2d, v31.2d, v28.2d - eor x28, x28, x25 - rax1 v27.2d, v27.2d, v29.2d - str x0, [x29, #32] - rax1 v28.2d, v28.2d, v30.2d - str x28, [x29, #24] - rax1 v29.2d, v29.2d, v31.2d - eor x1, x3, x8 - eor v0.16b, v0.16b, v25.16b - xar v30.2d, v1.2d, v26.2d, #63 - eor x28, x5, x10 - xar v1.2d, v6.2d, v26.2d, #20 - eor x1, x1, x13 - xar v6.2d, v9.2d, v29.2d, #44 - eor x28, x28, x15 - xar v9.2d, v22.2d, v27.2d, #3 - eor x1, x1, x19 - xar v22.2d, v14.2d, v29.2d, #25 - eor x28, x28, x21 - xar v14.2d, v20.2d, v25.2d, #46 - eor x1, x1, x24 - xar v20.2d, v2.2d, v27.2d, #2 - eor x28, x28, x26 - xar v2.2d, v12.2d, v27.2d, #21 - eor x0, x0, x1, ror 63 - xar v12.2d, v13.2d, v28.2d, #39 - eor x1, x1, x28, ror 63 - xar v13.2d, v19.2d, v29.2d, #56 - eor x2, x2, x0 - xar v19.2d, v23.2d, v28.2d, #8 - eor x7, x7, x0 - xar v23.2d, v15.2d, v25.2d, #23 - eor x12, x12, x0 - xar v15.2d, v4.2d, v29.2d, #37 - eor x17, x17, x0 - xar v4.2d, v24.2d, v29.2d, #50 - eor x23, x23, x0 - xar v24.2d, v21.2d, v26.2d, #62 - eor x4, x4, x1 - xar v21.2d, v8.2d, v28.2d, #9 - eor x9, x9, x1 - xar v8.2d, v16.2d, v26.2d, #19 - eor x14, x14, x1 - xar v16.2d, v5.2d, v25.2d, #28 - eor x20, x20, x1 - xar v5.2d, v3.2d, v28.2d, #36 - eor x25, x25, x1 - xar v3.2d, v18.2d, v28.2d, #43 - ldr x0, [x29, #32] - xar v18.2d, v17.2d, v27.2d, #49 - ldr x1, [x29, #24] - xar v17.2d, v11.2d, v26.2d, #54 - eor x28, x28, x30, ror 63 - xar v11.2d, v7.2d, v27.2d, #58 - eor x30, x30, x1, ror 63 - xar v7.2d, v10.2d, v25.2d, #61 - eor x1, x1, x0, ror 63 - # Row Mix - mov v25.16b, v0.16b - eor x6, x6, x28 - mov v26.16b, v1.16b - eor x11, x11, x28 - bcax v0.16b, v25.16b, v2.16b, v26.16b - eor x16, x16, x28 - bcax v1.16b, v26.16b, v3.16b, v2.16b - eor x22, x22, x28 - bcax v2.16b, v2.16b, v4.16b, v3.16b - eor x27, x27, x28 - bcax v3.16b, v3.16b, v25.16b, v4.16b - eor x3, x3, x30 - bcax v4.16b, v4.16b, v26.16b, v25.16b - eor x8, x8, x30 - mov v25.16b, v5.16b - eor x13, x13, x30 - mov v26.16b, v6.16b - eor x19, x19, x30 - bcax v5.16b, v25.16b, v7.16b, v26.16b - eor x24, x24, x30 - bcax v6.16b, v26.16b, v8.16b, v7.16b - eor x5, x5, x1 - bcax v7.16b, v7.16b, v9.16b, v8.16b - eor x10, x10, x1 - bcax v8.16b, v8.16b, v25.16b, v9.16b - eor x15, x15, x1 - bcax v9.16b, v9.16b, v26.16b, v25.16b - eor x21, x21, x1 - mov v26.16b, v11.16b - eor x26, x26, x1 - # Swap Rotate Base - bcax v10.16b, v30.16b, v12.16b, v26.16b - ror x0, x3, #63 - bcax v11.16b, v26.16b, v13.16b, v12.16b - ror x3, x8, #20 - bcax v12.16b, v12.16b, v14.16b, v13.16b - ror x8, x11, #44 - bcax v13.16b, v13.16b, v30.16b, v14.16b - ror x11, x25, #3 - bcax v14.16b, v14.16b, v26.16b, v30.16b - ror x25, x16, #25 - mov v25.16b, v15.16b - ror x16, x23, #46 - mov v26.16b, v16.16b - ror x23, x4, #2 - bcax v15.16b, v25.16b, v17.16b, v26.16b - ror x4, x14, #21 - bcax v16.16b, v26.16b, v18.16b, v17.16b - ror x14, x15, #39 - bcax v17.16b, v17.16b, v19.16b, v18.16b - ror x15, x22, #56 - bcax v18.16b, v18.16b, v25.16b, v19.16b - ror x22, x26, #8 - bcax v19.16b, v19.16b, v26.16b, v25.16b - ror x26, x17, #23 - mov v25.16b, v20.16b - ror x17, x6, #37 - mov v26.16b, v21.16b - ror x6, x27, #50 - bcax v20.16b, v25.16b, v22.16b, v26.16b - ror x27, x24, #62 - bcax v21.16b, v26.16b, v23.16b, v22.16b - ror x24, x10, #9 - bcax v22.16b, v22.16b, v24.16b, v23.16b - ror x10, x19, #19 - bcax v23.16b, v23.16b, v25.16b, v24.16b - ror x19, x7, #28 - bcax v24.16b, v24.16b, v26.16b, v25.16b - ror x7, x5, #36 - ror x5, x21, #43 - ror x21, x20, #49 - ror x20, x13, #54 - ror x13, x9, #58 - ror x9, x12, #61 - # Row Mix Base - bic x12, x4, x3 - bic x1, x5, x4 - bic x28, x2, x6 - bic x30, x3, x2 - eor x2, x2, x12 - eor x3, x3, x1 - bic x12, x6, x5 - eor x5, x5, x28 - eor x4, x4, x12 - eor x6, x6, x30 - bic x12, x9, x8 - bic x1, x10, x9 - bic x28, x7, x11 - bic x30, x8, x7 - eor x7, x7, x12 - eor x8, x8, x1 - bic x12, x11, x10 - eor x10, x10, x28 - eor x9, x9, x12 - eor x11, x11, x30 - bic x12, x14, x13 - bic x1, x15, x14 - bic x28, x0, x16 - bic x30, x13, x0 - eor x12, x0, x12 - eor x13, x13, x1 - bic x0, x16, x15 - eor x15, x15, x28 - eor x14, x14, x0 - eor x16, x16, x30 - bic x0, x20, x19 - bic x1, x21, x20 - bic x28, x17, x22 - bic x30, x19, x17 - eor x17, x17, x0 - eor x19, x19, x1 - bic x0, x22, x21 - eor x21, x21, x28 - eor x20, x20, x0 - eor x22, x22, x30 - bic x0, x25, x24 - bic x1, x26, x25 - bic x28, x23, x27 - bic x30, x24, x23 - eor x23, x23, x0 - eor x24, x24, x1 - bic x0, x27, x26 - eor x26, x26, x28 - eor x25, x25, x0 - eor x27, x27, x30 - # Done transforming - ldp x28, x1, [x29, #48] - ldr x0, [x28], #8 - subs x1, x1, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 - eor x2, x2, x0 - eor v0.16b, v0.16b, v30.16b - bne L_SHA3_shake128_blocksx3_seed_crypto_begin - ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] - add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] - add x0, x0, #8 - stp x2, x3, [x0] - stp x4, x5, [x0, #16] - stp x6, x7, [x0, #32] - stp x8, x9, [x0, #48] - stp x10, x11, [x0, #64] - stp x12, x13, [x0, #80] - stp x14, x15, [x0, #96] - stp x16, x17, [x0, #112] - stp x19, x20, [x0, #128] - stp x21, x22, [x0, #144] - stp x23, x24, [x0, #160] - stp x25, x26, [x0, #176] - str x27, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret -#ifndef __APPLE__ - .size mlkem_shake128_blocksx3_seed_crypto,.-mlkem_shake128_blocksx3_seed_crypto -#endif /* __APPLE__ */ -#ifndef __APPLE__ -.text -.globl mlkem_shake256_blocksx3_seed_crypto -WC_ASM_ATT_HIDDEN(mlkem_shake256_blocksx3_seed_crypto) -.type mlkem_shake256_blocksx3_seed_crypto,@function -.align 2 -mlkem_shake256_blocksx3_seed_crypto: -#else -.section __TEXT,__text -.globl _mlkem_shake256_blocksx3_seed_crypto -WC_ASM_ATT_HIDDEN(_mlkem_shake256_blocksx3_seed_crypto) -.p2align 2 -_mlkem_shake256_blocksx3_seed_crypto: -#endif /* __APPLE__ */ - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] -#ifndef __APPLE__ - adrp x28, L_sha3_aarch64_r - add x28, x28, :lo12:L_sha3_aarch64_r -#else - adrp x28, L_sha3_aarch64_r@PAGE - add x28, x28, L_sha3_aarch64_r@PAGEOFF -#endif /* __APPLE__ */ - str x0, [x29, #40] - add x0, x0, #32 - ld1 {v4.d}[0], [x0] - ldp x2, x3, [x1], #16 - add x0, x0, #0xc8 - ld1 {v4.d}[1], [x0] - ldp x4, x5, [x1], #16 - ldr x6, [x0, #200] - eor v5.16b, v5.16b, v5.16b - eor x7, x7, x7 - eor v6.16b, v6.16b, v6.16b - eor x8, x8, x8 - eor v7.16b, v7.16b, v7.16b - eor x9, x9, x9 - eor v8.16b, v8.16b, v8.16b - eor x10, x10, x10 - eor v9.16b, v9.16b, v9.16b - eor x11, x11, x11 - eor v10.16b, v10.16b, v10.16b - eor x12, x12, x12 - eor v11.16b, v11.16b, v11.16b - eor x13, x13, x13 - eor v12.16b, v12.16b, v12.16b - eor x14, x14, x14 - eor v13.16b, v13.16b, v13.16b - eor x15, x15, x15 - eor v14.16b, v14.16b, v14.16b - eor x16, x16, x16 - eor v15.16b, v15.16b, v15.16b - eor x17, x17, x17 - movz x19, #0x8000, lsl 48 - eor v17.16b, v17.16b, v17.16b - eor x20, x20, x20 - eor v18.16b, v18.16b, v18.16b - eor x21, x21, x21 - eor v19.16b, v19.16b, v19.16b - eor x22, x22, x22 - eor v20.16b, v20.16b, v20.16b - eor x23, x23, x23 - eor v21.16b, v21.16b, v21.16b - eor x24, x24, x24 - eor v22.16b, v22.16b, v22.16b - eor x25, x25, x25 - eor v23.16b, v23.16b, v23.16b - eor x26, x26, x26 - eor v24.16b, v24.16b, v24.16b - eor x27, x27, x27 - dup v0.2d, x2 - dup v1.2d, x3 - dup v2.2d, x4 - dup v3.2d, x5 - dup v16.2d, x19 - mov x1, #24 - # Start of 24 rounds -L_SHA3_shake256_blocksx3_seed_crypto_begin: - stp x28, x1, [x29, #48] - # Col Mix - eor3 v31.16b, v0.16b, v5.16b, v10.16b - eor x0, x6, x11 - eor3 v27.16b, v1.16b, v6.16b, v11.16b - eor x30, x2, x7 - eor3 v28.16b, v2.16b, v7.16b, v12.16b - eor x28, x4, x9 - eor3 v29.16b, v3.16b, v8.16b, v13.16b - eor x0, x0, x16 - eor3 v30.16b, v4.16b, v9.16b, v14.16b - eor x30, x30, x12 - eor3 v31.16b, v31.16b, v15.16b, v20.16b - eor x28, x28, x14 - eor3 v27.16b, v27.16b, v16.16b, v21.16b - eor x0, x0, x22 - eor3 v28.16b, v28.16b, v17.16b, v22.16b - eor x30, x30, x17 - eor3 v29.16b, v29.16b, v18.16b, v23.16b - eor x28, x28, x20 - eor3 v30.16b, v30.16b, v19.16b, v24.16b - eor x0, x0, x27 - rax1 v25.2d, v30.2d, v27.2d - eor x30, x30, x23 - rax1 v26.2d, v31.2d, v28.2d - eor x28, x28, x25 - rax1 v27.2d, v27.2d, v29.2d - str x0, [x29, #32] - rax1 v28.2d, v28.2d, v30.2d - str x28, [x29, #24] - rax1 v29.2d, v29.2d, v31.2d - eor x1, x3, x8 - eor v0.16b, v0.16b, v25.16b - xar v30.2d, v1.2d, v26.2d, #63 - eor x28, x5, x10 - xar v1.2d, v6.2d, v26.2d, #20 - eor x1, x1, x13 - xar v6.2d, v9.2d, v29.2d, #44 - eor x28, x28, x15 - xar v9.2d, v22.2d, v27.2d, #3 - eor x1, x1, x19 - xar v22.2d, v14.2d, v29.2d, #25 - eor x28, x28, x21 - xar v14.2d, v20.2d, v25.2d, #46 - eor x1, x1, x24 - xar v20.2d, v2.2d, v27.2d, #2 - eor x28, x28, x26 - xar v2.2d, v12.2d, v27.2d, #21 - eor x0, x0, x1, ror 63 - xar v12.2d, v13.2d, v28.2d, #39 - eor x1, x1, x28, ror 63 - xar v13.2d, v19.2d, v29.2d, #56 - eor x2, x2, x0 - xar v19.2d, v23.2d, v28.2d, #8 - eor x7, x7, x0 - xar v23.2d, v15.2d, v25.2d, #23 - eor x12, x12, x0 - xar v15.2d, v4.2d, v29.2d, #37 - eor x17, x17, x0 - xar v4.2d, v24.2d, v29.2d, #50 - eor x23, x23, x0 - xar v24.2d, v21.2d, v26.2d, #62 - eor x4, x4, x1 - xar v21.2d, v8.2d, v28.2d, #9 - eor x9, x9, x1 - xar v8.2d, v16.2d, v26.2d, #19 - eor x14, x14, x1 - xar v16.2d, v5.2d, v25.2d, #28 - eor x20, x20, x1 - xar v5.2d, v3.2d, v28.2d, #36 - eor x25, x25, x1 - xar v3.2d, v18.2d, v28.2d, #43 - ldr x0, [x29, #32] - xar v18.2d, v17.2d, v27.2d, #49 - ldr x1, [x29, #24] - xar v17.2d, v11.2d, v26.2d, #54 - eor x28, x28, x30, ror 63 - xar v11.2d, v7.2d, v27.2d, #58 - eor x30, x30, x1, ror 63 - xar v7.2d, v10.2d, v25.2d, #61 - eor x1, x1, x0, ror 63 - # Row Mix - mov v25.16b, v0.16b - eor x6, x6, x28 - mov v26.16b, v1.16b - eor x11, x11, x28 - bcax v0.16b, v25.16b, v2.16b, v26.16b - eor x16, x16, x28 - bcax v1.16b, v26.16b, v3.16b, v2.16b - eor x22, x22, x28 - bcax v2.16b, v2.16b, v4.16b, v3.16b - eor x27, x27, x28 - bcax v3.16b, v3.16b, v25.16b, v4.16b - eor x3, x3, x30 - bcax v4.16b, v4.16b, v26.16b, v25.16b - eor x8, x8, x30 - mov v25.16b, v5.16b - eor x13, x13, x30 - mov v26.16b, v6.16b - eor x19, x19, x30 - bcax v5.16b, v25.16b, v7.16b, v26.16b - eor x24, x24, x30 - bcax v6.16b, v26.16b, v8.16b, v7.16b - eor x5, x5, x1 - bcax v7.16b, v7.16b, v9.16b, v8.16b - eor x10, x10, x1 - bcax v8.16b, v8.16b, v25.16b, v9.16b - eor x15, x15, x1 - bcax v9.16b, v9.16b, v26.16b, v25.16b - eor x21, x21, x1 - mov v26.16b, v11.16b - eor x26, x26, x1 - # Swap Rotate Base - bcax v10.16b, v30.16b, v12.16b, v26.16b - ror x0, x3, #63 - bcax v11.16b, v26.16b, v13.16b, v12.16b - ror x3, x8, #20 - bcax v12.16b, v12.16b, v14.16b, v13.16b - ror x8, x11, #44 - bcax v13.16b, v13.16b, v30.16b, v14.16b - ror x11, x25, #3 - bcax v14.16b, v14.16b, v26.16b, v30.16b - ror x25, x16, #25 - mov v25.16b, v15.16b - ror x16, x23, #46 - mov v26.16b, v16.16b - ror x23, x4, #2 - bcax v15.16b, v25.16b, v17.16b, v26.16b - ror x4, x14, #21 - bcax v16.16b, v26.16b, v18.16b, v17.16b - ror x14, x15, #39 - bcax v17.16b, v17.16b, v19.16b, v18.16b - ror x15, x22, #56 - bcax v18.16b, v18.16b, v25.16b, v19.16b - ror x22, x26, #8 - bcax v19.16b, v19.16b, v26.16b, v25.16b - ror x26, x17, #23 - mov v25.16b, v20.16b - ror x17, x6, #37 - mov v26.16b, v21.16b - ror x6, x27, #50 - bcax v20.16b, v25.16b, v22.16b, v26.16b - ror x27, x24, #62 - bcax v21.16b, v26.16b, v23.16b, v22.16b - ror x24, x10, #9 - bcax v22.16b, v22.16b, v24.16b, v23.16b - ror x10, x19, #19 - bcax v23.16b, v23.16b, v25.16b, v24.16b - ror x19, x7, #28 - bcax v24.16b, v24.16b, v26.16b, v25.16b - ror x7, x5, #36 - ror x5, x21, #43 - ror x21, x20, #49 - ror x20, x13, #54 - ror x13, x9, #58 - ror x9, x12, #61 - # Row Mix Base - bic x12, x4, x3 - bic x1, x5, x4 - bic x28, x2, x6 - bic x30, x3, x2 - eor x2, x2, x12 - eor x3, x3, x1 - bic x12, x6, x5 - eor x5, x5, x28 - eor x4, x4, x12 - eor x6, x6, x30 - bic x12, x9, x8 - bic x1, x10, x9 - bic x28, x7, x11 - bic x30, x8, x7 - eor x7, x7, x12 - eor x8, x8, x1 - bic x12, x11, x10 - eor x10, x10, x28 - eor x9, x9, x12 - eor x11, x11, x30 - bic x12, x14, x13 - bic x1, x15, x14 - bic x28, x0, x16 - bic x30, x13, x0 - eor x12, x0, x12 - eor x13, x13, x1 - bic x0, x16, x15 - eor x15, x15, x28 - eor x14, x14, x0 - eor x16, x16, x30 - bic x0, x20, x19 - bic x1, x21, x20 - bic x28, x17, x22 - bic x30, x19, x17 - eor x17, x17, x0 - eor x19, x19, x1 - bic x0, x22, x21 - eor x21, x21, x28 - eor x20, x20, x0 - eor x22, x22, x30 - bic x0, x25, x24 - bic x1, x26, x25 - bic x28, x23, x27 - bic x30, x24, x23 - eor x23, x23, x0 - eor x24, x24, x1 - bic x0, x27, x26 - eor x26, x26, x28 - eor x25, x25, x0 - eor x27, x27, x30 - # Done transforming - ldp x28, x1, [x29, #48] - ldr x0, [x28], #8 - subs x1, x1, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 - eor x2, x2, x0 - eor v0.16b, v0.16b, v30.16b - bne L_SHA3_shake256_blocksx3_seed_crypto_begin - ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] - add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] - add x0, x0, #8 - stp x2, x3, [x0] - stp x4, x5, [x0, #16] - stp x6, x7, [x0, #32] - stp x8, x9, [x0, #48] - stp x10, x11, [x0, #64] - stp x12, x13, [x0, #80] - stp x14, x15, [x0, #96] - stp x16, x17, [x0, #112] - stp x19, x20, [x0, #128] - stp x21, x22, [x0, #144] - stp x23, x24, [x0, #160] - stp x25, x26, [x0, #176] - str x27, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret -#ifndef __APPLE__ - .size mlkem_shake256_blocksx3_seed_crypto,.-mlkem_shake256_blocksx3_seed_crypto -#endif /* __APPLE__ */ -#endif /* WOLFSSL_ARMASM_CRYPTO_SHA3 */ -#ifndef __APPLE__ -.text -.globl mlkem_sha3_blocksx3_neon -WC_ASM_ATT_HIDDEN(mlkem_sha3_blocksx3_neon) -.type mlkem_sha3_blocksx3_neon,@function -.align 2 -mlkem_sha3_blocksx3_neon: -#else -.section __TEXT,__text -.globl _mlkem_sha3_blocksx3_neon -WC_ASM_ATT_HIDDEN(_mlkem_sha3_blocksx3_neon) -.p2align 2 -_mlkem_sha3_blocksx3_neon: -#endif /* __APPLE__ */ - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] -#ifndef __APPLE__ - adrp x27, L_sha3_aarch64_r - add x27, x27, :lo12:L_sha3_aarch64_r -#else - adrp x27, L_sha3_aarch64_r@PAGE - add x27, x27, L_sha3_aarch64_r@PAGEOFF -#endif /* __APPLE__ */ - str x0, [x29, #40] - ld4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - ld4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - ld4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - ld4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - ld4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - ld4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - ld1 {v24.d}[0], [x0] - add x0, x0, #8 - ld4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - ld4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - ld4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - ld4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - ld4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - ld4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - ld1 {v24.d}[1], [x0] - add x0, x0, #8 - ldp x1, x2, [x0] - ldp x3, x4, [x0, #16] - ldp x5, x6, [x0, #32] - ldp x7, x8, [x0, #48] - ldp x9, x10, [x0, #64] - ldp x11, x12, [x0, #80] - ldp x13, x14, [x0, #96] - ldp x15, x16, [x0, #112] - ldp x17, x19, [x0, #128] - ldp x20, x21, [x0, #144] - ldp x22, x23, [x0, #160] - ldp x24, x25, [x0, #176] - ldr x26, [x0, #192] - mov x28, #24 - # Start of 24 rounds -L_SHA3_transform_blocksx3_neon_begin: - stp x27, x28, [x29, #48] - # Col Mix NEON - eor v30.16b, v4.16b, v9.16b - eor x0, x5, x10 - eor v27.16b, v1.16b, v6.16b - eor x30, x1, x6 - eor v30.16b, v30.16b, v14.16b - eor x28, x3, x8 - eor v27.16b, v27.16b, v11.16b - eor x0, x0, x15 - eor v30.16b, v30.16b, v19.16b - eor x30, x30, x11 - eor v27.16b, v27.16b, v16.16b - eor x28, x28, x13 - eor v30.16b, v30.16b, v24.16b - eor x0, x0, x21 - eor v27.16b, v27.16b, v21.16b - eor x30, x30, x16 - ushr v25.2d, v27.2d, #63 - eor x28, x28, x19 - sli v25.2d, v27.2d, #1 - eor x0, x0, x26 - eor v25.16b, v25.16b, v30.16b - eor x30, x30, x22 - eor v31.16b, v0.16b, v5.16b - eor x28, x28, x24 - eor v28.16b, v2.16b, v7.16b - str x0, [x29, #32] - eor v31.16b, v31.16b, v10.16b - str x28, [x29, #24] - eor v28.16b, v28.16b, v12.16b - eor x27, x2, x7 - eor v31.16b, v31.16b, v15.16b - eor x28, x4, x9 - eor v28.16b, v28.16b, v17.16b - eor x27, x27, x12 - eor v31.16b, v31.16b, v20.16b - eor x28, x28, x14 - eor v28.16b, v28.16b, v22.16b - eor x27, x27, x17 - ushr v29.2d, v30.2d, #63 - eor x28, x28, x20 - ushr v26.2d, v28.2d, #63 - eor x27, x27, x23 - sli v29.2d, v30.2d, #1 - eor x28, x28, x25 - sli v26.2d, v28.2d, #1 - eor x0, x0, x27, ror 63 - eor v28.16b, v28.16b, v29.16b - eor x27, x27, x28, ror 63 - eor v29.16b, v3.16b, v8.16b - eor x1, x1, x0 - eor v26.16b, v26.16b, v31.16b - eor x6, x6, x0 - eor v29.16b, v29.16b, v13.16b - eor x11, x11, x0 - eor v29.16b, v29.16b, v18.16b - eor x16, x16, x0 - eor v29.16b, v29.16b, v23.16b - eor x22, x22, x0 - ushr v30.2d, v29.2d, #63 - eor x3, x3, x27 - sli v30.2d, v29.2d, #1 - eor x8, x8, x27 - eor v27.16b, v27.16b, v30.16b - eor x13, x13, x27 - ushr v30.2d, v31.2d, #63 - eor x19, x19, x27 - sli v30.2d, v31.2d, #1 - eor x24, x24, x27 - eor v29.16b, v29.16b, v30.16b - ldr x0, [x29, #32] - # Swap Rotate NEON - eor v0.16b, v0.16b, v25.16b - eor v31.16b, v1.16b, v26.16b - ldr x27, [x29, #24] - eor v6.16b, v6.16b, v26.16b - eor x28, x28, x30, ror 63 - ushr v30.2d, v31.2d, #63 - eor x30, x30, x27, ror 63 - ushr v1.2d, v6.2d, #20 - eor x27, x27, x0, ror 63 - sli v30.2d, v31.2d, #1 - eor x5, x5, x28 - sli v1.2d, v6.2d, #44 - eor x10, x10, x28 - eor v31.16b, v9.16b, v29.16b - eor x15, x15, x28 - eor v22.16b, v22.16b, v27.16b - eor x21, x21, x28 - ushr v6.2d, v31.2d, #44 - eor x26, x26, x28 - ushr v9.2d, v22.2d, #3 - eor x2, x2, x30 - sli v6.2d, v31.2d, #20 - eor x7, x7, x30 - sli v9.2d, v22.2d, #61 - eor x12, x12, x30 - eor v31.16b, v14.16b, v29.16b - eor x17, x17, x30 - eor v20.16b, v20.16b, v25.16b - eor x23, x23, x30 - ushr v22.2d, v31.2d, #25 - eor x4, x4, x27 - ushr v14.2d, v20.2d, #46 - eor x9, x9, x27 - sli v22.2d, v31.2d, #39 - eor x14, x14, x27 - sli v14.2d, v20.2d, #18 - eor x20, x20, x27 - eor v31.16b, v2.16b, v27.16b - eor x25, x25, x27 - # Swap Rotate Base - eor v12.16b, v12.16b, v27.16b - ror x0, x2, #63 - ushr v20.2d, v31.2d, #2 - ror x2, x7, #20 - ushr v2.2d, v12.2d, #21 - ror x7, x10, #44 - sli v20.2d, v31.2d, #62 - ror x10, x24, #3 - sli v2.2d, v12.2d, #43 - ror x24, x15, #25 - eor v31.16b, v13.16b, v28.16b - ror x15, x22, #46 - eor v19.16b, v19.16b, v29.16b - ror x22, x3, #2 - ushr v12.2d, v31.2d, #39 - ror x3, x13, #21 - ushr v13.2d, v19.2d, #56 - ror x13, x14, #39 - sli v12.2d, v31.2d, #25 - ror x14, x21, #56 - sli v13.2d, v19.2d, #8 - ror x21, x25, #8 - eor v31.16b, v23.16b, v28.16b - ror x25, x16, #23 - eor v15.16b, v15.16b, v25.16b - ror x16, x5, #37 - ushr v19.2d, v31.2d, #8 - ror x5, x26, #50 - ushr v23.2d, v15.2d, #23 - ror x26, x23, #62 - sli v19.2d, v31.2d, #56 - ror x23, x9, #9 - sli v23.2d, v15.2d, #41 - ror x9, x17, #19 - eor v31.16b, v4.16b, v29.16b - ror x17, x6, #28 - eor v24.16b, v24.16b, v29.16b - ror x6, x4, #36 - ushr v15.2d, v31.2d, #37 - ror x4, x20, #43 - ushr v4.2d, v24.2d, #50 - ror x20, x19, #49 - sli v15.2d, v31.2d, #27 - ror x19, x12, #54 - sli v4.2d, v24.2d, #14 - ror x12, x8, #58 - eor v31.16b, v21.16b, v26.16b - ror x8, x11, #61 - # Row Mix Base - eor v8.16b, v8.16b, v28.16b - bic x11, x3, x2 - ushr v24.2d, v31.2d, #62 - bic x27, x4, x3 - ushr v21.2d, v8.2d, #9 - bic x28, x1, x5 - sli v24.2d, v31.2d, #2 - bic x30, x2, x1 - sli v21.2d, v8.2d, #55 - eor x1, x1, x11 - eor v31.16b, v16.16b, v26.16b - eor x2, x2, x27 - eor v5.16b, v5.16b, v25.16b - bic x11, x5, x4 - ushr v8.2d, v31.2d, #19 - eor x4, x4, x28 - ushr v16.2d, v5.2d, #28 - eor x3, x3, x11 - sli v8.2d, v31.2d, #45 - eor x5, x5, x30 - sli v16.2d, v5.2d, #36 - bic x11, x8, x7 - eor v31.16b, v3.16b, v28.16b - bic x27, x9, x8 - eor v18.16b, v18.16b, v28.16b - bic x28, x6, x10 - ushr v5.2d, v31.2d, #36 - bic x30, x7, x6 - ushr v3.2d, v18.2d, #43 - eor x6, x6, x11 - sli v5.2d, v31.2d, #28 - eor x7, x7, x27 - sli v3.2d, v18.2d, #21 - bic x11, x10, x9 - eor v31.16b, v17.16b, v27.16b - eor x9, x9, x28 - eor v11.16b, v11.16b, v26.16b - eor x8, x8, x11 - ushr v18.2d, v31.2d, #49 - eor x10, x10, x30 - ushr v17.2d, v11.2d, #54 - bic x11, x13, x12 - sli v18.2d, v31.2d, #15 - bic x27, x14, x13 - sli v17.2d, v11.2d, #10 - bic x28, x0, x15 - eor v31.16b, v7.16b, v27.16b - bic x30, x12, x0 - eor v10.16b, v10.16b, v25.16b - eor x11, x0, x11 - ushr v11.2d, v31.2d, #58 - eor x12, x12, x27 - ushr v7.2d, v10.2d, #61 - bic x0, x15, x14 - sli v11.2d, v31.2d, #6 - eor x14, x14, x28 - sli v7.2d, v10.2d, #3 - eor x13, x13, x0 - # Row Mix NEON - bic v25.16b, v2.16b, v1.16b - eor x15, x15, x30 - bic v26.16b, v3.16b, v2.16b - bic x0, x19, x17 - bic v27.16b, v4.16b, v3.16b - bic x27, x20, x19 - bic v28.16b, v0.16b, v4.16b - bic x28, x16, x21 - bic v29.16b, v1.16b, v0.16b - bic x30, x17, x16 - eor v0.16b, v0.16b, v25.16b - eor x16, x16, x0 - eor v1.16b, v1.16b, v26.16b - eor x17, x17, x27 - eor v2.16b, v2.16b, v27.16b - bic x0, x21, x20 - eor v3.16b, v3.16b, v28.16b - eor x20, x20, x28 - eor v4.16b, v4.16b, v29.16b - eor x19, x19, x0 - bic v25.16b, v7.16b, v6.16b - eor x21, x21, x30 - bic v26.16b, v8.16b, v7.16b - bic x0, x24, x23 - bic v27.16b, v9.16b, v8.16b - bic x27, x25, x24 - bic v28.16b, v5.16b, v9.16b - bic x28, x22, x26 - bic v29.16b, v6.16b, v5.16b - bic x30, x23, x22 - eor v5.16b, v5.16b, v25.16b - eor x22, x22, x0 - eor v6.16b, v6.16b, v26.16b - eor x23, x23, x27 - eor v7.16b, v7.16b, v27.16b - bic x0, x26, x25 - eor v8.16b, v8.16b, v28.16b - eor x25, x25, x28 - eor v9.16b, v9.16b, v29.16b - eor x24, x24, x0 - bic v25.16b, v12.16b, v11.16b - eor x26, x26, x30 - bic v26.16b, v13.16b, v12.16b - bic v27.16b, v14.16b, v13.16b - bic v28.16b, v30.16b, v14.16b - bic v29.16b, v11.16b, v30.16b - eor v10.16b, v30.16b, v25.16b - eor v11.16b, v11.16b, v26.16b - eor v12.16b, v12.16b, v27.16b - eor v13.16b, v13.16b, v28.16b - eor v14.16b, v14.16b, v29.16b - bic v25.16b, v17.16b, v16.16b - bic v26.16b, v18.16b, v17.16b - bic v27.16b, v19.16b, v18.16b - bic v28.16b, v15.16b, v19.16b - bic v29.16b, v16.16b, v15.16b - eor v15.16b, v15.16b, v25.16b - eor v16.16b, v16.16b, v26.16b - eor v17.16b, v17.16b, v27.16b - eor v18.16b, v18.16b, v28.16b - eor v19.16b, v19.16b, v29.16b - bic v25.16b, v22.16b, v21.16b - bic v26.16b, v23.16b, v22.16b - bic v27.16b, v24.16b, v23.16b - bic v28.16b, v20.16b, v24.16b - bic v29.16b, v21.16b, v20.16b - eor v20.16b, v20.16b, v25.16b - eor v21.16b, v21.16b, v26.16b - eor v22.16b, v22.16b, v27.16b - eor v23.16b, v23.16b, v28.16b - eor v24.16b, v24.16b, v29.16b - # Done transforming - ldp x27, x28, [x29, #48] - ldr x0, [x27], #8 - subs x28, x28, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 - eor x1, x1, x0 - eor v0.16b, v0.16b, v30.16b - bne L_SHA3_transform_blocksx3_neon_begin - ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] - add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] - add x0, x0, #8 - stp x1, x2, [x0] - stp x3, x4, [x0, #16] - stp x5, x6, [x0, #32] - stp x7, x8, [x0, #48] - stp x9, x10, [x0, #64] - stp x11, x12, [x0, #80] - stp x13, x14, [x0, #96] - stp x15, x16, [x0, #112] - stp x17, x19, [x0, #128] - stp x20, x21, [x0, #144] - stp x22, x23, [x0, #160] - stp x24, x25, [x0, #176] - str x26, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret -#ifndef __APPLE__ - .size mlkem_sha3_blocksx3_neon,.-mlkem_sha3_blocksx3_neon -#endif /* __APPLE__ */ -#ifndef __APPLE__ -.text -.globl mlkem_shake128_blocksx3_seed_neon -WC_ASM_ATT_HIDDEN(mlkem_shake128_blocksx3_seed_neon) -.type mlkem_shake128_blocksx3_seed_neon,@function -.align 2 -mlkem_shake128_blocksx3_seed_neon: -#else -.section __TEXT,__text -.globl _mlkem_shake128_blocksx3_seed_neon -WC_ASM_ATT_HIDDEN(_mlkem_shake128_blocksx3_seed_neon) -.p2align 2 -_mlkem_shake128_blocksx3_seed_neon: -#endif /* __APPLE__ */ - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] -#ifndef __APPLE__ - adrp x28, L_sha3_aarch64_r - add x28, x28, :lo12:L_sha3_aarch64_r -#else - adrp x28, L_sha3_aarch64_r@PAGE - add x28, x28, L_sha3_aarch64_r@PAGEOFF -#endif /* __APPLE__ */ - str x0, [x29, #40] - add x0, x0, #32 - ld1 {v4.d}[0], [x0] - ldp x2, x3, [x1], #16 - add x0, x0, #0xc8 - ld1 {v4.d}[1], [x0] - ldp x4, x5, [x1], #16 - ldr x6, [x0, #200] - eor v5.16b, v5.16b, v5.16b - eor x7, x7, x7 - eor v6.16b, v6.16b, v6.16b - eor x8, x8, x8 - eor v7.16b, v7.16b, v7.16b - eor x9, x9, x9 - eor v8.16b, v8.16b, v8.16b - eor x10, x10, x10 - eor v9.16b, v9.16b, v9.16b - eor x11, x11, x11 - eor v10.16b, v10.16b, v10.16b - eor x12, x12, x12 - eor v11.16b, v11.16b, v11.16b - eor x13, x13, x13 - eor v12.16b, v12.16b, v12.16b - eor x14, x14, x14 - eor v13.16b, v13.16b, v13.16b - eor x15, x15, x15 - eor v14.16b, v14.16b, v14.16b - eor x16, x16, x16 - eor v15.16b, v15.16b, v15.16b - eor x17, x17, x17 - eor v16.16b, v16.16b, v16.16b - eor x19, x19, x19 - eor v17.16b, v17.16b, v17.16b - eor x20, x20, x20 - eor v18.16b, v18.16b, v18.16b - eor x21, x21, x21 - eor v19.16b, v19.16b, v19.16b - eor x22, x22, x22 - movz x23, #0x8000, lsl 48 - eor v21.16b, v21.16b, v21.16b - eor x24, x24, x24 - eor v22.16b, v22.16b, v22.16b - eor x25, x25, x25 - eor v23.16b, v23.16b, v23.16b - eor x26, x26, x26 - eor v24.16b, v24.16b, v24.16b - eor x27, x27, x27 - dup v0.2d, x2 - dup v1.2d, x3 - dup v2.2d, x4 - dup v3.2d, x5 - dup v20.2d, x23 - mov x1, #24 - # Start of 24 rounds -L_SHA3_shake128_blocksx3_seed_neon_begin: - stp x28, x1, [x29, #48] - # Col Mix NEON - eor v30.16b, v4.16b, v9.16b - eor x0, x6, x11 - eor v27.16b, v1.16b, v6.16b - eor x30, x2, x7 - eor v30.16b, v30.16b, v14.16b - eor x28, x4, x9 - eor v27.16b, v27.16b, v11.16b - eor x0, x0, x16 - eor v30.16b, v30.16b, v19.16b - eor x30, x30, x12 - eor v27.16b, v27.16b, v16.16b - eor x28, x28, x14 - eor v30.16b, v30.16b, v24.16b - eor x0, x0, x22 - eor v27.16b, v27.16b, v21.16b - eor x30, x30, x17 - ushr v25.2d, v27.2d, #63 - eor x28, x28, x20 - sli v25.2d, v27.2d, #1 - eor x0, x0, x27 - eor v25.16b, v25.16b, v30.16b - eor x30, x30, x23 - eor v31.16b, v0.16b, v5.16b - eor x28, x28, x25 - eor v28.16b, v2.16b, v7.16b - str x0, [x29, #32] - eor v31.16b, v31.16b, v10.16b - str x28, [x29, #24] - eor v28.16b, v28.16b, v12.16b - eor x1, x3, x8 - eor v31.16b, v31.16b, v15.16b - eor x28, x5, x10 - eor v28.16b, v28.16b, v17.16b - eor x1, x1, x13 - eor v31.16b, v31.16b, v20.16b - eor x28, x28, x15 - eor v28.16b, v28.16b, v22.16b - eor x1, x1, x19 - ushr v29.2d, v30.2d, #63 - eor x28, x28, x21 - ushr v26.2d, v28.2d, #63 - eor x1, x1, x24 - sli v29.2d, v30.2d, #1 - eor x28, x28, x26 - sli v26.2d, v28.2d, #1 - eor x0, x0, x1, ror 63 - eor v28.16b, v28.16b, v29.16b - eor x1, x1, x28, ror 63 - eor v29.16b, v3.16b, v8.16b - eor x2, x2, x0 - eor v26.16b, v26.16b, v31.16b - eor x7, x7, x0 - eor v29.16b, v29.16b, v13.16b - eor x12, x12, x0 - eor v29.16b, v29.16b, v18.16b - eor x17, x17, x0 - eor v29.16b, v29.16b, v23.16b - eor x23, x23, x0 - ushr v30.2d, v29.2d, #63 - eor x4, x4, x1 - sli v30.2d, v29.2d, #1 - eor x9, x9, x1 - eor v27.16b, v27.16b, v30.16b - eor x14, x14, x1 - ushr v30.2d, v31.2d, #63 - eor x20, x20, x1 - sli v30.2d, v31.2d, #1 - eor x25, x25, x1 - eor v29.16b, v29.16b, v30.16b - ldr x0, [x29, #32] - # Swap Rotate NEON - eor v0.16b, v0.16b, v25.16b - eor v31.16b, v1.16b, v26.16b - ldr x1, [x29, #24] - eor v6.16b, v6.16b, v26.16b - eor x28, x28, x30, ror 63 - ushr v30.2d, v31.2d, #63 - eor x30, x30, x1, ror 63 - ushr v1.2d, v6.2d, #20 - eor x1, x1, x0, ror 63 - sli v30.2d, v31.2d, #1 - eor x6, x6, x28 - sli v1.2d, v6.2d, #44 - eor x11, x11, x28 - eor v31.16b, v9.16b, v29.16b - eor x16, x16, x28 - eor v22.16b, v22.16b, v27.16b - eor x22, x22, x28 - ushr v6.2d, v31.2d, #44 - eor x27, x27, x28 - ushr v9.2d, v22.2d, #3 - eor x3, x3, x30 - sli v6.2d, v31.2d, #20 - eor x8, x8, x30 - sli v9.2d, v22.2d, #61 - eor x13, x13, x30 - eor v31.16b, v14.16b, v29.16b - eor x19, x19, x30 - eor v20.16b, v20.16b, v25.16b - eor x24, x24, x30 - ushr v22.2d, v31.2d, #25 - eor x5, x5, x1 - ushr v14.2d, v20.2d, #46 - eor x10, x10, x1 - sli v22.2d, v31.2d, #39 - eor x15, x15, x1 - sli v14.2d, v20.2d, #18 - eor x21, x21, x1 - eor v31.16b, v2.16b, v27.16b - eor x26, x26, x1 - # Swap Rotate Base - eor v12.16b, v12.16b, v27.16b - ror x0, x3, #63 - ushr v20.2d, v31.2d, #2 - ror x3, x8, #20 - ushr v2.2d, v12.2d, #21 - ror x8, x11, #44 - sli v20.2d, v31.2d, #62 - ror x11, x25, #3 - sli v2.2d, v12.2d, #43 - ror x25, x16, #25 - eor v31.16b, v13.16b, v28.16b - ror x16, x23, #46 - eor v19.16b, v19.16b, v29.16b - ror x23, x4, #2 - ushr v12.2d, v31.2d, #39 - ror x4, x14, #21 - ushr v13.2d, v19.2d, #56 - ror x14, x15, #39 - sli v12.2d, v31.2d, #25 - ror x15, x22, #56 - sli v13.2d, v19.2d, #8 - ror x22, x26, #8 - eor v31.16b, v23.16b, v28.16b - ror x26, x17, #23 - eor v15.16b, v15.16b, v25.16b - ror x17, x6, #37 - ushr v19.2d, v31.2d, #8 - ror x6, x27, #50 - ushr v23.2d, v15.2d, #23 - ror x27, x24, #62 - sli v19.2d, v31.2d, #56 - ror x24, x10, #9 - sli v23.2d, v15.2d, #41 - ror x10, x19, #19 - eor v31.16b, v4.16b, v29.16b - ror x19, x7, #28 - eor v24.16b, v24.16b, v29.16b - ror x7, x5, #36 - ushr v15.2d, v31.2d, #37 - ror x5, x21, #43 - ushr v4.2d, v24.2d, #50 - ror x21, x20, #49 - sli v15.2d, v31.2d, #27 - ror x20, x13, #54 - sli v4.2d, v24.2d, #14 - ror x13, x9, #58 - eor v31.16b, v21.16b, v26.16b - ror x9, x12, #61 - # Row Mix Base - eor v8.16b, v8.16b, v28.16b - bic x12, x4, x3 - ushr v24.2d, v31.2d, #62 - bic x1, x5, x4 - ushr v21.2d, v8.2d, #9 - bic x28, x2, x6 - sli v24.2d, v31.2d, #2 - bic x30, x3, x2 - sli v21.2d, v8.2d, #55 - eor x2, x2, x12 - eor v31.16b, v16.16b, v26.16b - eor x3, x3, x1 - eor v5.16b, v5.16b, v25.16b - bic x12, x6, x5 - ushr v8.2d, v31.2d, #19 - eor x5, x5, x28 - ushr v16.2d, v5.2d, #28 - eor x4, x4, x12 - sli v8.2d, v31.2d, #45 - eor x6, x6, x30 - sli v16.2d, v5.2d, #36 - bic x12, x9, x8 - eor v31.16b, v3.16b, v28.16b - bic x1, x10, x9 - eor v18.16b, v18.16b, v28.16b - bic x28, x7, x11 - ushr v5.2d, v31.2d, #36 - bic x30, x8, x7 - ushr v3.2d, v18.2d, #43 - eor x7, x7, x12 - sli v5.2d, v31.2d, #28 - eor x8, x8, x1 - sli v3.2d, v18.2d, #21 - bic x12, x11, x10 - eor v31.16b, v17.16b, v27.16b - eor x10, x10, x28 - eor v11.16b, v11.16b, v26.16b - eor x9, x9, x12 - ushr v18.2d, v31.2d, #49 - eor x11, x11, x30 - ushr v17.2d, v11.2d, #54 - bic x12, x14, x13 - sli v18.2d, v31.2d, #15 - bic x1, x15, x14 - sli v17.2d, v11.2d, #10 - bic x28, x0, x16 - eor v31.16b, v7.16b, v27.16b - bic x30, x13, x0 - eor v10.16b, v10.16b, v25.16b - eor x12, x0, x12 - ushr v11.2d, v31.2d, #58 - eor x13, x13, x1 - ushr v7.2d, v10.2d, #61 - bic x0, x16, x15 - sli v11.2d, v31.2d, #6 - eor x15, x15, x28 - sli v7.2d, v10.2d, #3 - eor x14, x14, x0 - # Row Mix NEON - bic v25.16b, v2.16b, v1.16b - eor x16, x16, x30 - bic v26.16b, v3.16b, v2.16b - bic x0, x20, x19 - bic v27.16b, v4.16b, v3.16b - bic x1, x21, x20 - bic v28.16b, v0.16b, v4.16b - bic x28, x17, x22 - bic v29.16b, v1.16b, v0.16b - bic x30, x19, x17 - eor v0.16b, v0.16b, v25.16b - eor x17, x17, x0 - eor v1.16b, v1.16b, v26.16b - eor x19, x19, x1 - eor v2.16b, v2.16b, v27.16b - bic x0, x22, x21 - eor v3.16b, v3.16b, v28.16b - eor x21, x21, x28 - eor v4.16b, v4.16b, v29.16b - eor x20, x20, x0 - bic v25.16b, v7.16b, v6.16b - eor x22, x22, x30 - bic v26.16b, v8.16b, v7.16b - bic x0, x25, x24 - bic v27.16b, v9.16b, v8.16b - bic x1, x26, x25 - bic v28.16b, v5.16b, v9.16b - bic x28, x23, x27 - bic v29.16b, v6.16b, v5.16b - bic x30, x24, x23 - eor v5.16b, v5.16b, v25.16b - eor x23, x23, x0 - eor v6.16b, v6.16b, v26.16b - eor x24, x24, x1 - eor v7.16b, v7.16b, v27.16b - bic x0, x27, x26 - eor v8.16b, v8.16b, v28.16b - eor x26, x26, x28 - eor v9.16b, v9.16b, v29.16b - eor x25, x25, x0 - bic v25.16b, v12.16b, v11.16b - eor x27, x27, x30 - bic v26.16b, v13.16b, v12.16b - bic v27.16b, v14.16b, v13.16b - bic v28.16b, v30.16b, v14.16b - bic v29.16b, v11.16b, v30.16b - eor v10.16b, v30.16b, v25.16b - eor v11.16b, v11.16b, v26.16b - eor v12.16b, v12.16b, v27.16b - eor v13.16b, v13.16b, v28.16b - eor v14.16b, v14.16b, v29.16b - bic v25.16b, v17.16b, v16.16b - bic v26.16b, v18.16b, v17.16b - bic v27.16b, v19.16b, v18.16b - bic v28.16b, v15.16b, v19.16b - bic v29.16b, v16.16b, v15.16b - eor v15.16b, v15.16b, v25.16b - eor v16.16b, v16.16b, v26.16b - eor v17.16b, v17.16b, v27.16b - eor v18.16b, v18.16b, v28.16b - eor v19.16b, v19.16b, v29.16b - bic v25.16b, v22.16b, v21.16b - bic v26.16b, v23.16b, v22.16b - bic v27.16b, v24.16b, v23.16b - bic v28.16b, v20.16b, v24.16b - bic v29.16b, v21.16b, v20.16b - eor v20.16b, v20.16b, v25.16b - eor v21.16b, v21.16b, v26.16b - eor v22.16b, v22.16b, v27.16b - eor v23.16b, v23.16b, v28.16b - eor v24.16b, v24.16b, v29.16b - # Done transforming - ldp x28, x1, [x29, #48] - ldr x0, [x28], #8 - subs x1, x1, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 - eor x2, x2, x0 - eor v0.16b, v0.16b, v30.16b - bne L_SHA3_shake128_blocksx3_seed_neon_begin - ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] - add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] - add x0, x0, #8 - stp x2, x3, [x0] - stp x4, x5, [x0, #16] - stp x6, x7, [x0, #32] - stp x8, x9, [x0, #48] - stp x10, x11, [x0, #64] - stp x12, x13, [x0, #80] - stp x14, x15, [x0, #96] - stp x16, x17, [x0, #112] - stp x19, x20, [x0, #128] - stp x21, x22, [x0, #144] - stp x23, x24, [x0, #160] - stp x25, x26, [x0, #176] - str x27, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret -#ifndef __APPLE__ - .size mlkem_shake128_blocksx3_seed_neon,.-mlkem_shake128_blocksx3_seed_neon -#endif /* __APPLE__ */ -#ifndef __APPLE__ -.text -.globl mlkem_shake256_blocksx3_seed_neon -WC_ASM_ATT_HIDDEN(mlkem_shake256_blocksx3_seed_neon) -.type mlkem_shake256_blocksx3_seed_neon,@function -.align 2 -mlkem_shake256_blocksx3_seed_neon: -#else -.section __TEXT,__text -.globl _mlkem_shake256_blocksx3_seed_neon -WC_ASM_ATT_HIDDEN(_mlkem_shake256_blocksx3_seed_neon) -.p2align 2 -_mlkem_shake256_blocksx3_seed_neon: -#endif /* __APPLE__ */ - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp d8, d9, [x29, #160] - stp d10, d11, [x29, #176] - stp d12, d13, [x29, #192] - stp d14, d15, [x29, #208] -#ifndef __APPLE__ - adrp x28, L_sha3_aarch64_r - add x28, x28, :lo12:L_sha3_aarch64_r -#else - adrp x28, L_sha3_aarch64_r@PAGE - add x28, x28, L_sha3_aarch64_r@PAGEOFF -#endif /* __APPLE__ */ - str x0, [x29, #40] - add x0, x0, #32 - ld1 {v4.d}[0], [x0] - ldp x2, x3, [x1], #16 - add x0, x0, #0xc8 - ld1 {v4.d}[1], [x0] - ldp x4, x5, [x1], #16 - ldr x6, [x0, #200] - eor v5.16b, v5.16b, v5.16b - eor x7, x7, x7 - eor v6.16b, v6.16b, v6.16b - eor x8, x8, x8 - eor v7.16b, v7.16b, v7.16b - eor x9, x9, x9 - eor v8.16b, v8.16b, v8.16b - eor x10, x10, x10 - eor v9.16b, v9.16b, v9.16b - eor x11, x11, x11 - eor v10.16b, v10.16b, v10.16b - eor x12, x12, x12 - eor v11.16b, v11.16b, v11.16b - eor x13, x13, x13 - eor v12.16b, v12.16b, v12.16b - eor x14, x14, x14 - eor v13.16b, v13.16b, v13.16b - eor x15, x15, x15 - eor v14.16b, v14.16b, v14.16b - eor x16, x16, x16 - eor v15.16b, v15.16b, v15.16b - eor x17, x17, x17 - movz x19, #0x8000, lsl 48 - eor v17.16b, v17.16b, v17.16b - eor x20, x20, x20 - eor v18.16b, v18.16b, v18.16b - eor x21, x21, x21 - eor v19.16b, v19.16b, v19.16b - eor x22, x22, x22 - eor v20.16b, v20.16b, v20.16b - eor x23, x23, x23 - eor v21.16b, v21.16b, v21.16b - eor x24, x24, x24 - eor v22.16b, v22.16b, v22.16b - eor x25, x25, x25 - eor v23.16b, v23.16b, v23.16b - eor x26, x26, x26 - eor v24.16b, v24.16b, v24.16b - eor x27, x27, x27 - dup v0.2d, x2 - dup v1.2d, x3 - dup v2.2d, x4 - dup v3.2d, x5 - dup v16.2d, x19 - mov x1, #24 - # Start of 24 rounds -L_SHA3_shake256_blocksx3_seed_neon_begin: - stp x28, x1, [x29, #48] - # Col Mix NEON - eor v30.16b, v4.16b, v9.16b - eor x0, x6, x11 - eor v27.16b, v1.16b, v6.16b - eor x30, x2, x7 - eor v30.16b, v30.16b, v14.16b - eor x28, x4, x9 - eor v27.16b, v27.16b, v11.16b - eor x0, x0, x16 - eor v30.16b, v30.16b, v19.16b - eor x30, x30, x12 - eor v27.16b, v27.16b, v16.16b - eor x28, x28, x14 - eor v30.16b, v30.16b, v24.16b - eor x0, x0, x22 - eor v27.16b, v27.16b, v21.16b - eor x30, x30, x17 - ushr v25.2d, v27.2d, #63 - eor x28, x28, x20 - sli v25.2d, v27.2d, #1 - eor x0, x0, x27 - eor v25.16b, v25.16b, v30.16b - eor x30, x30, x23 - eor v31.16b, v0.16b, v5.16b - eor x28, x28, x25 - eor v28.16b, v2.16b, v7.16b - str x0, [x29, #32] - eor v31.16b, v31.16b, v10.16b - str x28, [x29, #24] - eor v28.16b, v28.16b, v12.16b - eor x1, x3, x8 - eor v31.16b, v31.16b, v15.16b - eor x28, x5, x10 - eor v28.16b, v28.16b, v17.16b - eor x1, x1, x13 - eor v31.16b, v31.16b, v20.16b - eor x28, x28, x15 - eor v28.16b, v28.16b, v22.16b - eor x1, x1, x19 - ushr v29.2d, v30.2d, #63 - eor x28, x28, x21 - ushr v26.2d, v28.2d, #63 - eor x1, x1, x24 - sli v29.2d, v30.2d, #1 - eor x28, x28, x26 - sli v26.2d, v28.2d, #1 - eor x0, x0, x1, ror 63 - eor v28.16b, v28.16b, v29.16b - eor x1, x1, x28, ror 63 - eor v29.16b, v3.16b, v8.16b - eor x2, x2, x0 - eor v26.16b, v26.16b, v31.16b - eor x7, x7, x0 - eor v29.16b, v29.16b, v13.16b - eor x12, x12, x0 - eor v29.16b, v29.16b, v18.16b - eor x17, x17, x0 - eor v29.16b, v29.16b, v23.16b - eor x23, x23, x0 - ushr v30.2d, v29.2d, #63 - eor x4, x4, x1 - sli v30.2d, v29.2d, #1 - eor x9, x9, x1 - eor v27.16b, v27.16b, v30.16b - eor x14, x14, x1 - ushr v30.2d, v31.2d, #63 - eor x20, x20, x1 - sli v30.2d, v31.2d, #1 - eor x25, x25, x1 - eor v29.16b, v29.16b, v30.16b - ldr x0, [x29, #32] - # Swap Rotate NEON - eor v0.16b, v0.16b, v25.16b - eor v31.16b, v1.16b, v26.16b - ldr x1, [x29, #24] - eor v6.16b, v6.16b, v26.16b - eor x28, x28, x30, ror 63 - ushr v30.2d, v31.2d, #63 - eor x30, x30, x1, ror 63 - ushr v1.2d, v6.2d, #20 - eor x1, x1, x0, ror 63 - sli v30.2d, v31.2d, #1 - eor x6, x6, x28 - sli v1.2d, v6.2d, #44 - eor x11, x11, x28 - eor v31.16b, v9.16b, v29.16b - eor x16, x16, x28 - eor v22.16b, v22.16b, v27.16b - eor x22, x22, x28 - ushr v6.2d, v31.2d, #44 - eor x27, x27, x28 - ushr v9.2d, v22.2d, #3 - eor x3, x3, x30 - sli v6.2d, v31.2d, #20 - eor x8, x8, x30 - sli v9.2d, v22.2d, #61 - eor x13, x13, x30 - eor v31.16b, v14.16b, v29.16b - eor x19, x19, x30 - eor v20.16b, v20.16b, v25.16b - eor x24, x24, x30 - ushr v22.2d, v31.2d, #25 - eor x5, x5, x1 - ushr v14.2d, v20.2d, #46 - eor x10, x10, x1 - sli v22.2d, v31.2d, #39 - eor x15, x15, x1 - sli v14.2d, v20.2d, #18 - eor x21, x21, x1 - eor v31.16b, v2.16b, v27.16b - eor x26, x26, x1 - # Swap Rotate Base - eor v12.16b, v12.16b, v27.16b - ror x0, x3, #63 - ushr v20.2d, v31.2d, #2 - ror x3, x8, #20 - ushr v2.2d, v12.2d, #21 - ror x8, x11, #44 - sli v20.2d, v31.2d, #62 - ror x11, x25, #3 - sli v2.2d, v12.2d, #43 - ror x25, x16, #25 - eor v31.16b, v13.16b, v28.16b - ror x16, x23, #46 - eor v19.16b, v19.16b, v29.16b - ror x23, x4, #2 - ushr v12.2d, v31.2d, #39 - ror x4, x14, #21 - ushr v13.2d, v19.2d, #56 - ror x14, x15, #39 - sli v12.2d, v31.2d, #25 - ror x15, x22, #56 - sli v13.2d, v19.2d, #8 - ror x22, x26, #8 - eor v31.16b, v23.16b, v28.16b - ror x26, x17, #23 - eor v15.16b, v15.16b, v25.16b - ror x17, x6, #37 - ushr v19.2d, v31.2d, #8 - ror x6, x27, #50 - ushr v23.2d, v15.2d, #23 - ror x27, x24, #62 - sli v19.2d, v31.2d, #56 - ror x24, x10, #9 - sli v23.2d, v15.2d, #41 - ror x10, x19, #19 - eor v31.16b, v4.16b, v29.16b - ror x19, x7, #28 - eor v24.16b, v24.16b, v29.16b - ror x7, x5, #36 - ushr v15.2d, v31.2d, #37 - ror x5, x21, #43 - ushr v4.2d, v24.2d, #50 - ror x21, x20, #49 - sli v15.2d, v31.2d, #27 - ror x20, x13, #54 - sli v4.2d, v24.2d, #14 - ror x13, x9, #58 - eor v31.16b, v21.16b, v26.16b - ror x9, x12, #61 - # Row Mix Base - eor v8.16b, v8.16b, v28.16b - bic x12, x4, x3 - ushr v24.2d, v31.2d, #62 - bic x1, x5, x4 - ushr v21.2d, v8.2d, #9 - bic x28, x2, x6 - sli v24.2d, v31.2d, #2 - bic x30, x3, x2 - sli v21.2d, v8.2d, #55 - eor x2, x2, x12 - eor v31.16b, v16.16b, v26.16b - eor x3, x3, x1 - eor v5.16b, v5.16b, v25.16b - bic x12, x6, x5 - ushr v8.2d, v31.2d, #19 - eor x5, x5, x28 - ushr v16.2d, v5.2d, #28 - eor x4, x4, x12 - sli v8.2d, v31.2d, #45 - eor x6, x6, x30 - sli v16.2d, v5.2d, #36 - bic x12, x9, x8 - eor v31.16b, v3.16b, v28.16b - bic x1, x10, x9 - eor v18.16b, v18.16b, v28.16b - bic x28, x7, x11 - ushr v5.2d, v31.2d, #36 - bic x30, x8, x7 - ushr v3.2d, v18.2d, #43 - eor x7, x7, x12 - sli v5.2d, v31.2d, #28 - eor x8, x8, x1 - sli v3.2d, v18.2d, #21 - bic x12, x11, x10 - eor v31.16b, v17.16b, v27.16b - eor x10, x10, x28 - eor v11.16b, v11.16b, v26.16b - eor x9, x9, x12 - ushr v18.2d, v31.2d, #49 - eor x11, x11, x30 - ushr v17.2d, v11.2d, #54 - bic x12, x14, x13 - sli v18.2d, v31.2d, #15 - bic x1, x15, x14 - sli v17.2d, v11.2d, #10 - bic x28, x0, x16 - eor v31.16b, v7.16b, v27.16b - bic x30, x13, x0 - eor v10.16b, v10.16b, v25.16b - eor x12, x0, x12 - ushr v11.2d, v31.2d, #58 - eor x13, x13, x1 - ushr v7.2d, v10.2d, #61 - bic x0, x16, x15 - sli v11.2d, v31.2d, #6 - eor x15, x15, x28 - sli v7.2d, v10.2d, #3 - eor x14, x14, x0 - # Row Mix NEON - bic v25.16b, v2.16b, v1.16b - eor x16, x16, x30 - bic v26.16b, v3.16b, v2.16b - bic x0, x20, x19 - bic v27.16b, v4.16b, v3.16b - bic x1, x21, x20 - bic v28.16b, v0.16b, v4.16b - bic x28, x17, x22 - bic v29.16b, v1.16b, v0.16b - bic x30, x19, x17 - eor v0.16b, v0.16b, v25.16b - eor x17, x17, x0 - eor v1.16b, v1.16b, v26.16b - eor x19, x19, x1 - eor v2.16b, v2.16b, v27.16b - bic x0, x22, x21 - eor v3.16b, v3.16b, v28.16b - eor x21, x21, x28 - eor v4.16b, v4.16b, v29.16b - eor x20, x20, x0 - bic v25.16b, v7.16b, v6.16b - eor x22, x22, x30 - bic v26.16b, v8.16b, v7.16b - bic x0, x25, x24 - bic v27.16b, v9.16b, v8.16b - bic x1, x26, x25 - bic v28.16b, v5.16b, v9.16b - bic x28, x23, x27 - bic v29.16b, v6.16b, v5.16b - bic x30, x24, x23 - eor v5.16b, v5.16b, v25.16b - eor x23, x23, x0 - eor v6.16b, v6.16b, v26.16b - eor x24, x24, x1 - eor v7.16b, v7.16b, v27.16b - bic x0, x27, x26 - eor v8.16b, v8.16b, v28.16b - eor x26, x26, x28 - eor v9.16b, v9.16b, v29.16b - eor x25, x25, x0 - bic v25.16b, v12.16b, v11.16b - eor x27, x27, x30 - bic v26.16b, v13.16b, v12.16b - bic v27.16b, v14.16b, v13.16b - bic v28.16b, v30.16b, v14.16b - bic v29.16b, v11.16b, v30.16b - eor v10.16b, v30.16b, v25.16b - eor v11.16b, v11.16b, v26.16b - eor v12.16b, v12.16b, v27.16b - eor v13.16b, v13.16b, v28.16b - eor v14.16b, v14.16b, v29.16b - bic v25.16b, v17.16b, v16.16b - bic v26.16b, v18.16b, v17.16b - bic v27.16b, v19.16b, v18.16b - bic v28.16b, v15.16b, v19.16b - bic v29.16b, v16.16b, v15.16b - eor v15.16b, v15.16b, v25.16b - eor v16.16b, v16.16b, v26.16b - eor v17.16b, v17.16b, v27.16b - eor v18.16b, v18.16b, v28.16b - eor v19.16b, v19.16b, v29.16b - bic v25.16b, v22.16b, v21.16b - bic v26.16b, v23.16b, v22.16b - bic v27.16b, v24.16b, v23.16b - bic v28.16b, v20.16b, v24.16b - bic v29.16b, v21.16b, v20.16b - eor v20.16b, v20.16b, v25.16b - eor v21.16b, v21.16b, v26.16b - eor v22.16b, v22.16b, v27.16b - eor v23.16b, v23.16b, v28.16b - eor v24.16b, v24.16b, v29.16b - # Done transforming - ldp x28, x1, [x29, #48] - ldr x0, [x28], #8 - subs x1, x1, #1 - mov v30.d[0], x0 - mov v30.d[1], x0 - eor x2, x2, x0 - eor v0.16b, v0.16b, v30.16b - bne L_SHA3_shake256_blocksx3_seed_neon_begin - ldr x0, [x29, #40] - st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 - st1 {v24.d}[0], [x0] - add x0, x0, #8 - st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 - st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 - st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 - st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 - st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 - st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 - st1 {v24.d}[1], [x0] - add x0, x0, #8 - stp x2, x3, [x0] - stp x4, x5, [x0, #16] - stp x6, x7, [x0, #32] - stp x8, x9, [x0, #48] - stp x10, x11, [x0, #64] - stp x12, x13, [x0, #80] - stp x14, x15, [x0, #96] - stp x16, x17, [x0, #112] - stp x19, x20, [x0, #128] - stp x21, x22, [x0, #144] - stp x23, x24, [x0, #160] - stp x25, x26, [x0, #176] - str x27, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp d8, d9, [x29, #160] - ldp d10, d11, [x29, #176] - ldp d12, d13, [x29, #192] - ldp d14, d15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret -#ifndef __APPLE__ - .size mlkem_shake256_blocksx3_seed_neon,.-mlkem_shake256_blocksx3_seed_neon -#endif /* __APPLE__ */ #endif /* WOLFSSL_HAVE_MLKEM */ #endif /* __aarch64__ */ #endif /* WOLFSSL_ARMASM */ diff --git a/wolfcrypt/src/port/arm/armv8-mlkem-asm.asm b/wolfcrypt/src/port/arm/armv8-mlkem-asm.asm index d72216929d1..cec02a19d8d 100644 --- a/wolfcrypt/src/port/arm/armv8-mlkem-asm.asm +++ b/wolfcrypt/src/port/arm/armv8-mlkem-asm.asm @@ -9200,2225 +9200,5 @@ L_mlkem_rej_uniform_done ldp x29, x30, [sp], #0x40 ret ENDP - AREA |.rodata|, DATA, READONLY, ALIGN=4 - ALIGN 16 -L_sha3_aarch64_r - DCQ 0x0000000000000001, 0x0000000000008082 - DCQ 0x800000000000808a, 0x8000000080008000 - DCQ 0x000000000000808b, 0x0000000080000001 - DCQ 0x8000000080008081, 0x8000000000008009 - DCQ 0x000000000000008a, 0x0000000000000088 - DCQ 0x0000000080008009, 0x000000008000000a - DCQ 0x000000008000808b, 0x800000000000008b - DCQ 0x8000000000008089, 0x8000000000008003 - DCQ 0x8000000000008002, 0x8000000000000080 - DCQ 0x000000000000800a, 0x800000008000000a - DCQ 0x8000000080008081, 0x8000000000008080 - DCQ 0x0000000080000001, 0x8000000080008008 - IF :DEF:WOLFSSL_ARMASM_CRYPTO_SHA3 - AREA |.text|, CODE, READONLY - ALIGN 4 - EXPORT mlkem_sha3_blocksx3_crypto -mlkem_sha3_blocksx3_crypto PROC - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp D8, D9, [x29, #160] - stp D10, D11, [x29, #176] - stp D12, D13, [x29, #192] - stp D14, D15, [x29, #208] - adrp x27, L_sha3_aarch64_r - add x27, x27, L_sha3_aarch64_r - str x0, [x29, #40] - ld4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 - ld4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 - ld4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 - ld4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 - ld4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 - ld4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 - ld1 {V24.D}[0], [x0] - add x0, x0, #8 - ld4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 - ld4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 - ld4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 - ld4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 - ld4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 - ld4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 - ld1 {V24.D}[1], [x0] - add x0, x0, #8 - ldp x1, x2, [x0] - ldp x3, x4, [x0, #16] - ldp x5, x6, [x0, #32] - ldp x7, x8, [x0, #48] - ldp x9, x10, [x0, #64] - ldp x11, x12, [x0, #80] - ldp x13, x14, [x0, #96] - ldp x15, x16, [x0, #112] - ldp x17, x19, [x0, #128] - ldp x20, x21, [x0, #144] - ldp x22, x23, [x0, #160] - ldp x24, x25, [x0, #176] - ldr x26, [x0, #192] - mov x28, #24 - ; Start of 24 rounds -L_SHA3_transform_blocksx3_crypto_begin - stp x27, x28, [x29, #48] - ; Col Mix - eor3 V31.16B, V0.16B, V5.16B, V10.16B - eor x0, x5, x10 - eor3 V27.16B, V1.16B, V6.16B, V11.16B - eor x30, x1, x6 - eor3 V28.16B, V2.16B, V7.16B, V12.16B - eor x28, x3, x8 - eor3 V29.16B, V3.16B, V8.16B, V13.16B - eor x0, x0, x15 - eor3 V30.16B, V4.16B, V9.16B, V14.16B - eor x30, x30, x11 - eor3 V31.16B, V31.16B, V15.16B, V20.16B - eor x28, x28, x13 - eor3 V27.16B, V27.16B, V16.16B, V21.16B - eor x0, x0, x21 - eor3 V28.16B, V28.16B, V17.16B, V22.16B - eor x30, x30, x16 - eor3 V29.16B, V29.16B, V18.16B, V23.16B - eor x28, x28, x19 - eor3 V30.16B, V30.16B, V19.16B, V24.16B - eor x0, x0, x26 - rax1 V25.2D, V30.2D, V27.2D - eor x30, x30, x22 - rax1 V26.2D, V31.2D, V28.2D - eor x28, x28, x24 - rax1 V27.2D, V27.2D, V29.2D - str x0, [x29, #32] - rax1 V28.2D, V28.2D, V30.2D - str x28, [x29, #24] - rax1 V29.2D, V29.2D, V31.2D - eor x27, x2, x7 - eor V0.16B, V0.16B, V25.16B - xar V30.2D, V1.2D, V26.2D, #63 - eor x28, x4, x9 - xar V1.2D, V6.2D, V26.2D, #20 - eor x27, x27, x12 - xar V6.2D, V9.2D, V29.2D, #44 - eor x28, x28, x14 - xar V9.2D, V22.2D, V27.2D, #3 - eor x27, x27, x17 - xar V22.2D, V14.2D, V29.2D, #25 - eor x28, x28, x20 - xar V14.2D, V20.2D, V25.2D, #46 - eor x27, x27, x23 - xar V20.2D, V2.2D, V27.2D, #2 - eor x28, x28, x25 - xar V2.2D, V12.2D, V27.2D, #21 - eor x0, x0, x27, ror 63 - xar V12.2D, V13.2D, V28.2D, #39 - eor x27, x27, x28, ror 63 - xar V13.2D, V19.2D, V29.2D, #56 - eor x1, x1, x0 - xar V19.2D, V23.2D, V28.2D, #8 - eor x6, x6, x0 - xar V23.2D, V15.2D, V25.2D, #23 - eor x11, x11, x0 - xar V15.2D, V4.2D, V29.2D, #37 - eor x16, x16, x0 - xar V4.2D, V24.2D, V29.2D, #50 - eor x22, x22, x0 - xar V24.2D, V21.2D, V26.2D, #62 - eor x3, x3, x27 - xar V21.2D, V8.2D, V28.2D, #9 - eor x8, x8, x27 - xar V8.2D, V16.2D, V26.2D, #19 - eor x13, x13, x27 - xar V16.2D, V5.2D, V25.2D, #28 - eor x19, x19, x27 - xar V5.2D, V3.2D, V28.2D, #36 - eor x24, x24, x27 - xar V3.2D, V18.2D, V28.2D, #43 - ldr x0, [x29, #32] - xar V18.2D, V17.2D, V27.2D, #49 - ldr x27, [x29, #24] - xar V17.2D, V11.2D, V26.2D, #54 - eor x28, x28, x30, ror 63 - xar V11.2D, V7.2D, V27.2D, #58 - eor x30, x30, x27, ror 63 - xar V7.2D, V10.2D, V25.2D, #61 - eor x27, x27, x0, ror 63 - ; Row Mix - mov V25.16B, V0.16B - eor x5, x5, x28 - mov V26.16B, V1.16B - eor x10, x10, x28 - bcax V0.16B, V25.16B, V2.16B, V26.16B - eor x15, x15, x28 - bcax V1.16B, V26.16B, V3.16B, V2.16B - eor x21, x21, x28 - bcax V2.16B, V2.16B, V4.16B, V3.16B - eor x26, x26, x28 - bcax V3.16B, V3.16B, V25.16B, V4.16B - eor x2, x2, x30 - bcax V4.16B, V4.16B, V26.16B, V25.16B - eor x7, x7, x30 - mov V25.16B, V5.16B - eor x12, x12, x30 - mov V26.16B, V6.16B - eor x17, x17, x30 - bcax V5.16B, V25.16B, V7.16B, V26.16B - eor x23, x23, x30 - bcax V6.16B, V26.16B, V8.16B, V7.16B - eor x4, x4, x27 - bcax V7.16B, V7.16B, V9.16B, V8.16B - eor x9, x9, x27 - bcax V8.16B, V8.16B, V25.16B, V9.16B - eor x14, x14, x27 - bcax V9.16B, V9.16B, V26.16B, V25.16B - eor x20, x20, x27 - mov V26.16B, V11.16B - eor x25, x25, x27 - ; Swap Rotate Base - bcax V10.16B, V30.16B, V12.16B, V26.16B - ror x0, x2, #63 - bcax V11.16B, V26.16B, V13.16B, V12.16B - ror x2, x7, #20 - bcax V12.16B, V12.16B, V14.16B, V13.16B - ror x7, x10, #44 - bcax V13.16B, V13.16B, V30.16B, V14.16B - ror x10, x24, #3 - bcax V14.16B, V14.16B, V26.16B, V30.16B - ror x24, x15, #25 - mov V25.16B, V15.16B - ror x15, x22, #46 - mov V26.16B, V16.16B - ror x22, x3, #2 - bcax V15.16B, V25.16B, V17.16B, V26.16B - ror x3, x13, #21 - bcax V16.16B, V26.16B, V18.16B, V17.16B - ror x13, x14, #39 - bcax V17.16B, V17.16B, V19.16B, V18.16B - ror x14, x21, #56 - bcax V18.16B, V18.16B, V25.16B, V19.16B - ror x21, x25, #8 - bcax V19.16B, V19.16B, V26.16B, V25.16B - ror x25, x16, #23 - mov V25.16B, V20.16B - ror x16, x5, #37 - mov V26.16B, V21.16B - ror x5, x26, #50 - bcax V20.16B, V25.16B, V22.16B, V26.16B - ror x26, x23, #62 - bcax V21.16B, V26.16B, V23.16B, V22.16B - ror x23, x9, #9 - bcax V22.16B, V22.16B, V24.16B, V23.16B - ror x9, x17, #19 - bcax V23.16B, V23.16B, V25.16B, V24.16B - ror x17, x6, #28 - bcax V24.16B, V24.16B, V26.16B, V25.16B - ror x6, x4, #36 - ror x4, x20, #43 - ror x20, x19, #49 - ror x19, x12, #54 - ror x12, x8, #58 - ror x8, x11, #61 - ; Row Mix Base - bic x11, x3, x2 - bic x27, x4, x3 - bic x28, x1, x5 - bic x30, x2, x1 - eor x1, x1, x11 - eor x2, x2, x27 - bic x11, x5, x4 - eor x4, x4, x28 - eor x3, x3, x11 - eor x5, x5, x30 - bic x11, x8, x7 - bic x27, x9, x8 - bic x28, x6, x10 - bic x30, x7, x6 - eor x6, x6, x11 - eor x7, x7, x27 - bic x11, x10, x9 - eor x9, x9, x28 - eor x8, x8, x11 - eor x10, x10, x30 - bic x11, x13, x12 - bic x27, x14, x13 - bic x28, x0, x15 - bic x30, x12, x0 - eor x11, x0, x11 - eor x12, x12, x27 - bic x0, x15, x14 - eor x14, x14, x28 - eor x13, x13, x0 - eor x15, x15, x30 - bic x0, x19, x17 - bic x27, x20, x19 - bic x28, x16, x21 - bic x30, x17, x16 - eor x16, x16, x0 - eor x17, x17, x27 - bic x0, x21, x20 - eor x20, x20, x28 - eor x19, x19, x0 - eor x21, x21, x30 - bic x0, x24, x23 - bic x27, x25, x24 - bic x28, x22, x26 - bic x30, x23, x22 - eor x22, x22, x0 - eor x23, x23, x27 - bic x0, x26, x25 - eor x25, x25, x28 - eor x24, x24, x0 - eor x26, x26, x30 - ; Done transforming - ldp x27, x28, [x29, #48] - ldr x0, [x27], #8 - subs x28, x28, #1 - mov V30.D[0], x0 - mov V30.D[1], x0 - eor x1, x1, x0 - eor V0.16B, V0.16B, V30.16B - bne L_SHA3_transform_blocksx3_crypto_begin - ldr x0, [x29, #40] - st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 - st1 {V24.D}[0], [x0] - add x0, x0, #8 - st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 - st1 {V24.D}[1], [x0] - add x0, x0, #8 - stp x1, x2, [x0] - stp x3, x4, [x0, #16] - stp x5, x6, [x0, #32] - stp x7, x8, [x0, #48] - stp x9, x10, [x0, #64] - stp x11, x12, [x0, #80] - stp x13, x14, [x0, #96] - stp x15, x16, [x0, #112] - stp x17, x19, [x0, #128] - stp x20, x21, [x0, #144] - stp x22, x23, [x0, #160] - stp x24, x25, [x0, #176] - str x26, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp D8, D9, [x29, #160] - ldp D10, D11, [x29, #176] - ldp D12, D13, [x29, #192] - ldp D14, D15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret - ENDP - AREA |.text|, CODE, READONLY - ALIGN 4 - EXPORT mlkem_shake128_blocksx3_seed_crypto -mlkem_shake128_blocksx3_seed_crypto PROC - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp D8, D9, [x29, #160] - stp D10, D11, [x29, #176] - stp D12, D13, [x29, #192] - stp D14, D15, [x29, #208] - adrp x28, L_sha3_aarch64_r - add x28, x28, L_sha3_aarch64_r - str x0, [x29, #40] - add x0, x0, #32 - ld1 {V4.D}[0], [x0] - ldp x2, x3, [x1], #16 - add x0, x0, #0xc8 - ld1 {V4.D}[1], [x0] - ldp x4, x5, [x1], #16 - ldr x6, [x0, #200] - eor V5.16B, V5.16B, V5.16B - eor x7, x7, x7 - eor V6.16B, V6.16B, V6.16B - eor x8, x8, x8 - eor V7.16B, V7.16B, V7.16B - eor x9, x9, x9 - eor V8.16B, V8.16B, V8.16B - eor x10, x10, x10 - eor V9.16B, V9.16B, V9.16B - eor x11, x11, x11 - eor V10.16B, V10.16B, V10.16B - eor x12, x12, x12 - eor V11.16B, V11.16B, V11.16B - eor x13, x13, x13 - eor V12.16B, V12.16B, V12.16B - eor x14, x14, x14 - eor V13.16B, V13.16B, V13.16B - eor x15, x15, x15 - eor V14.16B, V14.16B, V14.16B - eor x16, x16, x16 - eor V15.16B, V15.16B, V15.16B - eor x17, x17, x17 - eor V16.16B, V16.16B, V16.16B - eor x19, x19, x19 - eor V17.16B, V17.16B, V17.16B - eor x20, x20, x20 - eor V18.16B, V18.16B, V18.16B - eor x21, x21, x21 - eor V19.16B, V19.16B, V19.16B - eor x22, x22, x22 - movz x23, #0x8000, lsl 48 - eor V21.16B, V21.16B, V21.16B - eor x24, x24, x24 - eor V22.16B, V22.16B, V22.16B - eor x25, x25, x25 - eor V23.16B, V23.16B, V23.16B - eor x26, x26, x26 - eor V24.16B, V24.16B, V24.16B - eor x27, x27, x27 - dup V0.2D, x2 - dup V1.2D, x3 - dup V2.2D, x4 - dup V3.2D, x5 - dup V20.2D, x23 - mov x1, #24 - ; Start of 24 rounds -L_SHA3_shake128_blocksx3_seed_crypto_begin - stp x28, x1, [x29, #48] - ; Col Mix - eor3 V31.16B, V0.16B, V5.16B, V10.16B - eor x0, x6, x11 - eor3 V27.16B, V1.16B, V6.16B, V11.16B - eor x30, x2, x7 - eor3 V28.16B, V2.16B, V7.16B, V12.16B - eor x28, x4, x9 - eor3 V29.16B, V3.16B, V8.16B, V13.16B - eor x0, x0, x16 - eor3 V30.16B, V4.16B, V9.16B, V14.16B - eor x30, x30, x12 - eor3 V31.16B, V31.16B, V15.16B, V20.16B - eor x28, x28, x14 - eor3 V27.16B, V27.16B, V16.16B, V21.16B - eor x0, x0, x22 - eor3 V28.16B, V28.16B, V17.16B, V22.16B - eor x30, x30, x17 - eor3 V29.16B, V29.16B, V18.16B, V23.16B - eor x28, x28, x20 - eor3 V30.16B, V30.16B, V19.16B, V24.16B - eor x0, x0, x27 - rax1 V25.2D, V30.2D, V27.2D - eor x30, x30, x23 - rax1 V26.2D, V31.2D, V28.2D - eor x28, x28, x25 - rax1 V27.2D, V27.2D, V29.2D - str x0, [x29, #32] - rax1 V28.2D, V28.2D, V30.2D - str x28, [x29, #24] - rax1 V29.2D, V29.2D, V31.2D - eor x1, x3, x8 - eor V0.16B, V0.16B, V25.16B - xar V30.2D, V1.2D, V26.2D, #63 - eor x28, x5, x10 - xar V1.2D, V6.2D, V26.2D, #20 - eor x1, x1, x13 - xar V6.2D, V9.2D, V29.2D, #44 - eor x28, x28, x15 - xar V9.2D, V22.2D, V27.2D, #3 - eor x1, x1, x19 - xar V22.2D, V14.2D, V29.2D, #25 - eor x28, x28, x21 - xar V14.2D, V20.2D, V25.2D, #46 - eor x1, x1, x24 - xar V20.2D, V2.2D, V27.2D, #2 - eor x28, x28, x26 - xar V2.2D, V12.2D, V27.2D, #21 - eor x0, x0, x1, ror 63 - xar V12.2D, V13.2D, V28.2D, #39 - eor x1, x1, x28, ror 63 - xar V13.2D, V19.2D, V29.2D, #56 - eor x2, x2, x0 - xar V19.2D, V23.2D, V28.2D, #8 - eor x7, x7, x0 - xar V23.2D, V15.2D, V25.2D, #23 - eor x12, x12, x0 - xar V15.2D, V4.2D, V29.2D, #37 - eor x17, x17, x0 - xar V4.2D, V24.2D, V29.2D, #50 - eor x23, x23, x0 - xar V24.2D, V21.2D, V26.2D, #62 - eor x4, x4, x1 - xar V21.2D, V8.2D, V28.2D, #9 - eor x9, x9, x1 - xar V8.2D, V16.2D, V26.2D, #19 - eor x14, x14, x1 - xar V16.2D, V5.2D, V25.2D, #28 - eor x20, x20, x1 - xar V5.2D, V3.2D, V28.2D, #36 - eor x25, x25, x1 - xar V3.2D, V18.2D, V28.2D, #43 - ldr x0, [x29, #32] - xar V18.2D, V17.2D, V27.2D, #49 - ldr x1, [x29, #24] - xar V17.2D, V11.2D, V26.2D, #54 - eor x28, x28, x30, ror 63 - xar V11.2D, V7.2D, V27.2D, #58 - eor x30, x30, x1, ror 63 - xar V7.2D, V10.2D, V25.2D, #61 - eor x1, x1, x0, ror 63 - ; Row Mix - mov V25.16B, V0.16B - eor x6, x6, x28 - mov V26.16B, V1.16B - eor x11, x11, x28 - bcax V0.16B, V25.16B, V2.16B, V26.16B - eor x16, x16, x28 - bcax V1.16B, V26.16B, V3.16B, V2.16B - eor x22, x22, x28 - bcax V2.16B, V2.16B, V4.16B, V3.16B - eor x27, x27, x28 - bcax V3.16B, V3.16B, V25.16B, V4.16B - eor x3, x3, x30 - bcax V4.16B, V4.16B, V26.16B, V25.16B - eor x8, x8, x30 - mov V25.16B, V5.16B - eor x13, x13, x30 - mov V26.16B, V6.16B - eor x19, x19, x30 - bcax V5.16B, V25.16B, V7.16B, V26.16B - eor x24, x24, x30 - bcax V6.16B, V26.16B, V8.16B, V7.16B - eor x5, x5, x1 - bcax V7.16B, V7.16B, V9.16B, V8.16B - eor x10, x10, x1 - bcax V8.16B, V8.16B, V25.16B, V9.16B - eor x15, x15, x1 - bcax V9.16B, V9.16B, V26.16B, V25.16B - eor x21, x21, x1 - mov V26.16B, V11.16B - eor x26, x26, x1 - ; Swap Rotate Base - bcax V10.16B, V30.16B, V12.16B, V26.16B - ror x0, x3, #63 - bcax V11.16B, V26.16B, V13.16B, V12.16B - ror x3, x8, #20 - bcax V12.16B, V12.16B, V14.16B, V13.16B - ror x8, x11, #44 - bcax V13.16B, V13.16B, V30.16B, V14.16B - ror x11, x25, #3 - bcax V14.16B, V14.16B, V26.16B, V30.16B - ror x25, x16, #25 - mov V25.16B, V15.16B - ror x16, x23, #46 - mov V26.16B, V16.16B - ror x23, x4, #2 - bcax V15.16B, V25.16B, V17.16B, V26.16B - ror x4, x14, #21 - bcax V16.16B, V26.16B, V18.16B, V17.16B - ror x14, x15, #39 - bcax V17.16B, V17.16B, V19.16B, V18.16B - ror x15, x22, #56 - bcax V18.16B, V18.16B, V25.16B, V19.16B - ror x22, x26, #8 - bcax V19.16B, V19.16B, V26.16B, V25.16B - ror x26, x17, #23 - mov V25.16B, V20.16B - ror x17, x6, #37 - mov V26.16B, V21.16B - ror x6, x27, #50 - bcax V20.16B, V25.16B, V22.16B, V26.16B - ror x27, x24, #62 - bcax V21.16B, V26.16B, V23.16B, V22.16B - ror x24, x10, #9 - bcax V22.16B, V22.16B, V24.16B, V23.16B - ror x10, x19, #19 - bcax V23.16B, V23.16B, V25.16B, V24.16B - ror x19, x7, #28 - bcax V24.16B, V24.16B, V26.16B, V25.16B - ror x7, x5, #36 - ror x5, x21, #43 - ror x21, x20, #49 - ror x20, x13, #54 - ror x13, x9, #58 - ror x9, x12, #61 - ; Row Mix Base - bic x12, x4, x3 - bic x1, x5, x4 - bic x28, x2, x6 - bic x30, x3, x2 - eor x2, x2, x12 - eor x3, x3, x1 - bic x12, x6, x5 - eor x5, x5, x28 - eor x4, x4, x12 - eor x6, x6, x30 - bic x12, x9, x8 - bic x1, x10, x9 - bic x28, x7, x11 - bic x30, x8, x7 - eor x7, x7, x12 - eor x8, x8, x1 - bic x12, x11, x10 - eor x10, x10, x28 - eor x9, x9, x12 - eor x11, x11, x30 - bic x12, x14, x13 - bic x1, x15, x14 - bic x28, x0, x16 - bic x30, x13, x0 - eor x12, x0, x12 - eor x13, x13, x1 - bic x0, x16, x15 - eor x15, x15, x28 - eor x14, x14, x0 - eor x16, x16, x30 - bic x0, x20, x19 - bic x1, x21, x20 - bic x28, x17, x22 - bic x30, x19, x17 - eor x17, x17, x0 - eor x19, x19, x1 - bic x0, x22, x21 - eor x21, x21, x28 - eor x20, x20, x0 - eor x22, x22, x30 - bic x0, x25, x24 - bic x1, x26, x25 - bic x28, x23, x27 - bic x30, x24, x23 - eor x23, x23, x0 - eor x24, x24, x1 - bic x0, x27, x26 - eor x26, x26, x28 - eor x25, x25, x0 - eor x27, x27, x30 - ; Done transforming - ldp x28, x1, [x29, #48] - ldr x0, [x28], #8 - subs x1, x1, #1 - mov V30.D[0], x0 - mov V30.D[1], x0 - eor x2, x2, x0 - eor V0.16B, V0.16B, V30.16B - bne L_SHA3_shake128_blocksx3_seed_crypto_begin - ldr x0, [x29, #40] - st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 - st1 {V24.D}[0], [x0] - add x0, x0, #8 - st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 - st1 {V24.D}[1], [x0] - add x0, x0, #8 - stp x2, x3, [x0] - stp x4, x5, [x0, #16] - stp x6, x7, [x0, #32] - stp x8, x9, [x0, #48] - stp x10, x11, [x0, #64] - stp x12, x13, [x0, #80] - stp x14, x15, [x0, #96] - stp x16, x17, [x0, #112] - stp x19, x20, [x0, #128] - stp x21, x22, [x0, #144] - stp x23, x24, [x0, #160] - stp x25, x26, [x0, #176] - str x27, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp D8, D9, [x29, #160] - ldp D10, D11, [x29, #176] - ldp D12, D13, [x29, #192] - ldp D14, D15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret - ENDP - AREA |.text|, CODE, READONLY - ALIGN 4 - EXPORT mlkem_shake256_blocksx3_seed_crypto -mlkem_shake256_blocksx3_seed_crypto PROC - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp D8, D9, [x29, #160] - stp D10, D11, [x29, #176] - stp D12, D13, [x29, #192] - stp D14, D15, [x29, #208] - adrp x28, L_sha3_aarch64_r - add x28, x28, L_sha3_aarch64_r - str x0, [x29, #40] - add x0, x0, #32 - ld1 {V4.D}[0], [x0] - ldp x2, x3, [x1], #16 - add x0, x0, #0xc8 - ld1 {V4.D}[1], [x0] - ldp x4, x5, [x1], #16 - ldr x6, [x0, #200] - eor V5.16B, V5.16B, V5.16B - eor x7, x7, x7 - eor V6.16B, V6.16B, V6.16B - eor x8, x8, x8 - eor V7.16B, V7.16B, V7.16B - eor x9, x9, x9 - eor V8.16B, V8.16B, V8.16B - eor x10, x10, x10 - eor V9.16B, V9.16B, V9.16B - eor x11, x11, x11 - eor V10.16B, V10.16B, V10.16B - eor x12, x12, x12 - eor V11.16B, V11.16B, V11.16B - eor x13, x13, x13 - eor V12.16B, V12.16B, V12.16B - eor x14, x14, x14 - eor V13.16B, V13.16B, V13.16B - eor x15, x15, x15 - eor V14.16B, V14.16B, V14.16B - eor x16, x16, x16 - eor V15.16B, V15.16B, V15.16B - eor x17, x17, x17 - movz x19, #0x8000, lsl 48 - eor V17.16B, V17.16B, V17.16B - eor x20, x20, x20 - eor V18.16B, V18.16B, V18.16B - eor x21, x21, x21 - eor V19.16B, V19.16B, V19.16B - eor x22, x22, x22 - eor V20.16B, V20.16B, V20.16B - eor x23, x23, x23 - eor V21.16B, V21.16B, V21.16B - eor x24, x24, x24 - eor V22.16B, V22.16B, V22.16B - eor x25, x25, x25 - eor V23.16B, V23.16B, V23.16B - eor x26, x26, x26 - eor V24.16B, V24.16B, V24.16B - eor x27, x27, x27 - dup V0.2D, x2 - dup V1.2D, x3 - dup V2.2D, x4 - dup V3.2D, x5 - dup V16.2D, x19 - mov x1, #24 - ; Start of 24 rounds -L_SHA3_shake256_blocksx3_seed_crypto_begin - stp x28, x1, [x29, #48] - ; Col Mix - eor3 V31.16B, V0.16B, V5.16B, V10.16B - eor x0, x6, x11 - eor3 V27.16B, V1.16B, V6.16B, V11.16B - eor x30, x2, x7 - eor3 V28.16B, V2.16B, V7.16B, V12.16B - eor x28, x4, x9 - eor3 V29.16B, V3.16B, V8.16B, V13.16B - eor x0, x0, x16 - eor3 V30.16B, V4.16B, V9.16B, V14.16B - eor x30, x30, x12 - eor3 V31.16B, V31.16B, V15.16B, V20.16B - eor x28, x28, x14 - eor3 V27.16B, V27.16B, V16.16B, V21.16B - eor x0, x0, x22 - eor3 V28.16B, V28.16B, V17.16B, V22.16B - eor x30, x30, x17 - eor3 V29.16B, V29.16B, V18.16B, V23.16B - eor x28, x28, x20 - eor3 V30.16B, V30.16B, V19.16B, V24.16B - eor x0, x0, x27 - rax1 V25.2D, V30.2D, V27.2D - eor x30, x30, x23 - rax1 V26.2D, V31.2D, V28.2D - eor x28, x28, x25 - rax1 V27.2D, V27.2D, V29.2D - str x0, [x29, #32] - rax1 V28.2D, V28.2D, V30.2D - str x28, [x29, #24] - rax1 V29.2D, V29.2D, V31.2D - eor x1, x3, x8 - eor V0.16B, V0.16B, V25.16B - xar V30.2D, V1.2D, V26.2D, #63 - eor x28, x5, x10 - xar V1.2D, V6.2D, V26.2D, #20 - eor x1, x1, x13 - xar V6.2D, V9.2D, V29.2D, #44 - eor x28, x28, x15 - xar V9.2D, V22.2D, V27.2D, #3 - eor x1, x1, x19 - xar V22.2D, V14.2D, V29.2D, #25 - eor x28, x28, x21 - xar V14.2D, V20.2D, V25.2D, #46 - eor x1, x1, x24 - xar V20.2D, V2.2D, V27.2D, #2 - eor x28, x28, x26 - xar V2.2D, V12.2D, V27.2D, #21 - eor x0, x0, x1, ror 63 - xar V12.2D, V13.2D, V28.2D, #39 - eor x1, x1, x28, ror 63 - xar V13.2D, V19.2D, V29.2D, #56 - eor x2, x2, x0 - xar V19.2D, V23.2D, V28.2D, #8 - eor x7, x7, x0 - xar V23.2D, V15.2D, V25.2D, #23 - eor x12, x12, x0 - xar V15.2D, V4.2D, V29.2D, #37 - eor x17, x17, x0 - xar V4.2D, V24.2D, V29.2D, #50 - eor x23, x23, x0 - xar V24.2D, V21.2D, V26.2D, #62 - eor x4, x4, x1 - xar V21.2D, V8.2D, V28.2D, #9 - eor x9, x9, x1 - xar V8.2D, V16.2D, V26.2D, #19 - eor x14, x14, x1 - xar V16.2D, V5.2D, V25.2D, #28 - eor x20, x20, x1 - xar V5.2D, V3.2D, V28.2D, #36 - eor x25, x25, x1 - xar V3.2D, V18.2D, V28.2D, #43 - ldr x0, [x29, #32] - xar V18.2D, V17.2D, V27.2D, #49 - ldr x1, [x29, #24] - xar V17.2D, V11.2D, V26.2D, #54 - eor x28, x28, x30, ror 63 - xar V11.2D, V7.2D, V27.2D, #58 - eor x30, x30, x1, ror 63 - xar V7.2D, V10.2D, V25.2D, #61 - eor x1, x1, x0, ror 63 - ; Row Mix - mov V25.16B, V0.16B - eor x6, x6, x28 - mov V26.16B, V1.16B - eor x11, x11, x28 - bcax V0.16B, V25.16B, V2.16B, V26.16B - eor x16, x16, x28 - bcax V1.16B, V26.16B, V3.16B, V2.16B - eor x22, x22, x28 - bcax V2.16B, V2.16B, V4.16B, V3.16B - eor x27, x27, x28 - bcax V3.16B, V3.16B, V25.16B, V4.16B - eor x3, x3, x30 - bcax V4.16B, V4.16B, V26.16B, V25.16B - eor x8, x8, x30 - mov V25.16B, V5.16B - eor x13, x13, x30 - mov V26.16B, V6.16B - eor x19, x19, x30 - bcax V5.16B, V25.16B, V7.16B, V26.16B - eor x24, x24, x30 - bcax V6.16B, V26.16B, V8.16B, V7.16B - eor x5, x5, x1 - bcax V7.16B, V7.16B, V9.16B, V8.16B - eor x10, x10, x1 - bcax V8.16B, V8.16B, V25.16B, V9.16B - eor x15, x15, x1 - bcax V9.16B, V9.16B, V26.16B, V25.16B - eor x21, x21, x1 - mov V26.16B, V11.16B - eor x26, x26, x1 - ; Swap Rotate Base - bcax V10.16B, V30.16B, V12.16B, V26.16B - ror x0, x3, #63 - bcax V11.16B, V26.16B, V13.16B, V12.16B - ror x3, x8, #20 - bcax V12.16B, V12.16B, V14.16B, V13.16B - ror x8, x11, #44 - bcax V13.16B, V13.16B, V30.16B, V14.16B - ror x11, x25, #3 - bcax V14.16B, V14.16B, V26.16B, V30.16B - ror x25, x16, #25 - mov V25.16B, V15.16B - ror x16, x23, #46 - mov V26.16B, V16.16B - ror x23, x4, #2 - bcax V15.16B, V25.16B, V17.16B, V26.16B - ror x4, x14, #21 - bcax V16.16B, V26.16B, V18.16B, V17.16B - ror x14, x15, #39 - bcax V17.16B, V17.16B, V19.16B, V18.16B - ror x15, x22, #56 - bcax V18.16B, V18.16B, V25.16B, V19.16B - ror x22, x26, #8 - bcax V19.16B, V19.16B, V26.16B, V25.16B - ror x26, x17, #23 - mov V25.16B, V20.16B - ror x17, x6, #37 - mov V26.16B, V21.16B - ror x6, x27, #50 - bcax V20.16B, V25.16B, V22.16B, V26.16B - ror x27, x24, #62 - bcax V21.16B, V26.16B, V23.16B, V22.16B - ror x24, x10, #9 - bcax V22.16B, V22.16B, V24.16B, V23.16B - ror x10, x19, #19 - bcax V23.16B, V23.16B, V25.16B, V24.16B - ror x19, x7, #28 - bcax V24.16B, V24.16B, V26.16B, V25.16B - ror x7, x5, #36 - ror x5, x21, #43 - ror x21, x20, #49 - ror x20, x13, #54 - ror x13, x9, #58 - ror x9, x12, #61 - ; Row Mix Base - bic x12, x4, x3 - bic x1, x5, x4 - bic x28, x2, x6 - bic x30, x3, x2 - eor x2, x2, x12 - eor x3, x3, x1 - bic x12, x6, x5 - eor x5, x5, x28 - eor x4, x4, x12 - eor x6, x6, x30 - bic x12, x9, x8 - bic x1, x10, x9 - bic x28, x7, x11 - bic x30, x8, x7 - eor x7, x7, x12 - eor x8, x8, x1 - bic x12, x11, x10 - eor x10, x10, x28 - eor x9, x9, x12 - eor x11, x11, x30 - bic x12, x14, x13 - bic x1, x15, x14 - bic x28, x0, x16 - bic x30, x13, x0 - eor x12, x0, x12 - eor x13, x13, x1 - bic x0, x16, x15 - eor x15, x15, x28 - eor x14, x14, x0 - eor x16, x16, x30 - bic x0, x20, x19 - bic x1, x21, x20 - bic x28, x17, x22 - bic x30, x19, x17 - eor x17, x17, x0 - eor x19, x19, x1 - bic x0, x22, x21 - eor x21, x21, x28 - eor x20, x20, x0 - eor x22, x22, x30 - bic x0, x25, x24 - bic x1, x26, x25 - bic x28, x23, x27 - bic x30, x24, x23 - eor x23, x23, x0 - eor x24, x24, x1 - bic x0, x27, x26 - eor x26, x26, x28 - eor x25, x25, x0 - eor x27, x27, x30 - ; Done transforming - ldp x28, x1, [x29, #48] - ldr x0, [x28], #8 - subs x1, x1, #1 - mov V30.D[0], x0 - mov V30.D[1], x0 - eor x2, x2, x0 - eor V0.16B, V0.16B, V30.16B - bne L_SHA3_shake256_blocksx3_seed_crypto_begin - ldr x0, [x29, #40] - st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 - st1 {V24.D}[0], [x0] - add x0, x0, #8 - st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 - st1 {V24.D}[1], [x0] - add x0, x0, #8 - stp x2, x3, [x0] - stp x4, x5, [x0, #16] - stp x6, x7, [x0, #32] - stp x8, x9, [x0, #48] - stp x10, x11, [x0, #64] - stp x12, x13, [x0, #80] - stp x14, x15, [x0, #96] - stp x16, x17, [x0, #112] - stp x19, x20, [x0, #128] - stp x21, x22, [x0, #144] - stp x23, x24, [x0, #160] - stp x25, x26, [x0, #176] - str x27, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp D8, D9, [x29, #160] - ldp D10, D11, [x29, #176] - ldp D12, D13, [x29, #192] - ldp D14, D15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret - ENDP - ENDIF - AREA |.text|, CODE, READONLY - ALIGN 4 - EXPORT mlkem_sha3_blocksx3_neon -mlkem_sha3_blocksx3_neon PROC - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp D8, D9, [x29, #160] - stp D10, D11, [x29, #176] - stp D12, D13, [x29, #192] - stp D14, D15, [x29, #208] - adrp x27, L_sha3_aarch64_r - add x27, x27, L_sha3_aarch64_r - str x0, [x29, #40] - ld4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 - ld4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 - ld4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 - ld4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 - ld4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 - ld4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 - ld1 {V24.D}[0], [x0] - add x0, x0, #8 - ld4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 - ld4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 - ld4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 - ld4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 - ld4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 - ld4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 - ld1 {V24.D}[1], [x0] - add x0, x0, #8 - ldp x1, x2, [x0] - ldp x3, x4, [x0, #16] - ldp x5, x6, [x0, #32] - ldp x7, x8, [x0, #48] - ldp x9, x10, [x0, #64] - ldp x11, x12, [x0, #80] - ldp x13, x14, [x0, #96] - ldp x15, x16, [x0, #112] - ldp x17, x19, [x0, #128] - ldp x20, x21, [x0, #144] - ldp x22, x23, [x0, #160] - ldp x24, x25, [x0, #176] - ldr x26, [x0, #192] - mov x28, #24 - ; Start of 24 rounds -L_SHA3_transform_blocksx3_neon_begin - stp x27, x28, [x29, #48] - ; Col Mix NEON - eor V30.16B, V4.16B, V9.16B - eor x0, x5, x10 - eor V27.16B, V1.16B, V6.16B - eor x30, x1, x6 - eor V30.16B, V30.16B, V14.16B - eor x28, x3, x8 - eor V27.16B, V27.16B, V11.16B - eor x0, x0, x15 - eor V30.16B, V30.16B, V19.16B - eor x30, x30, x11 - eor V27.16B, V27.16B, V16.16B - eor x28, x28, x13 - eor V30.16B, V30.16B, V24.16B - eor x0, x0, x21 - eor V27.16B, V27.16B, V21.16B - eor x30, x30, x16 - ushr V25.2D, V27.2D, #63 - eor x28, x28, x19 - sli V25.2D, V27.2D, #1 - eor x0, x0, x26 - eor V25.16B, V25.16B, V30.16B - eor x30, x30, x22 - eor V31.16B, V0.16B, V5.16B - eor x28, x28, x24 - eor V28.16B, V2.16B, V7.16B - str x0, [x29, #32] - eor V31.16B, V31.16B, V10.16B - str x28, [x29, #24] - eor V28.16B, V28.16B, V12.16B - eor x27, x2, x7 - eor V31.16B, V31.16B, V15.16B - eor x28, x4, x9 - eor V28.16B, V28.16B, V17.16B - eor x27, x27, x12 - eor V31.16B, V31.16B, V20.16B - eor x28, x28, x14 - eor V28.16B, V28.16B, V22.16B - eor x27, x27, x17 - ushr V29.2D, V30.2D, #63 - eor x28, x28, x20 - ushr V26.2D, V28.2D, #63 - eor x27, x27, x23 - sli V29.2D, V30.2D, #1 - eor x28, x28, x25 - sli V26.2D, V28.2D, #1 - eor x0, x0, x27, ror 63 - eor V28.16B, V28.16B, V29.16B - eor x27, x27, x28, ror 63 - eor V29.16B, V3.16B, V8.16B - eor x1, x1, x0 - eor V26.16B, V26.16B, V31.16B - eor x6, x6, x0 - eor V29.16B, V29.16B, V13.16B - eor x11, x11, x0 - eor V29.16B, V29.16B, V18.16B - eor x16, x16, x0 - eor V29.16B, V29.16B, V23.16B - eor x22, x22, x0 - ushr V30.2D, V29.2D, #63 - eor x3, x3, x27 - sli V30.2D, V29.2D, #1 - eor x8, x8, x27 - eor V27.16B, V27.16B, V30.16B - eor x13, x13, x27 - ushr V30.2D, V31.2D, #63 - eor x19, x19, x27 - sli V30.2D, V31.2D, #1 - eor x24, x24, x27 - eor V29.16B, V29.16B, V30.16B - ldr x0, [x29, #32] - ; Swap Rotate NEON - eor V0.16B, V0.16B, V25.16B - eor V31.16B, V1.16B, V26.16B - ldr x27, [x29, #24] - eor V6.16B, V6.16B, V26.16B - eor x28, x28, x30, ror 63 - ushr V30.2D, V31.2D, #63 - eor x30, x30, x27, ror 63 - ushr V1.2D, V6.2D, #20 - eor x27, x27, x0, ror 63 - sli V30.2D, V31.2D, #1 - eor x5, x5, x28 - sli V1.2D, V6.2D, #44 - eor x10, x10, x28 - eor V31.16B, V9.16B, V29.16B - eor x15, x15, x28 - eor V22.16B, V22.16B, V27.16B - eor x21, x21, x28 - ushr V6.2D, V31.2D, #44 - eor x26, x26, x28 - ushr V9.2D, V22.2D, #3 - eor x2, x2, x30 - sli V6.2D, V31.2D, #20 - eor x7, x7, x30 - sli V9.2D, V22.2D, #61 - eor x12, x12, x30 - eor V31.16B, V14.16B, V29.16B - eor x17, x17, x30 - eor V20.16B, V20.16B, V25.16B - eor x23, x23, x30 - ushr V22.2D, V31.2D, #25 - eor x4, x4, x27 - ushr V14.2D, V20.2D, #46 - eor x9, x9, x27 - sli V22.2D, V31.2D, #39 - eor x14, x14, x27 - sli V14.2D, V20.2D, #18 - eor x20, x20, x27 - eor V31.16B, V2.16B, V27.16B - eor x25, x25, x27 - ; Swap Rotate Base - eor V12.16B, V12.16B, V27.16B - ror x0, x2, #63 - ushr V20.2D, V31.2D, #2 - ror x2, x7, #20 - ushr V2.2D, V12.2D, #21 - ror x7, x10, #44 - sli V20.2D, V31.2D, #62 - ror x10, x24, #3 - sli V2.2D, V12.2D, #43 - ror x24, x15, #25 - eor V31.16B, V13.16B, V28.16B - ror x15, x22, #46 - eor V19.16B, V19.16B, V29.16B - ror x22, x3, #2 - ushr V12.2D, V31.2D, #39 - ror x3, x13, #21 - ushr V13.2D, V19.2D, #56 - ror x13, x14, #39 - sli V12.2D, V31.2D, #25 - ror x14, x21, #56 - sli V13.2D, V19.2D, #8 - ror x21, x25, #8 - eor V31.16B, V23.16B, V28.16B - ror x25, x16, #23 - eor V15.16B, V15.16B, V25.16B - ror x16, x5, #37 - ushr V19.2D, V31.2D, #8 - ror x5, x26, #50 - ushr V23.2D, V15.2D, #23 - ror x26, x23, #62 - sli V19.2D, V31.2D, #56 - ror x23, x9, #9 - sli V23.2D, V15.2D, #41 - ror x9, x17, #19 - eor V31.16B, V4.16B, V29.16B - ror x17, x6, #28 - eor V24.16B, V24.16B, V29.16B - ror x6, x4, #36 - ushr V15.2D, V31.2D, #37 - ror x4, x20, #43 - ushr V4.2D, V24.2D, #50 - ror x20, x19, #49 - sli V15.2D, V31.2D, #27 - ror x19, x12, #54 - sli V4.2D, V24.2D, #14 - ror x12, x8, #58 - eor V31.16B, V21.16B, V26.16B - ror x8, x11, #61 - ; Row Mix Base - eor V8.16B, V8.16B, V28.16B - bic x11, x3, x2 - ushr V24.2D, V31.2D, #62 - bic x27, x4, x3 - ushr V21.2D, V8.2D, #9 - bic x28, x1, x5 - sli V24.2D, V31.2D, #2 - bic x30, x2, x1 - sli V21.2D, V8.2D, #55 - eor x1, x1, x11 - eor V31.16B, V16.16B, V26.16B - eor x2, x2, x27 - eor V5.16B, V5.16B, V25.16B - bic x11, x5, x4 - ushr V8.2D, V31.2D, #19 - eor x4, x4, x28 - ushr V16.2D, V5.2D, #28 - eor x3, x3, x11 - sli V8.2D, V31.2D, #45 - eor x5, x5, x30 - sli V16.2D, V5.2D, #36 - bic x11, x8, x7 - eor V31.16B, V3.16B, V28.16B - bic x27, x9, x8 - eor V18.16B, V18.16B, V28.16B - bic x28, x6, x10 - ushr V5.2D, V31.2D, #36 - bic x30, x7, x6 - ushr V3.2D, V18.2D, #43 - eor x6, x6, x11 - sli V5.2D, V31.2D, #28 - eor x7, x7, x27 - sli V3.2D, V18.2D, #21 - bic x11, x10, x9 - eor V31.16B, V17.16B, V27.16B - eor x9, x9, x28 - eor V11.16B, V11.16B, V26.16B - eor x8, x8, x11 - ushr V18.2D, V31.2D, #49 - eor x10, x10, x30 - ushr V17.2D, V11.2D, #54 - bic x11, x13, x12 - sli V18.2D, V31.2D, #15 - bic x27, x14, x13 - sli V17.2D, V11.2D, #10 - bic x28, x0, x15 - eor V31.16B, V7.16B, V27.16B - bic x30, x12, x0 - eor V10.16B, V10.16B, V25.16B - eor x11, x0, x11 - ushr V11.2D, V31.2D, #58 - eor x12, x12, x27 - ushr V7.2D, V10.2D, #61 - bic x0, x15, x14 - sli V11.2D, V31.2D, #6 - eor x14, x14, x28 - sli V7.2D, V10.2D, #3 - eor x13, x13, x0 - ; Row Mix NEON - bic V25.16B, V2.16B, V1.16B - eor x15, x15, x30 - bic V26.16B, V3.16B, V2.16B - bic x0, x19, x17 - bic V27.16B, V4.16B, V3.16B - bic x27, x20, x19 - bic V28.16B, V0.16B, V4.16B - bic x28, x16, x21 - bic V29.16B, V1.16B, V0.16B - bic x30, x17, x16 - eor V0.16B, V0.16B, V25.16B - eor x16, x16, x0 - eor V1.16B, V1.16B, V26.16B - eor x17, x17, x27 - eor V2.16B, V2.16B, V27.16B - bic x0, x21, x20 - eor V3.16B, V3.16B, V28.16B - eor x20, x20, x28 - eor V4.16B, V4.16B, V29.16B - eor x19, x19, x0 - bic V25.16B, V7.16B, V6.16B - eor x21, x21, x30 - bic V26.16B, V8.16B, V7.16B - bic x0, x24, x23 - bic V27.16B, V9.16B, V8.16B - bic x27, x25, x24 - bic V28.16B, V5.16B, V9.16B - bic x28, x22, x26 - bic V29.16B, V6.16B, V5.16B - bic x30, x23, x22 - eor V5.16B, V5.16B, V25.16B - eor x22, x22, x0 - eor V6.16B, V6.16B, V26.16B - eor x23, x23, x27 - eor V7.16B, V7.16B, V27.16B - bic x0, x26, x25 - eor V8.16B, V8.16B, V28.16B - eor x25, x25, x28 - eor V9.16B, V9.16B, V29.16B - eor x24, x24, x0 - bic V25.16B, V12.16B, V11.16B - eor x26, x26, x30 - bic V26.16B, V13.16B, V12.16B - bic V27.16B, V14.16B, V13.16B - bic V28.16B, V30.16B, V14.16B - bic V29.16B, V11.16B, V30.16B - eor V10.16B, V30.16B, V25.16B - eor V11.16B, V11.16B, V26.16B - eor V12.16B, V12.16B, V27.16B - eor V13.16B, V13.16B, V28.16B - eor V14.16B, V14.16B, V29.16B - bic V25.16B, V17.16B, V16.16B - bic V26.16B, V18.16B, V17.16B - bic V27.16B, V19.16B, V18.16B - bic V28.16B, V15.16B, V19.16B - bic V29.16B, V16.16B, V15.16B - eor V15.16B, V15.16B, V25.16B - eor V16.16B, V16.16B, V26.16B - eor V17.16B, V17.16B, V27.16B - eor V18.16B, V18.16B, V28.16B - eor V19.16B, V19.16B, V29.16B - bic V25.16B, V22.16B, V21.16B - bic V26.16B, V23.16B, V22.16B - bic V27.16B, V24.16B, V23.16B - bic V28.16B, V20.16B, V24.16B - bic V29.16B, V21.16B, V20.16B - eor V20.16B, V20.16B, V25.16B - eor V21.16B, V21.16B, V26.16B - eor V22.16B, V22.16B, V27.16B - eor V23.16B, V23.16B, V28.16B - eor V24.16B, V24.16B, V29.16B - ; Done transforming - ldp x27, x28, [x29, #48] - ldr x0, [x27], #8 - subs x28, x28, #1 - mov V30.D[0], x0 - mov V30.D[1], x0 - eor x1, x1, x0 - eor V0.16B, V0.16B, V30.16B - bne L_SHA3_transform_blocksx3_neon_begin - ldr x0, [x29, #40] - st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 - st1 {V24.D}[0], [x0] - add x0, x0, #8 - st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 - st1 {V24.D}[1], [x0] - add x0, x0, #8 - stp x1, x2, [x0] - stp x3, x4, [x0, #16] - stp x5, x6, [x0, #32] - stp x7, x8, [x0, #48] - stp x9, x10, [x0, #64] - stp x11, x12, [x0, #80] - stp x13, x14, [x0, #96] - stp x15, x16, [x0, #112] - stp x17, x19, [x0, #128] - stp x20, x21, [x0, #144] - stp x22, x23, [x0, #160] - stp x24, x25, [x0, #176] - str x26, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp D8, D9, [x29, #160] - ldp D10, D11, [x29, #176] - ldp D12, D13, [x29, #192] - ldp D14, D15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret - ENDP - AREA |.text|, CODE, READONLY - ALIGN 4 - EXPORT mlkem_shake128_blocksx3_seed_neon -mlkem_shake128_blocksx3_seed_neon PROC - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp D8, D9, [x29, #160] - stp D10, D11, [x29, #176] - stp D12, D13, [x29, #192] - stp D14, D15, [x29, #208] - adrp x28, L_sha3_aarch64_r - add x28, x28, L_sha3_aarch64_r - str x0, [x29, #40] - add x0, x0, #32 - ld1 {V4.D}[0], [x0] - ldp x2, x3, [x1], #16 - add x0, x0, #0xc8 - ld1 {V4.D}[1], [x0] - ldp x4, x5, [x1], #16 - ldr x6, [x0, #200] - eor V5.16B, V5.16B, V5.16B - eor x7, x7, x7 - eor V6.16B, V6.16B, V6.16B - eor x8, x8, x8 - eor V7.16B, V7.16B, V7.16B - eor x9, x9, x9 - eor V8.16B, V8.16B, V8.16B - eor x10, x10, x10 - eor V9.16B, V9.16B, V9.16B - eor x11, x11, x11 - eor V10.16B, V10.16B, V10.16B - eor x12, x12, x12 - eor V11.16B, V11.16B, V11.16B - eor x13, x13, x13 - eor V12.16B, V12.16B, V12.16B - eor x14, x14, x14 - eor V13.16B, V13.16B, V13.16B - eor x15, x15, x15 - eor V14.16B, V14.16B, V14.16B - eor x16, x16, x16 - eor V15.16B, V15.16B, V15.16B - eor x17, x17, x17 - eor V16.16B, V16.16B, V16.16B - eor x19, x19, x19 - eor V17.16B, V17.16B, V17.16B - eor x20, x20, x20 - eor V18.16B, V18.16B, V18.16B - eor x21, x21, x21 - eor V19.16B, V19.16B, V19.16B - eor x22, x22, x22 - movz x23, #0x8000, lsl 48 - eor V21.16B, V21.16B, V21.16B - eor x24, x24, x24 - eor V22.16B, V22.16B, V22.16B - eor x25, x25, x25 - eor V23.16B, V23.16B, V23.16B - eor x26, x26, x26 - eor V24.16B, V24.16B, V24.16B - eor x27, x27, x27 - dup V0.2D, x2 - dup V1.2D, x3 - dup V2.2D, x4 - dup V3.2D, x5 - dup V20.2D, x23 - mov x1, #24 - ; Start of 24 rounds -L_SHA3_shake128_blocksx3_seed_neon_begin - stp x28, x1, [x29, #48] - ; Col Mix NEON - eor V30.16B, V4.16B, V9.16B - eor x0, x6, x11 - eor V27.16B, V1.16B, V6.16B - eor x30, x2, x7 - eor V30.16B, V30.16B, V14.16B - eor x28, x4, x9 - eor V27.16B, V27.16B, V11.16B - eor x0, x0, x16 - eor V30.16B, V30.16B, V19.16B - eor x30, x30, x12 - eor V27.16B, V27.16B, V16.16B - eor x28, x28, x14 - eor V30.16B, V30.16B, V24.16B - eor x0, x0, x22 - eor V27.16B, V27.16B, V21.16B - eor x30, x30, x17 - ushr V25.2D, V27.2D, #63 - eor x28, x28, x20 - sli V25.2D, V27.2D, #1 - eor x0, x0, x27 - eor V25.16B, V25.16B, V30.16B - eor x30, x30, x23 - eor V31.16B, V0.16B, V5.16B - eor x28, x28, x25 - eor V28.16B, V2.16B, V7.16B - str x0, [x29, #32] - eor V31.16B, V31.16B, V10.16B - str x28, [x29, #24] - eor V28.16B, V28.16B, V12.16B - eor x1, x3, x8 - eor V31.16B, V31.16B, V15.16B - eor x28, x5, x10 - eor V28.16B, V28.16B, V17.16B - eor x1, x1, x13 - eor V31.16B, V31.16B, V20.16B - eor x28, x28, x15 - eor V28.16B, V28.16B, V22.16B - eor x1, x1, x19 - ushr V29.2D, V30.2D, #63 - eor x28, x28, x21 - ushr V26.2D, V28.2D, #63 - eor x1, x1, x24 - sli V29.2D, V30.2D, #1 - eor x28, x28, x26 - sli V26.2D, V28.2D, #1 - eor x0, x0, x1, ror 63 - eor V28.16B, V28.16B, V29.16B - eor x1, x1, x28, ror 63 - eor V29.16B, V3.16B, V8.16B - eor x2, x2, x0 - eor V26.16B, V26.16B, V31.16B - eor x7, x7, x0 - eor V29.16B, V29.16B, V13.16B - eor x12, x12, x0 - eor V29.16B, V29.16B, V18.16B - eor x17, x17, x0 - eor V29.16B, V29.16B, V23.16B - eor x23, x23, x0 - ushr V30.2D, V29.2D, #63 - eor x4, x4, x1 - sli V30.2D, V29.2D, #1 - eor x9, x9, x1 - eor V27.16B, V27.16B, V30.16B - eor x14, x14, x1 - ushr V30.2D, V31.2D, #63 - eor x20, x20, x1 - sli V30.2D, V31.2D, #1 - eor x25, x25, x1 - eor V29.16B, V29.16B, V30.16B - ldr x0, [x29, #32] - ; Swap Rotate NEON - eor V0.16B, V0.16B, V25.16B - eor V31.16B, V1.16B, V26.16B - ldr x1, [x29, #24] - eor V6.16B, V6.16B, V26.16B - eor x28, x28, x30, ror 63 - ushr V30.2D, V31.2D, #63 - eor x30, x30, x1, ror 63 - ushr V1.2D, V6.2D, #20 - eor x1, x1, x0, ror 63 - sli V30.2D, V31.2D, #1 - eor x6, x6, x28 - sli V1.2D, V6.2D, #44 - eor x11, x11, x28 - eor V31.16B, V9.16B, V29.16B - eor x16, x16, x28 - eor V22.16B, V22.16B, V27.16B - eor x22, x22, x28 - ushr V6.2D, V31.2D, #44 - eor x27, x27, x28 - ushr V9.2D, V22.2D, #3 - eor x3, x3, x30 - sli V6.2D, V31.2D, #20 - eor x8, x8, x30 - sli V9.2D, V22.2D, #61 - eor x13, x13, x30 - eor V31.16B, V14.16B, V29.16B - eor x19, x19, x30 - eor V20.16B, V20.16B, V25.16B - eor x24, x24, x30 - ushr V22.2D, V31.2D, #25 - eor x5, x5, x1 - ushr V14.2D, V20.2D, #46 - eor x10, x10, x1 - sli V22.2D, V31.2D, #39 - eor x15, x15, x1 - sli V14.2D, V20.2D, #18 - eor x21, x21, x1 - eor V31.16B, V2.16B, V27.16B - eor x26, x26, x1 - ; Swap Rotate Base - eor V12.16B, V12.16B, V27.16B - ror x0, x3, #63 - ushr V20.2D, V31.2D, #2 - ror x3, x8, #20 - ushr V2.2D, V12.2D, #21 - ror x8, x11, #44 - sli V20.2D, V31.2D, #62 - ror x11, x25, #3 - sli V2.2D, V12.2D, #43 - ror x25, x16, #25 - eor V31.16B, V13.16B, V28.16B - ror x16, x23, #46 - eor V19.16B, V19.16B, V29.16B - ror x23, x4, #2 - ushr V12.2D, V31.2D, #39 - ror x4, x14, #21 - ushr V13.2D, V19.2D, #56 - ror x14, x15, #39 - sli V12.2D, V31.2D, #25 - ror x15, x22, #56 - sli V13.2D, V19.2D, #8 - ror x22, x26, #8 - eor V31.16B, V23.16B, V28.16B - ror x26, x17, #23 - eor V15.16B, V15.16B, V25.16B - ror x17, x6, #37 - ushr V19.2D, V31.2D, #8 - ror x6, x27, #50 - ushr V23.2D, V15.2D, #23 - ror x27, x24, #62 - sli V19.2D, V31.2D, #56 - ror x24, x10, #9 - sli V23.2D, V15.2D, #41 - ror x10, x19, #19 - eor V31.16B, V4.16B, V29.16B - ror x19, x7, #28 - eor V24.16B, V24.16B, V29.16B - ror x7, x5, #36 - ushr V15.2D, V31.2D, #37 - ror x5, x21, #43 - ushr V4.2D, V24.2D, #50 - ror x21, x20, #49 - sli V15.2D, V31.2D, #27 - ror x20, x13, #54 - sli V4.2D, V24.2D, #14 - ror x13, x9, #58 - eor V31.16B, V21.16B, V26.16B - ror x9, x12, #61 - ; Row Mix Base - eor V8.16B, V8.16B, V28.16B - bic x12, x4, x3 - ushr V24.2D, V31.2D, #62 - bic x1, x5, x4 - ushr V21.2D, V8.2D, #9 - bic x28, x2, x6 - sli V24.2D, V31.2D, #2 - bic x30, x3, x2 - sli V21.2D, V8.2D, #55 - eor x2, x2, x12 - eor V31.16B, V16.16B, V26.16B - eor x3, x3, x1 - eor V5.16B, V5.16B, V25.16B - bic x12, x6, x5 - ushr V8.2D, V31.2D, #19 - eor x5, x5, x28 - ushr V16.2D, V5.2D, #28 - eor x4, x4, x12 - sli V8.2D, V31.2D, #45 - eor x6, x6, x30 - sli V16.2D, V5.2D, #36 - bic x12, x9, x8 - eor V31.16B, V3.16B, V28.16B - bic x1, x10, x9 - eor V18.16B, V18.16B, V28.16B - bic x28, x7, x11 - ushr V5.2D, V31.2D, #36 - bic x30, x8, x7 - ushr V3.2D, V18.2D, #43 - eor x7, x7, x12 - sli V5.2D, V31.2D, #28 - eor x8, x8, x1 - sli V3.2D, V18.2D, #21 - bic x12, x11, x10 - eor V31.16B, V17.16B, V27.16B - eor x10, x10, x28 - eor V11.16B, V11.16B, V26.16B - eor x9, x9, x12 - ushr V18.2D, V31.2D, #49 - eor x11, x11, x30 - ushr V17.2D, V11.2D, #54 - bic x12, x14, x13 - sli V18.2D, V31.2D, #15 - bic x1, x15, x14 - sli V17.2D, V11.2D, #10 - bic x28, x0, x16 - eor V31.16B, V7.16B, V27.16B - bic x30, x13, x0 - eor V10.16B, V10.16B, V25.16B - eor x12, x0, x12 - ushr V11.2D, V31.2D, #58 - eor x13, x13, x1 - ushr V7.2D, V10.2D, #61 - bic x0, x16, x15 - sli V11.2D, V31.2D, #6 - eor x15, x15, x28 - sli V7.2D, V10.2D, #3 - eor x14, x14, x0 - ; Row Mix NEON - bic V25.16B, V2.16B, V1.16B - eor x16, x16, x30 - bic V26.16B, V3.16B, V2.16B - bic x0, x20, x19 - bic V27.16B, V4.16B, V3.16B - bic x1, x21, x20 - bic V28.16B, V0.16B, V4.16B - bic x28, x17, x22 - bic V29.16B, V1.16B, V0.16B - bic x30, x19, x17 - eor V0.16B, V0.16B, V25.16B - eor x17, x17, x0 - eor V1.16B, V1.16B, V26.16B - eor x19, x19, x1 - eor V2.16B, V2.16B, V27.16B - bic x0, x22, x21 - eor V3.16B, V3.16B, V28.16B - eor x21, x21, x28 - eor V4.16B, V4.16B, V29.16B - eor x20, x20, x0 - bic V25.16B, V7.16B, V6.16B - eor x22, x22, x30 - bic V26.16B, V8.16B, V7.16B - bic x0, x25, x24 - bic V27.16B, V9.16B, V8.16B - bic x1, x26, x25 - bic V28.16B, V5.16B, V9.16B - bic x28, x23, x27 - bic V29.16B, V6.16B, V5.16B - bic x30, x24, x23 - eor V5.16B, V5.16B, V25.16B - eor x23, x23, x0 - eor V6.16B, V6.16B, V26.16B - eor x24, x24, x1 - eor V7.16B, V7.16B, V27.16B - bic x0, x27, x26 - eor V8.16B, V8.16B, V28.16B - eor x26, x26, x28 - eor V9.16B, V9.16B, V29.16B - eor x25, x25, x0 - bic V25.16B, V12.16B, V11.16B - eor x27, x27, x30 - bic V26.16B, V13.16B, V12.16B - bic V27.16B, V14.16B, V13.16B - bic V28.16B, V30.16B, V14.16B - bic V29.16B, V11.16B, V30.16B - eor V10.16B, V30.16B, V25.16B - eor V11.16B, V11.16B, V26.16B - eor V12.16B, V12.16B, V27.16B - eor V13.16B, V13.16B, V28.16B - eor V14.16B, V14.16B, V29.16B - bic V25.16B, V17.16B, V16.16B - bic V26.16B, V18.16B, V17.16B - bic V27.16B, V19.16B, V18.16B - bic V28.16B, V15.16B, V19.16B - bic V29.16B, V16.16B, V15.16B - eor V15.16B, V15.16B, V25.16B - eor V16.16B, V16.16B, V26.16B - eor V17.16B, V17.16B, V27.16B - eor V18.16B, V18.16B, V28.16B - eor V19.16B, V19.16B, V29.16B - bic V25.16B, V22.16B, V21.16B - bic V26.16B, V23.16B, V22.16B - bic V27.16B, V24.16B, V23.16B - bic V28.16B, V20.16B, V24.16B - bic V29.16B, V21.16B, V20.16B - eor V20.16B, V20.16B, V25.16B - eor V21.16B, V21.16B, V26.16B - eor V22.16B, V22.16B, V27.16B - eor V23.16B, V23.16B, V28.16B - eor V24.16B, V24.16B, V29.16B - ; Done transforming - ldp x28, x1, [x29, #48] - ldr x0, [x28], #8 - subs x1, x1, #1 - mov V30.D[0], x0 - mov V30.D[1], x0 - eor x2, x2, x0 - eor V0.16B, V0.16B, V30.16B - bne L_SHA3_shake128_blocksx3_seed_neon_begin - ldr x0, [x29, #40] - st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 - st1 {V24.D}[0], [x0] - add x0, x0, #8 - st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 - st1 {V24.D}[1], [x0] - add x0, x0, #8 - stp x2, x3, [x0] - stp x4, x5, [x0, #16] - stp x6, x7, [x0, #32] - stp x8, x9, [x0, #48] - stp x10, x11, [x0, #64] - stp x12, x13, [x0, #80] - stp x14, x15, [x0, #96] - stp x16, x17, [x0, #112] - stp x19, x20, [x0, #128] - stp x21, x22, [x0, #144] - stp x23, x24, [x0, #160] - stp x25, x26, [x0, #176] - str x27, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp D8, D9, [x29, #160] - ldp D10, D11, [x29, #176] - ldp D12, D13, [x29, #192] - ldp D14, D15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret - ENDP - AREA |.text|, CODE, READONLY - ALIGN 4 - EXPORT mlkem_shake256_blocksx3_seed_neon -mlkem_shake256_blocksx3_seed_neon PROC - stp x29, x30, [sp, #-224]! - add x29, sp, #0 - stp x17, x19, [x29, #72] - stp x20, x21, [x29, #88] - stp x22, x23, [x29, #104] - stp x24, x25, [x29, #120] - stp x26, x27, [x29, #136] - str x28, [x29, #152] - stp D8, D9, [x29, #160] - stp D10, D11, [x29, #176] - stp D12, D13, [x29, #192] - stp D14, D15, [x29, #208] - adrp x28, L_sha3_aarch64_r - add x28, x28, L_sha3_aarch64_r - str x0, [x29, #40] - add x0, x0, #32 - ld1 {V4.D}[0], [x0] - ldp x2, x3, [x1], #16 - add x0, x0, #0xc8 - ld1 {V4.D}[1], [x0] - ldp x4, x5, [x1], #16 - ldr x6, [x0, #200] - eor V5.16B, V5.16B, V5.16B - eor x7, x7, x7 - eor V6.16B, V6.16B, V6.16B - eor x8, x8, x8 - eor V7.16B, V7.16B, V7.16B - eor x9, x9, x9 - eor V8.16B, V8.16B, V8.16B - eor x10, x10, x10 - eor V9.16B, V9.16B, V9.16B - eor x11, x11, x11 - eor V10.16B, V10.16B, V10.16B - eor x12, x12, x12 - eor V11.16B, V11.16B, V11.16B - eor x13, x13, x13 - eor V12.16B, V12.16B, V12.16B - eor x14, x14, x14 - eor V13.16B, V13.16B, V13.16B - eor x15, x15, x15 - eor V14.16B, V14.16B, V14.16B - eor x16, x16, x16 - eor V15.16B, V15.16B, V15.16B - eor x17, x17, x17 - movz x19, #0x8000, lsl 48 - eor V17.16B, V17.16B, V17.16B - eor x20, x20, x20 - eor V18.16B, V18.16B, V18.16B - eor x21, x21, x21 - eor V19.16B, V19.16B, V19.16B - eor x22, x22, x22 - eor V20.16B, V20.16B, V20.16B - eor x23, x23, x23 - eor V21.16B, V21.16B, V21.16B - eor x24, x24, x24 - eor V22.16B, V22.16B, V22.16B - eor x25, x25, x25 - eor V23.16B, V23.16B, V23.16B - eor x26, x26, x26 - eor V24.16B, V24.16B, V24.16B - eor x27, x27, x27 - dup V0.2D, x2 - dup V1.2D, x3 - dup V2.2D, x4 - dup V3.2D, x5 - dup V16.2D, x19 - mov x1, #24 - ; Start of 24 rounds -L_SHA3_shake256_blocksx3_seed_neon_begin - stp x28, x1, [x29, #48] - ; Col Mix NEON - eor V30.16B, V4.16B, V9.16B - eor x0, x6, x11 - eor V27.16B, V1.16B, V6.16B - eor x30, x2, x7 - eor V30.16B, V30.16B, V14.16B - eor x28, x4, x9 - eor V27.16B, V27.16B, V11.16B - eor x0, x0, x16 - eor V30.16B, V30.16B, V19.16B - eor x30, x30, x12 - eor V27.16B, V27.16B, V16.16B - eor x28, x28, x14 - eor V30.16B, V30.16B, V24.16B - eor x0, x0, x22 - eor V27.16B, V27.16B, V21.16B - eor x30, x30, x17 - ushr V25.2D, V27.2D, #63 - eor x28, x28, x20 - sli V25.2D, V27.2D, #1 - eor x0, x0, x27 - eor V25.16B, V25.16B, V30.16B - eor x30, x30, x23 - eor V31.16B, V0.16B, V5.16B - eor x28, x28, x25 - eor V28.16B, V2.16B, V7.16B - str x0, [x29, #32] - eor V31.16B, V31.16B, V10.16B - str x28, [x29, #24] - eor V28.16B, V28.16B, V12.16B - eor x1, x3, x8 - eor V31.16B, V31.16B, V15.16B - eor x28, x5, x10 - eor V28.16B, V28.16B, V17.16B - eor x1, x1, x13 - eor V31.16B, V31.16B, V20.16B - eor x28, x28, x15 - eor V28.16B, V28.16B, V22.16B - eor x1, x1, x19 - ushr V29.2D, V30.2D, #63 - eor x28, x28, x21 - ushr V26.2D, V28.2D, #63 - eor x1, x1, x24 - sli V29.2D, V30.2D, #1 - eor x28, x28, x26 - sli V26.2D, V28.2D, #1 - eor x0, x0, x1, ror 63 - eor V28.16B, V28.16B, V29.16B - eor x1, x1, x28, ror 63 - eor V29.16B, V3.16B, V8.16B - eor x2, x2, x0 - eor V26.16B, V26.16B, V31.16B - eor x7, x7, x0 - eor V29.16B, V29.16B, V13.16B - eor x12, x12, x0 - eor V29.16B, V29.16B, V18.16B - eor x17, x17, x0 - eor V29.16B, V29.16B, V23.16B - eor x23, x23, x0 - ushr V30.2D, V29.2D, #63 - eor x4, x4, x1 - sli V30.2D, V29.2D, #1 - eor x9, x9, x1 - eor V27.16B, V27.16B, V30.16B - eor x14, x14, x1 - ushr V30.2D, V31.2D, #63 - eor x20, x20, x1 - sli V30.2D, V31.2D, #1 - eor x25, x25, x1 - eor V29.16B, V29.16B, V30.16B - ldr x0, [x29, #32] - ; Swap Rotate NEON - eor V0.16B, V0.16B, V25.16B - eor V31.16B, V1.16B, V26.16B - ldr x1, [x29, #24] - eor V6.16B, V6.16B, V26.16B - eor x28, x28, x30, ror 63 - ushr V30.2D, V31.2D, #63 - eor x30, x30, x1, ror 63 - ushr V1.2D, V6.2D, #20 - eor x1, x1, x0, ror 63 - sli V30.2D, V31.2D, #1 - eor x6, x6, x28 - sli V1.2D, V6.2D, #44 - eor x11, x11, x28 - eor V31.16B, V9.16B, V29.16B - eor x16, x16, x28 - eor V22.16B, V22.16B, V27.16B - eor x22, x22, x28 - ushr V6.2D, V31.2D, #44 - eor x27, x27, x28 - ushr V9.2D, V22.2D, #3 - eor x3, x3, x30 - sli V6.2D, V31.2D, #20 - eor x8, x8, x30 - sli V9.2D, V22.2D, #61 - eor x13, x13, x30 - eor V31.16B, V14.16B, V29.16B - eor x19, x19, x30 - eor V20.16B, V20.16B, V25.16B - eor x24, x24, x30 - ushr V22.2D, V31.2D, #25 - eor x5, x5, x1 - ushr V14.2D, V20.2D, #46 - eor x10, x10, x1 - sli V22.2D, V31.2D, #39 - eor x15, x15, x1 - sli V14.2D, V20.2D, #18 - eor x21, x21, x1 - eor V31.16B, V2.16B, V27.16B - eor x26, x26, x1 - ; Swap Rotate Base - eor V12.16B, V12.16B, V27.16B - ror x0, x3, #63 - ushr V20.2D, V31.2D, #2 - ror x3, x8, #20 - ushr V2.2D, V12.2D, #21 - ror x8, x11, #44 - sli V20.2D, V31.2D, #62 - ror x11, x25, #3 - sli V2.2D, V12.2D, #43 - ror x25, x16, #25 - eor V31.16B, V13.16B, V28.16B - ror x16, x23, #46 - eor V19.16B, V19.16B, V29.16B - ror x23, x4, #2 - ushr V12.2D, V31.2D, #39 - ror x4, x14, #21 - ushr V13.2D, V19.2D, #56 - ror x14, x15, #39 - sli V12.2D, V31.2D, #25 - ror x15, x22, #56 - sli V13.2D, V19.2D, #8 - ror x22, x26, #8 - eor V31.16B, V23.16B, V28.16B - ror x26, x17, #23 - eor V15.16B, V15.16B, V25.16B - ror x17, x6, #37 - ushr V19.2D, V31.2D, #8 - ror x6, x27, #50 - ushr V23.2D, V15.2D, #23 - ror x27, x24, #62 - sli V19.2D, V31.2D, #56 - ror x24, x10, #9 - sli V23.2D, V15.2D, #41 - ror x10, x19, #19 - eor V31.16B, V4.16B, V29.16B - ror x19, x7, #28 - eor V24.16B, V24.16B, V29.16B - ror x7, x5, #36 - ushr V15.2D, V31.2D, #37 - ror x5, x21, #43 - ushr V4.2D, V24.2D, #50 - ror x21, x20, #49 - sli V15.2D, V31.2D, #27 - ror x20, x13, #54 - sli V4.2D, V24.2D, #14 - ror x13, x9, #58 - eor V31.16B, V21.16B, V26.16B - ror x9, x12, #61 - ; Row Mix Base - eor V8.16B, V8.16B, V28.16B - bic x12, x4, x3 - ushr V24.2D, V31.2D, #62 - bic x1, x5, x4 - ushr V21.2D, V8.2D, #9 - bic x28, x2, x6 - sli V24.2D, V31.2D, #2 - bic x30, x3, x2 - sli V21.2D, V8.2D, #55 - eor x2, x2, x12 - eor V31.16B, V16.16B, V26.16B - eor x3, x3, x1 - eor V5.16B, V5.16B, V25.16B - bic x12, x6, x5 - ushr V8.2D, V31.2D, #19 - eor x5, x5, x28 - ushr V16.2D, V5.2D, #28 - eor x4, x4, x12 - sli V8.2D, V31.2D, #45 - eor x6, x6, x30 - sli V16.2D, V5.2D, #36 - bic x12, x9, x8 - eor V31.16B, V3.16B, V28.16B - bic x1, x10, x9 - eor V18.16B, V18.16B, V28.16B - bic x28, x7, x11 - ushr V5.2D, V31.2D, #36 - bic x30, x8, x7 - ushr V3.2D, V18.2D, #43 - eor x7, x7, x12 - sli V5.2D, V31.2D, #28 - eor x8, x8, x1 - sli V3.2D, V18.2D, #21 - bic x12, x11, x10 - eor V31.16B, V17.16B, V27.16B - eor x10, x10, x28 - eor V11.16B, V11.16B, V26.16B - eor x9, x9, x12 - ushr V18.2D, V31.2D, #49 - eor x11, x11, x30 - ushr V17.2D, V11.2D, #54 - bic x12, x14, x13 - sli V18.2D, V31.2D, #15 - bic x1, x15, x14 - sli V17.2D, V11.2D, #10 - bic x28, x0, x16 - eor V31.16B, V7.16B, V27.16B - bic x30, x13, x0 - eor V10.16B, V10.16B, V25.16B - eor x12, x0, x12 - ushr V11.2D, V31.2D, #58 - eor x13, x13, x1 - ushr V7.2D, V10.2D, #61 - bic x0, x16, x15 - sli V11.2D, V31.2D, #6 - eor x15, x15, x28 - sli V7.2D, V10.2D, #3 - eor x14, x14, x0 - ; Row Mix NEON - bic V25.16B, V2.16B, V1.16B - eor x16, x16, x30 - bic V26.16B, V3.16B, V2.16B - bic x0, x20, x19 - bic V27.16B, V4.16B, V3.16B - bic x1, x21, x20 - bic V28.16B, V0.16B, V4.16B - bic x28, x17, x22 - bic V29.16B, V1.16B, V0.16B - bic x30, x19, x17 - eor V0.16B, V0.16B, V25.16B - eor x17, x17, x0 - eor V1.16B, V1.16B, V26.16B - eor x19, x19, x1 - eor V2.16B, V2.16B, V27.16B - bic x0, x22, x21 - eor V3.16B, V3.16B, V28.16B - eor x21, x21, x28 - eor V4.16B, V4.16B, V29.16B - eor x20, x20, x0 - bic V25.16B, V7.16B, V6.16B - eor x22, x22, x30 - bic V26.16B, V8.16B, V7.16B - bic x0, x25, x24 - bic V27.16B, V9.16B, V8.16B - bic x1, x26, x25 - bic V28.16B, V5.16B, V9.16B - bic x28, x23, x27 - bic V29.16B, V6.16B, V5.16B - bic x30, x24, x23 - eor V5.16B, V5.16B, V25.16B - eor x23, x23, x0 - eor V6.16B, V6.16B, V26.16B - eor x24, x24, x1 - eor V7.16B, V7.16B, V27.16B - bic x0, x27, x26 - eor V8.16B, V8.16B, V28.16B - eor x26, x26, x28 - eor V9.16B, V9.16B, V29.16B - eor x25, x25, x0 - bic V25.16B, V12.16B, V11.16B - eor x27, x27, x30 - bic V26.16B, V13.16B, V12.16B - bic V27.16B, V14.16B, V13.16B - bic V28.16B, V30.16B, V14.16B - bic V29.16B, V11.16B, V30.16B - eor V10.16B, V30.16B, V25.16B - eor V11.16B, V11.16B, V26.16B - eor V12.16B, V12.16B, V27.16B - eor V13.16B, V13.16B, V28.16B - eor V14.16B, V14.16B, V29.16B - bic V25.16B, V17.16B, V16.16B - bic V26.16B, V18.16B, V17.16B - bic V27.16B, V19.16B, V18.16B - bic V28.16B, V15.16B, V19.16B - bic V29.16B, V16.16B, V15.16B - eor V15.16B, V15.16B, V25.16B - eor V16.16B, V16.16B, V26.16B - eor V17.16B, V17.16B, V27.16B - eor V18.16B, V18.16B, V28.16B - eor V19.16B, V19.16B, V29.16B - bic V25.16B, V22.16B, V21.16B - bic V26.16B, V23.16B, V22.16B - bic V27.16B, V24.16B, V23.16B - bic V28.16B, V20.16B, V24.16B - bic V29.16B, V21.16B, V20.16B - eor V20.16B, V20.16B, V25.16B - eor V21.16B, V21.16B, V26.16B - eor V22.16B, V22.16B, V27.16B - eor V23.16B, V23.16B, V28.16B - eor V24.16B, V24.16B, V29.16B - ; Done transforming - ldp x28, x1, [x29, #48] - ldr x0, [x28], #8 - subs x1, x1, #1 - mov V30.D[0], x0 - mov V30.D[1], x0 - eor x2, x2, x0 - eor V0.16B, V0.16B, V30.16B - bne L_SHA3_shake256_blocksx3_seed_neon_begin - ldr x0, [x29, #40] - st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 - st1 {V24.D}[0], [x0] - add x0, x0, #8 - st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 - st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 - st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 - st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 - st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 - st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 - st1 {V24.D}[1], [x0] - add x0, x0, #8 - stp x2, x3, [x0] - stp x4, x5, [x0, #16] - stp x6, x7, [x0, #32] - stp x8, x9, [x0, #48] - stp x10, x11, [x0, #64] - stp x12, x13, [x0, #80] - stp x14, x15, [x0, #96] - stp x16, x17, [x0, #112] - stp x19, x20, [x0, #128] - stp x21, x22, [x0, #144] - stp x23, x24, [x0, #160] - stp x25, x26, [x0, #176] - str x27, [x0, #192] - ldp x17, x19, [x29, #72] - ldp x20, x21, [x29, #88] - ldp x22, x23, [x29, #104] - ldp x24, x25, [x29, #120] - ldp x26, x27, [x29, #136] - ldr x28, [x29, #152] - ldp D8, D9, [x29, #160] - ldp D10, D11, [x29, #176] - ldp D12, D13, [x29, #192] - ldp D14, D15, [x29, #208] - ldp x29, x30, [sp], #0xe0 - ret - ENDP ENDIF END diff --git a/wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c b/wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c index 2b1f5b40b0f..f2938adeb77 100644 --- a/wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-mlkem-asm_c.c @@ -9104,2160 +9104,6 @@ unsigned int mlkem_rej_uniform_neon(sword16* p, unsigned int len, const byte* r, return (word32)(size_t)p; } -XALIGNED(16) static const word64 L_sha3_aarch64_r[] = { - 0x0000000000000001UL, 0x0000000000008082UL, - 0x800000000000808aUL, 0x8000000080008000UL, - 0x000000000000808bUL, 0x0000000080000001UL, - 0x8000000080008081UL, 0x8000000000008009UL, - 0x000000000000008aUL, 0x0000000000000088UL, - 0x0000000080008009UL, 0x000000008000000aUL, - 0x000000008000808bUL, 0x800000000000008bUL, - 0x8000000000008089UL, 0x8000000000008003UL, - 0x8000000000008002UL, 0x8000000000000080UL, - 0x000000000000800aUL, 0x800000008000000aUL, - 0x8000000080008081UL, 0x8000000000008080UL, - 0x0000000080000001UL, 0x8000000080008008UL, -}; - -#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 -void mlkem_sha3_blocksx3_crypto(word64* state) -{ - const word64* r = L_sha3_aarch64_r; - __asm__ __volatile__ ( - "stp x29, x30, [sp, #-64]!\n\t" - "add x29, sp, #0\n\t" - "str %x[state], [x29, #40]\n\t" - "ld4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" - "ld4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" - "ld4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" - "ld4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" - "ld4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" - "ld4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" - "ld1 {v24.d}[0], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "ld4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" - "ld4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" - "ld4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" - "ld4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" - "ld4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" - "ld4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" - "ld1 {v24.d}[1], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "ldp x1, x2, [%x[state]]\n\t" - "ldp x3, x4, [%x[state], #16]\n\t" - "ldp x5, x6, [%x[state], #32]\n\t" - "ldp x7, x8, [%x[state], #48]\n\t" - "ldp x9, x10, [%x[state], #64]\n\t" - "ldp x11, x12, [%x[state], #80]\n\t" - "ldp x13, x14, [%x[state], #96]\n\t" - "ldp x15, x16, [%x[state], #112]\n\t" - "ldp x17, x19, [%x[state], #128]\n\t" - "ldp x20, x21, [%x[state], #144]\n\t" - "ldp x22, x23, [%x[state], #160]\n\t" - "ldp x24, x25, [%x[state], #176]\n\t" - "ldr x26, [%x[state], #192]\n\t" - "mov x28, #24\n\t" - /* Start of 24 rounds */ - "\n" - "L_SHA3_transform_blocksx3_crypto_begin_%=:\n\t" - "stp %[r], x28, [x29, #48]\n\t" - /* Col Mix */ - "eor3 v31.16b, v0.16b, v5.16b, v10.16b\n\t" - "eor %x[state], x5, x10\n\t" - "eor3 v27.16b, v1.16b, v6.16b, v11.16b\n\t" - "eor x30, x1, x6\n\t" - "eor3 v28.16b, v2.16b, v7.16b, v12.16b\n\t" - "eor x28, x3, x8\n\t" - "eor3 v29.16b, v3.16b, v8.16b, v13.16b\n\t" - "eor %x[state], %x[state], x15\n\t" - "eor3 v30.16b, v4.16b, v9.16b, v14.16b\n\t" - "eor x30, x30, x11\n\t" - "eor3 v31.16b, v31.16b, v15.16b, v20.16b\n\t" - "eor x28, x28, x13\n\t" - "eor3 v27.16b, v27.16b, v16.16b, v21.16b\n\t" - "eor %x[state], %x[state], x21\n\t" - "eor3 v28.16b, v28.16b, v17.16b, v22.16b\n\t" - "eor x30, x30, x16\n\t" - "eor3 v29.16b, v29.16b, v18.16b, v23.16b\n\t" - "eor x28, x28, x19\n\t" - "eor3 v30.16b, v30.16b, v19.16b, v24.16b\n\t" - "eor %x[state], %x[state], x26\n\t" - "rax1 v25.2d, v30.2d, v27.2d\n\t" - "eor x30, x30, x22\n\t" - "rax1 v26.2d, v31.2d, v28.2d\n\t" - "eor x28, x28, x24\n\t" - "rax1 v27.2d, v27.2d, v29.2d\n\t" - "str %x[state], [x29, #32]\n\t" - "rax1 v28.2d, v28.2d, v30.2d\n\t" - "str x28, [x29, #24]\n\t" - "rax1 v29.2d, v29.2d, v31.2d\n\t" - "eor %[r], x2, x7\n\t" - "eor v0.16b, v0.16b, v25.16b\n\t" - "xar v30.2d, v1.2d, v26.2d, #63\n\t" - "eor x28, x4, x9\n\t" - "xar v1.2d, v6.2d, v26.2d, #20\n\t" - "eor %[r], %[r], x12\n\t" - "xar v6.2d, v9.2d, v29.2d, #44\n\t" - "eor x28, x28, x14\n\t" - "xar v9.2d, v22.2d, v27.2d, #3\n\t" - "eor %[r], %[r], x17\n\t" - "xar v22.2d, v14.2d, v29.2d, #25\n\t" - "eor x28, x28, x20\n\t" - "xar v14.2d, v20.2d, v25.2d, #46\n\t" - "eor %[r], %[r], x23\n\t" - "xar v20.2d, v2.2d, v27.2d, #2\n\t" - "eor x28, x28, x25\n\t" - "xar v2.2d, v12.2d, v27.2d, #21\n\t" - "eor %x[state], %x[state], %[r], ror 63\n\t" - "xar v12.2d, v13.2d, v28.2d, #39\n\t" - "eor %[r], %[r], x28, ror 63\n\t" - "xar v13.2d, v19.2d, v29.2d, #56\n\t" - "eor x1, x1, %x[state]\n\t" - "xar v19.2d, v23.2d, v28.2d, #8\n\t" - "eor x6, x6, %x[state]\n\t" - "xar v23.2d, v15.2d, v25.2d, #23\n\t" - "eor x11, x11, %x[state]\n\t" - "xar v15.2d, v4.2d, v29.2d, #37\n\t" - "eor x16, x16, %x[state]\n\t" - "xar v4.2d, v24.2d, v29.2d, #50\n\t" - "eor x22, x22, %x[state]\n\t" - "xar v24.2d, v21.2d, v26.2d, #62\n\t" - "eor x3, x3, %[r]\n\t" - "xar v21.2d, v8.2d, v28.2d, #9\n\t" - "eor x8, x8, %[r]\n\t" - "xar v8.2d, v16.2d, v26.2d, #19\n\t" - "eor x13, x13, %[r]\n\t" - "xar v16.2d, v5.2d, v25.2d, #28\n\t" - "eor x19, x19, %[r]\n\t" - "xar v5.2d, v3.2d, v28.2d, #36\n\t" - "eor x24, x24, %[r]\n\t" - "xar v3.2d, v18.2d, v28.2d, #43\n\t" - "ldr %x[state], [x29, #32]\n\t" - "xar v18.2d, v17.2d, v27.2d, #49\n\t" - "ldr %[r], [x29, #24]\n\t" - "xar v17.2d, v11.2d, v26.2d, #54\n\t" - "eor x28, x28, x30, ror 63\n\t" - "xar v11.2d, v7.2d, v27.2d, #58\n\t" - "eor x30, x30, %[r], ror 63\n\t" - "xar v7.2d, v10.2d, v25.2d, #61\n\t" - "eor %[r], %[r], %x[state], ror 63\n\t" - /* Row Mix */ - "mov v25.16b, v0.16b\n\t" - "eor x5, x5, x28\n\t" - "mov v26.16b, v1.16b\n\t" - "eor x10, x10, x28\n\t" - "bcax v0.16b, v25.16b, v2.16b, v26.16b\n\t" - "eor x15, x15, x28\n\t" - "bcax v1.16b, v26.16b, v3.16b, v2.16b\n\t" - "eor x21, x21, x28\n\t" - "bcax v2.16b, v2.16b, v4.16b, v3.16b\n\t" - "eor x26, x26, x28\n\t" - "bcax v3.16b, v3.16b, v25.16b, v4.16b\n\t" - "eor x2, x2, x30\n\t" - "bcax v4.16b, v4.16b, v26.16b, v25.16b\n\t" - "eor x7, x7, x30\n\t" - "mov v25.16b, v5.16b\n\t" - "eor x12, x12, x30\n\t" - "mov v26.16b, v6.16b\n\t" - "eor x17, x17, x30\n\t" - "bcax v5.16b, v25.16b, v7.16b, v26.16b\n\t" - "eor x23, x23, x30\n\t" - "bcax v6.16b, v26.16b, v8.16b, v7.16b\n\t" - "eor x4, x4, %[r]\n\t" - "bcax v7.16b, v7.16b, v9.16b, v8.16b\n\t" - "eor x9, x9, %[r]\n\t" - "bcax v8.16b, v8.16b, v25.16b, v9.16b\n\t" - "eor x14, x14, %[r]\n\t" - "bcax v9.16b, v9.16b, v26.16b, v25.16b\n\t" - "eor x20, x20, %[r]\n\t" - "mov v26.16b, v11.16b\n\t" - "eor x25, x25, %[r]\n\t" - /* Swap Rotate Base */ - "bcax v10.16b, v30.16b, v12.16b, v26.16b\n\t" - "ror %x[state], x2, #63\n\t" - "bcax v11.16b, v26.16b, v13.16b, v12.16b\n\t" - "ror x2, x7, #20\n\t" - "bcax v12.16b, v12.16b, v14.16b, v13.16b\n\t" - "ror x7, x10, #44\n\t" - "bcax v13.16b, v13.16b, v30.16b, v14.16b\n\t" - "ror x10, x24, #3\n\t" - "bcax v14.16b, v14.16b, v26.16b, v30.16b\n\t" - "ror x24, x15, #25\n\t" - "mov v25.16b, v15.16b\n\t" - "ror x15, x22, #46\n\t" - "mov v26.16b, v16.16b\n\t" - "ror x22, x3, #2\n\t" - "bcax v15.16b, v25.16b, v17.16b, v26.16b\n\t" - "ror x3, x13, #21\n\t" - "bcax v16.16b, v26.16b, v18.16b, v17.16b\n\t" - "ror x13, x14, #39\n\t" - "bcax v17.16b, v17.16b, v19.16b, v18.16b\n\t" - "ror x14, x21, #56\n\t" - "bcax v18.16b, v18.16b, v25.16b, v19.16b\n\t" - "ror x21, x25, #8\n\t" - "bcax v19.16b, v19.16b, v26.16b, v25.16b\n\t" - "ror x25, x16, #23\n\t" - "mov v25.16b, v20.16b\n\t" - "ror x16, x5, #37\n\t" - "mov v26.16b, v21.16b\n\t" - "ror x5, x26, #50\n\t" - "bcax v20.16b, v25.16b, v22.16b, v26.16b\n\t" - "ror x26, x23, #62\n\t" - "bcax v21.16b, v26.16b, v23.16b, v22.16b\n\t" - "ror x23, x9, #9\n\t" - "bcax v22.16b, v22.16b, v24.16b, v23.16b\n\t" - "ror x9, x17, #19\n\t" - "bcax v23.16b, v23.16b, v25.16b, v24.16b\n\t" - "ror x17, x6, #28\n\t" - "bcax v24.16b, v24.16b, v26.16b, v25.16b\n\t" - "ror x6, x4, #36\n\t" - "ror x4, x20, #43\n\t" - "ror x20, x19, #49\n\t" - "ror x19, x12, #54\n\t" - "ror x12, x8, #58\n\t" - "ror x8, x11, #61\n\t" - /* Row Mix Base */ - "bic x11, x3, x2\n\t" - "bic %[r], x4, x3\n\t" - "bic x28, x1, x5\n\t" - "bic x30, x2, x1\n\t" - "eor x1, x1, x11\n\t" - "eor x2, x2, %[r]\n\t" - "bic x11, x5, x4\n\t" - "eor x4, x4, x28\n\t" - "eor x3, x3, x11\n\t" - "eor x5, x5, x30\n\t" - "bic x11, x8, x7\n\t" - "bic %[r], x9, x8\n\t" - "bic x28, x6, x10\n\t" - "bic x30, x7, x6\n\t" - "eor x6, x6, x11\n\t" - "eor x7, x7, %[r]\n\t" - "bic x11, x10, x9\n\t" - "eor x9, x9, x28\n\t" - "eor x8, x8, x11\n\t" - "eor x10, x10, x30\n\t" - "bic x11, x13, x12\n\t" - "bic %[r], x14, x13\n\t" - "bic x28, %x[state], x15\n\t" - "bic x30, x12, %x[state]\n\t" - "eor x11, %x[state], x11\n\t" - "eor x12, x12, %[r]\n\t" - "bic %x[state], x15, x14\n\t" - "eor x14, x14, x28\n\t" - "eor x13, x13, %x[state]\n\t" - "eor x15, x15, x30\n\t" - "bic %x[state], x19, x17\n\t" - "bic %[r], x20, x19\n\t" - "bic x28, x16, x21\n\t" - "bic x30, x17, x16\n\t" - "eor x16, x16, %x[state]\n\t" - "eor x17, x17, %[r]\n\t" - "bic %x[state], x21, x20\n\t" - "eor x20, x20, x28\n\t" - "eor x19, x19, %x[state]\n\t" - "eor x21, x21, x30\n\t" - "bic %x[state], x24, x23\n\t" - "bic %[r], x25, x24\n\t" - "bic x28, x22, x26\n\t" - "bic x30, x23, x22\n\t" - "eor x22, x22, %x[state]\n\t" - "eor x23, x23, %[r]\n\t" - "bic %x[state], x26, x25\n\t" - "eor x25, x25, x28\n\t" - "eor x24, x24, %x[state]\n\t" - "eor x26, x26, x30\n\t" - /* Done transforming */ - "ldp %[r], x28, [x29, #48]\n\t" - "ldr %x[state], [%[r]], #8\n\t" - "subs x28, x28, #1\n\t" - "mov v30.d[0], %x[state]\n\t" - "mov v30.d[1], %x[state]\n\t" - "eor x1, x1, %x[state]\n\t" - "eor v0.16b, v0.16b, v30.16b\n\t" - "b.ne L_SHA3_transform_blocksx3_crypto_begin_%=\n\t" - "ldr %x[state], [x29, #40]\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" - "st1 {v24.d}[0], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" - "st1 {v24.d}[1], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "stp x1, x2, [%x[state]]\n\t" - "stp x3, x4, [%x[state], #16]\n\t" - "stp x5, x6, [%x[state], #32]\n\t" - "stp x7, x8, [%x[state], #48]\n\t" - "stp x9, x10, [%x[state], #64]\n\t" - "stp x11, x12, [%x[state], #80]\n\t" - "stp x13, x14, [%x[state], #96]\n\t" - "stp x15, x16, [%x[state], #112]\n\t" - "stp x17, x19, [%x[state], #128]\n\t" - "stp x20, x21, [%x[state], #144]\n\t" - "stp x22, x23, [%x[state], #160]\n\t" - "stp x24, x25, [%x[state], #176]\n\t" - "str x26, [%x[state], #192]\n\t" - "ldp x29, x30, [sp], #0x40\n\t" - : [state] "+r" (state) - : [r] "r" (r) - : "memory", "cc", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", - "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", - "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x28", "v0", "v1", - "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", - "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", - "v31" - ); -} - -void mlkem_shake128_blocksx3_seed_crypto(word64* state, byte* seed) -{ - const word64* r = L_sha3_aarch64_r; - __asm__ __volatile__ ( - "stp x29, x30, [sp, #-64]!\n\t" - "add x29, sp, #0\n\t" - "str %x[state], [x29, #40]\n\t" - "add %x[state], %x[state], #32\n\t" - "ld1 {v4.d}[0], [%x[state]]\n\t" - "ldp x2, x3, [%x[seed]], #16\n\t" - "add %x[state], %x[state], #0xc8\n\t" - "ld1 {v4.d}[1], [%x[state]]\n\t" - "ldp x4, x5, [%x[seed]], #16\n\t" - "ldr x6, [%x[state], #200]\n\t" - "eor v5.16b, v5.16b, v5.16b\n\t" - "eor x7, x7, x7\n\t" - "eor v6.16b, v6.16b, v6.16b\n\t" - "eor x8, x8, x8\n\t" - "eor v7.16b, v7.16b, v7.16b\n\t" - "eor x9, x9, x9\n\t" - "eor v8.16b, v8.16b, v8.16b\n\t" - "eor x10, x10, x10\n\t" - "eor v9.16b, v9.16b, v9.16b\n\t" - "eor x11, x11, x11\n\t" - "eor v10.16b, v10.16b, v10.16b\n\t" - "eor x12, x12, x12\n\t" - "eor v11.16b, v11.16b, v11.16b\n\t" - "eor x13, x13, x13\n\t" - "eor v12.16b, v12.16b, v12.16b\n\t" - "eor x14, x14, x14\n\t" - "eor v13.16b, v13.16b, v13.16b\n\t" - "eor x15, x15, x15\n\t" - "eor v14.16b, v14.16b, v14.16b\n\t" - "eor x16, x16, x16\n\t" - "eor v15.16b, v15.16b, v15.16b\n\t" - "eor x17, x17, x17\n\t" - "eor v16.16b, v16.16b, v16.16b\n\t" - "eor x19, x19, x19\n\t" - "eor v17.16b, v17.16b, v17.16b\n\t" - "eor x20, x20, x20\n\t" - "eor v18.16b, v18.16b, v18.16b\n\t" - "eor x21, x21, x21\n\t" - "eor v19.16b, v19.16b, v19.16b\n\t" - "eor x22, x22, x22\n\t" - "movz x23, #0x8000, lsl 48\n\t" - "eor v21.16b, v21.16b, v21.16b\n\t" - "eor x24, x24, x24\n\t" - "eor v22.16b, v22.16b, v22.16b\n\t" - "eor x25, x25, x25\n\t" - "eor v23.16b, v23.16b, v23.16b\n\t" - "eor x26, x26, x26\n\t" - "eor v24.16b, v24.16b, v24.16b\n\t" - "eor x27, x27, x27\n\t" - "dup v0.2d, x2\n\t" - "dup v1.2d, x3\n\t" - "dup v2.2d, x4\n\t" - "dup v3.2d, x5\n\t" - "dup v20.2d, x23\n\t" - "mov %x[seed], #24\n\t" - /* Start of 24 rounds */ - "\n" - "L_SHA3_shake128_blocksx3_seed_crypto_begin_%=:\n\t" - "stp %[r], %x[seed], [x29, #48]\n\t" - /* Col Mix */ - "eor3 v31.16b, v0.16b, v5.16b, v10.16b\n\t" - "eor %x[state], x6, x11\n\t" - "eor3 v27.16b, v1.16b, v6.16b, v11.16b\n\t" - "eor x30, x2, x7\n\t" - "eor3 v28.16b, v2.16b, v7.16b, v12.16b\n\t" - "eor %[r], x4, x9\n\t" - "eor3 v29.16b, v3.16b, v8.16b, v13.16b\n\t" - "eor %x[state], %x[state], x16\n\t" - "eor3 v30.16b, v4.16b, v9.16b, v14.16b\n\t" - "eor x30, x30, x12\n\t" - "eor3 v31.16b, v31.16b, v15.16b, v20.16b\n\t" - "eor %[r], %[r], x14\n\t" - "eor3 v27.16b, v27.16b, v16.16b, v21.16b\n\t" - "eor %x[state], %x[state], x22\n\t" - "eor3 v28.16b, v28.16b, v17.16b, v22.16b\n\t" - "eor x30, x30, x17\n\t" - "eor3 v29.16b, v29.16b, v18.16b, v23.16b\n\t" - "eor %[r], %[r], x20\n\t" - "eor3 v30.16b, v30.16b, v19.16b, v24.16b\n\t" - "eor %x[state], %x[state], x27\n\t" - "rax1 v25.2d, v30.2d, v27.2d\n\t" - "eor x30, x30, x23\n\t" - "rax1 v26.2d, v31.2d, v28.2d\n\t" - "eor %[r], %[r], x25\n\t" - "rax1 v27.2d, v27.2d, v29.2d\n\t" - "str %x[state], [x29, #32]\n\t" - "rax1 v28.2d, v28.2d, v30.2d\n\t" - "str %[r], [x29, #24]\n\t" - "rax1 v29.2d, v29.2d, v31.2d\n\t" - "eor %x[seed], x3, x8\n\t" - "eor v0.16b, v0.16b, v25.16b\n\t" - "xar v30.2d, v1.2d, v26.2d, #63\n\t" - "eor %[r], x5, x10\n\t" - "xar v1.2d, v6.2d, v26.2d, #20\n\t" - "eor %x[seed], %x[seed], x13\n\t" - "xar v6.2d, v9.2d, v29.2d, #44\n\t" - "eor %[r], %[r], x15\n\t" - "xar v9.2d, v22.2d, v27.2d, #3\n\t" - "eor %x[seed], %x[seed], x19\n\t" - "xar v22.2d, v14.2d, v29.2d, #25\n\t" - "eor %[r], %[r], x21\n\t" - "xar v14.2d, v20.2d, v25.2d, #46\n\t" - "eor %x[seed], %x[seed], x24\n\t" - "xar v20.2d, v2.2d, v27.2d, #2\n\t" - "eor %[r], %[r], x26\n\t" - "xar v2.2d, v12.2d, v27.2d, #21\n\t" - "eor %x[state], %x[state], %x[seed], ror 63\n\t" - "xar v12.2d, v13.2d, v28.2d, #39\n\t" - "eor %x[seed], %x[seed], %[r], ror 63\n\t" - "xar v13.2d, v19.2d, v29.2d, #56\n\t" - "eor x2, x2, %x[state]\n\t" - "xar v19.2d, v23.2d, v28.2d, #8\n\t" - "eor x7, x7, %x[state]\n\t" - "xar v23.2d, v15.2d, v25.2d, #23\n\t" - "eor x12, x12, %x[state]\n\t" - "xar v15.2d, v4.2d, v29.2d, #37\n\t" - "eor x17, x17, %x[state]\n\t" - "xar v4.2d, v24.2d, v29.2d, #50\n\t" - "eor x23, x23, %x[state]\n\t" - "xar v24.2d, v21.2d, v26.2d, #62\n\t" - "eor x4, x4, %x[seed]\n\t" - "xar v21.2d, v8.2d, v28.2d, #9\n\t" - "eor x9, x9, %x[seed]\n\t" - "xar v8.2d, v16.2d, v26.2d, #19\n\t" - "eor x14, x14, %x[seed]\n\t" - "xar v16.2d, v5.2d, v25.2d, #28\n\t" - "eor x20, x20, %x[seed]\n\t" - "xar v5.2d, v3.2d, v28.2d, #36\n\t" - "eor x25, x25, %x[seed]\n\t" - "xar v3.2d, v18.2d, v28.2d, #43\n\t" - "ldr %x[state], [x29, #32]\n\t" - "xar v18.2d, v17.2d, v27.2d, #49\n\t" - "ldr %x[seed], [x29, #24]\n\t" - "xar v17.2d, v11.2d, v26.2d, #54\n\t" - "eor %[r], %[r], x30, ror 63\n\t" - "xar v11.2d, v7.2d, v27.2d, #58\n\t" - "eor x30, x30, %x[seed], ror 63\n\t" - "xar v7.2d, v10.2d, v25.2d, #61\n\t" - "eor %x[seed], %x[seed], %x[state], ror 63\n\t" - /* Row Mix */ - "mov v25.16b, v0.16b\n\t" - "eor x6, x6, %[r]\n\t" - "mov v26.16b, v1.16b\n\t" - "eor x11, x11, %[r]\n\t" - "bcax v0.16b, v25.16b, v2.16b, v26.16b\n\t" - "eor x16, x16, %[r]\n\t" - "bcax v1.16b, v26.16b, v3.16b, v2.16b\n\t" - "eor x22, x22, %[r]\n\t" - "bcax v2.16b, v2.16b, v4.16b, v3.16b\n\t" - "eor x27, x27, %[r]\n\t" - "bcax v3.16b, v3.16b, v25.16b, v4.16b\n\t" - "eor x3, x3, x30\n\t" - "bcax v4.16b, v4.16b, v26.16b, v25.16b\n\t" - "eor x8, x8, x30\n\t" - "mov v25.16b, v5.16b\n\t" - "eor x13, x13, x30\n\t" - "mov v26.16b, v6.16b\n\t" - "eor x19, x19, x30\n\t" - "bcax v5.16b, v25.16b, v7.16b, v26.16b\n\t" - "eor x24, x24, x30\n\t" - "bcax v6.16b, v26.16b, v8.16b, v7.16b\n\t" - "eor x5, x5, %x[seed]\n\t" - "bcax v7.16b, v7.16b, v9.16b, v8.16b\n\t" - "eor x10, x10, %x[seed]\n\t" - "bcax v8.16b, v8.16b, v25.16b, v9.16b\n\t" - "eor x15, x15, %x[seed]\n\t" - "bcax v9.16b, v9.16b, v26.16b, v25.16b\n\t" - "eor x21, x21, %x[seed]\n\t" - "mov v26.16b, v11.16b\n\t" - "eor x26, x26, %x[seed]\n\t" - /* Swap Rotate Base */ - "bcax v10.16b, v30.16b, v12.16b, v26.16b\n\t" - "ror %x[state], x3, #63\n\t" - "bcax v11.16b, v26.16b, v13.16b, v12.16b\n\t" - "ror x3, x8, #20\n\t" - "bcax v12.16b, v12.16b, v14.16b, v13.16b\n\t" - "ror x8, x11, #44\n\t" - "bcax v13.16b, v13.16b, v30.16b, v14.16b\n\t" - "ror x11, x25, #3\n\t" - "bcax v14.16b, v14.16b, v26.16b, v30.16b\n\t" - "ror x25, x16, #25\n\t" - "mov v25.16b, v15.16b\n\t" - "ror x16, x23, #46\n\t" - "mov v26.16b, v16.16b\n\t" - "ror x23, x4, #2\n\t" - "bcax v15.16b, v25.16b, v17.16b, v26.16b\n\t" - "ror x4, x14, #21\n\t" - "bcax v16.16b, v26.16b, v18.16b, v17.16b\n\t" - "ror x14, x15, #39\n\t" - "bcax v17.16b, v17.16b, v19.16b, v18.16b\n\t" - "ror x15, x22, #56\n\t" - "bcax v18.16b, v18.16b, v25.16b, v19.16b\n\t" - "ror x22, x26, #8\n\t" - "bcax v19.16b, v19.16b, v26.16b, v25.16b\n\t" - "ror x26, x17, #23\n\t" - "mov v25.16b, v20.16b\n\t" - "ror x17, x6, #37\n\t" - "mov v26.16b, v21.16b\n\t" - "ror x6, x27, #50\n\t" - "bcax v20.16b, v25.16b, v22.16b, v26.16b\n\t" - "ror x27, x24, #62\n\t" - "bcax v21.16b, v26.16b, v23.16b, v22.16b\n\t" - "ror x24, x10, #9\n\t" - "bcax v22.16b, v22.16b, v24.16b, v23.16b\n\t" - "ror x10, x19, #19\n\t" - "bcax v23.16b, v23.16b, v25.16b, v24.16b\n\t" - "ror x19, x7, #28\n\t" - "bcax v24.16b, v24.16b, v26.16b, v25.16b\n\t" - "ror x7, x5, #36\n\t" - "ror x5, x21, #43\n\t" - "ror x21, x20, #49\n\t" - "ror x20, x13, #54\n\t" - "ror x13, x9, #58\n\t" - "ror x9, x12, #61\n\t" - /* Row Mix Base */ - "bic x12, x4, x3\n\t" - "bic %x[seed], x5, x4\n\t" - "bic %[r], x2, x6\n\t" - "bic x30, x3, x2\n\t" - "eor x2, x2, x12\n\t" - "eor x3, x3, %x[seed]\n\t" - "bic x12, x6, x5\n\t" - "eor x5, x5, %[r]\n\t" - "eor x4, x4, x12\n\t" - "eor x6, x6, x30\n\t" - "bic x12, x9, x8\n\t" - "bic %x[seed], x10, x9\n\t" - "bic %[r], x7, x11\n\t" - "bic x30, x8, x7\n\t" - "eor x7, x7, x12\n\t" - "eor x8, x8, %x[seed]\n\t" - "bic x12, x11, x10\n\t" - "eor x10, x10, %[r]\n\t" - "eor x9, x9, x12\n\t" - "eor x11, x11, x30\n\t" - "bic x12, x14, x13\n\t" - "bic %x[seed], x15, x14\n\t" - "bic %[r], %x[state], x16\n\t" - "bic x30, x13, %x[state]\n\t" - "eor x12, %x[state], x12\n\t" - "eor x13, x13, %x[seed]\n\t" - "bic %x[state], x16, x15\n\t" - "eor x15, x15, %[r]\n\t" - "eor x14, x14, %x[state]\n\t" - "eor x16, x16, x30\n\t" - "bic %x[state], x20, x19\n\t" - "bic %x[seed], x21, x20\n\t" - "bic %[r], x17, x22\n\t" - "bic x30, x19, x17\n\t" - "eor x17, x17, %x[state]\n\t" - "eor x19, x19, %x[seed]\n\t" - "bic %x[state], x22, x21\n\t" - "eor x21, x21, %[r]\n\t" - "eor x20, x20, %x[state]\n\t" - "eor x22, x22, x30\n\t" - "bic %x[state], x25, x24\n\t" - "bic %x[seed], x26, x25\n\t" - "bic %[r], x23, x27\n\t" - "bic x30, x24, x23\n\t" - "eor x23, x23, %x[state]\n\t" - "eor x24, x24, %x[seed]\n\t" - "bic %x[state], x27, x26\n\t" - "eor x26, x26, %[r]\n\t" - "eor x25, x25, %x[state]\n\t" - "eor x27, x27, x30\n\t" - /* Done transforming */ - "ldp %[r], %x[seed], [x29, #48]\n\t" - "ldr %x[state], [%[r]], #8\n\t" - "subs %x[seed], %x[seed], #1\n\t" - "mov v30.d[0], %x[state]\n\t" - "mov v30.d[1], %x[state]\n\t" - "eor x2, x2, %x[state]\n\t" - "eor v0.16b, v0.16b, v30.16b\n\t" - "b.ne L_SHA3_shake128_blocksx3_seed_crypto_begin_%=\n\t" - "ldr %x[state], [x29, #40]\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" - "st1 {v24.d}[0], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" - "st1 {v24.d}[1], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "stp x2, x3, [%x[state]]\n\t" - "stp x4, x5, [%x[state], #16]\n\t" - "stp x6, x7, [%x[state], #32]\n\t" - "stp x8, x9, [%x[state], #48]\n\t" - "stp x10, x11, [%x[state], #64]\n\t" - "stp x12, x13, [%x[state], #80]\n\t" - "stp x14, x15, [%x[state], #96]\n\t" - "stp x16, x17, [%x[state], #112]\n\t" - "stp x19, x20, [%x[state], #128]\n\t" - "stp x21, x22, [%x[state], #144]\n\t" - "stp x23, x24, [%x[state], #160]\n\t" - "stp x25, x26, [%x[state], #176]\n\t" - "str x27, [%x[state], #192]\n\t" - "ldp x29, x30, [sp], #0x40\n\t" - : [state] "+r" (state), [seed] "+r" (seed) - : [r] "r" (r) - : "memory", "cc", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", - "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", - "x21", "x22", "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", - "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", - "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", - "v31" - ); -} - -void mlkem_shake256_blocksx3_seed_crypto(word64* state, byte* seed) -{ - const word64* r = L_sha3_aarch64_r; - __asm__ __volatile__ ( - "stp x29, x30, [sp, #-64]!\n\t" - "add x29, sp, #0\n\t" - "str %x[state], [x29, #40]\n\t" - "add %x[state], %x[state], #32\n\t" - "ld1 {v4.d}[0], [%x[state]]\n\t" - "ldp x2, x3, [%x[seed]], #16\n\t" - "add %x[state], %x[state], #0xc8\n\t" - "ld1 {v4.d}[1], [%x[state]]\n\t" - "ldp x4, x5, [%x[seed]], #16\n\t" - "ldr x6, [%x[state], #200]\n\t" - "eor v5.16b, v5.16b, v5.16b\n\t" - "eor x7, x7, x7\n\t" - "eor v6.16b, v6.16b, v6.16b\n\t" - "eor x8, x8, x8\n\t" - "eor v7.16b, v7.16b, v7.16b\n\t" - "eor x9, x9, x9\n\t" - "eor v8.16b, v8.16b, v8.16b\n\t" - "eor x10, x10, x10\n\t" - "eor v9.16b, v9.16b, v9.16b\n\t" - "eor x11, x11, x11\n\t" - "eor v10.16b, v10.16b, v10.16b\n\t" - "eor x12, x12, x12\n\t" - "eor v11.16b, v11.16b, v11.16b\n\t" - "eor x13, x13, x13\n\t" - "eor v12.16b, v12.16b, v12.16b\n\t" - "eor x14, x14, x14\n\t" - "eor v13.16b, v13.16b, v13.16b\n\t" - "eor x15, x15, x15\n\t" - "eor v14.16b, v14.16b, v14.16b\n\t" - "eor x16, x16, x16\n\t" - "eor v15.16b, v15.16b, v15.16b\n\t" - "eor x17, x17, x17\n\t" - "movz x19, #0x8000, lsl 48\n\t" - "eor v17.16b, v17.16b, v17.16b\n\t" - "eor x20, x20, x20\n\t" - "eor v18.16b, v18.16b, v18.16b\n\t" - "eor x21, x21, x21\n\t" - "eor v19.16b, v19.16b, v19.16b\n\t" - "eor x22, x22, x22\n\t" - "eor v20.16b, v20.16b, v20.16b\n\t" - "eor x23, x23, x23\n\t" - "eor v21.16b, v21.16b, v21.16b\n\t" - "eor x24, x24, x24\n\t" - "eor v22.16b, v22.16b, v22.16b\n\t" - "eor x25, x25, x25\n\t" - "eor v23.16b, v23.16b, v23.16b\n\t" - "eor x26, x26, x26\n\t" - "eor v24.16b, v24.16b, v24.16b\n\t" - "eor x27, x27, x27\n\t" - "dup v0.2d, x2\n\t" - "dup v1.2d, x3\n\t" - "dup v2.2d, x4\n\t" - "dup v3.2d, x5\n\t" - "dup v16.2d, x19\n\t" - "mov %x[seed], #24\n\t" - /* Start of 24 rounds */ - "\n" - "L_SHA3_shake256_blocksx3_seed_crypto_begin_%=:\n\t" - "stp %[r], %x[seed], [x29, #48]\n\t" - /* Col Mix */ - "eor3 v31.16b, v0.16b, v5.16b, v10.16b\n\t" - "eor %x[state], x6, x11\n\t" - "eor3 v27.16b, v1.16b, v6.16b, v11.16b\n\t" - "eor x30, x2, x7\n\t" - "eor3 v28.16b, v2.16b, v7.16b, v12.16b\n\t" - "eor %[r], x4, x9\n\t" - "eor3 v29.16b, v3.16b, v8.16b, v13.16b\n\t" - "eor %x[state], %x[state], x16\n\t" - "eor3 v30.16b, v4.16b, v9.16b, v14.16b\n\t" - "eor x30, x30, x12\n\t" - "eor3 v31.16b, v31.16b, v15.16b, v20.16b\n\t" - "eor %[r], %[r], x14\n\t" - "eor3 v27.16b, v27.16b, v16.16b, v21.16b\n\t" - "eor %x[state], %x[state], x22\n\t" - "eor3 v28.16b, v28.16b, v17.16b, v22.16b\n\t" - "eor x30, x30, x17\n\t" - "eor3 v29.16b, v29.16b, v18.16b, v23.16b\n\t" - "eor %[r], %[r], x20\n\t" - "eor3 v30.16b, v30.16b, v19.16b, v24.16b\n\t" - "eor %x[state], %x[state], x27\n\t" - "rax1 v25.2d, v30.2d, v27.2d\n\t" - "eor x30, x30, x23\n\t" - "rax1 v26.2d, v31.2d, v28.2d\n\t" - "eor %[r], %[r], x25\n\t" - "rax1 v27.2d, v27.2d, v29.2d\n\t" - "str %x[state], [x29, #32]\n\t" - "rax1 v28.2d, v28.2d, v30.2d\n\t" - "str %[r], [x29, #24]\n\t" - "rax1 v29.2d, v29.2d, v31.2d\n\t" - "eor %x[seed], x3, x8\n\t" - "eor v0.16b, v0.16b, v25.16b\n\t" - "xar v30.2d, v1.2d, v26.2d, #63\n\t" - "eor %[r], x5, x10\n\t" - "xar v1.2d, v6.2d, v26.2d, #20\n\t" - "eor %x[seed], %x[seed], x13\n\t" - "xar v6.2d, v9.2d, v29.2d, #44\n\t" - "eor %[r], %[r], x15\n\t" - "xar v9.2d, v22.2d, v27.2d, #3\n\t" - "eor %x[seed], %x[seed], x19\n\t" - "xar v22.2d, v14.2d, v29.2d, #25\n\t" - "eor %[r], %[r], x21\n\t" - "xar v14.2d, v20.2d, v25.2d, #46\n\t" - "eor %x[seed], %x[seed], x24\n\t" - "xar v20.2d, v2.2d, v27.2d, #2\n\t" - "eor %[r], %[r], x26\n\t" - "xar v2.2d, v12.2d, v27.2d, #21\n\t" - "eor %x[state], %x[state], %x[seed], ror 63\n\t" - "xar v12.2d, v13.2d, v28.2d, #39\n\t" - "eor %x[seed], %x[seed], %[r], ror 63\n\t" - "xar v13.2d, v19.2d, v29.2d, #56\n\t" - "eor x2, x2, %x[state]\n\t" - "xar v19.2d, v23.2d, v28.2d, #8\n\t" - "eor x7, x7, %x[state]\n\t" - "xar v23.2d, v15.2d, v25.2d, #23\n\t" - "eor x12, x12, %x[state]\n\t" - "xar v15.2d, v4.2d, v29.2d, #37\n\t" - "eor x17, x17, %x[state]\n\t" - "xar v4.2d, v24.2d, v29.2d, #50\n\t" - "eor x23, x23, %x[state]\n\t" - "xar v24.2d, v21.2d, v26.2d, #62\n\t" - "eor x4, x4, %x[seed]\n\t" - "xar v21.2d, v8.2d, v28.2d, #9\n\t" - "eor x9, x9, %x[seed]\n\t" - "xar v8.2d, v16.2d, v26.2d, #19\n\t" - "eor x14, x14, %x[seed]\n\t" - "xar v16.2d, v5.2d, v25.2d, #28\n\t" - "eor x20, x20, %x[seed]\n\t" - "xar v5.2d, v3.2d, v28.2d, #36\n\t" - "eor x25, x25, %x[seed]\n\t" - "xar v3.2d, v18.2d, v28.2d, #43\n\t" - "ldr %x[state], [x29, #32]\n\t" - "xar v18.2d, v17.2d, v27.2d, #49\n\t" - "ldr %x[seed], [x29, #24]\n\t" - "xar v17.2d, v11.2d, v26.2d, #54\n\t" - "eor %[r], %[r], x30, ror 63\n\t" - "xar v11.2d, v7.2d, v27.2d, #58\n\t" - "eor x30, x30, %x[seed], ror 63\n\t" - "xar v7.2d, v10.2d, v25.2d, #61\n\t" - "eor %x[seed], %x[seed], %x[state], ror 63\n\t" - /* Row Mix */ - "mov v25.16b, v0.16b\n\t" - "eor x6, x6, %[r]\n\t" - "mov v26.16b, v1.16b\n\t" - "eor x11, x11, %[r]\n\t" - "bcax v0.16b, v25.16b, v2.16b, v26.16b\n\t" - "eor x16, x16, %[r]\n\t" - "bcax v1.16b, v26.16b, v3.16b, v2.16b\n\t" - "eor x22, x22, %[r]\n\t" - "bcax v2.16b, v2.16b, v4.16b, v3.16b\n\t" - "eor x27, x27, %[r]\n\t" - "bcax v3.16b, v3.16b, v25.16b, v4.16b\n\t" - "eor x3, x3, x30\n\t" - "bcax v4.16b, v4.16b, v26.16b, v25.16b\n\t" - "eor x8, x8, x30\n\t" - "mov v25.16b, v5.16b\n\t" - "eor x13, x13, x30\n\t" - "mov v26.16b, v6.16b\n\t" - "eor x19, x19, x30\n\t" - "bcax v5.16b, v25.16b, v7.16b, v26.16b\n\t" - "eor x24, x24, x30\n\t" - "bcax v6.16b, v26.16b, v8.16b, v7.16b\n\t" - "eor x5, x5, %x[seed]\n\t" - "bcax v7.16b, v7.16b, v9.16b, v8.16b\n\t" - "eor x10, x10, %x[seed]\n\t" - "bcax v8.16b, v8.16b, v25.16b, v9.16b\n\t" - "eor x15, x15, %x[seed]\n\t" - "bcax v9.16b, v9.16b, v26.16b, v25.16b\n\t" - "eor x21, x21, %x[seed]\n\t" - "mov v26.16b, v11.16b\n\t" - "eor x26, x26, %x[seed]\n\t" - /* Swap Rotate Base */ - "bcax v10.16b, v30.16b, v12.16b, v26.16b\n\t" - "ror %x[state], x3, #63\n\t" - "bcax v11.16b, v26.16b, v13.16b, v12.16b\n\t" - "ror x3, x8, #20\n\t" - "bcax v12.16b, v12.16b, v14.16b, v13.16b\n\t" - "ror x8, x11, #44\n\t" - "bcax v13.16b, v13.16b, v30.16b, v14.16b\n\t" - "ror x11, x25, #3\n\t" - "bcax v14.16b, v14.16b, v26.16b, v30.16b\n\t" - "ror x25, x16, #25\n\t" - "mov v25.16b, v15.16b\n\t" - "ror x16, x23, #46\n\t" - "mov v26.16b, v16.16b\n\t" - "ror x23, x4, #2\n\t" - "bcax v15.16b, v25.16b, v17.16b, v26.16b\n\t" - "ror x4, x14, #21\n\t" - "bcax v16.16b, v26.16b, v18.16b, v17.16b\n\t" - "ror x14, x15, #39\n\t" - "bcax v17.16b, v17.16b, v19.16b, v18.16b\n\t" - "ror x15, x22, #56\n\t" - "bcax v18.16b, v18.16b, v25.16b, v19.16b\n\t" - "ror x22, x26, #8\n\t" - "bcax v19.16b, v19.16b, v26.16b, v25.16b\n\t" - "ror x26, x17, #23\n\t" - "mov v25.16b, v20.16b\n\t" - "ror x17, x6, #37\n\t" - "mov v26.16b, v21.16b\n\t" - "ror x6, x27, #50\n\t" - "bcax v20.16b, v25.16b, v22.16b, v26.16b\n\t" - "ror x27, x24, #62\n\t" - "bcax v21.16b, v26.16b, v23.16b, v22.16b\n\t" - "ror x24, x10, #9\n\t" - "bcax v22.16b, v22.16b, v24.16b, v23.16b\n\t" - "ror x10, x19, #19\n\t" - "bcax v23.16b, v23.16b, v25.16b, v24.16b\n\t" - "ror x19, x7, #28\n\t" - "bcax v24.16b, v24.16b, v26.16b, v25.16b\n\t" - "ror x7, x5, #36\n\t" - "ror x5, x21, #43\n\t" - "ror x21, x20, #49\n\t" - "ror x20, x13, #54\n\t" - "ror x13, x9, #58\n\t" - "ror x9, x12, #61\n\t" - /* Row Mix Base */ - "bic x12, x4, x3\n\t" - "bic %x[seed], x5, x4\n\t" - "bic %[r], x2, x6\n\t" - "bic x30, x3, x2\n\t" - "eor x2, x2, x12\n\t" - "eor x3, x3, %x[seed]\n\t" - "bic x12, x6, x5\n\t" - "eor x5, x5, %[r]\n\t" - "eor x4, x4, x12\n\t" - "eor x6, x6, x30\n\t" - "bic x12, x9, x8\n\t" - "bic %x[seed], x10, x9\n\t" - "bic %[r], x7, x11\n\t" - "bic x30, x8, x7\n\t" - "eor x7, x7, x12\n\t" - "eor x8, x8, %x[seed]\n\t" - "bic x12, x11, x10\n\t" - "eor x10, x10, %[r]\n\t" - "eor x9, x9, x12\n\t" - "eor x11, x11, x30\n\t" - "bic x12, x14, x13\n\t" - "bic %x[seed], x15, x14\n\t" - "bic %[r], %x[state], x16\n\t" - "bic x30, x13, %x[state]\n\t" - "eor x12, %x[state], x12\n\t" - "eor x13, x13, %x[seed]\n\t" - "bic %x[state], x16, x15\n\t" - "eor x15, x15, %[r]\n\t" - "eor x14, x14, %x[state]\n\t" - "eor x16, x16, x30\n\t" - "bic %x[state], x20, x19\n\t" - "bic %x[seed], x21, x20\n\t" - "bic %[r], x17, x22\n\t" - "bic x30, x19, x17\n\t" - "eor x17, x17, %x[state]\n\t" - "eor x19, x19, %x[seed]\n\t" - "bic %x[state], x22, x21\n\t" - "eor x21, x21, %[r]\n\t" - "eor x20, x20, %x[state]\n\t" - "eor x22, x22, x30\n\t" - "bic %x[state], x25, x24\n\t" - "bic %x[seed], x26, x25\n\t" - "bic %[r], x23, x27\n\t" - "bic x30, x24, x23\n\t" - "eor x23, x23, %x[state]\n\t" - "eor x24, x24, %x[seed]\n\t" - "bic %x[state], x27, x26\n\t" - "eor x26, x26, %[r]\n\t" - "eor x25, x25, %x[state]\n\t" - "eor x27, x27, x30\n\t" - /* Done transforming */ - "ldp %[r], %x[seed], [x29, #48]\n\t" - "ldr %x[state], [%[r]], #8\n\t" - "subs %x[seed], %x[seed], #1\n\t" - "mov v30.d[0], %x[state]\n\t" - "mov v30.d[1], %x[state]\n\t" - "eor x2, x2, %x[state]\n\t" - "eor v0.16b, v0.16b, v30.16b\n\t" - "b.ne L_SHA3_shake256_blocksx3_seed_crypto_begin_%=\n\t" - "ldr %x[state], [x29, #40]\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" - "st1 {v24.d}[0], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" - "st1 {v24.d}[1], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "stp x2, x3, [%x[state]]\n\t" - "stp x4, x5, [%x[state], #16]\n\t" - "stp x6, x7, [%x[state], #32]\n\t" - "stp x8, x9, [%x[state], #48]\n\t" - "stp x10, x11, [%x[state], #64]\n\t" - "stp x12, x13, [%x[state], #80]\n\t" - "stp x14, x15, [%x[state], #96]\n\t" - "stp x16, x17, [%x[state], #112]\n\t" - "stp x19, x20, [%x[state], #128]\n\t" - "stp x21, x22, [%x[state], #144]\n\t" - "stp x23, x24, [%x[state], #160]\n\t" - "stp x25, x26, [%x[state], #176]\n\t" - "str x27, [%x[state], #192]\n\t" - "ldp x29, x30, [sp], #0x40\n\t" - : [state] "+r" (state), [seed] "+r" (seed) - : [r] "r" (r) - : "memory", "cc", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", - "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", - "x21", "x22", "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", - "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", - "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", - "v31" - ); -} - -#endif /* WOLFSSL_ARMASM_CRYPTO_SHA3 */ -void mlkem_sha3_blocksx3_neon(word64* state) -{ - const word64* r = L_sha3_aarch64_r; - __asm__ __volatile__ ( - "stp x29, x30, [sp, #-64]!\n\t" - "add x29, sp, #0\n\t" - "str %x[state], [x29, #40]\n\t" - "ld4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" - "ld4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" - "ld4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" - "ld4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" - "ld4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" - "ld4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" - "ld1 {v24.d}[0], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "ld4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" - "ld4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" - "ld4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" - "ld4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" - "ld4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" - "ld4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" - "ld1 {v24.d}[1], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "ldp x1, x2, [%x[state]]\n\t" - "ldp x3, x4, [%x[state], #16]\n\t" - "ldp x5, x6, [%x[state], #32]\n\t" - "ldp x7, x8, [%x[state], #48]\n\t" - "ldp x9, x10, [%x[state], #64]\n\t" - "ldp x11, x12, [%x[state], #80]\n\t" - "ldp x13, x14, [%x[state], #96]\n\t" - "ldp x15, x16, [%x[state], #112]\n\t" - "ldp x17, x19, [%x[state], #128]\n\t" - "ldp x20, x21, [%x[state], #144]\n\t" - "ldp x22, x23, [%x[state], #160]\n\t" - "ldp x24, x25, [%x[state], #176]\n\t" - "ldr x26, [%x[state], #192]\n\t" - "mov x28, #24\n\t" - /* Start of 24 rounds */ - "\n" - "L_SHA3_transform_blocksx3_neon_begin_%=:\n\t" - "stp %[r], x28, [x29, #48]\n\t" - /* Col Mix NEON */ - "eor v30.16b, v4.16b, v9.16b\n\t" - "eor %x[state], x5, x10\n\t" - "eor v27.16b, v1.16b, v6.16b\n\t" - "eor x30, x1, x6\n\t" - "eor v30.16b, v30.16b, v14.16b\n\t" - "eor x28, x3, x8\n\t" - "eor v27.16b, v27.16b, v11.16b\n\t" - "eor %x[state], %x[state], x15\n\t" - "eor v30.16b, v30.16b, v19.16b\n\t" - "eor x30, x30, x11\n\t" - "eor v27.16b, v27.16b, v16.16b\n\t" - "eor x28, x28, x13\n\t" - "eor v30.16b, v30.16b, v24.16b\n\t" - "eor %x[state], %x[state], x21\n\t" - "eor v27.16b, v27.16b, v21.16b\n\t" - "eor x30, x30, x16\n\t" - "ushr v25.2d, v27.2d, #63\n\t" - "eor x28, x28, x19\n\t" - "sli v25.2d, v27.2d, #1\n\t" - "eor %x[state], %x[state], x26\n\t" - "eor v25.16b, v25.16b, v30.16b\n\t" - "eor x30, x30, x22\n\t" - "eor v31.16b, v0.16b, v5.16b\n\t" - "eor x28, x28, x24\n\t" - "eor v28.16b, v2.16b, v7.16b\n\t" - "str %x[state], [x29, #32]\n\t" - "eor v31.16b, v31.16b, v10.16b\n\t" - "str x28, [x29, #24]\n\t" - "eor v28.16b, v28.16b, v12.16b\n\t" - "eor %[r], x2, x7\n\t" - "eor v31.16b, v31.16b, v15.16b\n\t" - "eor x28, x4, x9\n\t" - "eor v28.16b, v28.16b, v17.16b\n\t" - "eor %[r], %[r], x12\n\t" - "eor v31.16b, v31.16b, v20.16b\n\t" - "eor x28, x28, x14\n\t" - "eor v28.16b, v28.16b, v22.16b\n\t" - "eor %[r], %[r], x17\n\t" - "ushr v29.2d, v30.2d, #63\n\t" - "eor x28, x28, x20\n\t" - "ushr v26.2d, v28.2d, #63\n\t" - "eor %[r], %[r], x23\n\t" - "sli v29.2d, v30.2d, #1\n\t" - "eor x28, x28, x25\n\t" - "sli v26.2d, v28.2d, #1\n\t" - "eor %x[state], %x[state], %[r], ror 63\n\t" - "eor v28.16b, v28.16b, v29.16b\n\t" - "eor %[r], %[r], x28, ror 63\n\t" - "eor v29.16b, v3.16b, v8.16b\n\t" - "eor x1, x1, %x[state]\n\t" - "eor v26.16b, v26.16b, v31.16b\n\t" - "eor x6, x6, %x[state]\n\t" - "eor v29.16b, v29.16b, v13.16b\n\t" - "eor x11, x11, %x[state]\n\t" - "eor v29.16b, v29.16b, v18.16b\n\t" - "eor x16, x16, %x[state]\n\t" - "eor v29.16b, v29.16b, v23.16b\n\t" - "eor x22, x22, %x[state]\n\t" - "ushr v30.2d, v29.2d, #63\n\t" - "eor x3, x3, %[r]\n\t" - "sli v30.2d, v29.2d, #1\n\t" - "eor x8, x8, %[r]\n\t" - "eor v27.16b, v27.16b, v30.16b\n\t" - "eor x13, x13, %[r]\n\t" - "ushr v30.2d, v31.2d, #63\n\t" - "eor x19, x19, %[r]\n\t" - "sli v30.2d, v31.2d, #1\n\t" - "eor x24, x24, %[r]\n\t" - "eor v29.16b, v29.16b, v30.16b\n\t" - "ldr %x[state], [x29, #32]\n\t" - /* Swap Rotate NEON */ - "eor v0.16b, v0.16b, v25.16b\n\t" - "eor v31.16b, v1.16b, v26.16b\n\t" - "ldr %[r], [x29, #24]\n\t" - "eor v6.16b, v6.16b, v26.16b\n\t" - "eor x28, x28, x30, ror 63\n\t" - "ushr v30.2d, v31.2d, #63\n\t" - "eor x30, x30, %[r], ror 63\n\t" - "ushr v1.2d, v6.2d, #20\n\t" - "eor %[r], %[r], %x[state], ror 63\n\t" - "sli v30.2d, v31.2d, #1\n\t" - "eor x5, x5, x28\n\t" - "sli v1.2d, v6.2d, #44\n\t" - "eor x10, x10, x28\n\t" - "eor v31.16b, v9.16b, v29.16b\n\t" - "eor x15, x15, x28\n\t" - "eor v22.16b, v22.16b, v27.16b\n\t" - "eor x21, x21, x28\n\t" - "ushr v6.2d, v31.2d, #44\n\t" - "eor x26, x26, x28\n\t" - "ushr v9.2d, v22.2d, #3\n\t" - "eor x2, x2, x30\n\t" - "sli v6.2d, v31.2d, #20\n\t" - "eor x7, x7, x30\n\t" - "sli v9.2d, v22.2d, #61\n\t" - "eor x12, x12, x30\n\t" - "eor v31.16b, v14.16b, v29.16b\n\t" - "eor x17, x17, x30\n\t" - "eor v20.16b, v20.16b, v25.16b\n\t" - "eor x23, x23, x30\n\t" - "ushr v22.2d, v31.2d, #25\n\t" - "eor x4, x4, %[r]\n\t" - "ushr v14.2d, v20.2d, #46\n\t" - "eor x9, x9, %[r]\n\t" - "sli v22.2d, v31.2d, #39\n\t" - "eor x14, x14, %[r]\n\t" - "sli v14.2d, v20.2d, #18\n\t" - "eor x20, x20, %[r]\n\t" - "eor v31.16b, v2.16b, v27.16b\n\t" - "eor x25, x25, %[r]\n\t" - /* Swap Rotate Base */ - "eor v12.16b, v12.16b, v27.16b\n\t" - "ror %x[state], x2, #63\n\t" - "ushr v20.2d, v31.2d, #2\n\t" - "ror x2, x7, #20\n\t" - "ushr v2.2d, v12.2d, #21\n\t" - "ror x7, x10, #44\n\t" - "sli v20.2d, v31.2d, #62\n\t" - "ror x10, x24, #3\n\t" - "sli v2.2d, v12.2d, #43\n\t" - "ror x24, x15, #25\n\t" - "eor v31.16b, v13.16b, v28.16b\n\t" - "ror x15, x22, #46\n\t" - "eor v19.16b, v19.16b, v29.16b\n\t" - "ror x22, x3, #2\n\t" - "ushr v12.2d, v31.2d, #39\n\t" - "ror x3, x13, #21\n\t" - "ushr v13.2d, v19.2d, #56\n\t" - "ror x13, x14, #39\n\t" - "sli v12.2d, v31.2d, #25\n\t" - "ror x14, x21, #56\n\t" - "sli v13.2d, v19.2d, #8\n\t" - "ror x21, x25, #8\n\t" - "eor v31.16b, v23.16b, v28.16b\n\t" - "ror x25, x16, #23\n\t" - "eor v15.16b, v15.16b, v25.16b\n\t" - "ror x16, x5, #37\n\t" - "ushr v19.2d, v31.2d, #8\n\t" - "ror x5, x26, #50\n\t" - "ushr v23.2d, v15.2d, #23\n\t" - "ror x26, x23, #62\n\t" - "sli v19.2d, v31.2d, #56\n\t" - "ror x23, x9, #9\n\t" - "sli v23.2d, v15.2d, #41\n\t" - "ror x9, x17, #19\n\t" - "eor v31.16b, v4.16b, v29.16b\n\t" - "ror x17, x6, #28\n\t" - "eor v24.16b, v24.16b, v29.16b\n\t" - "ror x6, x4, #36\n\t" - "ushr v15.2d, v31.2d, #37\n\t" - "ror x4, x20, #43\n\t" - "ushr v4.2d, v24.2d, #50\n\t" - "ror x20, x19, #49\n\t" - "sli v15.2d, v31.2d, #27\n\t" - "ror x19, x12, #54\n\t" - "sli v4.2d, v24.2d, #14\n\t" - "ror x12, x8, #58\n\t" - "eor v31.16b, v21.16b, v26.16b\n\t" - "ror x8, x11, #61\n\t" - /* Row Mix Base */ - "eor v8.16b, v8.16b, v28.16b\n\t" - "bic x11, x3, x2\n\t" - "ushr v24.2d, v31.2d, #62\n\t" - "bic %[r], x4, x3\n\t" - "ushr v21.2d, v8.2d, #9\n\t" - "bic x28, x1, x5\n\t" - "sli v24.2d, v31.2d, #2\n\t" - "bic x30, x2, x1\n\t" - "sli v21.2d, v8.2d, #55\n\t" - "eor x1, x1, x11\n\t" - "eor v31.16b, v16.16b, v26.16b\n\t" - "eor x2, x2, %[r]\n\t" - "eor v5.16b, v5.16b, v25.16b\n\t" - "bic x11, x5, x4\n\t" - "ushr v8.2d, v31.2d, #19\n\t" - "eor x4, x4, x28\n\t" - "ushr v16.2d, v5.2d, #28\n\t" - "eor x3, x3, x11\n\t" - "sli v8.2d, v31.2d, #45\n\t" - "eor x5, x5, x30\n\t" - "sli v16.2d, v5.2d, #36\n\t" - "bic x11, x8, x7\n\t" - "eor v31.16b, v3.16b, v28.16b\n\t" - "bic %[r], x9, x8\n\t" - "eor v18.16b, v18.16b, v28.16b\n\t" - "bic x28, x6, x10\n\t" - "ushr v5.2d, v31.2d, #36\n\t" - "bic x30, x7, x6\n\t" - "ushr v3.2d, v18.2d, #43\n\t" - "eor x6, x6, x11\n\t" - "sli v5.2d, v31.2d, #28\n\t" - "eor x7, x7, %[r]\n\t" - "sli v3.2d, v18.2d, #21\n\t" - "bic x11, x10, x9\n\t" - "eor v31.16b, v17.16b, v27.16b\n\t" - "eor x9, x9, x28\n\t" - "eor v11.16b, v11.16b, v26.16b\n\t" - "eor x8, x8, x11\n\t" - "ushr v18.2d, v31.2d, #49\n\t" - "eor x10, x10, x30\n\t" - "ushr v17.2d, v11.2d, #54\n\t" - "bic x11, x13, x12\n\t" - "sli v18.2d, v31.2d, #15\n\t" - "bic %[r], x14, x13\n\t" - "sli v17.2d, v11.2d, #10\n\t" - "bic x28, %x[state], x15\n\t" - "eor v31.16b, v7.16b, v27.16b\n\t" - "bic x30, x12, %x[state]\n\t" - "eor v10.16b, v10.16b, v25.16b\n\t" - "eor x11, %x[state], x11\n\t" - "ushr v11.2d, v31.2d, #58\n\t" - "eor x12, x12, %[r]\n\t" - "ushr v7.2d, v10.2d, #61\n\t" - "bic %x[state], x15, x14\n\t" - "sli v11.2d, v31.2d, #6\n\t" - "eor x14, x14, x28\n\t" - "sli v7.2d, v10.2d, #3\n\t" - "eor x13, x13, %x[state]\n\t" - /* Row Mix NEON */ - "bic v25.16b, v2.16b, v1.16b\n\t" - "eor x15, x15, x30\n\t" - "bic v26.16b, v3.16b, v2.16b\n\t" - "bic %x[state], x19, x17\n\t" - "bic v27.16b, v4.16b, v3.16b\n\t" - "bic %[r], x20, x19\n\t" - "bic v28.16b, v0.16b, v4.16b\n\t" - "bic x28, x16, x21\n\t" - "bic v29.16b, v1.16b, v0.16b\n\t" - "bic x30, x17, x16\n\t" - "eor v0.16b, v0.16b, v25.16b\n\t" - "eor x16, x16, %x[state]\n\t" - "eor v1.16b, v1.16b, v26.16b\n\t" - "eor x17, x17, %[r]\n\t" - "eor v2.16b, v2.16b, v27.16b\n\t" - "bic %x[state], x21, x20\n\t" - "eor v3.16b, v3.16b, v28.16b\n\t" - "eor x20, x20, x28\n\t" - "eor v4.16b, v4.16b, v29.16b\n\t" - "eor x19, x19, %x[state]\n\t" - "bic v25.16b, v7.16b, v6.16b\n\t" - "eor x21, x21, x30\n\t" - "bic v26.16b, v8.16b, v7.16b\n\t" - "bic %x[state], x24, x23\n\t" - "bic v27.16b, v9.16b, v8.16b\n\t" - "bic %[r], x25, x24\n\t" - "bic v28.16b, v5.16b, v9.16b\n\t" - "bic x28, x22, x26\n\t" - "bic v29.16b, v6.16b, v5.16b\n\t" - "bic x30, x23, x22\n\t" - "eor v5.16b, v5.16b, v25.16b\n\t" - "eor x22, x22, %x[state]\n\t" - "eor v6.16b, v6.16b, v26.16b\n\t" - "eor x23, x23, %[r]\n\t" - "eor v7.16b, v7.16b, v27.16b\n\t" - "bic %x[state], x26, x25\n\t" - "eor v8.16b, v8.16b, v28.16b\n\t" - "eor x25, x25, x28\n\t" - "eor v9.16b, v9.16b, v29.16b\n\t" - "eor x24, x24, %x[state]\n\t" - "bic v25.16b, v12.16b, v11.16b\n\t" - "eor x26, x26, x30\n\t" - "bic v26.16b, v13.16b, v12.16b\n\t" - "bic v27.16b, v14.16b, v13.16b\n\t" - "bic v28.16b, v30.16b, v14.16b\n\t" - "bic v29.16b, v11.16b, v30.16b\n\t" - "eor v10.16b, v30.16b, v25.16b\n\t" - "eor v11.16b, v11.16b, v26.16b\n\t" - "eor v12.16b, v12.16b, v27.16b\n\t" - "eor v13.16b, v13.16b, v28.16b\n\t" - "eor v14.16b, v14.16b, v29.16b\n\t" - "bic v25.16b, v17.16b, v16.16b\n\t" - "bic v26.16b, v18.16b, v17.16b\n\t" - "bic v27.16b, v19.16b, v18.16b\n\t" - "bic v28.16b, v15.16b, v19.16b\n\t" - "bic v29.16b, v16.16b, v15.16b\n\t" - "eor v15.16b, v15.16b, v25.16b\n\t" - "eor v16.16b, v16.16b, v26.16b\n\t" - "eor v17.16b, v17.16b, v27.16b\n\t" - "eor v18.16b, v18.16b, v28.16b\n\t" - "eor v19.16b, v19.16b, v29.16b\n\t" - "bic v25.16b, v22.16b, v21.16b\n\t" - "bic v26.16b, v23.16b, v22.16b\n\t" - "bic v27.16b, v24.16b, v23.16b\n\t" - "bic v28.16b, v20.16b, v24.16b\n\t" - "bic v29.16b, v21.16b, v20.16b\n\t" - "eor v20.16b, v20.16b, v25.16b\n\t" - "eor v21.16b, v21.16b, v26.16b\n\t" - "eor v22.16b, v22.16b, v27.16b\n\t" - "eor v23.16b, v23.16b, v28.16b\n\t" - "eor v24.16b, v24.16b, v29.16b\n\t" - /* Done transforming */ - "ldp %[r], x28, [x29, #48]\n\t" - "ldr %x[state], [%[r]], #8\n\t" - "subs x28, x28, #1\n\t" - "mov v30.d[0], %x[state]\n\t" - "mov v30.d[1], %x[state]\n\t" - "eor x1, x1, %x[state]\n\t" - "eor v0.16b, v0.16b, v30.16b\n\t" - "b.ne L_SHA3_transform_blocksx3_neon_begin_%=\n\t" - "ldr %x[state], [x29, #40]\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" - "st1 {v24.d}[0], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" - "st1 {v24.d}[1], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "stp x1, x2, [%x[state]]\n\t" - "stp x3, x4, [%x[state], #16]\n\t" - "stp x5, x6, [%x[state], #32]\n\t" - "stp x7, x8, [%x[state], #48]\n\t" - "stp x9, x10, [%x[state], #64]\n\t" - "stp x11, x12, [%x[state], #80]\n\t" - "stp x13, x14, [%x[state], #96]\n\t" - "stp x15, x16, [%x[state], #112]\n\t" - "stp x17, x19, [%x[state], #128]\n\t" - "stp x20, x21, [%x[state], #144]\n\t" - "stp x22, x23, [%x[state], #160]\n\t" - "stp x24, x25, [%x[state], #176]\n\t" - "str x26, [%x[state], #192]\n\t" - "ldp x29, x30, [sp], #0x40\n\t" - : [state] "+r" (state) - : [r] "r" (r) - : "memory", "cc", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", - "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", - "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x28", "v0", "v1", - "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", - "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", - "v31" - ); -} - -void mlkem_shake128_blocksx3_seed_neon(word64* state, byte* seed) -{ - const word64* r = L_sha3_aarch64_r; - __asm__ __volatile__ ( - "stp x29, x30, [sp, #-64]!\n\t" - "add x29, sp, #0\n\t" - "str %x[state], [x29, #40]\n\t" - "add %x[state], %x[state], #32\n\t" - "ld1 {v4.d}[0], [%x[state]]\n\t" - "ldp x2, x3, [%x[seed]], #16\n\t" - "add %x[state], %x[state], #0xc8\n\t" - "ld1 {v4.d}[1], [%x[state]]\n\t" - "ldp x4, x5, [%x[seed]], #16\n\t" - "ldr x6, [%x[state], #200]\n\t" - "eor v5.16b, v5.16b, v5.16b\n\t" - "eor x7, x7, x7\n\t" - "eor v6.16b, v6.16b, v6.16b\n\t" - "eor x8, x8, x8\n\t" - "eor v7.16b, v7.16b, v7.16b\n\t" - "eor x9, x9, x9\n\t" - "eor v8.16b, v8.16b, v8.16b\n\t" - "eor x10, x10, x10\n\t" - "eor v9.16b, v9.16b, v9.16b\n\t" - "eor x11, x11, x11\n\t" - "eor v10.16b, v10.16b, v10.16b\n\t" - "eor x12, x12, x12\n\t" - "eor v11.16b, v11.16b, v11.16b\n\t" - "eor x13, x13, x13\n\t" - "eor v12.16b, v12.16b, v12.16b\n\t" - "eor x14, x14, x14\n\t" - "eor v13.16b, v13.16b, v13.16b\n\t" - "eor x15, x15, x15\n\t" - "eor v14.16b, v14.16b, v14.16b\n\t" - "eor x16, x16, x16\n\t" - "eor v15.16b, v15.16b, v15.16b\n\t" - "eor x17, x17, x17\n\t" - "eor v16.16b, v16.16b, v16.16b\n\t" - "eor x19, x19, x19\n\t" - "eor v17.16b, v17.16b, v17.16b\n\t" - "eor x20, x20, x20\n\t" - "eor v18.16b, v18.16b, v18.16b\n\t" - "eor x21, x21, x21\n\t" - "eor v19.16b, v19.16b, v19.16b\n\t" - "eor x22, x22, x22\n\t" - "movz x23, #0x8000, lsl 48\n\t" - "eor v21.16b, v21.16b, v21.16b\n\t" - "eor x24, x24, x24\n\t" - "eor v22.16b, v22.16b, v22.16b\n\t" - "eor x25, x25, x25\n\t" - "eor v23.16b, v23.16b, v23.16b\n\t" - "eor x26, x26, x26\n\t" - "eor v24.16b, v24.16b, v24.16b\n\t" - "eor x27, x27, x27\n\t" - "dup v0.2d, x2\n\t" - "dup v1.2d, x3\n\t" - "dup v2.2d, x4\n\t" - "dup v3.2d, x5\n\t" - "dup v20.2d, x23\n\t" - "mov %x[seed], #24\n\t" - /* Start of 24 rounds */ - "\n" - "L_SHA3_shake128_blocksx3_seed_neon_begin_%=:\n\t" - "stp %[r], %x[seed], [x29, #48]\n\t" - /* Col Mix NEON */ - "eor v30.16b, v4.16b, v9.16b\n\t" - "eor %x[state], x6, x11\n\t" - "eor v27.16b, v1.16b, v6.16b\n\t" - "eor x30, x2, x7\n\t" - "eor v30.16b, v30.16b, v14.16b\n\t" - "eor %[r], x4, x9\n\t" - "eor v27.16b, v27.16b, v11.16b\n\t" - "eor %x[state], %x[state], x16\n\t" - "eor v30.16b, v30.16b, v19.16b\n\t" - "eor x30, x30, x12\n\t" - "eor v27.16b, v27.16b, v16.16b\n\t" - "eor %[r], %[r], x14\n\t" - "eor v30.16b, v30.16b, v24.16b\n\t" - "eor %x[state], %x[state], x22\n\t" - "eor v27.16b, v27.16b, v21.16b\n\t" - "eor x30, x30, x17\n\t" - "ushr v25.2d, v27.2d, #63\n\t" - "eor %[r], %[r], x20\n\t" - "sli v25.2d, v27.2d, #1\n\t" - "eor %x[state], %x[state], x27\n\t" - "eor v25.16b, v25.16b, v30.16b\n\t" - "eor x30, x30, x23\n\t" - "eor v31.16b, v0.16b, v5.16b\n\t" - "eor %[r], %[r], x25\n\t" - "eor v28.16b, v2.16b, v7.16b\n\t" - "str %x[state], [x29, #32]\n\t" - "eor v31.16b, v31.16b, v10.16b\n\t" - "str %[r], [x29, #24]\n\t" - "eor v28.16b, v28.16b, v12.16b\n\t" - "eor %x[seed], x3, x8\n\t" - "eor v31.16b, v31.16b, v15.16b\n\t" - "eor %[r], x5, x10\n\t" - "eor v28.16b, v28.16b, v17.16b\n\t" - "eor %x[seed], %x[seed], x13\n\t" - "eor v31.16b, v31.16b, v20.16b\n\t" - "eor %[r], %[r], x15\n\t" - "eor v28.16b, v28.16b, v22.16b\n\t" - "eor %x[seed], %x[seed], x19\n\t" - "ushr v29.2d, v30.2d, #63\n\t" - "eor %[r], %[r], x21\n\t" - "ushr v26.2d, v28.2d, #63\n\t" - "eor %x[seed], %x[seed], x24\n\t" - "sli v29.2d, v30.2d, #1\n\t" - "eor %[r], %[r], x26\n\t" - "sli v26.2d, v28.2d, #1\n\t" - "eor %x[state], %x[state], %x[seed], ror 63\n\t" - "eor v28.16b, v28.16b, v29.16b\n\t" - "eor %x[seed], %x[seed], %[r], ror 63\n\t" - "eor v29.16b, v3.16b, v8.16b\n\t" - "eor x2, x2, %x[state]\n\t" - "eor v26.16b, v26.16b, v31.16b\n\t" - "eor x7, x7, %x[state]\n\t" - "eor v29.16b, v29.16b, v13.16b\n\t" - "eor x12, x12, %x[state]\n\t" - "eor v29.16b, v29.16b, v18.16b\n\t" - "eor x17, x17, %x[state]\n\t" - "eor v29.16b, v29.16b, v23.16b\n\t" - "eor x23, x23, %x[state]\n\t" - "ushr v30.2d, v29.2d, #63\n\t" - "eor x4, x4, %x[seed]\n\t" - "sli v30.2d, v29.2d, #1\n\t" - "eor x9, x9, %x[seed]\n\t" - "eor v27.16b, v27.16b, v30.16b\n\t" - "eor x14, x14, %x[seed]\n\t" - "ushr v30.2d, v31.2d, #63\n\t" - "eor x20, x20, %x[seed]\n\t" - "sli v30.2d, v31.2d, #1\n\t" - "eor x25, x25, %x[seed]\n\t" - "eor v29.16b, v29.16b, v30.16b\n\t" - "ldr %x[state], [x29, #32]\n\t" - /* Swap Rotate NEON */ - "eor v0.16b, v0.16b, v25.16b\n\t" - "eor v31.16b, v1.16b, v26.16b\n\t" - "ldr %x[seed], [x29, #24]\n\t" - "eor v6.16b, v6.16b, v26.16b\n\t" - "eor %[r], %[r], x30, ror 63\n\t" - "ushr v30.2d, v31.2d, #63\n\t" - "eor x30, x30, %x[seed], ror 63\n\t" - "ushr v1.2d, v6.2d, #20\n\t" - "eor %x[seed], %x[seed], %x[state], ror 63\n\t" - "sli v30.2d, v31.2d, #1\n\t" - "eor x6, x6, %[r]\n\t" - "sli v1.2d, v6.2d, #44\n\t" - "eor x11, x11, %[r]\n\t" - "eor v31.16b, v9.16b, v29.16b\n\t" - "eor x16, x16, %[r]\n\t" - "eor v22.16b, v22.16b, v27.16b\n\t" - "eor x22, x22, %[r]\n\t" - "ushr v6.2d, v31.2d, #44\n\t" - "eor x27, x27, %[r]\n\t" - "ushr v9.2d, v22.2d, #3\n\t" - "eor x3, x3, x30\n\t" - "sli v6.2d, v31.2d, #20\n\t" - "eor x8, x8, x30\n\t" - "sli v9.2d, v22.2d, #61\n\t" - "eor x13, x13, x30\n\t" - "eor v31.16b, v14.16b, v29.16b\n\t" - "eor x19, x19, x30\n\t" - "eor v20.16b, v20.16b, v25.16b\n\t" - "eor x24, x24, x30\n\t" - "ushr v22.2d, v31.2d, #25\n\t" - "eor x5, x5, %x[seed]\n\t" - "ushr v14.2d, v20.2d, #46\n\t" - "eor x10, x10, %x[seed]\n\t" - "sli v22.2d, v31.2d, #39\n\t" - "eor x15, x15, %x[seed]\n\t" - "sli v14.2d, v20.2d, #18\n\t" - "eor x21, x21, %x[seed]\n\t" - "eor v31.16b, v2.16b, v27.16b\n\t" - "eor x26, x26, %x[seed]\n\t" - /* Swap Rotate Base */ - "eor v12.16b, v12.16b, v27.16b\n\t" - "ror %x[state], x3, #63\n\t" - "ushr v20.2d, v31.2d, #2\n\t" - "ror x3, x8, #20\n\t" - "ushr v2.2d, v12.2d, #21\n\t" - "ror x8, x11, #44\n\t" - "sli v20.2d, v31.2d, #62\n\t" - "ror x11, x25, #3\n\t" - "sli v2.2d, v12.2d, #43\n\t" - "ror x25, x16, #25\n\t" - "eor v31.16b, v13.16b, v28.16b\n\t" - "ror x16, x23, #46\n\t" - "eor v19.16b, v19.16b, v29.16b\n\t" - "ror x23, x4, #2\n\t" - "ushr v12.2d, v31.2d, #39\n\t" - "ror x4, x14, #21\n\t" - "ushr v13.2d, v19.2d, #56\n\t" - "ror x14, x15, #39\n\t" - "sli v12.2d, v31.2d, #25\n\t" - "ror x15, x22, #56\n\t" - "sli v13.2d, v19.2d, #8\n\t" - "ror x22, x26, #8\n\t" - "eor v31.16b, v23.16b, v28.16b\n\t" - "ror x26, x17, #23\n\t" - "eor v15.16b, v15.16b, v25.16b\n\t" - "ror x17, x6, #37\n\t" - "ushr v19.2d, v31.2d, #8\n\t" - "ror x6, x27, #50\n\t" - "ushr v23.2d, v15.2d, #23\n\t" - "ror x27, x24, #62\n\t" - "sli v19.2d, v31.2d, #56\n\t" - "ror x24, x10, #9\n\t" - "sli v23.2d, v15.2d, #41\n\t" - "ror x10, x19, #19\n\t" - "eor v31.16b, v4.16b, v29.16b\n\t" - "ror x19, x7, #28\n\t" - "eor v24.16b, v24.16b, v29.16b\n\t" - "ror x7, x5, #36\n\t" - "ushr v15.2d, v31.2d, #37\n\t" - "ror x5, x21, #43\n\t" - "ushr v4.2d, v24.2d, #50\n\t" - "ror x21, x20, #49\n\t" - "sli v15.2d, v31.2d, #27\n\t" - "ror x20, x13, #54\n\t" - "sli v4.2d, v24.2d, #14\n\t" - "ror x13, x9, #58\n\t" - "eor v31.16b, v21.16b, v26.16b\n\t" - "ror x9, x12, #61\n\t" - /* Row Mix Base */ - "eor v8.16b, v8.16b, v28.16b\n\t" - "bic x12, x4, x3\n\t" - "ushr v24.2d, v31.2d, #62\n\t" - "bic %x[seed], x5, x4\n\t" - "ushr v21.2d, v8.2d, #9\n\t" - "bic %[r], x2, x6\n\t" - "sli v24.2d, v31.2d, #2\n\t" - "bic x30, x3, x2\n\t" - "sli v21.2d, v8.2d, #55\n\t" - "eor x2, x2, x12\n\t" - "eor v31.16b, v16.16b, v26.16b\n\t" - "eor x3, x3, %x[seed]\n\t" - "eor v5.16b, v5.16b, v25.16b\n\t" - "bic x12, x6, x5\n\t" - "ushr v8.2d, v31.2d, #19\n\t" - "eor x5, x5, %[r]\n\t" - "ushr v16.2d, v5.2d, #28\n\t" - "eor x4, x4, x12\n\t" - "sli v8.2d, v31.2d, #45\n\t" - "eor x6, x6, x30\n\t" - "sli v16.2d, v5.2d, #36\n\t" - "bic x12, x9, x8\n\t" - "eor v31.16b, v3.16b, v28.16b\n\t" - "bic %x[seed], x10, x9\n\t" - "eor v18.16b, v18.16b, v28.16b\n\t" - "bic %[r], x7, x11\n\t" - "ushr v5.2d, v31.2d, #36\n\t" - "bic x30, x8, x7\n\t" - "ushr v3.2d, v18.2d, #43\n\t" - "eor x7, x7, x12\n\t" - "sli v5.2d, v31.2d, #28\n\t" - "eor x8, x8, %x[seed]\n\t" - "sli v3.2d, v18.2d, #21\n\t" - "bic x12, x11, x10\n\t" - "eor v31.16b, v17.16b, v27.16b\n\t" - "eor x10, x10, %[r]\n\t" - "eor v11.16b, v11.16b, v26.16b\n\t" - "eor x9, x9, x12\n\t" - "ushr v18.2d, v31.2d, #49\n\t" - "eor x11, x11, x30\n\t" - "ushr v17.2d, v11.2d, #54\n\t" - "bic x12, x14, x13\n\t" - "sli v18.2d, v31.2d, #15\n\t" - "bic %x[seed], x15, x14\n\t" - "sli v17.2d, v11.2d, #10\n\t" - "bic %[r], %x[state], x16\n\t" - "eor v31.16b, v7.16b, v27.16b\n\t" - "bic x30, x13, %x[state]\n\t" - "eor v10.16b, v10.16b, v25.16b\n\t" - "eor x12, %x[state], x12\n\t" - "ushr v11.2d, v31.2d, #58\n\t" - "eor x13, x13, %x[seed]\n\t" - "ushr v7.2d, v10.2d, #61\n\t" - "bic %x[state], x16, x15\n\t" - "sli v11.2d, v31.2d, #6\n\t" - "eor x15, x15, %[r]\n\t" - "sli v7.2d, v10.2d, #3\n\t" - "eor x14, x14, %x[state]\n\t" - /* Row Mix NEON */ - "bic v25.16b, v2.16b, v1.16b\n\t" - "eor x16, x16, x30\n\t" - "bic v26.16b, v3.16b, v2.16b\n\t" - "bic %x[state], x20, x19\n\t" - "bic v27.16b, v4.16b, v3.16b\n\t" - "bic %x[seed], x21, x20\n\t" - "bic v28.16b, v0.16b, v4.16b\n\t" - "bic %[r], x17, x22\n\t" - "bic v29.16b, v1.16b, v0.16b\n\t" - "bic x30, x19, x17\n\t" - "eor v0.16b, v0.16b, v25.16b\n\t" - "eor x17, x17, %x[state]\n\t" - "eor v1.16b, v1.16b, v26.16b\n\t" - "eor x19, x19, %x[seed]\n\t" - "eor v2.16b, v2.16b, v27.16b\n\t" - "bic %x[state], x22, x21\n\t" - "eor v3.16b, v3.16b, v28.16b\n\t" - "eor x21, x21, %[r]\n\t" - "eor v4.16b, v4.16b, v29.16b\n\t" - "eor x20, x20, %x[state]\n\t" - "bic v25.16b, v7.16b, v6.16b\n\t" - "eor x22, x22, x30\n\t" - "bic v26.16b, v8.16b, v7.16b\n\t" - "bic %x[state], x25, x24\n\t" - "bic v27.16b, v9.16b, v8.16b\n\t" - "bic %x[seed], x26, x25\n\t" - "bic v28.16b, v5.16b, v9.16b\n\t" - "bic %[r], x23, x27\n\t" - "bic v29.16b, v6.16b, v5.16b\n\t" - "bic x30, x24, x23\n\t" - "eor v5.16b, v5.16b, v25.16b\n\t" - "eor x23, x23, %x[state]\n\t" - "eor v6.16b, v6.16b, v26.16b\n\t" - "eor x24, x24, %x[seed]\n\t" - "eor v7.16b, v7.16b, v27.16b\n\t" - "bic %x[state], x27, x26\n\t" - "eor v8.16b, v8.16b, v28.16b\n\t" - "eor x26, x26, %[r]\n\t" - "eor v9.16b, v9.16b, v29.16b\n\t" - "eor x25, x25, %x[state]\n\t" - "bic v25.16b, v12.16b, v11.16b\n\t" - "eor x27, x27, x30\n\t" - "bic v26.16b, v13.16b, v12.16b\n\t" - "bic v27.16b, v14.16b, v13.16b\n\t" - "bic v28.16b, v30.16b, v14.16b\n\t" - "bic v29.16b, v11.16b, v30.16b\n\t" - "eor v10.16b, v30.16b, v25.16b\n\t" - "eor v11.16b, v11.16b, v26.16b\n\t" - "eor v12.16b, v12.16b, v27.16b\n\t" - "eor v13.16b, v13.16b, v28.16b\n\t" - "eor v14.16b, v14.16b, v29.16b\n\t" - "bic v25.16b, v17.16b, v16.16b\n\t" - "bic v26.16b, v18.16b, v17.16b\n\t" - "bic v27.16b, v19.16b, v18.16b\n\t" - "bic v28.16b, v15.16b, v19.16b\n\t" - "bic v29.16b, v16.16b, v15.16b\n\t" - "eor v15.16b, v15.16b, v25.16b\n\t" - "eor v16.16b, v16.16b, v26.16b\n\t" - "eor v17.16b, v17.16b, v27.16b\n\t" - "eor v18.16b, v18.16b, v28.16b\n\t" - "eor v19.16b, v19.16b, v29.16b\n\t" - "bic v25.16b, v22.16b, v21.16b\n\t" - "bic v26.16b, v23.16b, v22.16b\n\t" - "bic v27.16b, v24.16b, v23.16b\n\t" - "bic v28.16b, v20.16b, v24.16b\n\t" - "bic v29.16b, v21.16b, v20.16b\n\t" - "eor v20.16b, v20.16b, v25.16b\n\t" - "eor v21.16b, v21.16b, v26.16b\n\t" - "eor v22.16b, v22.16b, v27.16b\n\t" - "eor v23.16b, v23.16b, v28.16b\n\t" - "eor v24.16b, v24.16b, v29.16b\n\t" - /* Done transforming */ - "ldp %[r], %x[seed], [x29, #48]\n\t" - "ldr %x[state], [%[r]], #8\n\t" - "subs %x[seed], %x[seed], #1\n\t" - "mov v30.d[0], %x[state]\n\t" - "mov v30.d[1], %x[state]\n\t" - "eor x2, x2, %x[state]\n\t" - "eor v0.16b, v0.16b, v30.16b\n\t" - "b.ne L_SHA3_shake128_blocksx3_seed_neon_begin_%=\n\t" - "ldr %x[state], [x29, #40]\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" - "st1 {v24.d}[0], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" - "st1 {v24.d}[1], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "stp x2, x3, [%x[state]]\n\t" - "stp x4, x5, [%x[state], #16]\n\t" - "stp x6, x7, [%x[state], #32]\n\t" - "stp x8, x9, [%x[state], #48]\n\t" - "stp x10, x11, [%x[state], #64]\n\t" - "stp x12, x13, [%x[state], #80]\n\t" - "stp x14, x15, [%x[state], #96]\n\t" - "stp x16, x17, [%x[state], #112]\n\t" - "stp x19, x20, [%x[state], #128]\n\t" - "stp x21, x22, [%x[state], #144]\n\t" - "stp x23, x24, [%x[state], #160]\n\t" - "stp x25, x26, [%x[state], #176]\n\t" - "str x27, [%x[state], #192]\n\t" - "ldp x29, x30, [sp], #0x40\n\t" - : [state] "+r" (state), [seed] "+r" (seed) - : [r] "r" (r) - : "memory", "cc", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", - "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", - "x21", "x22", "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", - "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", - "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", - "v31" - ); -} - -void mlkem_shake256_blocksx3_seed_neon(word64* state, byte* seed) -{ - const word64* r = L_sha3_aarch64_r; - __asm__ __volatile__ ( - "stp x29, x30, [sp, #-64]!\n\t" - "add x29, sp, #0\n\t" - "str %x[state], [x29, #40]\n\t" - "add %x[state], %x[state], #32\n\t" - "ld1 {v4.d}[0], [%x[state]]\n\t" - "ldp x2, x3, [%x[seed]], #16\n\t" - "add %x[state], %x[state], #0xc8\n\t" - "ld1 {v4.d}[1], [%x[state]]\n\t" - "ldp x4, x5, [%x[seed]], #16\n\t" - "ldr x6, [%x[state], #200]\n\t" - "eor v5.16b, v5.16b, v5.16b\n\t" - "eor x7, x7, x7\n\t" - "eor v6.16b, v6.16b, v6.16b\n\t" - "eor x8, x8, x8\n\t" - "eor v7.16b, v7.16b, v7.16b\n\t" - "eor x9, x9, x9\n\t" - "eor v8.16b, v8.16b, v8.16b\n\t" - "eor x10, x10, x10\n\t" - "eor v9.16b, v9.16b, v9.16b\n\t" - "eor x11, x11, x11\n\t" - "eor v10.16b, v10.16b, v10.16b\n\t" - "eor x12, x12, x12\n\t" - "eor v11.16b, v11.16b, v11.16b\n\t" - "eor x13, x13, x13\n\t" - "eor v12.16b, v12.16b, v12.16b\n\t" - "eor x14, x14, x14\n\t" - "eor v13.16b, v13.16b, v13.16b\n\t" - "eor x15, x15, x15\n\t" - "eor v14.16b, v14.16b, v14.16b\n\t" - "eor x16, x16, x16\n\t" - "eor v15.16b, v15.16b, v15.16b\n\t" - "eor x17, x17, x17\n\t" - "movz x19, #0x8000, lsl 48\n\t" - "eor v17.16b, v17.16b, v17.16b\n\t" - "eor x20, x20, x20\n\t" - "eor v18.16b, v18.16b, v18.16b\n\t" - "eor x21, x21, x21\n\t" - "eor v19.16b, v19.16b, v19.16b\n\t" - "eor x22, x22, x22\n\t" - "eor v20.16b, v20.16b, v20.16b\n\t" - "eor x23, x23, x23\n\t" - "eor v21.16b, v21.16b, v21.16b\n\t" - "eor x24, x24, x24\n\t" - "eor v22.16b, v22.16b, v22.16b\n\t" - "eor x25, x25, x25\n\t" - "eor v23.16b, v23.16b, v23.16b\n\t" - "eor x26, x26, x26\n\t" - "eor v24.16b, v24.16b, v24.16b\n\t" - "eor x27, x27, x27\n\t" - "dup v0.2d, x2\n\t" - "dup v1.2d, x3\n\t" - "dup v2.2d, x4\n\t" - "dup v3.2d, x5\n\t" - "dup v16.2d, x19\n\t" - "mov %x[seed], #24\n\t" - /* Start of 24 rounds */ - "\n" - "L_SHA3_shake256_blocksx3_seed_neon_begin_%=:\n\t" - "stp %[r], %x[seed], [x29, #48]\n\t" - /* Col Mix NEON */ - "eor v30.16b, v4.16b, v9.16b\n\t" - "eor %x[state], x6, x11\n\t" - "eor v27.16b, v1.16b, v6.16b\n\t" - "eor x30, x2, x7\n\t" - "eor v30.16b, v30.16b, v14.16b\n\t" - "eor %[r], x4, x9\n\t" - "eor v27.16b, v27.16b, v11.16b\n\t" - "eor %x[state], %x[state], x16\n\t" - "eor v30.16b, v30.16b, v19.16b\n\t" - "eor x30, x30, x12\n\t" - "eor v27.16b, v27.16b, v16.16b\n\t" - "eor %[r], %[r], x14\n\t" - "eor v30.16b, v30.16b, v24.16b\n\t" - "eor %x[state], %x[state], x22\n\t" - "eor v27.16b, v27.16b, v21.16b\n\t" - "eor x30, x30, x17\n\t" - "ushr v25.2d, v27.2d, #63\n\t" - "eor %[r], %[r], x20\n\t" - "sli v25.2d, v27.2d, #1\n\t" - "eor %x[state], %x[state], x27\n\t" - "eor v25.16b, v25.16b, v30.16b\n\t" - "eor x30, x30, x23\n\t" - "eor v31.16b, v0.16b, v5.16b\n\t" - "eor %[r], %[r], x25\n\t" - "eor v28.16b, v2.16b, v7.16b\n\t" - "str %x[state], [x29, #32]\n\t" - "eor v31.16b, v31.16b, v10.16b\n\t" - "str %[r], [x29, #24]\n\t" - "eor v28.16b, v28.16b, v12.16b\n\t" - "eor %x[seed], x3, x8\n\t" - "eor v31.16b, v31.16b, v15.16b\n\t" - "eor %[r], x5, x10\n\t" - "eor v28.16b, v28.16b, v17.16b\n\t" - "eor %x[seed], %x[seed], x13\n\t" - "eor v31.16b, v31.16b, v20.16b\n\t" - "eor %[r], %[r], x15\n\t" - "eor v28.16b, v28.16b, v22.16b\n\t" - "eor %x[seed], %x[seed], x19\n\t" - "ushr v29.2d, v30.2d, #63\n\t" - "eor %[r], %[r], x21\n\t" - "ushr v26.2d, v28.2d, #63\n\t" - "eor %x[seed], %x[seed], x24\n\t" - "sli v29.2d, v30.2d, #1\n\t" - "eor %[r], %[r], x26\n\t" - "sli v26.2d, v28.2d, #1\n\t" - "eor %x[state], %x[state], %x[seed], ror 63\n\t" - "eor v28.16b, v28.16b, v29.16b\n\t" - "eor %x[seed], %x[seed], %[r], ror 63\n\t" - "eor v29.16b, v3.16b, v8.16b\n\t" - "eor x2, x2, %x[state]\n\t" - "eor v26.16b, v26.16b, v31.16b\n\t" - "eor x7, x7, %x[state]\n\t" - "eor v29.16b, v29.16b, v13.16b\n\t" - "eor x12, x12, %x[state]\n\t" - "eor v29.16b, v29.16b, v18.16b\n\t" - "eor x17, x17, %x[state]\n\t" - "eor v29.16b, v29.16b, v23.16b\n\t" - "eor x23, x23, %x[state]\n\t" - "ushr v30.2d, v29.2d, #63\n\t" - "eor x4, x4, %x[seed]\n\t" - "sli v30.2d, v29.2d, #1\n\t" - "eor x9, x9, %x[seed]\n\t" - "eor v27.16b, v27.16b, v30.16b\n\t" - "eor x14, x14, %x[seed]\n\t" - "ushr v30.2d, v31.2d, #63\n\t" - "eor x20, x20, %x[seed]\n\t" - "sli v30.2d, v31.2d, #1\n\t" - "eor x25, x25, %x[seed]\n\t" - "eor v29.16b, v29.16b, v30.16b\n\t" - "ldr %x[state], [x29, #32]\n\t" - /* Swap Rotate NEON */ - "eor v0.16b, v0.16b, v25.16b\n\t" - "eor v31.16b, v1.16b, v26.16b\n\t" - "ldr %x[seed], [x29, #24]\n\t" - "eor v6.16b, v6.16b, v26.16b\n\t" - "eor %[r], %[r], x30, ror 63\n\t" - "ushr v30.2d, v31.2d, #63\n\t" - "eor x30, x30, %x[seed], ror 63\n\t" - "ushr v1.2d, v6.2d, #20\n\t" - "eor %x[seed], %x[seed], %x[state], ror 63\n\t" - "sli v30.2d, v31.2d, #1\n\t" - "eor x6, x6, %[r]\n\t" - "sli v1.2d, v6.2d, #44\n\t" - "eor x11, x11, %[r]\n\t" - "eor v31.16b, v9.16b, v29.16b\n\t" - "eor x16, x16, %[r]\n\t" - "eor v22.16b, v22.16b, v27.16b\n\t" - "eor x22, x22, %[r]\n\t" - "ushr v6.2d, v31.2d, #44\n\t" - "eor x27, x27, %[r]\n\t" - "ushr v9.2d, v22.2d, #3\n\t" - "eor x3, x3, x30\n\t" - "sli v6.2d, v31.2d, #20\n\t" - "eor x8, x8, x30\n\t" - "sli v9.2d, v22.2d, #61\n\t" - "eor x13, x13, x30\n\t" - "eor v31.16b, v14.16b, v29.16b\n\t" - "eor x19, x19, x30\n\t" - "eor v20.16b, v20.16b, v25.16b\n\t" - "eor x24, x24, x30\n\t" - "ushr v22.2d, v31.2d, #25\n\t" - "eor x5, x5, %x[seed]\n\t" - "ushr v14.2d, v20.2d, #46\n\t" - "eor x10, x10, %x[seed]\n\t" - "sli v22.2d, v31.2d, #39\n\t" - "eor x15, x15, %x[seed]\n\t" - "sli v14.2d, v20.2d, #18\n\t" - "eor x21, x21, %x[seed]\n\t" - "eor v31.16b, v2.16b, v27.16b\n\t" - "eor x26, x26, %x[seed]\n\t" - /* Swap Rotate Base */ - "eor v12.16b, v12.16b, v27.16b\n\t" - "ror %x[state], x3, #63\n\t" - "ushr v20.2d, v31.2d, #2\n\t" - "ror x3, x8, #20\n\t" - "ushr v2.2d, v12.2d, #21\n\t" - "ror x8, x11, #44\n\t" - "sli v20.2d, v31.2d, #62\n\t" - "ror x11, x25, #3\n\t" - "sli v2.2d, v12.2d, #43\n\t" - "ror x25, x16, #25\n\t" - "eor v31.16b, v13.16b, v28.16b\n\t" - "ror x16, x23, #46\n\t" - "eor v19.16b, v19.16b, v29.16b\n\t" - "ror x23, x4, #2\n\t" - "ushr v12.2d, v31.2d, #39\n\t" - "ror x4, x14, #21\n\t" - "ushr v13.2d, v19.2d, #56\n\t" - "ror x14, x15, #39\n\t" - "sli v12.2d, v31.2d, #25\n\t" - "ror x15, x22, #56\n\t" - "sli v13.2d, v19.2d, #8\n\t" - "ror x22, x26, #8\n\t" - "eor v31.16b, v23.16b, v28.16b\n\t" - "ror x26, x17, #23\n\t" - "eor v15.16b, v15.16b, v25.16b\n\t" - "ror x17, x6, #37\n\t" - "ushr v19.2d, v31.2d, #8\n\t" - "ror x6, x27, #50\n\t" - "ushr v23.2d, v15.2d, #23\n\t" - "ror x27, x24, #62\n\t" - "sli v19.2d, v31.2d, #56\n\t" - "ror x24, x10, #9\n\t" - "sli v23.2d, v15.2d, #41\n\t" - "ror x10, x19, #19\n\t" - "eor v31.16b, v4.16b, v29.16b\n\t" - "ror x19, x7, #28\n\t" - "eor v24.16b, v24.16b, v29.16b\n\t" - "ror x7, x5, #36\n\t" - "ushr v15.2d, v31.2d, #37\n\t" - "ror x5, x21, #43\n\t" - "ushr v4.2d, v24.2d, #50\n\t" - "ror x21, x20, #49\n\t" - "sli v15.2d, v31.2d, #27\n\t" - "ror x20, x13, #54\n\t" - "sli v4.2d, v24.2d, #14\n\t" - "ror x13, x9, #58\n\t" - "eor v31.16b, v21.16b, v26.16b\n\t" - "ror x9, x12, #61\n\t" - /* Row Mix Base */ - "eor v8.16b, v8.16b, v28.16b\n\t" - "bic x12, x4, x3\n\t" - "ushr v24.2d, v31.2d, #62\n\t" - "bic %x[seed], x5, x4\n\t" - "ushr v21.2d, v8.2d, #9\n\t" - "bic %[r], x2, x6\n\t" - "sli v24.2d, v31.2d, #2\n\t" - "bic x30, x3, x2\n\t" - "sli v21.2d, v8.2d, #55\n\t" - "eor x2, x2, x12\n\t" - "eor v31.16b, v16.16b, v26.16b\n\t" - "eor x3, x3, %x[seed]\n\t" - "eor v5.16b, v5.16b, v25.16b\n\t" - "bic x12, x6, x5\n\t" - "ushr v8.2d, v31.2d, #19\n\t" - "eor x5, x5, %[r]\n\t" - "ushr v16.2d, v5.2d, #28\n\t" - "eor x4, x4, x12\n\t" - "sli v8.2d, v31.2d, #45\n\t" - "eor x6, x6, x30\n\t" - "sli v16.2d, v5.2d, #36\n\t" - "bic x12, x9, x8\n\t" - "eor v31.16b, v3.16b, v28.16b\n\t" - "bic %x[seed], x10, x9\n\t" - "eor v18.16b, v18.16b, v28.16b\n\t" - "bic %[r], x7, x11\n\t" - "ushr v5.2d, v31.2d, #36\n\t" - "bic x30, x8, x7\n\t" - "ushr v3.2d, v18.2d, #43\n\t" - "eor x7, x7, x12\n\t" - "sli v5.2d, v31.2d, #28\n\t" - "eor x8, x8, %x[seed]\n\t" - "sli v3.2d, v18.2d, #21\n\t" - "bic x12, x11, x10\n\t" - "eor v31.16b, v17.16b, v27.16b\n\t" - "eor x10, x10, %[r]\n\t" - "eor v11.16b, v11.16b, v26.16b\n\t" - "eor x9, x9, x12\n\t" - "ushr v18.2d, v31.2d, #49\n\t" - "eor x11, x11, x30\n\t" - "ushr v17.2d, v11.2d, #54\n\t" - "bic x12, x14, x13\n\t" - "sli v18.2d, v31.2d, #15\n\t" - "bic %x[seed], x15, x14\n\t" - "sli v17.2d, v11.2d, #10\n\t" - "bic %[r], %x[state], x16\n\t" - "eor v31.16b, v7.16b, v27.16b\n\t" - "bic x30, x13, %x[state]\n\t" - "eor v10.16b, v10.16b, v25.16b\n\t" - "eor x12, %x[state], x12\n\t" - "ushr v11.2d, v31.2d, #58\n\t" - "eor x13, x13, %x[seed]\n\t" - "ushr v7.2d, v10.2d, #61\n\t" - "bic %x[state], x16, x15\n\t" - "sli v11.2d, v31.2d, #6\n\t" - "eor x15, x15, %[r]\n\t" - "sli v7.2d, v10.2d, #3\n\t" - "eor x14, x14, %x[state]\n\t" - /* Row Mix NEON */ - "bic v25.16b, v2.16b, v1.16b\n\t" - "eor x16, x16, x30\n\t" - "bic v26.16b, v3.16b, v2.16b\n\t" - "bic %x[state], x20, x19\n\t" - "bic v27.16b, v4.16b, v3.16b\n\t" - "bic %x[seed], x21, x20\n\t" - "bic v28.16b, v0.16b, v4.16b\n\t" - "bic %[r], x17, x22\n\t" - "bic v29.16b, v1.16b, v0.16b\n\t" - "bic x30, x19, x17\n\t" - "eor v0.16b, v0.16b, v25.16b\n\t" - "eor x17, x17, %x[state]\n\t" - "eor v1.16b, v1.16b, v26.16b\n\t" - "eor x19, x19, %x[seed]\n\t" - "eor v2.16b, v2.16b, v27.16b\n\t" - "bic %x[state], x22, x21\n\t" - "eor v3.16b, v3.16b, v28.16b\n\t" - "eor x21, x21, %[r]\n\t" - "eor v4.16b, v4.16b, v29.16b\n\t" - "eor x20, x20, %x[state]\n\t" - "bic v25.16b, v7.16b, v6.16b\n\t" - "eor x22, x22, x30\n\t" - "bic v26.16b, v8.16b, v7.16b\n\t" - "bic %x[state], x25, x24\n\t" - "bic v27.16b, v9.16b, v8.16b\n\t" - "bic %x[seed], x26, x25\n\t" - "bic v28.16b, v5.16b, v9.16b\n\t" - "bic %[r], x23, x27\n\t" - "bic v29.16b, v6.16b, v5.16b\n\t" - "bic x30, x24, x23\n\t" - "eor v5.16b, v5.16b, v25.16b\n\t" - "eor x23, x23, %x[state]\n\t" - "eor v6.16b, v6.16b, v26.16b\n\t" - "eor x24, x24, %x[seed]\n\t" - "eor v7.16b, v7.16b, v27.16b\n\t" - "bic %x[state], x27, x26\n\t" - "eor v8.16b, v8.16b, v28.16b\n\t" - "eor x26, x26, %[r]\n\t" - "eor v9.16b, v9.16b, v29.16b\n\t" - "eor x25, x25, %x[state]\n\t" - "bic v25.16b, v12.16b, v11.16b\n\t" - "eor x27, x27, x30\n\t" - "bic v26.16b, v13.16b, v12.16b\n\t" - "bic v27.16b, v14.16b, v13.16b\n\t" - "bic v28.16b, v30.16b, v14.16b\n\t" - "bic v29.16b, v11.16b, v30.16b\n\t" - "eor v10.16b, v30.16b, v25.16b\n\t" - "eor v11.16b, v11.16b, v26.16b\n\t" - "eor v12.16b, v12.16b, v27.16b\n\t" - "eor v13.16b, v13.16b, v28.16b\n\t" - "eor v14.16b, v14.16b, v29.16b\n\t" - "bic v25.16b, v17.16b, v16.16b\n\t" - "bic v26.16b, v18.16b, v17.16b\n\t" - "bic v27.16b, v19.16b, v18.16b\n\t" - "bic v28.16b, v15.16b, v19.16b\n\t" - "bic v29.16b, v16.16b, v15.16b\n\t" - "eor v15.16b, v15.16b, v25.16b\n\t" - "eor v16.16b, v16.16b, v26.16b\n\t" - "eor v17.16b, v17.16b, v27.16b\n\t" - "eor v18.16b, v18.16b, v28.16b\n\t" - "eor v19.16b, v19.16b, v29.16b\n\t" - "bic v25.16b, v22.16b, v21.16b\n\t" - "bic v26.16b, v23.16b, v22.16b\n\t" - "bic v27.16b, v24.16b, v23.16b\n\t" - "bic v28.16b, v20.16b, v24.16b\n\t" - "bic v29.16b, v21.16b, v20.16b\n\t" - "eor v20.16b, v20.16b, v25.16b\n\t" - "eor v21.16b, v21.16b, v26.16b\n\t" - "eor v22.16b, v22.16b, v27.16b\n\t" - "eor v23.16b, v23.16b, v28.16b\n\t" - "eor v24.16b, v24.16b, v29.16b\n\t" - /* Done transforming */ - "ldp %[r], %x[seed], [x29, #48]\n\t" - "ldr %x[state], [%[r]], #8\n\t" - "subs %x[seed], %x[seed], #1\n\t" - "mov v30.d[0], %x[state]\n\t" - "mov v30.d[1], %x[state]\n\t" - "eor x2, x2, %x[state]\n\t" - "eor v0.16b, v0.16b, v30.16b\n\t" - "b.ne L_SHA3_shake256_blocksx3_seed_neon_begin_%=\n\t" - "ldr %x[state], [x29, #40]\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" - "st1 {v24.d}[0], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" - "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" - "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" - "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" - "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" - "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" - "st1 {v24.d}[1], [%x[state]]\n\t" - "add %x[state], %x[state], #8\n\t" - "stp x2, x3, [%x[state]]\n\t" - "stp x4, x5, [%x[state], #16]\n\t" - "stp x6, x7, [%x[state], #32]\n\t" - "stp x8, x9, [%x[state], #48]\n\t" - "stp x10, x11, [%x[state], #64]\n\t" - "stp x12, x13, [%x[state], #80]\n\t" - "stp x14, x15, [%x[state], #96]\n\t" - "stp x16, x17, [%x[state], #112]\n\t" - "stp x19, x20, [%x[state], #128]\n\t" - "stp x21, x22, [%x[state], #144]\n\t" - "stp x23, x24, [%x[state], #160]\n\t" - "stp x25, x26, [%x[state], #176]\n\t" - "str x27, [%x[state], #192]\n\t" - "ldp x29, x30, [sp], #0x40\n\t" - : [state] "+r" (state), [seed] "+r" (seed) - : [r] "r" (r) - : "memory", "cc", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", - "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", - "x21", "x22", "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", - "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", - "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", - "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", - "v31" - ); -} - #endif /* WOLFSSL_HAVE_MLKEM */ #endif /* __aarch64__ */ #endif /* WOLFSSL_ARMASM */ diff --git a/wolfcrypt/src/port/arm/armv8-sha3-asm.S b/wolfcrypt/src/port/arm/armv8-sha3-asm.S index c9ad4a5f333..82b24c79470 100644 --- a/wolfcrypt/src/port/arm/armv8-sha3-asm.S +++ b/wolfcrypt/src/port/arm/armv8-sha3-asm.S @@ -435,6 +435,3133 @@ L_SHA3_transform_base_begin: #ifndef __APPLE__ .size BlockSha3_base,.-BlockSha3_base #endif /* __APPLE__ */ +#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) +#ifndef __APPLE__ + .text + .section .rodata + .type L_sha3_aarch64_r, %object + .size L_sha3_aarch64_r, 192 +#else + .section __DATA,__data +#endif /* __APPLE__ */ + # 16-byte aligned, 128-bit aligned +#ifndef __APPLE__ + .align 4 +#else + .p2align 4 +#endif /* __APPLE__ */ +L_sha3_aarch64_r: + .quad 0x0000000000000001,0x0000000000008082 + .quad 0x800000000000808a,0x8000000080008000 + .quad 0x000000000000808b,0x0000000080000001 + .quad 0x8000000080008081,0x8000000000008009 + .quad 0x000000000000008a,0x0000000000000088 + .quad 0x0000000080008009,0x000000008000000a + .quad 0x000000008000808b,0x800000000000008b + .quad 0x8000000000008089,0x8000000000008003 + .quad 0x8000000000008002,0x8000000000000080 + .quad 0x000000000000800a,0x800000008000000a + .quad 0x8000000080008081,0x8000000000008080 + .quad 0x0000000080000001,0x8000000080008008 +#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 +#ifndef __APPLE__ +.text +.globl sha3_blocksx3_crypto +WC_ASM_ATT_HIDDEN(sha3_blocksx3_crypto) +.type sha3_blocksx3_crypto,@function +.align 2 +sha3_blocksx3_crypto: +#else +.section __TEXT,__text +.globl _sha3_blocksx3_crypto +WC_ASM_ATT_HIDDEN(_sha3_blocksx3_crypto) +.p2align 2 +_sha3_blocksx3_crypto: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp d8, d9, [x29, #160] + stp d10, d11, [x29, #176] + stp d12, d13, [x29, #192] + stp d14, d15, [x29, #208] +#ifndef __APPLE__ + adrp x27, L_sha3_aarch64_r + add x27, x27, :lo12:L_sha3_aarch64_r +#else + adrp x27, L_sha3_aarch64_r@PAGE + add x27, x27, L_sha3_aarch64_r@PAGEOFF +#endif /* __APPLE__ */ +#ifdef __APPLE__ +.arch_extension sha3 +#endif /* __APPLE__ */ + str x0, [x29, #40] + ld4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 + ld4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 + ld4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 + ld4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 + ld4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 + ld4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 + ld1 {v24.d}[0], [x0] + add x0, x0, #8 + ld4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 + ld4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 + ld4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 + ld4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 + ld4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 + ld4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 + ld1 {v24.d}[1], [x0] + add x0, x0, #8 + ldp x1, x2, [x0] + ldp x3, x4, [x0, #16] + ldp x5, x6, [x0, #32] + ldp x7, x8, [x0, #48] + ldp x9, x10, [x0, #64] + ldp x11, x12, [x0, #80] + ldp x13, x14, [x0, #96] + ldp x15, x16, [x0, #112] + ldp x17, x19, [x0, #128] + ldp x20, x21, [x0, #144] + ldp x22, x23, [x0, #160] + ldp x24, x25, [x0, #176] + ldr x26, [x0, #192] + mov x28, #24 + # Start of 24 rounds +L_SHA3_transform_blocksx3_crypto_begin: + stp x27, x28, [x29, #48] + # Col Mix + eor3 v31.16b, v0.16b, v5.16b, v10.16b + eor x0, x5, x10 + eor3 v27.16b, v1.16b, v6.16b, v11.16b + eor x30, x1, x6 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x28, x3, x8 + eor3 v29.16b, v3.16b, v8.16b, v13.16b + eor x0, x0, x15 + eor3 v30.16b, v4.16b, v9.16b, v14.16b + eor x30, x30, x11 + eor3 v31.16b, v31.16b, v15.16b, v20.16b + eor x28, x28, x13 + eor3 v27.16b, v27.16b, v16.16b, v21.16b + eor x0, x0, x21 + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor x30, x30, x16 + eor3 v29.16b, v29.16b, v18.16b, v23.16b + eor x28, x28, x19 + eor3 v30.16b, v30.16b, v19.16b, v24.16b + eor x0, x0, x26 + rax1 v25.2d, v30.2d, v27.2d + eor x30, x30, x22 + rax1 v26.2d, v31.2d, v28.2d + eor x28, x28, x24 + rax1 v27.2d, v27.2d, v29.2d + str x0, [x29, #32] + rax1 v28.2d, v28.2d, v30.2d + str x28, [x29, #24] + rax1 v29.2d, v29.2d, v31.2d + eor x27, x2, x7 + eor v0.16b, v0.16b, v25.16b + xar v30.2d, v1.2d, v26.2d, #63 + eor x28, x4, x9 + xar v1.2d, v6.2d, v26.2d, #20 + eor x27, x27, x12 + xar v6.2d, v9.2d, v29.2d, #44 + eor x28, x28, x14 + xar v9.2d, v22.2d, v27.2d, #3 + eor x27, x27, x17 + xar v22.2d, v14.2d, v29.2d, #25 + eor x28, x28, x20 + xar v14.2d, v20.2d, v25.2d, #46 + eor x27, x27, x23 + xar v20.2d, v2.2d, v27.2d, #2 + eor x28, x28, x25 + xar v2.2d, v12.2d, v27.2d, #21 + eor x0, x0, x27, ror 63 + xar v12.2d, v13.2d, v28.2d, #39 + eor x27, x27, x28, ror 63 + xar v13.2d, v19.2d, v29.2d, #56 + eor x1, x1, x0 + xar v19.2d, v23.2d, v28.2d, #8 + eor x6, x6, x0 + xar v23.2d, v15.2d, v25.2d, #23 + eor x11, x11, x0 + xar v15.2d, v4.2d, v29.2d, #37 + eor x16, x16, x0 + xar v4.2d, v24.2d, v29.2d, #50 + eor x22, x22, x0 + xar v24.2d, v21.2d, v26.2d, #62 + eor x3, x3, x27 + xar v21.2d, v8.2d, v28.2d, #9 + eor x8, x8, x27 + xar v8.2d, v16.2d, v26.2d, #19 + eor x13, x13, x27 + xar v16.2d, v5.2d, v25.2d, #28 + eor x19, x19, x27 + xar v5.2d, v3.2d, v28.2d, #36 + eor x24, x24, x27 + xar v3.2d, v18.2d, v28.2d, #43 + ldr x0, [x29, #32] + xar v18.2d, v17.2d, v27.2d, #49 + ldr x27, [x29, #24] + xar v17.2d, v11.2d, v26.2d, #54 + eor x28, x28, x30, ror 63 + xar v11.2d, v7.2d, v27.2d, #58 + eor x30, x30, x27, ror 63 + xar v7.2d, v10.2d, v25.2d, #61 + eor x27, x27, x0, ror 63 + # Row Mix + mov v25.16b, v0.16b + eor x5, x5, x28 + mov v26.16b, v1.16b + eor x10, x10, x28 + bcax v0.16b, v25.16b, v2.16b, v26.16b + eor x15, x15, x28 + bcax v1.16b, v26.16b, v3.16b, v2.16b + eor x21, x21, x28 + bcax v2.16b, v2.16b, v4.16b, v3.16b + eor x26, x26, x28 + bcax v3.16b, v3.16b, v25.16b, v4.16b + eor x2, x2, x30 + bcax v4.16b, v4.16b, v26.16b, v25.16b + eor x7, x7, x30 + mov v25.16b, v5.16b + eor x12, x12, x30 + mov v26.16b, v6.16b + eor x17, x17, x30 + bcax v5.16b, v25.16b, v7.16b, v26.16b + eor x23, x23, x30 + bcax v6.16b, v26.16b, v8.16b, v7.16b + eor x4, x4, x27 + bcax v7.16b, v7.16b, v9.16b, v8.16b + eor x9, x9, x27 + bcax v8.16b, v8.16b, v25.16b, v9.16b + eor x14, x14, x27 + bcax v9.16b, v9.16b, v26.16b, v25.16b + eor x20, x20, x27 + mov v26.16b, v11.16b + eor x25, x25, x27 + # Swap Rotate Base + bcax v10.16b, v30.16b, v12.16b, v26.16b + ror x0, x2, #63 + bcax v11.16b, v26.16b, v13.16b, v12.16b + ror x2, x7, #20 + bcax v12.16b, v12.16b, v14.16b, v13.16b + ror x7, x10, #44 + bcax v13.16b, v13.16b, v30.16b, v14.16b + ror x10, x24, #3 + bcax v14.16b, v14.16b, v26.16b, v30.16b + ror x24, x15, #25 + mov v25.16b, v15.16b + ror x15, x22, #46 + mov v26.16b, v16.16b + ror x22, x3, #2 + bcax v15.16b, v25.16b, v17.16b, v26.16b + ror x3, x13, #21 + bcax v16.16b, v26.16b, v18.16b, v17.16b + ror x13, x14, #39 + bcax v17.16b, v17.16b, v19.16b, v18.16b + ror x14, x21, #56 + bcax v18.16b, v18.16b, v25.16b, v19.16b + ror x21, x25, #8 + bcax v19.16b, v19.16b, v26.16b, v25.16b + ror x25, x16, #23 + mov v25.16b, v20.16b + ror x16, x5, #37 + mov v26.16b, v21.16b + ror x5, x26, #50 + bcax v20.16b, v25.16b, v22.16b, v26.16b + ror x26, x23, #62 + bcax v21.16b, v26.16b, v23.16b, v22.16b + ror x23, x9, #9 + bcax v22.16b, v22.16b, v24.16b, v23.16b + ror x9, x17, #19 + bcax v23.16b, v23.16b, v25.16b, v24.16b + ror x17, x6, #28 + bcax v24.16b, v24.16b, v26.16b, v25.16b + ror x6, x4, #36 + ror x4, x20, #43 + ror x20, x19, #49 + ror x19, x12, #54 + ror x12, x8, #58 + ror x8, x11, #61 + # Row Mix Base + bic x11, x3, x2 + bic x27, x4, x3 + bic x28, x1, x5 + bic x30, x2, x1 + eor x1, x1, x11 + eor x2, x2, x27 + bic x11, x5, x4 + eor x4, x4, x28 + eor x3, x3, x11 + eor x5, x5, x30 + bic x11, x8, x7 + bic x27, x9, x8 + bic x28, x6, x10 + bic x30, x7, x6 + eor x6, x6, x11 + eor x7, x7, x27 + bic x11, x10, x9 + eor x9, x9, x28 + eor x8, x8, x11 + eor x10, x10, x30 + bic x11, x13, x12 + bic x27, x14, x13 + bic x28, x0, x15 + bic x30, x12, x0 + eor x11, x0, x11 + eor x12, x12, x27 + bic x0, x15, x14 + eor x14, x14, x28 + eor x13, x13, x0 + eor x15, x15, x30 + bic x0, x19, x17 + bic x27, x20, x19 + bic x28, x16, x21 + bic x30, x17, x16 + eor x16, x16, x0 + eor x17, x17, x27 + bic x0, x21, x20 + eor x20, x20, x28 + eor x19, x19, x0 + eor x21, x21, x30 + bic x0, x24, x23 + bic x27, x25, x24 + bic x28, x22, x26 + bic x30, x23, x22 + eor x22, x22, x0 + eor x23, x23, x27 + bic x0, x26, x25 + eor x25, x25, x28 + eor x24, x24, x0 + eor x26, x26, x30 + # Done transforming + ldp x27, x28, [x29, #48] + ldr x0, [x27], #8 + subs x28, x28, #1 + mov v30.d[0], x0 + mov v30.d[1], x0 + eor x1, x1, x0 + eor v0.16b, v0.16b, v30.16b + bne L_SHA3_transform_blocksx3_crypto_begin + ldr x0, [x29, #40] + st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 + st1 {v24.d}[0], [x0] + add x0, x0, #8 + st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 + st1 {v24.d}[1], [x0] + add x0, x0, #8 + stp x1, x2, [x0] + stp x3, x4, [x0, #16] + stp x5, x6, [x0, #32] + stp x7, x8, [x0, #48] + stp x9, x10, [x0, #64] + stp x11, x12, [x0, #80] + stp x13, x14, [x0, #96] + stp x15, x16, [x0, #112] + stp x17, x19, [x0, #128] + stp x20, x21, [x0, #144] + stp x22, x23, [x0, #160] + stp x24, x25, [x0, #176] + str x26, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp d8, d9, [x29, #160] + ldp d10, d11, [x29, #176] + ldp d12, d13, [x29, #192] + ldp d14, d15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret +#ifndef __APPLE__ + .size sha3_blocksx3_crypto,.-sha3_blocksx3_crypto +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl sha3_128_blocksx3_seed_crypto +WC_ASM_ATT_HIDDEN(sha3_128_blocksx3_seed_crypto) +.type sha3_128_blocksx3_seed_crypto,@function +.align 2 +sha3_128_blocksx3_seed_crypto: +#else +.section __TEXT,__text +.globl _sha3_128_blocksx3_seed_crypto +WC_ASM_ATT_HIDDEN(_sha3_128_blocksx3_seed_crypto) +.p2align 2 +_sha3_128_blocksx3_seed_crypto: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp d8, d9, [x29, #160] + stp d10, d11, [x29, #176] + stp d12, d13, [x29, #192] + stp d14, d15, [x29, #208] +#ifndef __APPLE__ + adrp x28, L_sha3_aarch64_r + add x28, x28, :lo12:L_sha3_aarch64_r +#else + adrp x28, L_sha3_aarch64_r@PAGE + add x28, x28, L_sha3_aarch64_r@PAGEOFF +#endif /* __APPLE__ */ +#ifdef __APPLE__ +.arch_extension sha3 +#endif /* __APPLE__ */ + str x0, [x29, #40] + add x0, x0, #32 + ld1 {v4.d}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {v4.d}[1], [x0] + ldp x4, x5, [x1], #16 + ldr x6, [x0, #200] + eor v5.16b, v5.16b, v5.16b + eor x7, x7, x7 + eor v6.16b, v6.16b, v6.16b + eor x8, x8, x8 + eor v7.16b, v7.16b, v7.16b + eor x9, x9, x9 + eor v8.16b, v8.16b, v8.16b + eor x10, x10, x10 + eor v9.16b, v9.16b, v9.16b + eor x11, x11, x11 + eor v10.16b, v10.16b, v10.16b + eor x12, x12, x12 + eor v11.16b, v11.16b, v11.16b + eor x13, x13, x13 + eor v12.16b, v12.16b, v12.16b + eor x14, x14, x14 + eor v13.16b, v13.16b, v13.16b + eor x15, x15, x15 + eor v14.16b, v14.16b, v14.16b + eor x16, x16, x16 + eor v15.16b, v15.16b, v15.16b + eor x17, x17, x17 + eor v16.16b, v16.16b, v16.16b + eor x19, x19, x19 + eor v17.16b, v17.16b, v17.16b + eor x20, x20, x20 + eor v18.16b, v18.16b, v18.16b + eor x21, x21, x21 + eor v19.16b, v19.16b, v19.16b + eor x22, x22, x22 + movz x23, #0x8000, lsl 48 + eor v21.16b, v21.16b, v21.16b + eor x24, x24, x24 + eor v22.16b, v22.16b, v22.16b + eor x25, x25, x25 + eor v23.16b, v23.16b, v23.16b + eor x26, x26, x26 + eor v24.16b, v24.16b, v24.16b + eor x27, x27, x27 + dup v0.2d, x2 + dup v1.2d, x3 + dup v2.2d, x4 + dup v3.2d, x5 + dup v20.2d, x23 + mov x1, #24 + # Start of 24 rounds +L_SHA3_shake128_blocksx3_seed_crypto_begin: + stp x28, x1, [x29, #48] + # Col Mix + eor3 v31.16b, v0.16b, v5.16b, v10.16b + eor x0, x6, x11 + eor3 v27.16b, v1.16b, v6.16b, v11.16b + eor x30, x2, x7 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x28, x4, x9 + eor3 v29.16b, v3.16b, v8.16b, v13.16b + eor x0, x0, x16 + eor3 v30.16b, v4.16b, v9.16b, v14.16b + eor x30, x30, x12 + eor3 v31.16b, v31.16b, v15.16b, v20.16b + eor x28, x28, x14 + eor3 v27.16b, v27.16b, v16.16b, v21.16b + eor x0, x0, x22 + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor x30, x30, x17 + eor3 v29.16b, v29.16b, v18.16b, v23.16b + eor x28, x28, x20 + eor3 v30.16b, v30.16b, v19.16b, v24.16b + eor x0, x0, x27 + rax1 v25.2d, v30.2d, v27.2d + eor x30, x30, x23 + rax1 v26.2d, v31.2d, v28.2d + eor x28, x28, x25 + rax1 v27.2d, v27.2d, v29.2d + str x0, [x29, #32] + rax1 v28.2d, v28.2d, v30.2d + str x28, [x29, #24] + rax1 v29.2d, v29.2d, v31.2d + eor x1, x3, x8 + eor v0.16b, v0.16b, v25.16b + xar v30.2d, v1.2d, v26.2d, #63 + eor x28, x5, x10 + xar v1.2d, v6.2d, v26.2d, #20 + eor x1, x1, x13 + xar v6.2d, v9.2d, v29.2d, #44 + eor x28, x28, x15 + xar v9.2d, v22.2d, v27.2d, #3 + eor x1, x1, x19 + xar v22.2d, v14.2d, v29.2d, #25 + eor x28, x28, x21 + xar v14.2d, v20.2d, v25.2d, #46 + eor x1, x1, x24 + xar v20.2d, v2.2d, v27.2d, #2 + eor x28, x28, x26 + xar v2.2d, v12.2d, v27.2d, #21 + eor x0, x0, x1, ror 63 + xar v12.2d, v13.2d, v28.2d, #39 + eor x1, x1, x28, ror 63 + xar v13.2d, v19.2d, v29.2d, #56 + eor x2, x2, x0 + xar v19.2d, v23.2d, v28.2d, #8 + eor x7, x7, x0 + xar v23.2d, v15.2d, v25.2d, #23 + eor x12, x12, x0 + xar v15.2d, v4.2d, v29.2d, #37 + eor x17, x17, x0 + xar v4.2d, v24.2d, v29.2d, #50 + eor x23, x23, x0 + xar v24.2d, v21.2d, v26.2d, #62 + eor x4, x4, x1 + xar v21.2d, v8.2d, v28.2d, #9 + eor x9, x9, x1 + xar v8.2d, v16.2d, v26.2d, #19 + eor x14, x14, x1 + xar v16.2d, v5.2d, v25.2d, #28 + eor x20, x20, x1 + xar v5.2d, v3.2d, v28.2d, #36 + eor x25, x25, x1 + xar v3.2d, v18.2d, v28.2d, #43 + ldr x0, [x29, #32] + xar v18.2d, v17.2d, v27.2d, #49 + ldr x1, [x29, #24] + xar v17.2d, v11.2d, v26.2d, #54 + eor x28, x28, x30, ror 63 + xar v11.2d, v7.2d, v27.2d, #58 + eor x30, x30, x1, ror 63 + xar v7.2d, v10.2d, v25.2d, #61 + eor x1, x1, x0, ror 63 + # Row Mix + mov v25.16b, v0.16b + eor x6, x6, x28 + mov v26.16b, v1.16b + eor x11, x11, x28 + bcax v0.16b, v25.16b, v2.16b, v26.16b + eor x16, x16, x28 + bcax v1.16b, v26.16b, v3.16b, v2.16b + eor x22, x22, x28 + bcax v2.16b, v2.16b, v4.16b, v3.16b + eor x27, x27, x28 + bcax v3.16b, v3.16b, v25.16b, v4.16b + eor x3, x3, x30 + bcax v4.16b, v4.16b, v26.16b, v25.16b + eor x8, x8, x30 + mov v25.16b, v5.16b + eor x13, x13, x30 + mov v26.16b, v6.16b + eor x19, x19, x30 + bcax v5.16b, v25.16b, v7.16b, v26.16b + eor x24, x24, x30 + bcax v6.16b, v26.16b, v8.16b, v7.16b + eor x5, x5, x1 + bcax v7.16b, v7.16b, v9.16b, v8.16b + eor x10, x10, x1 + bcax v8.16b, v8.16b, v25.16b, v9.16b + eor x15, x15, x1 + bcax v9.16b, v9.16b, v26.16b, v25.16b + eor x21, x21, x1 + mov v26.16b, v11.16b + eor x26, x26, x1 + # Swap Rotate Base + bcax v10.16b, v30.16b, v12.16b, v26.16b + ror x0, x3, #63 + bcax v11.16b, v26.16b, v13.16b, v12.16b + ror x3, x8, #20 + bcax v12.16b, v12.16b, v14.16b, v13.16b + ror x8, x11, #44 + bcax v13.16b, v13.16b, v30.16b, v14.16b + ror x11, x25, #3 + bcax v14.16b, v14.16b, v26.16b, v30.16b + ror x25, x16, #25 + mov v25.16b, v15.16b + ror x16, x23, #46 + mov v26.16b, v16.16b + ror x23, x4, #2 + bcax v15.16b, v25.16b, v17.16b, v26.16b + ror x4, x14, #21 + bcax v16.16b, v26.16b, v18.16b, v17.16b + ror x14, x15, #39 + bcax v17.16b, v17.16b, v19.16b, v18.16b + ror x15, x22, #56 + bcax v18.16b, v18.16b, v25.16b, v19.16b + ror x22, x26, #8 + bcax v19.16b, v19.16b, v26.16b, v25.16b + ror x26, x17, #23 + mov v25.16b, v20.16b + ror x17, x6, #37 + mov v26.16b, v21.16b + ror x6, x27, #50 + bcax v20.16b, v25.16b, v22.16b, v26.16b + ror x27, x24, #62 + bcax v21.16b, v26.16b, v23.16b, v22.16b + ror x24, x10, #9 + bcax v22.16b, v22.16b, v24.16b, v23.16b + ror x10, x19, #19 + bcax v23.16b, v23.16b, v25.16b, v24.16b + ror x19, x7, #28 + bcax v24.16b, v24.16b, v26.16b, v25.16b + ror x7, x5, #36 + ror x5, x21, #43 + ror x21, x20, #49 + ror x20, x13, #54 + ror x13, x9, #58 + ror x9, x12, #61 + # Row Mix Base + bic x12, x4, x3 + bic x1, x5, x4 + bic x28, x2, x6 + bic x30, x3, x2 + eor x2, x2, x12 + eor x3, x3, x1 + bic x12, x6, x5 + eor x5, x5, x28 + eor x4, x4, x12 + eor x6, x6, x30 + bic x12, x9, x8 + bic x1, x10, x9 + bic x28, x7, x11 + bic x30, x8, x7 + eor x7, x7, x12 + eor x8, x8, x1 + bic x12, x11, x10 + eor x10, x10, x28 + eor x9, x9, x12 + eor x11, x11, x30 + bic x12, x14, x13 + bic x1, x15, x14 + bic x28, x0, x16 + bic x30, x13, x0 + eor x12, x0, x12 + eor x13, x13, x1 + bic x0, x16, x15 + eor x15, x15, x28 + eor x14, x14, x0 + eor x16, x16, x30 + bic x0, x20, x19 + bic x1, x21, x20 + bic x28, x17, x22 + bic x30, x19, x17 + eor x17, x17, x0 + eor x19, x19, x1 + bic x0, x22, x21 + eor x21, x21, x28 + eor x20, x20, x0 + eor x22, x22, x30 + bic x0, x25, x24 + bic x1, x26, x25 + bic x28, x23, x27 + bic x30, x24, x23 + eor x23, x23, x0 + eor x24, x24, x1 + bic x0, x27, x26 + eor x26, x26, x28 + eor x25, x25, x0 + eor x27, x27, x30 + # Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov v30.d[0], x0 + mov v30.d[1], x0 + eor x2, x2, x0 + eor v0.16b, v0.16b, v30.16b + bne L_SHA3_shake128_blocksx3_seed_crypto_begin + ldr x0, [x29, #40] + st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 + st1 {v24.d}[0], [x0] + add x0, x0, #8 + st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 + st1 {v24.d}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp d8, d9, [x29, #160] + ldp d10, d11, [x29, #176] + ldp d12, d13, [x29, #192] + ldp d14, d15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret +#ifndef __APPLE__ + .size sha3_128_blocksx3_seed_crypto,.-sha3_128_blocksx3_seed_crypto +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl sha3_256_blocksx3_seed_crypto +WC_ASM_ATT_HIDDEN(sha3_256_blocksx3_seed_crypto) +.type sha3_256_blocksx3_seed_crypto,@function +.align 2 +sha3_256_blocksx3_seed_crypto: +#else +.section __TEXT,__text +.globl _sha3_256_blocksx3_seed_crypto +WC_ASM_ATT_HIDDEN(_sha3_256_blocksx3_seed_crypto) +.p2align 2 +_sha3_256_blocksx3_seed_crypto: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp d8, d9, [x29, #160] + stp d10, d11, [x29, #176] + stp d12, d13, [x29, #192] + stp d14, d15, [x29, #208] +#ifndef __APPLE__ + adrp x28, L_sha3_aarch64_r + add x28, x28, :lo12:L_sha3_aarch64_r +#else + adrp x28, L_sha3_aarch64_r@PAGE + add x28, x28, L_sha3_aarch64_r@PAGEOFF +#endif /* __APPLE__ */ +#ifdef __APPLE__ +.arch_extension sha3 +#endif /* __APPLE__ */ + str x0, [x29, #40] + add x0, x0, #32 + ld1 {v4.d}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {v4.d}[1], [x0] + ldp x4, x5, [x1], #16 + ldr x6, [x0, #200] + eor v5.16b, v5.16b, v5.16b + eor x7, x7, x7 + eor v6.16b, v6.16b, v6.16b + eor x8, x8, x8 + eor v7.16b, v7.16b, v7.16b + eor x9, x9, x9 + eor v8.16b, v8.16b, v8.16b + eor x10, x10, x10 + eor v9.16b, v9.16b, v9.16b + eor x11, x11, x11 + eor v10.16b, v10.16b, v10.16b + eor x12, x12, x12 + eor v11.16b, v11.16b, v11.16b + eor x13, x13, x13 + eor v12.16b, v12.16b, v12.16b + eor x14, x14, x14 + eor v13.16b, v13.16b, v13.16b + eor x15, x15, x15 + eor v14.16b, v14.16b, v14.16b + eor x16, x16, x16 + eor v15.16b, v15.16b, v15.16b + eor x17, x17, x17 + movz x19, #0x8000, lsl 48 + eor v17.16b, v17.16b, v17.16b + eor x20, x20, x20 + eor v18.16b, v18.16b, v18.16b + eor x21, x21, x21 + eor v19.16b, v19.16b, v19.16b + eor x22, x22, x22 + eor v20.16b, v20.16b, v20.16b + eor x23, x23, x23 + eor v21.16b, v21.16b, v21.16b + eor x24, x24, x24 + eor v22.16b, v22.16b, v22.16b + eor x25, x25, x25 + eor v23.16b, v23.16b, v23.16b + eor x26, x26, x26 + eor v24.16b, v24.16b, v24.16b + eor x27, x27, x27 + dup v0.2d, x2 + dup v1.2d, x3 + dup v2.2d, x4 + dup v3.2d, x5 + dup v16.2d, x19 + mov x1, #24 + # Start of 24 rounds +L_SHA3_shake256_blocksx3_seed_crypto_begin: + stp x28, x1, [x29, #48] + # Col Mix + eor3 v31.16b, v0.16b, v5.16b, v10.16b + eor x0, x6, x11 + eor3 v27.16b, v1.16b, v6.16b, v11.16b + eor x30, x2, x7 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x28, x4, x9 + eor3 v29.16b, v3.16b, v8.16b, v13.16b + eor x0, x0, x16 + eor3 v30.16b, v4.16b, v9.16b, v14.16b + eor x30, x30, x12 + eor3 v31.16b, v31.16b, v15.16b, v20.16b + eor x28, x28, x14 + eor3 v27.16b, v27.16b, v16.16b, v21.16b + eor x0, x0, x22 + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor x30, x30, x17 + eor3 v29.16b, v29.16b, v18.16b, v23.16b + eor x28, x28, x20 + eor3 v30.16b, v30.16b, v19.16b, v24.16b + eor x0, x0, x27 + rax1 v25.2d, v30.2d, v27.2d + eor x30, x30, x23 + rax1 v26.2d, v31.2d, v28.2d + eor x28, x28, x25 + rax1 v27.2d, v27.2d, v29.2d + str x0, [x29, #32] + rax1 v28.2d, v28.2d, v30.2d + str x28, [x29, #24] + rax1 v29.2d, v29.2d, v31.2d + eor x1, x3, x8 + eor v0.16b, v0.16b, v25.16b + xar v30.2d, v1.2d, v26.2d, #63 + eor x28, x5, x10 + xar v1.2d, v6.2d, v26.2d, #20 + eor x1, x1, x13 + xar v6.2d, v9.2d, v29.2d, #44 + eor x28, x28, x15 + xar v9.2d, v22.2d, v27.2d, #3 + eor x1, x1, x19 + xar v22.2d, v14.2d, v29.2d, #25 + eor x28, x28, x21 + xar v14.2d, v20.2d, v25.2d, #46 + eor x1, x1, x24 + xar v20.2d, v2.2d, v27.2d, #2 + eor x28, x28, x26 + xar v2.2d, v12.2d, v27.2d, #21 + eor x0, x0, x1, ror 63 + xar v12.2d, v13.2d, v28.2d, #39 + eor x1, x1, x28, ror 63 + xar v13.2d, v19.2d, v29.2d, #56 + eor x2, x2, x0 + xar v19.2d, v23.2d, v28.2d, #8 + eor x7, x7, x0 + xar v23.2d, v15.2d, v25.2d, #23 + eor x12, x12, x0 + xar v15.2d, v4.2d, v29.2d, #37 + eor x17, x17, x0 + xar v4.2d, v24.2d, v29.2d, #50 + eor x23, x23, x0 + xar v24.2d, v21.2d, v26.2d, #62 + eor x4, x4, x1 + xar v21.2d, v8.2d, v28.2d, #9 + eor x9, x9, x1 + xar v8.2d, v16.2d, v26.2d, #19 + eor x14, x14, x1 + xar v16.2d, v5.2d, v25.2d, #28 + eor x20, x20, x1 + xar v5.2d, v3.2d, v28.2d, #36 + eor x25, x25, x1 + xar v3.2d, v18.2d, v28.2d, #43 + ldr x0, [x29, #32] + xar v18.2d, v17.2d, v27.2d, #49 + ldr x1, [x29, #24] + xar v17.2d, v11.2d, v26.2d, #54 + eor x28, x28, x30, ror 63 + xar v11.2d, v7.2d, v27.2d, #58 + eor x30, x30, x1, ror 63 + xar v7.2d, v10.2d, v25.2d, #61 + eor x1, x1, x0, ror 63 + # Row Mix + mov v25.16b, v0.16b + eor x6, x6, x28 + mov v26.16b, v1.16b + eor x11, x11, x28 + bcax v0.16b, v25.16b, v2.16b, v26.16b + eor x16, x16, x28 + bcax v1.16b, v26.16b, v3.16b, v2.16b + eor x22, x22, x28 + bcax v2.16b, v2.16b, v4.16b, v3.16b + eor x27, x27, x28 + bcax v3.16b, v3.16b, v25.16b, v4.16b + eor x3, x3, x30 + bcax v4.16b, v4.16b, v26.16b, v25.16b + eor x8, x8, x30 + mov v25.16b, v5.16b + eor x13, x13, x30 + mov v26.16b, v6.16b + eor x19, x19, x30 + bcax v5.16b, v25.16b, v7.16b, v26.16b + eor x24, x24, x30 + bcax v6.16b, v26.16b, v8.16b, v7.16b + eor x5, x5, x1 + bcax v7.16b, v7.16b, v9.16b, v8.16b + eor x10, x10, x1 + bcax v8.16b, v8.16b, v25.16b, v9.16b + eor x15, x15, x1 + bcax v9.16b, v9.16b, v26.16b, v25.16b + eor x21, x21, x1 + mov v26.16b, v11.16b + eor x26, x26, x1 + # Swap Rotate Base + bcax v10.16b, v30.16b, v12.16b, v26.16b + ror x0, x3, #63 + bcax v11.16b, v26.16b, v13.16b, v12.16b + ror x3, x8, #20 + bcax v12.16b, v12.16b, v14.16b, v13.16b + ror x8, x11, #44 + bcax v13.16b, v13.16b, v30.16b, v14.16b + ror x11, x25, #3 + bcax v14.16b, v14.16b, v26.16b, v30.16b + ror x25, x16, #25 + mov v25.16b, v15.16b + ror x16, x23, #46 + mov v26.16b, v16.16b + ror x23, x4, #2 + bcax v15.16b, v25.16b, v17.16b, v26.16b + ror x4, x14, #21 + bcax v16.16b, v26.16b, v18.16b, v17.16b + ror x14, x15, #39 + bcax v17.16b, v17.16b, v19.16b, v18.16b + ror x15, x22, #56 + bcax v18.16b, v18.16b, v25.16b, v19.16b + ror x22, x26, #8 + bcax v19.16b, v19.16b, v26.16b, v25.16b + ror x26, x17, #23 + mov v25.16b, v20.16b + ror x17, x6, #37 + mov v26.16b, v21.16b + ror x6, x27, #50 + bcax v20.16b, v25.16b, v22.16b, v26.16b + ror x27, x24, #62 + bcax v21.16b, v26.16b, v23.16b, v22.16b + ror x24, x10, #9 + bcax v22.16b, v22.16b, v24.16b, v23.16b + ror x10, x19, #19 + bcax v23.16b, v23.16b, v25.16b, v24.16b + ror x19, x7, #28 + bcax v24.16b, v24.16b, v26.16b, v25.16b + ror x7, x5, #36 + ror x5, x21, #43 + ror x21, x20, #49 + ror x20, x13, #54 + ror x13, x9, #58 + ror x9, x12, #61 + # Row Mix Base + bic x12, x4, x3 + bic x1, x5, x4 + bic x28, x2, x6 + bic x30, x3, x2 + eor x2, x2, x12 + eor x3, x3, x1 + bic x12, x6, x5 + eor x5, x5, x28 + eor x4, x4, x12 + eor x6, x6, x30 + bic x12, x9, x8 + bic x1, x10, x9 + bic x28, x7, x11 + bic x30, x8, x7 + eor x7, x7, x12 + eor x8, x8, x1 + bic x12, x11, x10 + eor x10, x10, x28 + eor x9, x9, x12 + eor x11, x11, x30 + bic x12, x14, x13 + bic x1, x15, x14 + bic x28, x0, x16 + bic x30, x13, x0 + eor x12, x0, x12 + eor x13, x13, x1 + bic x0, x16, x15 + eor x15, x15, x28 + eor x14, x14, x0 + eor x16, x16, x30 + bic x0, x20, x19 + bic x1, x21, x20 + bic x28, x17, x22 + bic x30, x19, x17 + eor x17, x17, x0 + eor x19, x19, x1 + bic x0, x22, x21 + eor x21, x21, x28 + eor x20, x20, x0 + eor x22, x22, x30 + bic x0, x25, x24 + bic x1, x26, x25 + bic x28, x23, x27 + bic x30, x24, x23 + eor x23, x23, x0 + eor x24, x24, x1 + bic x0, x27, x26 + eor x26, x26, x28 + eor x25, x25, x0 + eor x27, x27, x30 + # Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov v30.d[0], x0 + mov v30.d[1], x0 + eor x2, x2, x0 + eor v0.16b, v0.16b, v30.16b + bne L_SHA3_shake256_blocksx3_seed_crypto_begin + ldr x0, [x29, #40] + st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 + st1 {v24.d}[0], [x0] + add x0, x0, #8 + st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 + st1 {v24.d}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp d8, d9, [x29, #160] + ldp d10, d11, [x29, #176] + ldp d12, d13, [x29, #192] + ldp d14, d15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret +#ifndef __APPLE__ + .size sha3_256_blocksx3_seed_crypto,.-sha3_256_blocksx3_seed_crypto +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl sha3_256_blocksx3_seed_64_crypto +WC_ASM_ATT_HIDDEN(sha3_256_blocksx3_seed_64_crypto) +.type sha3_256_blocksx3_seed_64_crypto,@function +.align 2 +sha3_256_blocksx3_seed_64_crypto: +#else +.section __TEXT,__text +.globl _sha3_256_blocksx3_seed_64_crypto +WC_ASM_ATT_HIDDEN(_sha3_256_blocksx3_seed_64_crypto) +.p2align 2 +_sha3_256_blocksx3_seed_64_crypto: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp d8, d9, [x29, #160] + stp d10, d11, [x29, #176] + stp d12, d13, [x29, #192] + stp d14, d15, [x29, #208] +#ifndef __APPLE__ + adrp x28, L_sha3_aarch64_r + add x28, x28, :lo12:L_sha3_aarch64_r +#else + adrp x28, L_sha3_aarch64_r@PAGE + add x28, x28, L_sha3_aarch64_r@PAGEOFF +#endif /* __APPLE__ */ +#ifdef __APPLE__ +.arch_extension sha3 +#endif /* __APPLE__ */ + str x0, [x29, #40] + add x0, x0, #0x40 + ld1 {v8.d}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {v8.d}[1], [x0] + ldp x4, x5, [x1], #16 + ldp x6, x7, [x1], #16 + ldp x8, x9, [x1], #16 + ldr x10, [x0, #200] + eor v9.16b, v9.16b, v9.16b + eor x11, x11, x11 + eor v10.16b, v10.16b, v10.16b + eor x12, x12, x12 + eor v11.16b, v11.16b, v11.16b + eor x13, x13, x13 + eor v12.16b, v12.16b, v12.16b + eor x14, x14, x14 + eor v13.16b, v13.16b, v13.16b + eor x15, x15, x15 + eor v14.16b, v14.16b, v14.16b + eor x16, x16, x16 + eor v15.16b, v15.16b, v15.16b + eor x17, x17, x17 + movz x19, #0x8000, lsl 48 + eor v17.16b, v17.16b, v17.16b + eor x20, x20, x20 + eor v18.16b, v18.16b, v18.16b + eor x21, x21, x21 + eor v19.16b, v19.16b, v19.16b + eor x22, x22, x22 + eor v20.16b, v20.16b, v20.16b + eor x23, x23, x23 + eor v21.16b, v21.16b, v21.16b + eor x24, x24, x24 + eor v22.16b, v22.16b, v22.16b + eor x25, x25, x25 + eor v23.16b, v23.16b, v23.16b + eor x26, x26, x26 + eor v24.16b, v24.16b, v24.16b + eor x27, x27, x27 + dup v0.2d, x2 + dup v1.2d, x3 + dup v2.2d, x4 + dup v3.2d, x5 + dup v4.2d, x6 + dup v5.2d, x7 + dup v6.2d, x8 + dup v7.2d, x9 + dup v16.2d, x19 + mov x1, #24 + # Start of 24 rounds +L_SHA3_shake256_blocksx3_seed_64_crypto_begin: + stp x28, x1, [x29, #48] + # Col Mix + eor3 v31.16b, v0.16b, v5.16b, v10.16b + eor x0, x6, x11 + eor3 v27.16b, v1.16b, v6.16b, v11.16b + eor x30, x2, x7 + eor3 v28.16b, v2.16b, v7.16b, v12.16b + eor x28, x4, x9 + eor3 v29.16b, v3.16b, v8.16b, v13.16b + eor x0, x0, x16 + eor3 v30.16b, v4.16b, v9.16b, v14.16b + eor x30, x30, x12 + eor3 v31.16b, v31.16b, v15.16b, v20.16b + eor x28, x28, x14 + eor3 v27.16b, v27.16b, v16.16b, v21.16b + eor x0, x0, x22 + eor3 v28.16b, v28.16b, v17.16b, v22.16b + eor x30, x30, x17 + eor3 v29.16b, v29.16b, v18.16b, v23.16b + eor x28, x28, x20 + eor3 v30.16b, v30.16b, v19.16b, v24.16b + eor x0, x0, x27 + rax1 v25.2d, v30.2d, v27.2d + eor x30, x30, x23 + rax1 v26.2d, v31.2d, v28.2d + eor x28, x28, x25 + rax1 v27.2d, v27.2d, v29.2d + str x0, [x29, #32] + rax1 v28.2d, v28.2d, v30.2d + str x28, [x29, #24] + rax1 v29.2d, v29.2d, v31.2d + eor x1, x3, x8 + eor v0.16b, v0.16b, v25.16b + xar v30.2d, v1.2d, v26.2d, #63 + eor x28, x5, x10 + xar v1.2d, v6.2d, v26.2d, #20 + eor x1, x1, x13 + xar v6.2d, v9.2d, v29.2d, #44 + eor x28, x28, x15 + xar v9.2d, v22.2d, v27.2d, #3 + eor x1, x1, x19 + xar v22.2d, v14.2d, v29.2d, #25 + eor x28, x28, x21 + xar v14.2d, v20.2d, v25.2d, #46 + eor x1, x1, x24 + xar v20.2d, v2.2d, v27.2d, #2 + eor x28, x28, x26 + xar v2.2d, v12.2d, v27.2d, #21 + eor x0, x0, x1, ror 63 + xar v12.2d, v13.2d, v28.2d, #39 + eor x1, x1, x28, ror 63 + xar v13.2d, v19.2d, v29.2d, #56 + eor x2, x2, x0 + xar v19.2d, v23.2d, v28.2d, #8 + eor x7, x7, x0 + xar v23.2d, v15.2d, v25.2d, #23 + eor x12, x12, x0 + xar v15.2d, v4.2d, v29.2d, #37 + eor x17, x17, x0 + xar v4.2d, v24.2d, v29.2d, #50 + eor x23, x23, x0 + xar v24.2d, v21.2d, v26.2d, #62 + eor x4, x4, x1 + xar v21.2d, v8.2d, v28.2d, #9 + eor x9, x9, x1 + xar v8.2d, v16.2d, v26.2d, #19 + eor x14, x14, x1 + xar v16.2d, v5.2d, v25.2d, #28 + eor x20, x20, x1 + xar v5.2d, v3.2d, v28.2d, #36 + eor x25, x25, x1 + xar v3.2d, v18.2d, v28.2d, #43 + ldr x0, [x29, #32] + xar v18.2d, v17.2d, v27.2d, #49 + ldr x1, [x29, #24] + xar v17.2d, v11.2d, v26.2d, #54 + eor x28, x28, x30, ror 63 + xar v11.2d, v7.2d, v27.2d, #58 + eor x30, x30, x1, ror 63 + xar v7.2d, v10.2d, v25.2d, #61 + eor x1, x1, x0, ror 63 + # Row Mix + mov v25.16b, v0.16b + eor x6, x6, x28 + mov v26.16b, v1.16b + eor x11, x11, x28 + bcax v0.16b, v25.16b, v2.16b, v26.16b + eor x16, x16, x28 + bcax v1.16b, v26.16b, v3.16b, v2.16b + eor x22, x22, x28 + bcax v2.16b, v2.16b, v4.16b, v3.16b + eor x27, x27, x28 + bcax v3.16b, v3.16b, v25.16b, v4.16b + eor x3, x3, x30 + bcax v4.16b, v4.16b, v26.16b, v25.16b + eor x8, x8, x30 + mov v25.16b, v5.16b + eor x13, x13, x30 + mov v26.16b, v6.16b + eor x19, x19, x30 + bcax v5.16b, v25.16b, v7.16b, v26.16b + eor x24, x24, x30 + bcax v6.16b, v26.16b, v8.16b, v7.16b + eor x5, x5, x1 + bcax v7.16b, v7.16b, v9.16b, v8.16b + eor x10, x10, x1 + bcax v8.16b, v8.16b, v25.16b, v9.16b + eor x15, x15, x1 + bcax v9.16b, v9.16b, v26.16b, v25.16b + eor x21, x21, x1 + mov v26.16b, v11.16b + eor x26, x26, x1 + # Swap Rotate Base + bcax v10.16b, v30.16b, v12.16b, v26.16b + ror x0, x3, #63 + bcax v11.16b, v26.16b, v13.16b, v12.16b + ror x3, x8, #20 + bcax v12.16b, v12.16b, v14.16b, v13.16b + ror x8, x11, #44 + bcax v13.16b, v13.16b, v30.16b, v14.16b + ror x11, x25, #3 + bcax v14.16b, v14.16b, v26.16b, v30.16b + ror x25, x16, #25 + mov v25.16b, v15.16b + ror x16, x23, #46 + mov v26.16b, v16.16b + ror x23, x4, #2 + bcax v15.16b, v25.16b, v17.16b, v26.16b + ror x4, x14, #21 + bcax v16.16b, v26.16b, v18.16b, v17.16b + ror x14, x15, #39 + bcax v17.16b, v17.16b, v19.16b, v18.16b + ror x15, x22, #56 + bcax v18.16b, v18.16b, v25.16b, v19.16b + ror x22, x26, #8 + bcax v19.16b, v19.16b, v26.16b, v25.16b + ror x26, x17, #23 + mov v25.16b, v20.16b + ror x17, x6, #37 + mov v26.16b, v21.16b + ror x6, x27, #50 + bcax v20.16b, v25.16b, v22.16b, v26.16b + ror x27, x24, #62 + bcax v21.16b, v26.16b, v23.16b, v22.16b + ror x24, x10, #9 + bcax v22.16b, v22.16b, v24.16b, v23.16b + ror x10, x19, #19 + bcax v23.16b, v23.16b, v25.16b, v24.16b + ror x19, x7, #28 + bcax v24.16b, v24.16b, v26.16b, v25.16b + ror x7, x5, #36 + ror x5, x21, #43 + ror x21, x20, #49 + ror x20, x13, #54 + ror x13, x9, #58 + ror x9, x12, #61 + # Row Mix Base + bic x12, x4, x3 + bic x1, x5, x4 + bic x28, x2, x6 + bic x30, x3, x2 + eor x2, x2, x12 + eor x3, x3, x1 + bic x12, x6, x5 + eor x5, x5, x28 + eor x4, x4, x12 + eor x6, x6, x30 + bic x12, x9, x8 + bic x1, x10, x9 + bic x28, x7, x11 + bic x30, x8, x7 + eor x7, x7, x12 + eor x8, x8, x1 + bic x12, x11, x10 + eor x10, x10, x28 + eor x9, x9, x12 + eor x11, x11, x30 + bic x12, x14, x13 + bic x1, x15, x14 + bic x28, x0, x16 + bic x30, x13, x0 + eor x12, x0, x12 + eor x13, x13, x1 + bic x0, x16, x15 + eor x15, x15, x28 + eor x14, x14, x0 + eor x16, x16, x30 + bic x0, x20, x19 + bic x1, x21, x20 + bic x28, x17, x22 + bic x30, x19, x17 + eor x17, x17, x0 + eor x19, x19, x1 + bic x0, x22, x21 + eor x21, x21, x28 + eor x20, x20, x0 + eor x22, x22, x30 + bic x0, x25, x24 + bic x1, x26, x25 + bic x28, x23, x27 + bic x30, x24, x23 + eor x23, x23, x0 + eor x24, x24, x1 + bic x0, x27, x26 + eor x26, x26, x28 + eor x25, x25, x0 + eor x27, x27, x30 + # Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov v30.d[0], x0 + mov v30.d[1], x0 + eor x2, x2, x0 + eor v0.16b, v0.16b, v30.16b + bne L_SHA3_shake256_blocksx3_seed_64_crypto_begin + ldr x0, [x29, #40] + st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 + st1 {v24.d}[0], [x0] + add x0, x0, #8 + st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 + st1 {v24.d}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp d8, d9, [x29, #160] + ldp d10, d11, [x29, #176] + ldp d12, d13, [x29, #192] + ldp d14, d15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret +#ifndef __APPLE__ + .size sha3_256_blocksx3_seed_64_crypto,.-sha3_256_blocksx3_seed_64_crypto +#endif /* __APPLE__ */ +#endif /* WOLFSSL_ARMASM_CRYPTO_SHA3 */ +#ifndef __APPLE__ +.text +.globl sha3_blocksx3_neon +WC_ASM_ATT_HIDDEN(sha3_blocksx3_neon) +.type sha3_blocksx3_neon,@function +.align 2 +sha3_blocksx3_neon: +#else +.section __TEXT,__text +.globl _sha3_blocksx3_neon +WC_ASM_ATT_HIDDEN(_sha3_blocksx3_neon) +.p2align 2 +_sha3_blocksx3_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp d8, d9, [x29, #160] + stp d10, d11, [x29, #176] + stp d12, d13, [x29, #192] + stp d14, d15, [x29, #208] +#ifndef __APPLE__ + adrp x27, L_sha3_aarch64_r + add x27, x27, :lo12:L_sha3_aarch64_r +#else + adrp x27, L_sha3_aarch64_r@PAGE + add x27, x27, L_sha3_aarch64_r@PAGEOFF +#endif /* __APPLE__ */ + str x0, [x29, #40] + ld4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 + ld4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 + ld4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 + ld4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 + ld4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 + ld4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 + ld1 {v24.d}[0], [x0] + add x0, x0, #8 + ld4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 + ld4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 + ld4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 + ld4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 + ld4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 + ld4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 + ld1 {v24.d}[1], [x0] + add x0, x0, #8 + ldp x1, x2, [x0] + ldp x3, x4, [x0, #16] + ldp x5, x6, [x0, #32] + ldp x7, x8, [x0, #48] + ldp x9, x10, [x0, #64] + ldp x11, x12, [x0, #80] + ldp x13, x14, [x0, #96] + ldp x15, x16, [x0, #112] + ldp x17, x19, [x0, #128] + ldp x20, x21, [x0, #144] + ldp x22, x23, [x0, #160] + ldp x24, x25, [x0, #176] + ldr x26, [x0, #192] + mov x28, #24 + # Start of 24 rounds +L_SHA3_transform_blocksx3_neon_begin: + stp x27, x28, [x29, #48] + # Col Mix NEON + eor v30.16b, v4.16b, v9.16b + eor x0, x5, x10 + eor v27.16b, v1.16b, v6.16b + eor x30, x1, x6 + eor v30.16b, v30.16b, v14.16b + eor x28, x3, x8 + eor v27.16b, v27.16b, v11.16b + eor x0, x0, x15 + eor v30.16b, v30.16b, v19.16b + eor x30, x30, x11 + eor v27.16b, v27.16b, v16.16b + eor x28, x28, x13 + eor v30.16b, v30.16b, v24.16b + eor x0, x0, x21 + eor v27.16b, v27.16b, v21.16b + eor x30, x30, x16 + ushr v25.2d, v27.2d, #63 + eor x28, x28, x19 + sli v25.2d, v27.2d, #1 + eor x0, x0, x26 + eor v25.16b, v25.16b, v30.16b + eor x30, x30, x22 + eor v31.16b, v0.16b, v5.16b + eor x28, x28, x24 + eor v28.16b, v2.16b, v7.16b + str x0, [x29, #32] + eor v31.16b, v31.16b, v10.16b + str x28, [x29, #24] + eor v28.16b, v28.16b, v12.16b + eor x27, x2, x7 + eor v31.16b, v31.16b, v15.16b + eor x28, x4, x9 + eor v28.16b, v28.16b, v17.16b + eor x27, x27, x12 + eor v31.16b, v31.16b, v20.16b + eor x28, x28, x14 + eor v28.16b, v28.16b, v22.16b + eor x27, x27, x17 + ushr v29.2d, v30.2d, #63 + eor x28, x28, x20 + ushr v26.2d, v28.2d, #63 + eor x27, x27, x23 + sli v29.2d, v30.2d, #1 + eor x28, x28, x25 + sli v26.2d, v28.2d, #1 + eor x0, x0, x27, ror 63 + eor v28.16b, v28.16b, v29.16b + eor x27, x27, x28, ror 63 + eor v29.16b, v3.16b, v8.16b + eor x1, x1, x0 + eor v26.16b, v26.16b, v31.16b + eor x6, x6, x0 + eor v29.16b, v29.16b, v13.16b + eor x11, x11, x0 + eor v29.16b, v29.16b, v18.16b + eor x16, x16, x0 + eor v29.16b, v29.16b, v23.16b + eor x22, x22, x0 + ushr v30.2d, v29.2d, #63 + eor x3, x3, x27 + sli v30.2d, v29.2d, #1 + eor x8, x8, x27 + eor v27.16b, v27.16b, v30.16b + eor x13, x13, x27 + ushr v30.2d, v31.2d, #63 + eor x19, x19, x27 + sli v30.2d, v31.2d, #1 + eor x24, x24, x27 + eor v29.16b, v29.16b, v30.16b + ldr x0, [x29, #32] + # Swap Rotate NEON + eor v0.16b, v0.16b, v25.16b + eor v31.16b, v1.16b, v26.16b + ldr x27, [x29, #24] + eor v6.16b, v6.16b, v26.16b + eor x28, x28, x30, ror 63 + ushr v30.2d, v31.2d, #63 + eor x30, x30, x27, ror 63 + ushr v1.2d, v6.2d, #20 + eor x27, x27, x0, ror 63 + sli v30.2d, v31.2d, #1 + eor x5, x5, x28 + sli v1.2d, v6.2d, #44 + eor x10, x10, x28 + eor v31.16b, v9.16b, v29.16b + eor x15, x15, x28 + eor v22.16b, v22.16b, v27.16b + eor x21, x21, x28 + ushr v6.2d, v31.2d, #44 + eor x26, x26, x28 + ushr v9.2d, v22.2d, #3 + eor x2, x2, x30 + sli v6.2d, v31.2d, #20 + eor x7, x7, x30 + sli v9.2d, v22.2d, #61 + eor x12, x12, x30 + eor v31.16b, v14.16b, v29.16b + eor x17, x17, x30 + eor v20.16b, v20.16b, v25.16b + eor x23, x23, x30 + ushr v22.2d, v31.2d, #25 + eor x4, x4, x27 + ushr v14.2d, v20.2d, #46 + eor x9, x9, x27 + sli v22.2d, v31.2d, #39 + eor x14, x14, x27 + sli v14.2d, v20.2d, #18 + eor x20, x20, x27 + eor v31.16b, v2.16b, v27.16b + eor x25, x25, x27 + # Swap Rotate Base + eor v12.16b, v12.16b, v27.16b + ror x0, x2, #63 + ushr v20.2d, v31.2d, #2 + ror x2, x7, #20 + ushr v2.2d, v12.2d, #21 + ror x7, x10, #44 + sli v20.2d, v31.2d, #62 + ror x10, x24, #3 + sli v2.2d, v12.2d, #43 + ror x24, x15, #25 + eor v31.16b, v13.16b, v28.16b + ror x15, x22, #46 + eor v19.16b, v19.16b, v29.16b + ror x22, x3, #2 + ushr v12.2d, v31.2d, #39 + ror x3, x13, #21 + ushr v13.2d, v19.2d, #56 + ror x13, x14, #39 + sli v12.2d, v31.2d, #25 + ror x14, x21, #56 + sli v13.2d, v19.2d, #8 + ror x21, x25, #8 + eor v31.16b, v23.16b, v28.16b + ror x25, x16, #23 + eor v15.16b, v15.16b, v25.16b + ror x16, x5, #37 + ushr v19.2d, v31.2d, #8 + ror x5, x26, #50 + ushr v23.2d, v15.2d, #23 + ror x26, x23, #62 + sli v19.2d, v31.2d, #56 + ror x23, x9, #9 + sli v23.2d, v15.2d, #41 + ror x9, x17, #19 + eor v31.16b, v4.16b, v29.16b + ror x17, x6, #28 + eor v24.16b, v24.16b, v29.16b + ror x6, x4, #36 + ushr v15.2d, v31.2d, #37 + ror x4, x20, #43 + ushr v4.2d, v24.2d, #50 + ror x20, x19, #49 + sli v15.2d, v31.2d, #27 + ror x19, x12, #54 + sli v4.2d, v24.2d, #14 + ror x12, x8, #58 + eor v31.16b, v21.16b, v26.16b + ror x8, x11, #61 + # Row Mix Base + eor v8.16b, v8.16b, v28.16b + bic x11, x3, x2 + ushr v24.2d, v31.2d, #62 + bic x27, x4, x3 + ushr v21.2d, v8.2d, #9 + bic x28, x1, x5 + sli v24.2d, v31.2d, #2 + bic x30, x2, x1 + sli v21.2d, v8.2d, #55 + eor x1, x1, x11 + eor v31.16b, v16.16b, v26.16b + eor x2, x2, x27 + eor v5.16b, v5.16b, v25.16b + bic x11, x5, x4 + ushr v8.2d, v31.2d, #19 + eor x4, x4, x28 + ushr v16.2d, v5.2d, #28 + eor x3, x3, x11 + sli v8.2d, v31.2d, #45 + eor x5, x5, x30 + sli v16.2d, v5.2d, #36 + bic x11, x8, x7 + eor v31.16b, v3.16b, v28.16b + bic x27, x9, x8 + eor v18.16b, v18.16b, v28.16b + bic x28, x6, x10 + ushr v5.2d, v31.2d, #36 + bic x30, x7, x6 + ushr v3.2d, v18.2d, #43 + eor x6, x6, x11 + sli v5.2d, v31.2d, #28 + eor x7, x7, x27 + sli v3.2d, v18.2d, #21 + bic x11, x10, x9 + eor v31.16b, v17.16b, v27.16b + eor x9, x9, x28 + eor v11.16b, v11.16b, v26.16b + eor x8, x8, x11 + ushr v18.2d, v31.2d, #49 + eor x10, x10, x30 + ushr v17.2d, v11.2d, #54 + bic x11, x13, x12 + sli v18.2d, v31.2d, #15 + bic x27, x14, x13 + sli v17.2d, v11.2d, #10 + bic x28, x0, x15 + eor v31.16b, v7.16b, v27.16b + bic x30, x12, x0 + eor v10.16b, v10.16b, v25.16b + eor x11, x0, x11 + ushr v11.2d, v31.2d, #58 + eor x12, x12, x27 + ushr v7.2d, v10.2d, #61 + bic x0, x15, x14 + sli v11.2d, v31.2d, #6 + eor x14, x14, x28 + sli v7.2d, v10.2d, #3 + eor x13, x13, x0 + # Row Mix NEON + bic v25.16b, v2.16b, v1.16b + eor x15, x15, x30 + bic v26.16b, v3.16b, v2.16b + bic x0, x19, x17 + bic v27.16b, v4.16b, v3.16b + bic x27, x20, x19 + bic v28.16b, v0.16b, v4.16b + bic x28, x16, x21 + bic v29.16b, v1.16b, v0.16b + bic x30, x17, x16 + eor v0.16b, v0.16b, v25.16b + eor x16, x16, x0 + eor v1.16b, v1.16b, v26.16b + eor x17, x17, x27 + eor v2.16b, v2.16b, v27.16b + bic x0, x21, x20 + eor v3.16b, v3.16b, v28.16b + eor x20, x20, x28 + eor v4.16b, v4.16b, v29.16b + eor x19, x19, x0 + bic v25.16b, v7.16b, v6.16b + eor x21, x21, x30 + bic v26.16b, v8.16b, v7.16b + bic x0, x24, x23 + bic v27.16b, v9.16b, v8.16b + bic x27, x25, x24 + bic v28.16b, v5.16b, v9.16b + bic x28, x22, x26 + bic v29.16b, v6.16b, v5.16b + bic x30, x23, x22 + eor v5.16b, v5.16b, v25.16b + eor x22, x22, x0 + eor v6.16b, v6.16b, v26.16b + eor x23, x23, x27 + eor v7.16b, v7.16b, v27.16b + bic x0, x26, x25 + eor v8.16b, v8.16b, v28.16b + eor x25, x25, x28 + eor v9.16b, v9.16b, v29.16b + eor x24, x24, x0 + bic v25.16b, v12.16b, v11.16b + eor x26, x26, x30 + bic v26.16b, v13.16b, v12.16b + bic v27.16b, v14.16b, v13.16b + bic v28.16b, v30.16b, v14.16b + bic v29.16b, v11.16b, v30.16b + eor v10.16b, v30.16b, v25.16b + eor v11.16b, v11.16b, v26.16b + eor v12.16b, v12.16b, v27.16b + eor v13.16b, v13.16b, v28.16b + eor v14.16b, v14.16b, v29.16b + bic v25.16b, v17.16b, v16.16b + bic v26.16b, v18.16b, v17.16b + bic v27.16b, v19.16b, v18.16b + bic v28.16b, v15.16b, v19.16b + bic v29.16b, v16.16b, v15.16b + eor v15.16b, v15.16b, v25.16b + eor v16.16b, v16.16b, v26.16b + eor v17.16b, v17.16b, v27.16b + eor v18.16b, v18.16b, v28.16b + eor v19.16b, v19.16b, v29.16b + bic v25.16b, v22.16b, v21.16b + bic v26.16b, v23.16b, v22.16b + bic v27.16b, v24.16b, v23.16b + bic v28.16b, v20.16b, v24.16b + bic v29.16b, v21.16b, v20.16b + eor v20.16b, v20.16b, v25.16b + eor v21.16b, v21.16b, v26.16b + eor v22.16b, v22.16b, v27.16b + eor v23.16b, v23.16b, v28.16b + eor v24.16b, v24.16b, v29.16b + # Done transforming + ldp x27, x28, [x29, #48] + ldr x0, [x27], #8 + subs x28, x28, #1 + mov v30.d[0], x0 + mov v30.d[1], x0 + eor x1, x1, x0 + eor v0.16b, v0.16b, v30.16b + bne L_SHA3_transform_blocksx3_neon_begin + ldr x0, [x29, #40] + st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 + st1 {v24.d}[0], [x0] + add x0, x0, #8 + st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 + st1 {v24.d}[1], [x0] + add x0, x0, #8 + stp x1, x2, [x0] + stp x3, x4, [x0, #16] + stp x5, x6, [x0, #32] + stp x7, x8, [x0, #48] + stp x9, x10, [x0, #64] + stp x11, x12, [x0, #80] + stp x13, x14, [x0, #96] + stp x15, x16, [x0, #112] + stp x17, x19, [x0, #128] + stp x20, x21, [x0, #144] + stp x22, x23, [x0, #160] + stp x24, x25, [x0, #176] + str x26, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp d8, d9, [x29, #160] + ldp d10, d11, [x29, #176] + ldp d12, d13, [x29, #192] + ldp d14, d15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret +#ifndef __APPLE__ + .size sha3_blocksx3_neon,.-sha3_blocksx3_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl sha3_128_blocksx3_seed_neon +WC_ASM_ATT_HIDDEN(sha3_128_blocksx3_seed_neon) +.type sha3_128_blocksx3_seed_neon,@function +.align 2 +sha3_128_blocksx3_seed_neon: +#else +.section __TEXT,__text +.globl _sha3_128_blocksx3_seed_neon +WC_ASM_ATT_HIDDEN(_sha3_128_blocksx3_seed_neon) +.p2align 2 +_sha3_128_blocksx3_seed_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp d8, d9, [x29, #160] + stp d10, d11, [x29, #176] + stp d12, d13, [x29, #192] + stp d14, d15, [x29, #208] +#ifndef __APPLE__ + adrp x28, L_sha3_aarch64_r + add x28, x28, :lo12:L_sha3_aarch64_r +#else + adrp x28, L_sha3_aarch64_r@PAGE + add x28, x28, L_sha3_aarch64_r@PAGEOFF +#endif /* __APPLE__ */ + str x0, [x29, #40] + add x0, x0, #32 + ld1 {v4.d}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {v4.d}[1], [x0] + ldp x4, x5, [x1], #16 + ldr x6, [x0, #200] + eor v5.16b, v5.16b, v5.16b + eor x7, x7, x7 + eor v6.16b, v6.16b, v6.16b + eor x8, x8, x8 + eor v7.16b, v7.16b, v7.16b + eor x9, x9, x9 + eor v8.16b, v8.16b, v8.16b + eor x10, x10, x10 + eor v9.16b, v9.16b, v9.16b + eor x11, x11, x11 + eor v10.16b, v10.16b, v10.16b + eor x12, x12, x12 + eor v11.16b, v11.16b, v11.16b + eor x13, x13, x13 + eor v12.16b, v12.16b, v12.16b + eor x14, x14, x14 + eor v13.16b, v13.16b, v13.16b + eor x15, x15, x15 + eor v14.16b, v14.16b, v14.16b + eor x16, x16, x16 + eor v15.16b, v15.16b, v15.16b + eor x17, x17, x17 + eor v16.16b, v16.16b, v16.16b + eor x19, x19, x19 + eor v17.16b, v17.16b, v17.16b + eor x20, x20, x20 + eor v18.16b, v18.16b, v18.16b + eor x21, x21, x21 + eor v19.16b, v19.16b, v19.16b + eor x22, x22, x22 + movz x23, #0x8000, lsl 48 + eor v21.16b, v21.16b, v21.16b + eor x24, x24, x24 + eor v22.16b, v22.16b, v22.16b + eor x25, x25, x25 + eor v23.16b, v23.16b, v23.16b + eor x26, x26, x26 + eor v24.16b, v24.16b, v24.16b + eor x27, x27, x27 + dup v0.2d, x2 + dup v1.2d, x3 + dup v2.2d, x4 + dup v3.2d, x5 + dup v20.2d, x23 + mov x1, #24 + # Start of 24 rounds +L_SHA3_shake128_blocksx3_seed_neon_begin: + stp x28, x1, [x29, #48] + # Col Mix NEON + eor v30.16b, v4.16b, v9.16b + eor x0, x6, x11 + eor v27.16b, v1.16b, v6.16b + eor x30, x2, x7 + eor v30.16b, v30.16b, v14.16b + eor x28, x4, x9 + eor v27.16b, v27.16b, v11.16b + eor x0, x0, x16 + eor v30.16b, v30.16b, v19.16b + eor x30, x30, x12 + eor v27.16b, v27.16b, v16.16b + eor x28, x28, x14 + eor v30.16b, v30.16b, v24.16b + eor x0, x0, x22 + eor v27.16b, v27.16b, v21.16b + eor x30, x30, x17 + ushr v25.2d, v27.2d, #63 + eor x28, x28, x20 + sli v25.2d, v27.2d, #1 + eor x0, x0, x27 + eor v25.16b, v25.16b, v30.16b + eor x30, x30, x23 + eor v31.16b, v0.16b, v5.16b + eor x28, x28, x25 + eor v28.16b, v2.16b, v7.16b + str x0, [x29, #32] + eor v31.16b, v31.16b, v10.16b + str x28, [x29, #24] + eor v28.16b, v28.16b, v12.16b + eor x1, x3, x8 + eor v31.16b, v31.16b, v15.16b + eor x28, x5, x10 + eor v28.16b, v28.16b, v17.16b + eor x1, x1, x13 + eor v31.16b, v31.16b, v20.16b + eor x28, x28, x15 + eor v28.16b, v28.16b, v22.16b + eor x1, x1, x19 + ushr v29.2d, v30.2d, #63 + eor x28, x28, x21 + ushr v26.2d, v28.2d, #63 + eor x1, x1, x24 + sli v29.2d, v30.2d, #1 + eor x28, x28, x26 + sli v26.2d, v28.2d, #1 + eor x0, x0, x1, ror 63 + eor v28.16b, v28.16b, v29.16b + eor x1, x1, x28, ror 63 + eor v29.16b, v3.16b, v8.16b + eor x2, x2, x0 + eor v26.16b, v26.16b, v31.16b + eor x7, x7, x0 + eor v29.16b, v29.16b, v13.16b + eor x12, x12, x0 + eor v29.16b, v29.16b, v18.16b + eor x17, x17, x0 + eor v29.16b, v29.16b, v23.16b + eor x23, x23, x0 + ushr v30.2d, v29.2d, #63 + eor x4, x4, x1 + sli v30.2d, v29.2d, #1 + eor x9, x9, x1 + eor v27.16b, v27.16b, v30.16b + eor x14, x14, x1 + ushr v30.2d, v31.2d, #63 + eor x20, x20, x1 + sli v30.2d, v31.2d, #1 + eor x25, x25, x1 + eor v29.16b, v29.16b, v30.16b + ldr x0, [x29, #32] + # Swap Rotate NEON + eor v0.16b, v0.16b, v25.16b + eor v31.16b, v1.16b, v26.16b + ldr x1, [x29, #24] + eor v6.16b, v6.16b, v26.16b + eor x28, x28, x30, ror 63 + ushr v30.2d, v31.2d, #63 + eor x30, x30, x1, ror 63 + ushr v1.2d, v6.2d, #20 + eor x1, x1, x0, ror 63 + sli v30.2d, v31.2d, #1 + eor x6, x6, x28 + sli v1.2d, v6.2d, #44 + eor x11, x11, x28 + eor v31.16b, v9.16b, v29.16b + eor x16, x16, x28 + eor v22.16b, v22.16b, v27.16b + eor x22, x22, x28 + ushr v6.2d, v31.2d, #44 + eor x27, x27, x28 + ushr v9.2d, v22.2d, #3 + eor x3, x3, x30 + sli v6.2d, v31.2d, #20 + eor x8, x8, x30 + sli v9.2d, v22.2d, #61 + eor x13, x13, x30 + eor v31.16b, v14.16b, v29.16b + eor x19, x19, x30 + eor v20.16b, v20.16b, v25.16b + eor x24, x24, x30 + ushr v22.2d, v31.2d, #25 + eor x5, x5, x1 + ushr v14.2d, v20.2d, #46 + eor x10, x10, x1 + sli v22.2d, v31.2d, #39 + eor x15, x15, x1 + sli v14.2d, v20.2d, #18 + eor x21, x21, x1 + eor v31.16b, v2.16b, v27.16b + eor x26, x26, x1 + # Swap Rotate Base + eor v12.16b, v12.16b, v27.16b + ror x0, x3, #63 + ushr v20.2d, v31.2d, #2 + ror x3, x8, #20 + ushr v2.2d, v12.2d, #21 + ror x8, x11, #44 + sli v20.2d, v31.2d, #62 + ror x11, x25, #3 + sli v2.2d, v12.2d, #43 + ror x25, x16, #25 + eor v31.16b, v13.16b, v28.16b + ror x16, x23, #46 + eor v19.16b, v19.16b, v29.16b + ror x23, x4, #2 + ushr v12.2d, v31.2d, #39 + ror x4, x14, #21 + ushr v13.2d, v19.2d, #56 + ror x14, x15, #39 + sli v12.2d, v31.2d, #25 + ror x15, x22, #56 + sli v13.2d, v19.2d, #8 + ror x22, x26, #8 + eor v31.16b, v23.16b, v28.16b + ror x26, x17, #23 + eor v15.16b, v15.16b, v25.16b + ror x17, x6, #37 + ushr v19.2d, v31.2d, #8 + ror x6, x27, #50 + ushr v23.2d, v15.2d, #23 + ror x27, x24, #62 + sli v19.2d, v31.2d, #56 + ror x24, x10, #9 + sli v23.2d, v15.2d, #41 + ror x10, x19, #19 + eor v31.16b, v4.16b, v29.16b + ror x19, x7, #28 + eor v24.16b, v24.16b, v29.16b + ror x7, x5, #36 + ushr v15.2d, v31.2d, #37 + ror x5, x21, #43 + ushr v4.2d, v24.2d, #50 + ror x21, x20, #49 + sli v15.2d, v31.2d, #27 + ror x20, x13, #54 + sli v4.2d, v24.2d, #14 + ror x13, x9, #58 + eor v31.16b, v21.16b, v26.16b + ror x9, x12, #61 + # Row Mix Base + eor v8.16b, v8.16b, v28.16b + bic x12, x4, x3 + ushr v24.2d, v31.2d, #62 + bic x1, x5, x4 + ushr v21.2d, v8.2d, #9 + bic x28, x2, x6 + sli v24.2d, v31.2d, #2 + bic x30, x3, x2 + sli v21.2d, v8.2d, #55 + eor x2, x2, x12 + eor v31.16b, v16.16b, v26.16b + eor x3, x3, x1 + eor v5.16b, v5.16b, v25.16b + bic x12, x6, x5 + ushr v8.2d, v31.2d, #19 + eor x5, x5, x28 + ushr v16.2d, v5.2d, #28 + eor x4, x4, x12 + sli v8.2d, v31.2d, #45 + eor x6, x6, x30 + sli v16.2d, v5.2d, #36 + bic x12, x9, x8 + eor v31.16b, v3.16b, v28.16b + bic x1, x10, x9 + eor v18.16b, v18.16b, v28.16b + bic x28, x7, x11 + ushr v5.2d, v31.2d, #36 + bic x30, x8, x7 + ushr v3.2d, v18.2d, #43 + eor x7, x7, x12 + sli v5.2d, v31.2d, #28 + eor x8, x8, x1 + sli v3.2d, v18.2d, #21 + bic x12, x11, x10 + eor v31.16b, v17.16b, v27.16b + eor x10, x10, x28 + eor v11.16b, v11.16b, v26.16b + eor x9, x9, x12 + ushr v18.2d, v31.2d, #49 + eor x11, x11, x30 + ushr v17.2d, v11.2d, #54 + bic x12, x14, x13 + sli v18.2d, v31.2d, #15 + bic x1, x15, x14 + sli v17.2d, v11.2d, #10 + bic x28, x0, x16 + eor v31.16b, v7.16b, v27.16b + bic x30, x13, x0 + eor v10.16b, v10.16b, v25.16b + eor x12, x0, x12 + ushr v11.2d, v31.2d, #58 + eor x13, x13, x1 + ushr v7.2d, v10.2d, #61 + bic x0, x16, x15 + sli v11.2d, v31.2d, #6 + eor x15, x15, x28 + sli v7.2d, v10.2d, #3 + eor x14, x14, x0 + # Row Mix NEON + bic v25.16b, v2.16b, v1.16b + eor x16, x16, x30 + bic v26.16b, v3.16b, v2.16b + bic x0, x20, x19 + bic v27.16b, v4.16b, v3.16b + bic x1, x21, x20 + bic v28.16b, v0.16b, v4.16b + bic x28, x17, x22 + bic v29.16b, v1.16b, v0.16b + bic x30, x19, x17 + eor v0.16b, v0.16b, v25.16b + eor x17, x17, x0 + eor v1.16b, v1.16b, v26.16b + eor x19, x19, x1 + eor v2.16b, v2.16b, v27.16b + bic x0, x22, x21 + eor v3.16b, v3.16b, v28.16b + eor x21, x21, x28 + eor v4.16b, v4.16b, v29.16b + eor x20, x20, x0 + bic v25.16b, v7.16b, v6.16b + eor x22, x22, x30 + bic v26.16b, v8.16b, v7.16b + bic x0, x25, x24 + bic v27.16b, v9.16b, v8.16b + bic x1, x26, x25 + bic v28.16b, v5.16b, v9.16b + bic x28, x23, x27 + bic v29.16b, v6.16b, v5.16b + bic x30, x24, x23 + eor v5.16b, v5.16b, v25.16b + eor x23, x23, x0 + eor v6.16b, v6.16b, v26.16b + eor x24, x24, x1 + eor v7.16b, v7.16b, v27.16b + bic x0, x27, x26 + eor v8.16b, v8.16b, v28.16b + eor x26, x26, x28 + eor v9.16b, v9.16b, v29.16b + eor x25, x25, x0 + bic v25.16b, v12.16b, v11.16b + eor x27, x27, x30 + bic v26.16b, v13.16b, v12.16b + bic v27.16b, v14.16b, v13.16b + bic v28.16b, v30.16b, v14.16b + bic v29.16b, v11.16b, v30.16b + eor v10.16b, v30.16b, v25.16b + eor v11.16b, v11.16b, v26.16b + eor v12.16b, v12.16b, v27.16b + eor v13.16b, v13.16b, v28.16b + eor v14.16b, v14.16b, v29.16b + bic v25.16b, v17.16b, v16.16b + bic v26.16b, v18.16b, v17.16b + bic v27.16b, v19.16b, v18.16b + bic v28.16b, v15.16b, v19.16b + bic v29.16b, v16.16b, v15.16b + eor v15.16b, v15.16b, v25.16b + eor v16.16b, v16.16b, v26.16b + eor v17.16b, v17.16b, v27.16b + eor v18.16b, v18.16b, v28.16b + eor v19.16b, v19.16b, v29.16b + bic v25.16b, v22.16b, v21.16b + bic v26.16b, v23.16b, v22.16b + bic v27.16b, v24.16b, v23.16b + bic v28.16b, v20.16b, v24.16b + bic v29.16b, v21.16b, v20.16b + eor v20.16b, v20.16b, v25.16b + eor v21.16b, v21.16b, v26.16b + eor v22.16b, v22.16b, v27.16b + eor v23.16b, v23.16b, v28.16b + eor v24.16b, v24.16b, v29.16b + # Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov v30.d[0], x0 + mov v30.d[1], x0 + eor x2, x2, x0 + eor v0.16b, v0.16b, v30.16b + bne L_SHA3_shake128_blocksx3_seed_neon_begin + ldr x0, [x29, #40] + st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 + st1 {v24.d}[0], [x0] + add x0, x0, #8 + st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 + st1 {v24.d}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp d8, d9, [x29, #160] + ldp d10, d11, [x29, #176] + ldp d12, d13, [x29, #192] + ldp d14, d15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret +#ifndef __APPLE__ + .size sha3_128_blocksx3_seed_neon,.-sha3_128_blocksx3_seed_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl sha3_256_blocksx3_seed_neon +WC_ASM_ATT_HIDDEN(sha3_256_blocksx3_seed_neon) +.type sha3_256_blocksx3_seed_neon,@function +.align 2 +sha3_256_blocksx3_seed_neon: +#else +.section __TEXT,__text +.globl _sha3_256_blocksx3_seed_neon +WC_ASM_ATT_HIDDEN(_sha3_256_blocksx3_seed_neon) +.p2align 2 +_sha3_256_blocksx3_seed_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp d8, d9, [x29, #160] + stp d10, d11, [x29, #176] + stp d12, d13, [x29, #192] + stp d14, d15, [x29, #208] +#ifndef __APPLE__ + adrp x28, L_sha3_aarch64_r + add x28, x28, :lo12:L_sha3_aarch64_r +#else + adrp x28, L_sha3_aarch64_r@PAGE + add x28, x28, L_sha3_aarch64_r@PAGEOFF +#endif /* __APPLE__ */ + str x0, [x29, #40] + add x0, x0, #32 + ld1 {v4.d}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {v4.d}[1], [x0] + ldp x4, x5, [x1], #16 + ldr x6, [x0, #200] + eor v5.16b, v5.16b, v5.16b + eor x7, x7, x7 + eor v6.16b, v6.16b, v6.16b + eor x8, x8, x8 + eor v7.16b, v7.16b, v7.16b + eor x9, x9, x9 + eor v8.16b, v8.16b, v8.16b + eor x10, x10, x10 + eor v9.16b, v9.16b, v9.16b + eor x11, x11, x11 + eor v10.16b, v10.16b, v10.16b + eor x12, x12, x12 + eor v11.16b, v11.16b, v11.16b + eor x13, x13, x13 + eor v12.16b, v12.16b, v12.16b + eor x14, x14, x14 + eor v13.16b, v13.16b, v13.16b + eor x15, x15, x15 + eor v14.16b, v14.16b, v14.16b + eor x16, x16, x16 + eor v15.16b, v15.16b, v15.16b + eor x17, x17, x17 + movz x19, #0x8000, lsl 48 + eor v17.16b, v17.16b, v17.16b + eor x20, x20, x20 + eor v18.16b, v18.16b, v18.16b + eor x21, x21, x21 + eor v19.16b, v19.16b, v19.16b + eor x22, x22, x22 + eor v20.16b, v20.16b, v20.16b + eor x23, x23, x23 + eor v21.16b, v21.16b, v21.16b + eor x24, x24, x24 + eor v22.16b, v22.16b, v22.16b + eor x25, x25, x25 + eor v23.16b, v23.16b, v23.16b + eor x26, x26, x26 + eor v24.16b, v24.16b, v24.16b + eor x27, x27, x27 + dup v0.2d, x2 + dup v1.2d, x3 + dup v2.2d, x4 + dup v3.2d, x5 + dup v16.2d, x19 + mov x1, #24 + # Start of 24 rounds +L_SHA3_shake256_blocksx3_seed_neon_begin: + stp x28, x1, [x29, #48] + # Col Mix NEON + eor v30.16b, v4.16b, v9.16b + eor x0, x6, x11 + eor v27.16b, v1.16b, v6.16b + eor x30, x2, x7 + eor v30.16b, v30.16b, v14.16b + eor x28, x4, x9 + eor v27.16b, v27.16b, v11.16b + eor x0, x0, x16 + eor v30.16b, v30.16b, v19.16b + eor x30, x30, x12 + eor v27.16b, v27.16b, v16.16b + eor x28, x28, x14 + eor v30.16b, v30.16b, v24.16b + eor x0, x0, x22 + eor v27.16b, v27.16b, v21.16b + eor x30, x30, x17 + ushr v25.2d, v27.2d, #63 + eor x28, x28, x20 + sli v25.2d, v27.2d, #1 + eor x0, x0, x27 + eor v25.16b, v25.16b, v30.16b + eor x30, x30, x23 + eor v31.16b, v0.16b, v5.16b + eor x28, x28, x25 + eor v28.16b, v2.16b, v7.16b + str x0, [x29, #32] + eor v31.16b, v31.16b, v10.16b + str x28, [x29, #24] + eor v28.16b, v28.16b, v12.16b + eor x1, x3, x8 + eor v31.16b, v31.16b, v15.16b + eor x28, x5, x10 + eor v28.16b, v28.16b, v17.16b + eor x1, x1, x13 + eor v31.16b, v31.16b, v20.16b + eor x28, x28, x15 + eor v28.16b, v28.16b, v22.16b + eor x1, x1, x19 + ushr v29.2d, v30.2d, #63 + eor x28, x28, x21 + ushr v26.2d, v28.2d, #63 + eor x1, x1, x24 + sli v29.2d, v30.2d, #1 + eor x28, x28, x26 + sli v26.2d, v28.2d, #1 + eor x0, x0, x1, ror 63 + eor v28.16b, v28.16b, v29.16b + eor x1, x1, x28, ror 63 + eor v29.16b, v3.16b, v8.16b + eor x2, x2, x0 + eor v26.16b, v26.16b, v31.16b + eor x7, x7, x0 + eor v29.16b, v29.16b, v13.16b + eor x12, x12, x0 + eor v29.16b, v29.16b, v18.16b + eor x17, x17, x0 + eor v29.16b, v29.16b, v23.16b + eor x23, x23, x0 + ushr v30.2d, v29.2d, #63 + eor x4, x4, x1 + sli v30.2d, v29.2d, #1 + eor x9, x9, x1 + eor v27.16b, v27.16b, v30.16b + eor x14, x14, x1 + ushr v30.2d, v31.2d, #63 + eor x20, x20, x1 + sli v30.2d, v31.2d, #1 + eor x25, x25, x1 + eor v29.16b, v29.16b, v30.16b + ldr x0, [x29, #32] + # Swap Rotate NEON + eor v0.16b, v0.16b, v25.16b + eor v31.16b, v1.16b, v26.16b + ldr x1, [x29, #24] + eor v6.16b, v6.16b, v26.16b + eor x28, x28, x30, ror 63 + ushr v30.2d, v31.2d, #63 + eor x30, x30, x1, ror 63 + ushr v1.2d, v6.2d, #20 + eor x1, x1, x0, ror 63 + sli v30.2d, v31.2d, #1 + eor x6, x6, x28 + sli v1.2d, v6.2d, #44 + eor x11, x11, x28 + eor v31.16b, v9.16b, v29.16b + eor x16, x16, x28 + eor v22.16b, v22.16b, v27.16b + eor x22, x22, x28 + ushr v6.2d, v31.2d, #44 + eor x27, x27, x28 + ushr v9.2d, v22.2d, #3 + eor x3, x3, x30 + sli v6.2d, v31.2d, #20 + eor x8, x8, x30 + sli v9.2d, v22.2d, #61 + eor x13, x13, x30 + eor v31.16b, v14.16b, v29.16b + eor x19, x19, x30 + eor v20.16b, v20.16b, v25.16b + eor x24, x24, x30 + ushr v22.2d, v31.2d, #25 + eor x5, x5, x1 + ushr v14.2d, v20.2d, #46 + eor x10, x10, x1 + sli v22.2d, v31.2d, #39 + eor x15, x15, x1 + sli v14.2d, v20.2d, #18 + eor x21, x21, x1 + eor v31.16b, v2.16b, v27.16b + eor x26, x26, x1 + # Swap Rotate Base + eor v12.16b, v12.16b, v27.16b + ror x0, x3, #63 + ushr v20.2d, v31.2d, #2 + ror x3, x8, #20 + ushr v2.2d, v12.2d, #21 + ror x8, x11, #44 + sli v20.2d, v31.2d, #62 + ror x11, x25, #3 + sli v2.2d, v12.2d, #43 + ror x25, x16, #25 + eor v31.16b, v13.16b, v28.16b + ror x16, x23, #46 + eor v19.16b, v19.16b, v29.16b + ror x23, x4, #2 + ushr v12.2d, v31.2d, #39 + ror x4, x14, #21 + ushr v13.2d, v19.2d, #56 + ror x14, x15, #39 + sli v12.2d, v31.2d, #25 + ror x15, x22, #56 + sli v13.2d, v19.2d, #8 + ror x22, x26, #8 + eor v31.16b, v23.16b, v28.16b + ror x26, x17, #23 + eor v15.16b, v15.16b, v25.16b + ror x17, x6, #37 + ushr v19.2d, v31.2d, #8 + ror x6, x27, #50 + ushr v23.2d, v15.2d, #23 + ror x27, x24, #62 + sli v19.2d, v31.2d, #56 + ror x24, x10, #9 + sli v23.2d, v15.2d, #41 + ror x10, x19, #19 + eor v31.16b, v4.16b, v29.16b + ror x19, x7, #28 + eor v24.16b, v24.16b, v29.16b + ror x7, x5, #36 + ushr v15.2d, v31.2d, #37 + ror x5, x21, #43 + ushr v4.2d, v24.2d, #50 + ror x21, x20, #49 + sli v15.2d, v31.2d, #27 + ror x20, x13, #54 + sli v4.2d, v24.2d, #14 + ror x13, x9, #58 + eor v31.16b, v21.16b, v26.16b + ror x9, x12, #61 + # Row Mix Base + eor v8.16b, v8.16b, v28.16b + bic x12, x4, x3 + ushr v24.2d, v31.2d, #62 + bic x1, x5, x4 + ushr v21.2d, v8.2d, #9 + bic x28, x2, x6 + sli v24.2d, v31.2d, #2 + bic x30, x3, x2 + sli v21.2d, v8.2d, #55 + eor x2, x2, x12 + eor v31.16b, v16.16b, v26.16b + eor x3, x3, x1 + eor v5.16b, v5.16b, v25.16b + bic x12, x6, x5 + ushr v8.2d, v31.2d, #19 + eor x5, x5, x28 + ushr v16.2d, v5.2d, #28 + eor x4, x4, x12 + sli v8.2d, v31.2d, #45 + eor x6, x6, x30 + sli v16.2d, v5.2d, #36 + bic x12, x9, x8 + eor v31.16b, v3.16b, v28.16b + bic x1, x10, x9 + eor v18.16b, v18.16b, v28.16b + bic x28, x7, x11 + ushr v5.2d, v31.2d, #36 + bic x30, x8, x7 + ushr v3.2d, v18.2d, #43 + eor x7, x7, x12 + sli v5.2d, v31.2d, #28 + eor x8, x8, x1 + sli v3.2d, v18.2d, #21 + bic x12, x11, x10 + eor v31.16b, v17.16b, v27.16b + eor x10, x10, x28 + eor v11.16b, v11.16b, v26.16b + eor x9, x9, x12 + ushr v18.2d, v31.2d, #49 + eor x11, x11, x30 + ushr v17.2d, v11.2d, #54 + bic x12, x14, x13 + sli v18.2d, v31.2d, #15 + bic x1, x15, x14 + sli v17.2d, v11.2d, #10 + bic x28, x0, x16 + eor v31.16b, v7.16b, v27.16b + bic x30, x13, x0 + eor v10.16b, v10.16b, v25.16b + eor x12, x0, x12 + ushr v11.2d, v31.2d, #58 + eor x13, x13, x1 + ushr v7.2d, v10.2d, #61 + bic x0, x16, x15 + sli v11.2d, v31.2d, #6 + eor x15, x15, x28 + sli v7.2d, v10.2d, #3 + eor x14, x14, x0 + # Row Mix NEON + bic v25.16b, v2.16b, v1.16b + eor x16, x16, x30 + bic v26.16b, v3.16b, v2.16b + bic x0, x20, x19 + bic v27.16b, v4.16b, v3.16b + bic x1, x21, x20 + bic v28.16b, v0.16b, v4.16b + bic x28, x17, x22 + bic v29.16b, v1.16b, v0.16b + bic x30, x19, x17 + eor v0.16b, v0.16b, v25.16b + eor x17, x17, x0 + eor v1.16b, v1.16b, v26.16b + eor x19, x19, x1 + eor v2.16b, v2.16b, v27.16b + bic x0, x22, x21 + eor v3.16b, v3.16b, v28.16b + eor x21, x21, x28 + eor v4.16b, v4.16b, v29.16b + eor x20, x20, x0 + bic v25.16b, v7.16b, v6.16b + eor x22, x22, x30 + bic v26.16b, v8.16b, v7.16b + bic x0, x25, x24 + bic v27.16b, v9.16b, v8.16b + bic x1, x26, x25 + bic v28.16b, v5.16b, v9.16b + bic x28, x23, x27 + bic v29.16b, v6.16b, v5.16b + bic x30, x24, x23 + eor v5.16b, v5.16b, v25.16b + eor x23, x23, x0 + eor v6.16b, v6.16b, v26.16b + eor x24, x24, x1 + eor v7.16b, v7.16b, v27.16b + bic x0, x27, x26 + eor v8.16b, v8.16b, v28.16b + eor x26, x26, x28 + eor v9.16b, v9.16b, v29.16b + eor x25, x25, x0 + bic v25.16b, v12.16b, v11.16b + eor x27, x27, x30 + bic v26.16b, v13.16b, v12.16b + bic v27.16b, v14.16b, v13.16b + bic v28.16b, v30.16b, v14.16b + bic v29.16b, v11.16b, v30.16b + eor v10.16b, v30.16b, v25.16b + eor v11.16b, v11.16b, v26.16b + eor v12.16b, v12.16b, v27.16b + eor v13.16b, v13.16b, v28.16b + eor v14.16b, v14.16b, v29.16b + bic v25.16b, v17.16b, v16.16b + bic v26.16b, v18.16b, v17.16b + bic v27.16b, v19.16b, v18.16b + bic v28.16b, v15.16b, v19.16b + bic v29.16b, v16.16b, v15.16b + eor v15.16b, v15.16b, v25.16b + eor v16.16b, v16.16b, v26.16b + eor v17.16b, v17.16b, v27.16b + eor v18.16b, v18.16b, v28.16b + eor v19.16b, v19.16b, v29.16b + bic v25.16b, v22.16b, v21.16b + bic v26.16b, v23.16b, v22.16b + bic v27.16b, v24.16b, v23.16b + bic v28.16b, v20.16b, v24.16b + bic v29.16b, v21.16b, v20.16b + eor v20.16b, v20.16b, v25.16b + eor v21.16b, v21.16b, v26.16b + eor v22.16b, v22.16b, v27.16b + eor v23.16b, v23.16b, v28.16b + eor v24.16b, v24.16b, v29.16b + # Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov v30.d[0], x0 + mov v30.d[1], x0 + eor x2, x2, x0 + eor v0.16b, v0.16b, v30.16b + bne L_SHA3_shake256_blocksx3_seed_neon_begin + ldr x0, [x29, #40] + st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 + st1 {v24.d}[0], [x0] + add x0, x0, #8 + st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 + st1 {v24.d}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp d8, d9, [x29, #160] + ldp d10, d11, [x29, #176] + ldp d12, d13, [x29, #192] + ldp d14, d15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret +#ifndef __APPLE__ + .size sha3_256_blocksx3_seed_neon,.-sha3_256_blocksx3_seed_neon +#endif /* __APPLE__ */ +#ifndef __APPLE__ +.text +.globl sha3_256_blocksx3_seed_64_neon +WC_ASM_ATT_HIDDEN(sha3_256_blocksx3_seed_64_neon) +.type sha3_256_blocksx3_seed_64_neon,@function +.align 2 +sha3_256_blocksx3_seed_64_neon: +#else +.section __TEXT,__text +.globl _sha3_256_blocksx3_seed_64_neon +WC_ASM_ATT_HIDDEN(_sha3_256_blocksx3_seed_64_neon) +.p2align 2 +_sha3_256_blocksx3_seed_64_neon: +#endif /* __APPLE__ */ + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp d8, d9, [x29, #160] + stp d10, d11, [x29, #176] + stp d12, d13, [x29, #192] + stp d14, d15, [x29, #208] +#ifndef __APPLE__ + adrp x28, L_sha3_aarch64_r + add x28, x28, :lo12:L_sha3_aarch64_r +#else + adrp x28, L_sha3_aarch64_r@PAGE + add x28, x28, L_sha3_aarch64_r@PAGEOFF +#endif /* __APPLE__ */ + str x0, [x29, #40] + add x0, x0, #0x40 + ld1 {v8.d}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {v8.d}[1], [x0] + ldp x4, x5, [x1], #16 + ldp x6, x7, [x1], #16 + ldp x8, x9, [x1], #16 + ldr x10, [x0, #200] + eor v9.16b, v9.16b, v9.16b + eor x11, x11, x11 + eor v10.16b, v10.16b, v10.16b + eor x12, x12, x12 + eor v11.16b, v11.16b, v11.16b + eor x13, x13, x13 + eor v12.16b, v12.16b, v12.16b + eor x14, x14, x14 + eor v13.16b, v13.16b, v13.16b + eor x15, x15, x15 + eor v14.16b, v14.16b, v14.16b + eor x16, x16, x16 + eor v15.16b, v15.16b, v15.16b + eor x17, x17, x17 + movz x19, #0x8000, lsl 48 + eor v17.16b, v17.16b, v17.16b + eor x20, x20, x20 + eor v18.16b, v18.16b, v18.16b + eor x21, x21, x21 + eor v19.16b, v19.16b, v19.16b + eor x22, x22, x22 + eor v20.16b, v20.16b, v20.16b + eor x23, x23, x23 + eor v21.16b, v21.16b, v21.16b + eor x24, x24, x24 + eor v22.16b, v22.16b, v22.16b + eor x25, x25, x25 + eor v23.16b, v23.16b, v23.16b + eor x26, x26, x26 + eor v24.16b, v24.16b, v24.16b + eor x27, x27, x27 + dup v0.2d, x2 + dup v1.2d, x3 + dup v2.2d, x4 + dup v3.2d, x5 + dup v4.2d, x6 + dup v5.2d, x7 + dup v6.2d, x8 + dup v7.2d, x9 + dup v16.2d, x19 + mov x1, #24 + # Start of 24 rounds +L_SHA3_shake256_blocksx3_seed_64_neon_begin: + stp x28, x1, [x29, #48] + # Col Mix NEON + eor v30.16b, v4.16b, v9.16b + eor x0, x6, x11 + eor v27.16b, v1.16b, v6.16b + eor x30, x2, x7 + eor v30.16b, v30.16b, v14.16b + eor x28, x4, x9 + eor v27.16b, v27.16b, v11.16b + eor x0, x0, x16 + eor v30.16b, v30.16b, v19.16b + eor x30, x30, x12 + eor v27.16b, v27.16b, v16.16b + eor x28, x28, x14 + eor v30.16b, v30.16b, v24.16b + eor x0, x0, x22 + eor v27.16b, v27.16b, v21.16b + eor x30, x30, x17 + ushr v25.2d, v27.2d, #63 + eor x28, x28, x20 + sli v25.2d, v27.2d, #1 + eor x0, x0, x27 + eor v25.16b, v25.16b, v30.16b + eor x30, x30, x23 + eor v31.16b, v0.16b, v5.16b + eor x28, x28, x25 + eor v28.16b, v2.16b, v7.16b + str x0, [x29, #32] + eor v31.16b, v31.16b, v10.16b + str x28, [x29, #24] + eor v28.16b, v28.16b, v12.16b + eor x1, x3, x8 + eor v31.16b, v31.16b, v15.16b + eor x28, x5, x10 + eor v28.16b, v28.16b, v17.16b + eor x1, x1, x13 + eor v31.16b, v31.16b, v20.16b + eor x28, x28, x15 + eor v28.16b, v28.16b, v22.16b + eor x1, x1, x19 + ushr v29.2d, v30.2d, #63 + eor x28, x28, x21 + ushr v26.2d, v28.2d, #63 + eor x1, x1, x24 + sli v29.2d, v30.2d, #1 + eor x28, x28, x26 + sli v26.2d, v28.2d, #1 + eor x0, x0, x1, ror 63 + eor v28.16b, v28.16b, v29.16b + eor x1, x1, x28, ror 63 + eor v29.16b, v3.16b, v8.16b + eor x2, x2, x0 + eor v26.16b, v26.16b, v31.16b + eor x7, x7, x0 + eor v29.16b, v29.16b, v13.16b + eor x12, x12, x0 + eor v29.16b, v29.16b, v18.16b + eor x17, x17, x0 + eor v29.16b, v29.16b, v23.16b + eor x23, x23, x0 + ushr v30.2d, v29.2d, #63 + eor x4, x4, x1 + sli v30.2d, v29.2d, #1 + eor x9, x9, x1 + eor v27.16b, v27.16b, v30.16b + eor x14, x14, x1 + ushr v30.2d, v31.2d, #63 + eor x20, x20, x1 + sli v30.2d, v31.2d, #1 + eor x25, x25, x1 + eor v29.16b, v29.16b, v30.16b + ldr x0, [x29, #32] + # Swap Rotate NEON + eor v0.16b, v0.16b, v25.16b + eor v31.16b, v1.16b, v26.16b + ldr x1, [x29, #24] + eor v6.16b, v6.16b, v26.16b + eor x28, x28, x30, ror 63 + ushr v30.2d, v31.2d, #63 + eor x30, x30, x1, ror 63 + ushr v1.2d, v6.2d, #20 + eor x1, x1, x0, ror 63 + sli v30.2d, v31.2d, #1 + eor x6, x6, x28 + sli v1.2d, v6.2d, #44 + eor x11, x11, x28 + eor v31.16b, v9.16b, v29.16b + eor x16, x16, x28 + eor v22.16b, v22.16b, v27.16b + eor x22, x22, x28 + ushr v6.2d, v31.2d, #44 + eor x27, x27, x28 + ushr v9.2d, v22.2d, #3 + eor x3, x3, x30 + sli v6.2d, v31.2d, #20 + eor x8, x8, x30 + sli v9.2d, v22.2d, #61 + eor x13, x13, x30 + eor v31.16b, v14.16b, v29.16b + eor x19, x19, x30 + eor v20.16b, v20.16b, v25.16b + eor x24, x24, x30 + ushr v22.2d, v31.2d, #25 + eor x5, x5, x1 + ushr v14.2d, v20.2d, #46 + eor x10, x10, x1 + sli v22.2d, v31.2d, #39 + eor x15, x15, x1 + sli v14.2d, v20.2d, #18 + eor x21, x21, x1 + eor v31.16b, v2.16b, v27.16b + eor x26, x26, x1 + # Swap Rotate Base + eor v12.16b, v12.16b, v27.16b + ror x0, x3, #63 + ushr v20.2d, v31.2d, #2 + ror x3, x8, #20 + ushr v2.2d, v12.2d, #21 + ror x8, x11, #44 + sli v20.2d, v31.2d, #62 + ror x11, x25, #3 + sli v2.2d, v12.2d, #43 + ror x25, x16, #25 + eor v31.16b, v13.16b, v28.16b + ror x16, x23, #46 + eor v19.16b, v19.16b, v29.16b + ror x23, x4, #2 + ushr v12.2d, v31.2d, #39 + ror x4, x14, #21 + ushr v13.2d, v19.2d, #56 + ror x14, x15, #39 + sli v12.2d, v31.2d, #25 + ror x15, x22, #56 + sli v13.2d, v19.2d, #8 + ror x22, x26, #8 + eor v31.16b, v23.16b, v28.16b + ror x26, x17, #23 + eor v15.16b, v15.16b, v25.16b + ror x17, x6, #37 + ushr v19.2d, v31.2d, #8 + ror x6, x27, #50 + ushr v23.2d, v15.2d, #23 + ror x27, x24, #62 + sli v19.2d, v31.2d, #56 + ror x24, x10, #9 + sli v23.2d, v15.2d, #41 + ror x10, x19, #19 + eor v31.16b, v4.16b, v29.16b + ror x19, x7, #28 + eor v24.16b, v24.16b, v29.16b + ror x7, x5, #36 + ushr v15.2d, v31.2d, #37 + ror x5, x21, #43 + ushr v4.2d, v24.2d, #50 + ror x21, x20, #49 + sli v15.2d, v31.2d, #27 + ror x20, x13, #54 + sli v4.2d, v24.2d, #14 + ror x13, x9, #58 + eor v31.16b, v21.16b, v26.16b + ror x9, x12, #61 + # Row Mix Base + eor v8.16b, v8.16b, v28.16b + bic x12, x4, x3 + ushr v24.2d, v31.2d, #62 + bic x1, x5, x4 + ushr v21.2d, v8.2d, #9 + bic x28, x2, x6 + sli v24.2d, v31.2d, #2 + bic x30, x3, x2 + sli v21.2d, v8.2d, #55 + eor x2, x2, x12 + eor v31.16b, v16.16b, v26.16b + eor x3, x3, x1 + eor v5.16b, v5.16b, v25.16b + bic x12, x6, x5 + ushr v8.2d, v31.2d, #19 + eor x5, x5, x28 + ushr v16.2d, v5.2d, #28 + eor x4, x4, x12 + sli v8.2d, v31.2d, #45 + eor x6, x6, x30 + sli v16.2d, v5.2d, #36 + bic x12, x9, x8 + eor v31.16b, v3.16b, v28.16b + bic x1, x10, x9 + eor v18.16b, v18.16b, v28.16b + bic x28, x7, x11 + ushr v5.2d, v31.2d, #36 + bic x30, x8, x7 + ushr v3.2d, v18.2d, #43 + eor x7, x7, x12 + sli v5.2d, v31.2d, #28 + eor x8, x8, x1 + sli v3.2d, v18.2d, #21 + bic x12, x11, x10 + eor v31.16b, v17.16b, v27.16b + eor x10, x10, x28 + eor v11.16b, v11.16b, v26.16b + eor x9, x9, x12 + ushr v18.2d, v31.2d, #49 + eor x11, x11, x30 + ushr v17.2d, v11.2d, #54 + bic x12, x14, x13 + sli v18.2d, v31.2d, #15 + bic x1, x15, x14 + sli v17.2d, v11.2d, #10 + bic x28, x0, x16 + eor v31.16b, v7.16b, v27.16b + bic x30, x13, x0 + eor v10.16b, v10.16b, v25.16b + eor x12, x0, x12 + ushr v11.2d, v31.2d, #58 + eor x13, x13, x1 + ushr v7.2d, v10.2d, #61 + bic x0, x16, x15 + sli v11.2d, v31.2d, #6 + eor x15, x15, x28 + sli v7.2d, v10.2d, #3 + eor x14, x14, x0 + # Row Mix NEON + bic v25.16b, v2.16b, v1.16b + eor x16, x16, x30 + bic v26.16b, v3.16b, v2.16b + bic x0, x20, x19 + bic v27.16b, v4.16b, v3.16b + bic x1, x21, x20 + bic v28.16b, v0.16b, v4.16b + bic x28, x17, x22 + bic v29.16b, v1.16b, v0.16b + bic x30, x19, x17 + eor v0.16b, v0.16b, v25.16b + eor x17, x17, x0 + eor v1.16b, v1.16b, v26.16b + eor x19, x19, x1 + eor v2.16b, v2.16b, v27.16b + bic x0, x22, x21 + eor v3.16b, v3.16b, v28.16b + eor x21, x21, x28 + eor v4.16b, v4.16b, v29.16b + eor x20, x20, x0 + bic v25.16b, v7.16b, v6.16b + eor x22, x22, x30 + bic v26.16b, v8.16b, v7.16b + bic x0, x25, x24 + bic v27.16b, v9.16b, v8.16b + bic x1, x26, x25 + bic v28.16b, v5.16b, v9.16b + bic x28, x23, x27 + bic v29.16b, v6.16b, v5.16b + bic x30, x24, x23 + eor v5.16b, v5.16b, v25.16b + eor x23, x23, x0 + eor v6.16b, v6.16b, v26.16b + eor x24, x24, x1 + eor v7.16b, v7.16b, v27.16b + bic x0, x27, x26 + eor v8.16b, v8.16b, v28.16b + eor x26, x26, x28 + eor v9.16b, v9.16b, v29.16b + eor x25, x25, x0 + bic v25.16b, v12.16b, v11.16b + eor x27, x27, x30 + bic v26.16b, v13.16b, v12.16b + bic v27.16b, v14.16b, v13.16b + bic v28.16b, v30.16b, v14.16b + bic v29.16b, v11.16b, v30.16b + eor v10.16b, v30.16b, v25.16b + eor v11.16b, v11.16b, v26.16b + eor v12.16b, v12.16b, v27.16b + eor v13.16b, v13.16b, v28.16b + eor v14.16b, v14.16b, v29.16b + bic v25.16b, v17.16b, v16.16b + bic v26.16b, v18.16b, v17.16b + bic v27.16b, v19.16b, v18.16b + bic v28.16b, v15.16b, v19.16b + bic v29.16b, v16.16b, v15.16b + eor v15.16b, v15.16b, v25.16b + eor v16.16b, v16.16b, v26.16b + eor v17.16b, v17.16b, v27.16b + eor v18.16b, v18.16b, v28.16b + eor v19.16b, v19.16b, v29.16b + bic v25.16b, v22.16b, v21.16b + bic v26.16b, v23.16b, v22.16b + bic v27.16b, v24.16b, v23.16b + bic v28.16b, v20.16b, v24.16b + bic v29.16b, v21.16b, v20.16b + eor v20.16b, v20.16b, v25.16b + eor v21.16b, v21.16b, v26.16b + eor v22.16b, v22.16b, v27.16b + eor v23.16b, v23.16b, v28.16b + eor v24.16b, v24.16b, v29.16b + # Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov v30.d[0], x0 + mov v30.d[1], x0 + eor x2, x2, x0 + eor v0.16b, v0.16b, v30.16b + bne L_SHA3_shake256_blocksx3_seed_64_neon_begin + ldr x0, [x29, #40] + st4 {v0.d, v1.d, v2.d, v3.d}[0], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[0], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[0], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[0], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[0], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[0], [x0], #32 + st1 {v24.d}[0], [x0] + add x0, x0, #8 + st4 {v0.d, v1.d, v2.d, v3.d}[1], [x0], #32 + st4 {v4.d, v5.d, v6.d, v7.d}[1], [x0], #32 + st4 {v8.d, v9.d, v10.d, v11.d}[1], [x0], #32 + st4 {v12.d, v13.d, v14.d, v15.d}[1], [x0], #32 + st4 {v16.d, v17.d, v18.d, v19.d}[1], [x0], #32 + st4 {v20.d, v21.d, v22.d, v23.d}[1], [x0], #32 + st1 {v24.d}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp d8, d9, [x29, #160] + ldp d10, d11, [x29, #176] + ldp d12, d13, [x29, #192] + ldp d14, d15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret +#ifndef __APPLE__ + .size sha3_256_blocksx3_seed_64_neon,.-sha3_256_blocksx3_seed_64_neon +#endif /* __APPLE__ */ +#endif /* defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) */ #endif /* WOLFSSL_SHA3 */ #endif /* __aarch64__ */ #endif /* WOLFSSL_ARMASM */ diff --git a/wolfcrypt/src/port/arm/armv8-sha3-asm.asm b/wolfcrypt/src/port/arm/armv8-sha3-asm.asm index 08da6092f5d..378ed46054b 100644 --- a/wolfcrypt/src/port/arm/armv8-sha3-asm.asm +++ b/wolfcrypt/src/port/arm/armv8-sha3-asm.asm @@ -365,5 +365,2976 @@ L_SHA3_transform_base_begin ldp x29, x30, [sp], #0xa0 ret ENDP + IF :DEF:WOLFSSL_HAVE_MLKEM :LOR: :DEF:WOLFSSL_HAVE_MLDSA + AREA |.rodata|, DATA, READONLY, ALIGN=4 + ALIGN 16 +L_sha3_aarch64_r + DCQ 0x0000000000000001, 0x0000000000008082 + DCQ 0x800000000000808a, 0x8000000080008000 + DCQ 0x000000000000808b, 0x0000000080000001 + DCQ 0x8000000080008081, 0x8000000000008009 + DCQ 0x000000000000008a, 0x0000000000000088 + DCQ 0x0000000080008009, 0x000000008000000a + DCQ 0x000000008000808b, 0x800000000000008b + DCQ 0x8000000000008089, 0x8000000000008003 + DCQ 0x8000000000008002, 0x8000000000000080 + DCQ 0x000000000000800a, 0x800000008000000a + DCQ 0x8000000080008081, 0x8000000000008080 + DCQ 0x0000000080000001, 0x8000000080008008 + IF :DEF:WOLFSSL_ARMASM_CRYPTO_SHA3 + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT sha3_blocksx3_crypto +sha3_blocksx3_crypto PROC + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] + adrp x27, L_sha3_aarch64_r + add x27, x27, L_sha3_aarch64_r +; .arch_extension sha3 + str x0, [x29, #40] + ld4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + ld4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + ld4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + ld4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + ld4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + ld4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + ld1 {V24.D}[0], [x0] + add x0, x0, #8 + ld4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + ld4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + ld4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + ld4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + ld4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + ld4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + ld1 {V24.D}[1], [x0] + add x0, x0, #8 + ldp x1, x2, [x0] + ldp x3, x4, [x0, #16] + ldp x5, x6, [x0, #32] + ldp x7, x8, [x0, #48] + ldp x9, x10, [x0, #64] + ldp x11, x12, [x0, #80] + ldp x13, x14, [x0, #96] + ldp x15, x16, [x0, #112] + ldp x17, x19, [x0, #128] + ldp x20, x21, [x0, #144] + ldp x22, x23, [x0, #160] + ldp x24, x25, [x0, #176] + ldr x26, [x0, #192] + mov x28, #24 + ; Start of 24 rounds +L_SHA3_transform_blocksx3_crypto_begin + stp x27, x28, [x29, #48] + ; Col Mix + eor3 V31.16B, V0.16B, V5.16B, V10.16B + eor x0, x5, x10 + eor3 V27.16B, V1.16B, V6.16B, V11.16B + eor x30, x1, x6 + eor3 V28.16B, V2.16B, V7.16B, V12.16B + eor x28, x3, x8 + eor3 V29.16B, V3.16B, V8.16B, V13.16B + eor x0, x0, x15 + eor3 V30.16B, V4.16B, V9.16B, V14.16B + eor x30, x30, x11 + eor3 V31.16B, V31.16B, V15.16B, V20.16B + eor x28, x28, x13 + eor3 V27.16B, V27.16B, V16.16B, V21.16B + eor x0, x0, x21 + eor3 V28.16B, V28.16B, V17.16B, V22.16B + eor x30, x30, x16 + eor3 V29.16B, V29.16B, V18.16B, V23.16B + eor x28, x28, x19 + eor3 V30.16B, V30.16B, V19.16B, V24.16B + eor x0, x0, x26 + rax1 V25.2D, V30.2D, V27.2D + eor x30, x30, x22 + rax1 V26.2D, V31.2D, V28.2D + eor x28, x28, x24 + rax1 V27.2D, V27.2D, V29.2D + str x0, [x29, #32] + rax1 V28.2D, V28.2D, V30.2D + str x28, [x29, #24] + rax1 V29.2D, V29.2D, V31.2D + eor x27, x2, x7 + eor V0.16B, V0.16B, V25.16B + xar V30.2D, V1.2D, V26.2D, #63 + eor x28, x4, x9 + xar V1.2D, V6.2D, V26.2D, #20 + eor x27, x27, x12 + xar V6.2D, V9.2D, V29.2D, #44 + eor x28, x28, x14 + xar V9.2D, V22.2D, V27.2D, #3 + eor x27, x27, x17 + xar V22.2D, V14.2D, V29.2D, #25 + eor x28, x28, x20 + xar V14.2D, V20.2D, V25.2D, #46 + eor x27, x27, x23 + xar V20.2D, V2.2D, V27.2D, #2 + eor x28, x28, x25 + xar V2.2D, V12.2D, V27.2D, #21 + eor x0, x0, x27, ror 63 + xar V12.2D, V13.2D, V28.2D, #39 + eor x27, x27, x28, ror 63 + xar V13.2D, V19.2D, V29.2D, #56 + eor x1, x1, x0 + xar V19.2D, V23.2D, V28.2D, #8 + eor x6, x6, x0 + xar V23.2D, V15.2D, V25.2D, #23 + eor x11, x11, x0 + xar V15.2D, V4.2D, V29.2D, #37 + eor x16, x16, x0 + xar V4.2D, V24.2D, V29.2D, #50 + eor x22, x22, x0 + xar V24.2D, V21.2D, V26.2D, #62 + eor x3, x3, x27 + xar V21.2D, V8.2D, V28.2D, #9 + eor x8, x8, x27 + xar V8.2D, V16.2D, V26.2D, #19 + eor x13, x13, x27 + xar V16.2D, V5.2D, V25.2D, #28 + eor x19, x19, x27 + xar V5.2D, V3.2D, V28.2D, #36 + eor x24, x24, x27 + xar V3.2D, V18.2D, V28.2D, #43 + ldr x0, [x29, #32] + xar V18.2D, V17.2D, V27.2D, #49 + ldr x27, [x29, #24] + xar V17.2D, V11.2D, V26.2D, #54 + eor x28, x28, x30, ror 63 + xar V11.2D, V7.2D, V27.2D, #58 + eor x30, x30, x27, ror 63 + xar V7.2D, V10.2D, V25.2D, #61 + eor x27, x27, x0, ror 63 + ; Row Mix + mov V25.16B, V0.16B + eor x5, x5, x28 + mov V26.16B, V1.16B + eor x10, x10, x28 + bcax V0.16B, V25.16B, V2.16B, V26.16B + eor x15, x15, x28 + bcax V1.16B, V26.16B, V3.16B, V2.16B + eor x21, x21, x28 + bcax V2.16B, V2.16B, V4.16B, V3.16B + eor x26, x26, x28 + bcax V3.16B, V3.16B, V25.16B, V4.16B + eor x2, x2, x30 + bcax V4.16B, V4.16B, V26.16B, V25.16B + eor x7, x7, x30 + mov V25.16B, V5.16B + eor x12, x12, x30 + mov V26.16B, V6.16B + eor x17, x17, x30 + bcax V5.16B, V25.16B, V7.16B, V26.16B + eor x23, x23, x30 + bcax V6.16B, V26.16B, V8.16B, V7.16B + eor x4, x4, x27 + bcax V7.16B, V7.16B, V9.16B, V8.16B + eor x9, x9, x27 + bcax V8.16B, V8.16B, V25.16B, V9.16B + eor x14, x14, x27 + bcax V9.16B, V9.16B, V26.16B, V25.16B + eor x20, x20, x27 + mov V26.16B, V11.16B + eor x25, x25, x27 + ; Swap Rotate Base + bcax V10.16B, V30.16B, V12.16B, V26.16B + ror x0, x2, #63 + bcax V11.16B, V26.16B, V13.16B, V12.16B + ror x2, x7, #20 + bcax V12.16B, V12.16B, V14.16B, V13.16B + ror x7, x10, #44 + bcax V13.16B, V13.16B, V30.16B, V14.16B + ror x10, x24, #3 + bcax V14.16B, V14.16B, V26.16B, V30.16B + ror x24, x15, #25 + mov V25.16B, V15.16B + ror x15, x22, #46 + mov V26.16B, V16.16B + ror x22, x3, #2 + bcax V15.16B, V25.16B, V17.16B, V26.16B + ror x3, x13, #21 + bcax V16.16B, V26.16B, V18.16B, V17.16B + ror x13, x14, #39 + bcax V17.16B, V17.16B, V19.16B, V18.16B + ror x14, x21, #56 + bcax V18.16B, V18.16B, V25.16B, V19.16B + ror x21, x25, #8 + bcax V19.16B, V19.16B, V26.16B, V25.16B + ror x25, x16, #23 + mov V25.16B, V20.16B + ror x16, x5, #37 + mov V26.16B, V21.16B + ror x5, x26, #50 + bcax V20.16B, V25.16B, V22.16B, V26.16B + ror x26, x23, #62 + bcax V21.16B, V26.16B, V23.16B, V22.16B + ror x23, x9, #9 + bcax V22.16B, V22.16B, V24.16B, V23.16B + ror x9, x17, #19 + bcax V23.16B, V23.16B, V25.16B, V24.16B + ror x17, x6, #28 + bcax V24.16B, V24.16B, V26.16B, V25.16B + ror x6, x4, #36 + ror x4, x20, #43 + ror x20, x19, #49 + ror x19, x12, #54 + ror x12, x8, #58 + ror x8, x11, #61 + ; Row Mix Base + bic x11, x3, x2 + bic x27, x4, x3 + bic x28, x1, x5 + bic x30, x2, x1 + eor x1, x1, x11 + eor x2, x2, x27 + bic x11, x5, x4 + eor x4, x4, x28 + eor x3, x3, x11 + eor x5, x5, x30 + bic x11, x8, x7 + bic x27, x9, x8 + bic x28, x6, x10 + bic x30, x7, x6 + eor x6, x6, x11 + eor x7, x7, x27 + bic x11, x10, x9 + eor x9, x9, x28 + eor x8, x8, x11 + eor x10, x10, x30 + bic x11, x13, x12 + bic x27, x14, x13 + bic x28, x0, x15 + bic x30, x12, x0 + eor x11, x0, x11 + eor x12, x12, x27 + bic x0, x15, x14 + eor x14, x14, x28 + eor x13, x13, x0 + eor x15, x15, x30 + bic x0, x19, x17 + bic x27, x20, x19 + bic x28, x16, x21 + bic x30, x17, x16 + eor x16, x16, x0 + eor x17, x17, x27 + bic x0, x21, x20 + eor x20, x20, x28 + eor x19, x19, x0 + eor x21, x21, x30 + bic x0, x24, x23 + bic x27, x25, x24 + bic x28, x22, x26 + bic x30, x23, x22 + eor x22, x22, x0 + eor x23, x23, x27 + bic x0, x26, x25 + eor x25, x25, x28 + eor x24, x24, x0 + eor x26, x26, x30 + ; Done transforming + ldp x27, x28, [x29, #48] + ldr x0, [x27], #8 + subs x28, x28, #1 + mov V30.D[0], x0 + mov V30.D[1], x0 + eor x1, x1, x0 + eor V0.16B, V0.16B, V30.16B + bne L_SHA3_transform_blocksx3_crypto_begin + ldr x0, [x29, #40] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] + add x0, x0, #8 + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] + add x0, x0, #8 + stp x1, x2, [x0] + stp x3, x4, [x0, #16] + stp x5, x6, [x0, #32] + stp x7, x8, [x0, #48] + stp x9, x10, [x0, #64] + stp x11, x12, [x0, #80] + stp x13, x14, [x0, #96] + stp x15, x16, [x0, #112] + stp x17, x19, [x0, #128] + stp x20, x21, [x0, #144] + stp x22, x23, [x0, #160] + stp x24, x25, [x0, #176] + str x26, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT sha3_128_blocksx3_seed_crypto +sha3_128_blocksx3_seed_crypto PROC + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] + adrp x28, L_sha3_aarch64_r + add x28, x28, L_sha3_aarch64_r +; .arch_extension sha3 + str x0, [x29, #40] + add x0, x0, #32 + ld1 {V4.D}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {V4.D}[1], [x0] + ldp x4, x5, [x1], #16 + ldr x6, [x0, #200] + eor V5.16B, V5.16B, V5.16B + eor x7, x7, x7 + eor V6.16B, V6.16B, V6.16B + eor x8, x8, x8 + eor V7.16B, V7.16B, V7.16B + eor x9, x9, x9 + eor V8.16B, V8.16B, V8.16B + eor x10, x10, x10 + eor V9.16B, V9.16B, V9.16B + eor x11, x11, x11 + eor V10.16B, V10.16B, V10.16B + eor x12, x12, x12 + eor V11.16B, V11.16B, V11.16B + eor x13, x13, x13 + eor V12.16B, V12.16B, V12.16B + eor x14, x14, x14 + eor V13.16B, V13.16B, V13.16B + eor x15, x15, x15 + eor V14.16B, V14.16B, V14.16B + eor x16, x16, x16 + eor V15.16B, V15.16B, V15.16B + eor x17, x17, x17 + eor V16.16B, V16.16B, V16.16B + eor x19, x19, x19 + eor V17.16B, V17.16B, V17.16B + eor x20, x20, x20 + eor V18.16B, V18.16B, V18.16B + eor x21, x21, x21 + eor V19.16B, V19.16B, V19.16B + eor x22, x22, x22 + movz x23, #0x8000, lsl 48 + eor V21.16B, V21.16B, V21.16B + eor x24, x24, x24 + eor V22.16B, V22.16B, V22.16B + eor x25, x25, x25 + eor V23.16B, V23.16B, V23.16B + eor x26, x26, x26 + eor V24.16B, V24.16B, V24.16B + eor x27, x27, x27 + dup V0.2D, x2 + dup V1.2D, x3 + dup V2.2D, x4 + dup V3.2D, x5 + dup V20.2D, x23 + mov x1, #24 + ; Start of 24 rounds +L_SHA3_shake128_blocksx3_seed_crypto_begin + stp x28, x1, [x29, #48] + ; Col Mix + eor3 V31.16B, V0.16B, V5.16B, V10.16B + eor x0, x6, x11 + eor3 V27.16B, V1.16B, V6.16B, V11.16B + eor x30, x2, x7 + eor3 V28.16B, V2.16B, V7.16B, V12.16B + eor x28, x4, x9 + eor3 V29.16B, V3.16B, V8.16B, V13.16B + eor x0, x0, x16 + eor3 V30.16B, V4.16B, V9.16B, V14.16B + eor x30, x30, x12 + eor3 V31.16B, V31.16B, V15.16B, V20.16B + eor x28, x28, x14 + eor3 V27.16B, V27.16B, V16.16B, V21.16B + eor x0, x0, x22 + eor3 V28.16B, V28.16B, V17.16B, V22.16B + eor x30, x30, x17 + eor3 V29.16B, V29.16B, V18.16B, V23.16B + eor x28, x28, x20 + eor3 V30.16B, V30.16B, V19.16B, V24.16B + eor x0, x0, x27 + rax1 V25.2D, V30.2D, V27.2D + eor x30, x30, x23 + rax1 V26.2D, V31.2D, V28.2D + eor x28, x28, x25 + rax1 V27.2D, V27.2D, V29.2D + str x0, [x29, #32] + rax1 V28.2D, V28.2D, V30.2D + str x28, [x29, #24] + rax1 V29.2D, V29.2D, V31.2D + eor x1, x3, x8 + eor V0.16B, V0.16B, V25.16B + xar V30.2D, V1.2D, V26.2D, #63 + eor x28, x5, x10 + xar V1.2D, V6.2D, V26.2D, #20 + eor x1, x1, x13 + xar V6.2D, V9.2D, V29.2D, #44 + eor x28, x28, x15 + xar V9.2D, V22.2D, V27.2D, #3 + eor x1, x1, x19 + xar V22.2D, V14.2D, V29.2D, #25 + eor x28, x28, x21 + xar V14.2D, V20.2D, V25.2D, #46 + eor x1, x1, x24 + xar V20.2D, V2.2D, V27.2D, #2 + eor x28, x28, x26 + xar V2.2D, V12.2D, V27.2D, #21 + eor x0, x0, x1, ror 63 + xar V12.2D, V13.2D, V28.2D, #39 + eor x1, x1, x28, ror 63 + xar V13.2D, V19.2D, V29.2D, #56 + eor x2, x2, x0 + xar V19.2D, V23.2D, V28.2D, #8 + eor x7, x7, x0 + xar V23.2D, V15.2D, V25.2D, #23 + eor x12, x12, x0 + xar V15.2D, V4.2D, V29.2D, #37 + eor x17, x17, x0 + xar V4.2D, V24.2D, V29.2D, #50 + eor x23, x23, x0 + xar V24.2D, V21.2D, V26.2D, #62 + eor x4, x4, x1 + xar V21.2D, V8.2D, V28.2D, #9 + eor x9, x9, x1 + xar V8.2D, V16.2D, V26.2D, #19 + eor x14, x14, x1 + xar V16.2D, V5.2D, V25.2D, #28 + eor x20, x20, x1 + xar V5.2D, V3.2D, V28.2D, #36 + eor x25, x25, x1 + xar V3.2D, V18.2D, V28.2D, #43 + ldr x0, [x29, #32] + xar V18.2D, V17.2D, V27.2D, #49 + ldr x1, [x29, #24] + xar V17.2D, V11.2D, V26.2D, #54 + eor x28, x28, x30, ror 63 + xar V11.2D, V7.2D, V27.2D, #58 + eor x30, x30, x1, ror 63 + xar V7.2D, V10.2D, V25.2D, #61 + eor x1, x1, x0, ror 63 + ; Row Mix + mov V25.16B, V0.16B + eor x6, x6, x28 + mov V26.16B, V1.16B + eor x11, x11, x28 + bcax V0.16B, V25.16B, V2.16B, V26.16B + eor x16, x16, x28 + bcax V1.16B, V26.16B, V3.16B, V2.16B + eor x22, x22, x28 + bcax V2.16B, V2.16B, V4.16B, V3.16B + eor x27, x27, x28 + bcax V3.16B, V3.16B, V25.16B, V4.16B + eor x3, x3, x30 + bcax V4.16B, V4.16B, V26.16B, V25.16B + eor x8, x8, x30 + mov V25.16B, V5.16B + eor x13, x13, x30 + mov V26.16B, V6.16B + eor x19, x19, x30 + bcax V5.16B, V25.16B, V7.16B, V26.16B + eor x24, x24, x30 + bcax V6.16B, V26.16B, V8.16B, V7.16B + eor x5, x5, x1 + bcax V7.16B, V7.16B, V9.16B, V8.16B + eor x10, x10, x1 + bcax V8.16B, V8.16B, V25.16B, V9.16B + eor x15, x15, x1 + bcax V9.16B, V9.16B, V26.16B, V25.16B + eor x21, x21, x1 + mov V26.16B, V11.16B + eor x26, x26, x1 + ; Swap Rotate Base + bcax V10.16B, V30.16B, V12.16B, V26.16B + ror x0, x3, #63 + bcax V11.16B, V26.16B, V13.16B, V12.16B + ror x3, x8, #20 + bcax V12.16B, V12.16B, V14.16B, V13.16B + ror x8, x11, #44 + bcax V13.16B, V13.16B, V30.16B, V14.16B + ror x11, x25, #3 + bcax V14.16B, V14.16B, V26.16B, V30.16B + ror x25, x16, #25 + mov V25.16B, V15.16B + ror x16, x23, #46 + mov V26.16B, V16.16B + ror x23, x4, #2 + bcax V15.16B, V25.16B, V17.16B, V26.16B + ror x4, x14, #21 + bcax V16.16B, V26.16B, V18.16B, V17.16B + ror x14, x15, #39 + bcax V17.16B, V17.16B, V19.16B, V18.16B + ror x15, x22, #56 + bcax V18.16B, V18.16B, V25.16B, V19.16B + ror x22, x26, #8 + bcax V19.16B, V19.16B, V26.16B, V25.16B + ror x26, x17, #23 + mov V25.16B, V20.16B + ror x17, x6, #37 + mov V26.16B, V21.16B + ror x6, x27, #50 + bcax V20.16B, V25.16B, V22.16B, V26.16B + ror x27, x24, #62 + bcax V21.16B, V26.16B, V23.16B, V22.16B + ror x24, x10, #9 + bcax V22.16B, V22.16B, V24.16B, V23.16B + ror x10, x19, #19 + bcax V23.16B, V23.16B, V25.16B, V24.16B + ror x19, x7, #28 + bcax V24.16B, V24.16B, V26.16B, V25.16B + ror x7, x5, #36 + ror x5, x21, #43 + ror x21, x20, #49 + ror x20, x13, #54 + ror x13, x9, #58 + ror x9, x12, #61 + ; Row Mix Base + bic x12, x4, x3 + bic x1, x5, x4 + bic x28, x2, x6 + bic x30, x3, x2 + eor x2, x2, x12 + eor x3, x3, x1 + bic x12, x6, x5 + eor x5, x5, x28 + eor x4, x4, x12 + eor x6, x6, x30 + bic x12, x9, x8 + bic x1, x10, x9 + bic x28, x7, x11 + bic x30, x8, x7 + eor x7, x7, x12 + eor x8, x8, x1 + bic x12, x11, x10 + eor x10, x10, x28 + eor x9, x9, x12 + eor x11, x11, x30 + bic x12, x14, x13 + bic x1, x15, x14 + bic x28, x0, x16 + bic x30, x13, x0 + eor x12, x0, x12 + eor x13, x13, x1 + bic x0, x16, x15 + eor x15, x15, x28 + eor x14, x14, x0 + eor x16, x16, x30 + bic x0, x20, x19 + bic x1, x21, x20 + bic x28, x17, x22 + bic x30, x19, x17 + eor x17, x17, x0 + eor x19, x19, x1 + bic x0, x22, x21 + eor x21, x21, x28 + eor x20, x20, x0 + eor x22, x22, x30 + bic x0, x25, x24 + bic x1, x26, x25 + bic x28, x23, x27 + bic x30, x24, x23 + eor x23, x23, x0 + eor x24, x24, x1 + bic x0, x27, x26 + eor x26, x26, x28 + eor x25, x25, x0 + eor x27, x27, x30 + ; Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov V30.D[0], x0 + mov V30.D[1], x0 + eor x2, x2, x0 + eor V0.16B, V0.16B, V30.16B + bne L_SHA3_shake128_blocksx3_seed_crypto_begin + ldr x0, [x29, #40] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] + add x0, x0, #8 + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT sha3_256_blocksx3_seed_crypto +sha3_256_blocksx3_seed_crypto PROC + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] + adrp x28, L_sha3_aarch64_r + add x28, x28, L_sha3_aarch64_r +; .arch_extension sha3 + str x0, [x29, #40] + add x0, x0, #32 + ld1 {V4.D}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {V4.D}[1], [x0] + ldp x4, x5, [x1], #16 + ldr x6, [x0, #200] + eor V5.16B, V5.16B, V5.16B + eor x7, x7, x7 + eor V6.16B, V6.16B, V6.16B + eor x8, x8, x8 + eor V7.16B, V7.16B, V7.16B + eor x9, x9, x9 + eor V8.16B, V8.16B, V8.16B + eor x10, x10, x10 + eor V9.16B, V9.16B, V9.16B + eor x11, x11, x11 + eor V10.16B, V10.16B, V10.16B + eor x12, x12, x12 + eor V11.16B, V11.16B, V11.16B + eor x13, x13, x13 + eor V12.16B, V12.16B, V12.16B + eor x14, x14, x14 + eor V13.16B, V13.16B, V13.16B + eor x15, x15, x15 + eor V14.16B, V14.16B, V14.16B + eor x16, x16, x16 + eor V15.16B, V15.16B, V15.16B + eor x17, x17, x17 + movz x19, #0x8000, lsl 48 + eor V17.16B, V17.16B, V17.16B + eor x20, x20, x20 + eor V18.16B, V18.16B, V18.16B + eor x21, x21, x21 + eor V19.16B, V19.16B, V19.16B + eor x22, x22, x22 + eor V20.16B, V20.16B, V20.16B + eor x23, x23, x23 + eor V21.16B, V21.16B, V21.16B + eor x24, x24, x24 + eor V22.16B, V22.16B, V22.16B + eor x25, x25, x25 + eor V23.16B, V23.16B, V23.16B + eor x26, x26, x26 + eor V24.16B, V24.16B, V24.16B + eor x27, x27, x27 + dup V0.2D, x2 + dup V1.2D, x3 + dup V2.2D, x4 + dup V3.2D, x5 + dup V16.2D, x19 + mov x1, #24 + ; Start of 24 rounds +L_SHA3_shake256_blocksx3_seed_crypto_begin + stp x28, x1, [x29, #48] + ; Col Mix + eor3 V31.16B, V0.16B, V5.16B, V10.16B + eor x0, x6, x11 + eor3 V27.16B, V1.16B, V6.16B, V11.16B + eor x30, x2, x7 + eor3 V28.16B, V2.16B, V7.16B, V12.16B + eor x28, x4, x9 + eor3 V29.16B, V3.16B, V8.16B, V13.16B + eor x0, x0, x16 + eor3 V30.16B, V4.16B, V9.16B, V14.16B + eor x30, x30, x12 + eor3 V31.16B, V31.16B, V15.16B, V20.16B + eor x28, x28, x14 + eor3 V27.16B, V27.16B, V16.16B, V21.16B + eor x0, x0, x22 + eor3 V28.16B, V28.16B, V17.16B, V22.16B + eor x30, x30, x17 + eor3 V29.16B, V29.16B, V18.16B, V23.16B + eor x28, x28, x20 + eor3 V30.16B, V30.16B, V19.16B, V24.16B + eor x0, x0, x27 + rax1 V25.2D, V30.2D, V27.2D + eor x30, x30, x23 + rax1 V26.2D, V31.2D, V28.2D + eor x28, x28, x25 + rax1 V27.2D, V27.2D, V29.2D + str x0, [x29, #32] + rax1 V28.2D, V28.2D, V30.2D + str x28, [x29, #24] + rax1 V29.2D, V29.2D, V31.2D + eor x1, x3, x8 + eor V0.16B, V0.16B, V25.16B + xar V30.2D, V1.2D, V26.2D, #63 + eor x28, x5, x10 + xar V1.2D, V6.2D, V26.2D, #20 + eor x1, x1, x13 + xar V6.2D, V9.2D, V29.2D, #44 + eor x28, x28, x15 + xar V9.2D, V22.2D, V27.2D, #3 + eor x1, x1, x19 + xar V22.2D, V14.2D, V29.2D, #25 + eor x28, x28, x21 + xar V14.2D, V20.2D, V25.2D, #46 + eor x1, x1, x24 + xar V20.2D, V2.2D, V27.2D, #2 + eor x28, x28, x26 + xar V2.2D, V12.2D, V27.2D, #21 + eor x0, x0, x1, ror 63 + xar V12.2D, V13.2D, V28.2D, #39 + eor x1, x1, x28, ror 63 + xar V13.2D, V19.2D, V29.2D, #56 + eor x2, x2, x0 + xar V19.2D, V23.2D, V28.2D, #8 + eor x7, x7, x0 + xar V23.2D, V15.2D, V25.2D, #23 + eor x12, x12, x0 + xar V15.2D, V4.2D, V29.2D, #37 + eor x17, x17, x0 + xar V4.2D, V24.2D, V29.2D, #50 + eor x23, x23, x0 + xar V24.2D, V21.2D, V26.2D, #62 + eor x4, x4, x1 + xar V21.2D, V8.2D, V28.2D, #9 + eor x9, x9, x1 + xar V8.2D, V16.2D, V26.2D, #19 + eor x14, x14, x1 + xar V16.2D, V5.2D, V25.2D, #28 + eor x20, x20, x1 + xar V5.2D, V3.2D, V28.2D, #36 + eor x25, x25, x1 + xar V3.2D, V18.2D, V28.2D, #43 + ldr x0, [x29, #32] + xar V18.2D, V17.2D, V27.2D, #49 + ldr x1, [x29, #24] + xar V17.2D, V11.2D, V26.2D, #54 + eor x28, x28, x30, ror 63 + xar V11.2D, V7.2D, V27.2D, #58 + eor x30, x30, x1, ror 63 + xar V7.2D, V10.2D, V25.2D, #61 + eor x1, x1, x0, ror 63 + ; Row Mix + mov V25.16B, V0.16B + eor x6, x6, x28 + mov V26.16B, V1.16B + eor x11, x11, x28 + bcax V0.16B, V25.16B, V2.16B, V26.16B + eor x16, x16, x28 + bcax V1.16B, V26.16B, V3.16B, V2.16B + eor x22, x22, x28 + bcax V2.16B, V2.16B, V4.16B, V3.16B + eor x27, x27, x28 + bcax V3.16B, V3.16B, V25.16B, V4.16B + eor x3, x3, x30 + bcax V4.16B, V4.16B, V26.16B, V25.16B + eor x8, x8, x30 + mov V25.16B, V5.16B + eor x13, x13, x30 + mov V26.16B, V6.16B + eor x19, x19, x30 + bcax V5.16B, V25.16B, V7.16B, V26.16B + eor x24, x24, x30 + bcax V6.16B, V26.16B, V8.16B, V7.16B + eor x5, x5, x1 + bcax V7.16B, V7.16B, V9.16B, V8.16B + eor x10, x10, x1 + bcax V8.16B, V8.16B, V25.16B, V9.16B + eor x15, x15, x1 + bcax V9.16B, V9.16B, V26.16B, V25.16B + eor x21, x21, x1 + mov V26.16B, V11.16B + eor x26, x26, x1 + ; Swap Rotate Base + bcax V10.16B, V30.16B, V12.16B, V26.16B + ror x0, x3, #63 + bcax V11.16B, V26.16B, V13.16B, V12.16B + ror x3, x8, #20 + bcax V12.16B, V12.16B, V14.16B, V13.16B + ror x8, x11, #44 + bcax V13.16B, V13.16B, V30.16B, V14.16B + ror x11, x25, #3 + bcax V14.16B, V14.16B, V26.16B, V30.16B + ror x25, x16, #25 + mov V25.16B, V15.16B + ror x16, x23, #46 + mov V26.16B, V16.16B + ror x23, x4, #2 + bcax V15.16B, V25.16B, V17.16B, V26.16B + ror x4, x14, #21 + bcax V16.16B, V26.16B, V18.16B, V17.16B + ror x14, x15, #39 + bcax V17.16B, V17.16B, V19.16B, V18.16B + ror x15, x22, #56 + bcax V18.16B, V18.16B, V25.16B, V19.16B + ror x22, x26, #8 + bcax V19.16B, V19.16B, V26.16B, V25.16B + ror x26, x17, #23 + mov V25.16B, V20.16B + ror x17, x6, #37 + mov V26.16B, V21.16B + ror x6, x27, #50 + bcax V20.16B, V25.16B, V22.16B, V26.16B + ror x27, x24, #62 + bcax V21.16B, V26.16B, V23.16B, V22.16B + ror x24, x10, #9 + bcax V22.16B, V22.16B, V24.16B, V23.16B + ror x10, x19, #19 + bcax V23.16B, V23.16B, V25.16B, V24.16B + ror x19, x7, #28 + bcax V24.16B, V24.16B, V26.16B, V25.16B + ror x7, x5, #36 + ror x5, x21, #43 + ror x21, x20, #49 + ror x20, x13, #54 + ror x13, x9, #58 + ror x9, x12, #61 + ; Row Mix Base + bic x12, x4, x3 + bic x1, x5, x4 + bic x28, x2, x6 + bic x30, x3, x2 + eor x2, x2, x12 + eor x3, x3, x1 + bic x12, x6, x5 + eor x5, x5, x28 + eor x4, x4, x12 + eor x6, x6, x30 + bic x12, x9, x8 + bic x1, x10, x9 + bic x28, x7, x11 + bic x30, x8, x7 + eor x7, x7, x12 + eor x8, x8, x1 + bic x12, x11, x10 + eor x10, x10, x28 + eor x9, x9, x12 + eor x11, x11, x30 + bic x12, x14, x13 + bic x1, x15, x14 + bic x28, x0, x16 + bic x30, x13, x0 + eor x12, x0, x12 + eor x13, x13, x1 + bic x0, x16, x15 + eor x15, x15, x28 + eor x14, x14, x0 + eor x16, x16, x30 + bic x0, x20, x19 + bic x1, x21, x20 + bic x28, x17, x22 + bic x30, x19, x17 + eor x17, x17, x0 + eor x19, x19, x1 + bic x0, x22, x21 + eor x21, x21, x28 + eor x20, x20, x0 + eor x22, x22, x30 + bic x0, x25, x24 + bic x1, x26, x25 + bic x28, x23, x27 + bic x30, x24, x23 + eor x23, x23, x0 + eor x24, x24, x1 + bic x0, x27, x26 + eor x26, x26, x28 + eor x25, x25, x0 + eor x27, x27, x30 + ; Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov V30.D[0], x0 + mov V30.D[1], x0 + eor x2, x2, x0 + eor V0.16B, V0.16B, V30.16B + bne L_SHA3_shake256_blocksx3_seed_crypto_begin + ldr x0, [x29, #40] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] + add x0, x0, #8 + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT sha3_256_blocksx3_seed_64_crypto +sha3_256_blocksx3_seed_64_crypto PROC + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] + adrp x28, L_sha3_aarch64_r + add x28, x28, L_sha3_aarch64_r +; .arch_extension sha3 + str x0, [x29, #40] + add x0, x0, #0x40 + ld1 {V8.D}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {V8.D}[1], [x0] + ldp x4, x5, [x1], #16 + ldp x6, x7, [x1], #16 + ldp x8, x9, [x1], #16 + ldr x10, [x0, #200] + eor V9.16B, V9.16B, V9.16B + eor x11, x11, x11 + eor V10.16B, V10.16B, V10.16B + eor x12, x12, x12 + eor V11.16B, V11.16B, V11.16B + eor x13, x13, x13 + eor V12.16B, V12.16B, V12.16B + eor x14, x14, x14 + eor V13.16B, V13.16B, V13.16B + eor x15, x15, x15 + eor V14.16B, V14.16B, V14.16B + eor x16, x16, x16 + eor V15.16B, V15.16B, V15.16B + eor x17, x17, x17 + movz x19, #0x8000, lsl 48 + eor V17.16B, V17.16B, V17.16B + eor x20, x20, x20 + eor V18.16B, V18.16B, V18.16B + eor x21, x21, x21 + eor V19.16B, V19.16B, V19.16B + eor x22, x22, x22 + eor V20.16B, V20.16B, V20.16B + eor x23, x23, x23 + eor V21.16B, V21.16B, V21.16B + eor x24, x24, x24 + eor V22.16B, V22.16B, V22.16B + eor x25, x25, x25 + eor V23.16B, V23.16B, V23.16B + eor x26, x26, x26 + eor V24.16B, V24.16B, V24.16B + eor x27, x27, x27 + dup V0.2D, x2 + dup V1.2D, x3 + dup V2.2D, x4 + dup V3.2D, x5 + dup V4.2D, x6 + dup V5.2D, x7 + dup V6.2D, x8 + dup V7.2D, x9 + dup V16.2D, x19 + mov x1, #24 + ; Start of 24 rounds +L_SHA3_shake256_blocksx3_seed_64_crypto_begin + stp x28, x1, [x29, #48] + ; Col Mix + eor3 V31.16B, V0.16B, V5.16B, V10.16B + eor x0, x6, x11 + eor3 V27.16B, V1.16B, V6.16B, V11.16B + eor x30, x2, x7 + eor3 V28.16B, V2.16B, V7.16B, V12.16B + eor x28, x4, x9 + eor3 V29.16B, V3.16B, V8.16B, V13.16B + eor x0, x0, x16 + eor3 V30.16B, V4.16B, V9.16B, V14.16B + eor x30, x30, x12 + eor3 V31.16B, V31.16B, V15.16B, V20.16B + eor x28, x28, x14 + eor3 V27.16B, V27.16B, V16.16B, V21.16B + eor x0, x0, x22 + eor3 V28.16B, V28.16B, V17.16B, V22.16B + eor x30, x30, x17 + eor3 V29.16B, V29.16B, V18.16B, V23.16B + eor x28, x28, x20 + eor3 V30.16B, V30.16B, V19.16B, V24.16B + eor x0, x0, x27 + rax1 V25.2D, V30.2D, V27.2D + eor x30, x30, x23 + rax1 V26.2D, V31.2D, V28.2D + eor x28, x28, x25 + rax1 V27.2D, V27.2D, V29.2D + str x0, [x29, #32] + rax1 V28.2D, V28.2D, V30.2D + str x28, [x29, #24] + rax1 V29.2D, V29.2D, V31.2D + eor x1, x3, x8 + eor V0.16B, V0.16B, V25.16B + xar V30.2D, V1.2D, V26.2D, #63 + eor x28, x5, x10 + xar V1.2D, V6.2D, V26.2D, #20 + eor x1, x1, x13 + xar V6.2D, V9.2D, V29.2D, #44 + eor x28, x28, x15 + xar V9.2D, V22.2D, V27.2D, #3 + eor x1, x1, x19 + xar V22.2D, V14.2D, V29.2D, #25 + eor x28, x28, x21 + xar V14.2D, V20.2D, V25.2D, #46 + eor x1, x1, x24 + xar V20.2D, V2.2D, V27.2D, #2 + eor x28, x28, x26 + xar V2.2D, V12.2D, V27.2D, #21 + eor x0, x0, x1, ror 63 + xar V12.2D, V13.2D, V28.2D, #39 + eor x1, x1, x28, ror 63 + xar V13.2D, V19.2D, V29.2D, #56 + eor x2, x2, x0 + xar V19.2D, V23.2D, V28.2D, #8 + eor x7, x7, x0 + xar V23.2D, V15.2D, V25.2D, #23 + eor x12, x12, x0 + xar V15.2D, V4.2D, V29.2D, #37 + eor x17, x17, x0 + xar V4.2D, V24.2D, V29.2D, #50 + eor x23, x23, x0 + xar V24.2D, V21.2D, V26.2D, #62 + eor x4, x4, x1 + xar V21.2D, V8.2D, V28.2D, #9 + eor x9, x9, x1 + xar V8.2D, V16.2D, V26.2D, #19 + eor x14, x14, x1 + xar V16.2D, V5.2D, V25.2D, #28 + eor x20, x20, x1 + xar V5.2D, V3.2D, V28.2D, #36 + eor x25, x25, x1 + xar V3.2D, V18.2D, V28.2D, #43 + ldr x0, [x29, #32] + xar V18.2D, V17.2D, V27.2D, #49 + ldr x1, [x29, #24] + xar V17.2D, V11.2D, V26.2D, #54 + eor x28, x28, x30, ror 63 + xar V11.2D, V7.2D, V27.2D, #58 + eor x30, x30, x1, ror 63 + xar V7.2D, V10.2D, V25.2D, #61 + eor x1, x1, x0, ror 63 + ; Row Mix + mov V25.16B, V0.16B + eor x6, x6, x28 + mov V26.16B, V1.16B + eor x11, x11, x28 + bcax V0.16B, V25.16B, V2.16B, V26.16B + eor x16, x16, x28 + bcax V1.16B, V26.16B, V3.16B, V2.16B + eor x22, x22, x28 + bcax V2.16B, V2.16B, V4.16B, V3.16B + eor x27, x27, x28 + bcax V3.16B, V3.16B, V25.16B, V4.16B + eor x3, x3, x30 + bcax V4.16B, V4.16B, V26.16B, V25.16B + eor x8, x8, x30 + mov V25.16B, V5.16B + eor x13, x13, x30 + mov V26.16B, V6.16B + eor x19, x19, x30 + bcax V5.16B, V25.16B, V7.16B, V26.16B + eor x24, x24, x30 + bcax V6.16B, V26.16B, V8.16B, V7.16B + eor x5, x5, x1 + bcax V7.16B, V7.16B, V9.16B, V8.16B + eor x10, x10, x1 + bcax V8.16B, V8.16B, V25.16B, V9.16B + eor x15, x15, x1 + bcax V9.16B, V9.16B, V26.16B, V25.16B + eor x21, x21, x1 + mov V26.16B, V11.16B + eor x26, x26, x1 + ; Swap Rotate Base + bcax V10.16B, V30.16B, V12.16B, V26.16B + ror x0, x3, #63 + bcax V11.16B, V26.16B, V13.16B, V12.16B + ror x3, x8, #20 + bcax V12.16B, V12.16B, V14.16B, V13.16B + ror x8, x11, #44 + bcax V13.16B, V13.16B, V30.16B, V14.16B + ror x11, x25, #3 + bcax V14.16B, V14.16B, V26.16B, V30.16B + ror x25, x16, #25 + mov V25.16B, V15.16B + ror x16, x23, #46 + mov V26.16B, V16.16B + ror x23, x4, #2 + bcax V15.16B, V25.16B, V17.16B, V26.16B + ror x4, x14, #21 + bcax V16.16B, V26.16B, V18.16B, V17.16B + ror x14, x15, #39 + bcax V17.16B, V17.16B, V19.16B, V18.16B + ror x15, x22, #56 + bcax V18.16B, V18.16B, V25.16B, V19.16B + ror x22, x26, #8 + bcax V19.16B, V19.16B, V26.16B, V25.16B + ror x26, x17, #23 + mov V25.16B, V20.16B + ror x17, x6, #37 + mov V26.16B, V21.16B + ror x6, x27, #50 + bcax V20.16B, V25.16B, V22.16B, V26.16B + ror x27, x24, #62 + bcax V21.16B, V26.16B, V23.16B, V22.16B + ror x24, x10, #9 + bcax V22.16B, V22.16B, V24.16B, V23.16B + ror x10, x19, #19 + bcax V23.16B, V23.16B, V25.16B, V24.16B + ror x19, x7, #28 + bcax V24.16B, V24.16B, V26.16B, V25.16B + ror x7, x5, #36 + ror x5, x21, #43 + ror x21, x20, #49 + ror x20, x13, #54 + ror x13, x9, #58 + ror x9, x12, #61 + ; Row Mix Base + bic x12, x4, x3 + bic x1, x5, x4 + bic x28, x2, x6 + bic x30, x3, x2 + eor x2, x2, x12 + eor x3, x3, x1 + bic x12, x6, x5 + eor x5, x5, x28 + eor x4, x4, x12 + eor x6, x6, x30 + bic x12, x9, x8 + bic x1, x10, x9 + bic x28, x7, x11 + bic x30, x8, x7 + eor x7, x7, x12 + eor x8, x8, x1 + bic x12, x11, x10 + eor x10, x10, x28 + eor x9, x9, x12 + eor x11, x11, x30 + bic x12, x14, x13 + bic x1, x15, x14 + bic x28, x0, x16 + bic x30, x13, x0 + eor x12, x0, x12 + eor x13, x13, x1 + bic x0, x16, x15 + eor x15, x15, x28 + eor x14, x14, x0 + eor x16, x16, x30 + bic x0, x20, x19 + bic x1, x21, x20 + bic x28, x17, x22 + bic x30, x19, x17 + eor x17, x17, x0 + eor x19, x19, x1 + bic x0, x22, x21 + eor x21, x21, x28 + eor x20, x20, x0 + eor x22, x22, x30 + bic x0, x25, x24 + bic x1, x26, x25 + bic x28, x23, x27 + bic x30, x24, x23 + eor x23, x23, x0 + eor x24, x24, x1 + bic x0, x27, x26 + eor x26, x26, x28 + eor x25, x25, x0 + eor x27, x27, x30 + ; Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov V30.D[0], x0 + mov V30.D[1], x0 + eor x2, x2, x0 + eor V0.16B, V0.16B, V30.16B + bne L_SHA3_shake256_blocksx3_seed_64_crypto_begin + ldr x0, [x29, #40] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] + add x0, x0, #8 + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret + ENDP + ENDIF + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT sha3_blocksx3_neon +sha3_blocksx3_neon PROC + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] + adrp x27, L_sha3_aarch64_r + add x27, x27, L_sha3_aarch64_r + str x0, [x29, #40] + ld4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + ld4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + ld4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + ld4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + ld4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + ld4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + ld1 {V24.D}[0], [x0] + add x0, x0, #8 + ld4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + ld4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + ld4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + ld4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + ld4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + ld4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + ld1 {V24.D}[1], [x0] + add x0, x0, #8 + ldp x1, x2, [x0] + ldp x3, x4, [x0, #16] + ldp x5, x6, [x0, #32] + ldp x7, x8, [x0, #48] + ldp x9, x10, [x0, #64] + ldp x11, x12, [x0, #80] + ldp x13, x14, [x0, #96] + ldp x15, x16, [x0, #112] + ldp x17, x19, [x0, #128] + ldp x20, x21, [x0, #144] + ldp x22, x23, [x0, #160] + ldp x24, x25, [x0, #176] + ldr x26, [x0, #192] + mov x28, #24 + ; Start of 24 rounds +L_SHA3_transform_blocksx3_neon_begin + stp x27, x28, [x29, #48] + ; Col Mix NEON + eor V30.16B, V4.16B, V9.16B + eor x0, x5, x10 + eor V27.16B, V1.16B, V6.16B + eor x30, x1, x6 + eor V30.16B, V30.16B, V14.16B + eor x28, x3, x8 + eor V27.16B, V27.16B, V11.16B + eor x0, x0, x15 + eor V30.16B, V30.16B, V19.16B + eor x30, x30, x11 + eor V27.16B, V27.16B, V16.16B + eor x28, x28, x13 + eor V30.16B, V30.16B, V24.16B + eor x0, x0, x21 + eor V27.16B, V27.16B, V21.16B + eor x30, x30, x16 + ushr V25.2D, V27.2D, #63 + eor x28, x28, x19 + sli V25.2D, V27.2D, #1 + eor x0, x0, x26 + eor V25.16B, V25.16B, V30.16B + eor x30, x30, x22 + eor V31.16B, V0.16B, V5.16B + eor x28, x28, x24 + eor V28.16B, V2.16B, V7.16B + str x0, [x29, #32] + eor V31.16B, V31.16B, V10.16B + str x28, [x29, #24] + eor V28.16B, V28.16B, V12.16B + eor x27, x2, x7 + eor V31.16B, V31.16B, V15.16B + eor x28, x4, x9 + eor V28.16B, V28.16B, V17.16B + eor x27, x27, x12 + eor V31.16B, V31.16B, V20.16B + eor x28, x28, x14 + eor V28.16B, V28.16B, V22.16B + eor x27, x27, x17 + ushr V29.2D, V30.2D, #63 + eor x28, x28, x20 + ushr V26.2D, V28.2D, #63 + eor x27, x27, x23 + sli V29.2D, V30.2D, #1 + eor x28, x28, x25 + sli V26.2D, V28.2D, #1 + eor x0, x0, x27, ror 63 + eor V28.16B, V28.16B, V29.16B + eor x27, x27, x28, ror 63 + eor V29.16B, V3.16B, V8.16B + eor x1, x1, x0 + eor V26.16B, V26.16B, V31.16B + eor x6, x6, x0 + eor V29.16B, V29.16B, V13.16B + eor x11, x11, x0 + eor V29.16B, V29.16B, V18.16B + eor x16, x16, x0 + eor V29.16B, V29.16B, V23.16B + eor x22, x22, x0 + ushr V30.2D, V29.2D, #63 + eor x3, x3, x27 + sli V30.2D, V29.2D, #1 + eor x8, x8, x27 + eor V27.16B, V27.16B, V30.16B + eor x13, x13, x27 + ushr V30.2D, V31.2D, #63 + eor x19, x19, x27 + sli V30.2D, V31.2D, #1 + eor x24, x24, x27 + eor V29.16B, V29.16B, V30.16B + ldr x0, [x29, #32] + ; Swap Rotate NEON + eor V0.16B, V0.16B, V25.16B + eor V31.16B, V1.16B, V26.16B + ldr x27, [x29, #24] + eor V6.16B, V6.16B, V26.16B + eor x28, x28, x30, ror 63 + ushr V30.2D, V31.2D, #63 + eor x30, x30, x27, ror 63 + ushr V1.2D, V6.2D, #20 + eor x27, x27, x0, ror 63 + sli V30.2D, V31.2D, #1 + eor x5, x5, x28 + sli V1.2D, V6.2D, #44 + eor x10, x10, x28 + eor V31.16B, V9.16B, V29.16B + eor x15, x15, x28 + eor V22.16B, V22.16B, V27.16B + eor x21, x21, x28 + ushr V6.2D, V31.2D, #44 + eor x26, x26, x28 + ushr V9.2D, V22.2D, #3 + eor x2, x2, x30 + sli V6.2D, V31.2D, #20 + eor x7, x7, x30 + sli V9.2D, V22.2D, #61 + eor x12, x12, x30 + eor V31.16B, V14.16B, V29.16B + eor x17, x17, x30 + eor V20.16B, V20.16B, V25.16B + eor x23, x23, x30 + ushr V22.2D, V31.2D, #25 + eor x4, x4, x27 + ushr V14.2D, V20.2D, #46 + eor x9, x9, x27 + sli V22.2D, V31.2D, #39 + eor x14, x14, x27 + sli V14.2D, V20.2D, #18 + eor x20, x20, x27 + eor V31.16B, V2.16B, V27.16B + eor x25, x25, x27 + ; Swap Rotate Base + eor V12.16B, V12.16B, V27.16B + ror x0, x2, #63 + ushr V20.2D, V31.2D, #2 + ror x2, x7, #20 + ushr V2.2D, V12.2D, #21 + ror x7, x10, #44 + sli V20.2D, V31.2D, #62 + ror x10, x24, #3 + sli V2.2D, V12.2D, #43 + ror x24, x15, #25 + eor V31.16B, V13.16B, V28.16B + ror x15, x22, #46 + eor V19.16B, V19.16B, V29.16B + ror x22, x3, #2 + ushr V12.2D, V31.2D, #39 + ror x3, x13, #21 + ushr V13.2D, V19.2D, #56 + ror x13, x14, #39 + sli V12.2D, V31.2D, #25 + ror x14, x21, #56 + sli V13.2D, V19.2D, #8 + ror x21, x25, #8 + eor V31.16B, V23.16B, V28.16B + ror x25, x16, #23 + eor V15.16B, V15.16B, V25.16B + ror x16, x5, #37 + ushr V19.2D, V31.2D, #8 + ror x5, x26, #50 + ushr V23.2D, V15.2D, #23 + ror x26, x23, #62 + sli V19.2D, V31.2D, #56 + ror x23, x9, #9 + sli V23.2D, V15.2D, #41 + ror x9, x17, #19 + eor V31.16B, V4.16B, V29.16B + ror x17, x6, #28 + eor V24.16B, V24.16B, V29.16B + ror x6, x4, #36 + ushr V15.2D, V31.2D, #37 + ror x4, x20, #43 + ushr V4.2D, V24.2D, #50 + ror x20, x19, #49 + sli V15.2D, V31.2D, #27 + ror x19, x12, #54 + sli V4.2D, V24.2D, #14 + ror x12, x8, #58 + eor V31.16B, V21.16B, V26.16B + ror x8, x11, #61 + ; Row Mix Base + eor V8.16B, V8.16B, V28.16B + bic x11, x3, x2 + ushr V24.2D, V31.2D, #62 + bic x27, x4, x3 + ushr V21.2D, V8.2D, #9 + bic x28, x1, x5 + sli V24.2D, V31.2D, #2 + bic x30, x2, x1 + sli V21.2D, V8.2D, #55 + eor x1, x1, x11 + eor V31.16B, V16.16B, V26.16B + eor x2, x2, x27 + eor V5.16B, V5.16B, V25.16B + bic x11, x5, x4 + ushr V8.2D, V31.2D, #19 + eor x4, x4, x28 + ushr V16.2D, V5.2D, #28 + eor x3, x3, x11 + sli V8.2D, V31.2D, #45 + eor x5, x5, x30 + sli V16.2D, V5.2D, #36 + bic x11, x8, x7 + eor V31.16B, V3.16B, V28.16B + bic x27, x9, x8 + eor V18.16B, V18.16B, V28.16B + bic x28, x6, x10 + ushr V5.2D, V31.2D, #36 + bic x30, x7, x6 + ushr V3.2D, V18.2D, #43 + eor x6, x6, x11 + sli V5.2D, V31.2D, #28 + eor x7, x7, x27 + sli V3.2D, V18.2D, #21 + bic x11, x10, x9 + eor V31.16B, V17.16B, V27.16B + eor x9, x9, x28 + eor V11.16B, V11.16B, V26.16B + eor x8, x8, x11 + ushr V18.2D, V31.2D, #49 + eor x10, x10, x30 + ushr V17.2D, V11.2D, #54 + bic x11, x13, x12 + sli V18.2D, V31.2D, #15 + bic x27, x14, x13 + sli V17.2D, V11.2D, #10 + bic x28, x0, x15 + eor V31.16B, V7.16B, V27.16B + bic x30, x12, x0 + eor V10.16B, V10.16B, V25.16B + eor x11, x0, x11 + ushr V11.2D, V31.2D, #58 + eor x12, x12, x27 + ushr V7.2D, V10.2D, #61 + bic x0, x15, x14 + sli V11.2D, V31.2D, #6 + eor x14, x14, x28 + sli V7.2D, V10.2D, #3 + eor x13, x13, x0 + ; Row Mix NEON + bic V25.16B, V2.16B, V1.16B + eor x15, x15, x30 + bic V26.16B, V3.16B, V2.16B + bic x0, x19, x17 + bic V27.16B, V4.16B, V3.16B + bic x27, x20, x19 + bic V28.16B, V0.16B, V4.16B + bic x28, x16, x21 + bic V29.16B, V1.16B, V0.16B + bic x30, x17, x16 + eor V0.16B, V0.16B, V25.16B + eor x16, x16, x0 + eor V1.16B, V1.16B, V26.16B + eor x17, x17, x27 + eor V2.16B, V2.16B, V27.16B + bic x0, x21, x20 + eor V3.16B, V3.16B, V28.16B + eor x20, x20, x28 + eor V4.16B, V4.16B, V29.16B + eor x19, x19, x0 + bic V25.16B, V7.16B, V6.16B + eor x21, x21, x30 + bic V26.16B, V8.16B, V7.16B + bic x0, x24, x23 + bic V27.16B, V9.16B, V8.16B + bic x27, x25, x24 + bic V28.16B, V5.16B, V9.16B + bic x28, x22, x26 + bic V29.16B, V6.16B, V5.16B + bic x30, x23, x22 + eor V5.16B, V5.16B, V25.16B + eor x22, x22, x0 + eor V6.16B, V6.16B, V26.16B + eor x23, x23, x27 + eor V7.16B, V7.16B, V27.16B + bic x0, x26, x25 + eor V8.16B, V8.16B, V28.16B + eor x25, x25, x28 + eor V9.16B, V9.16B, V29.16B + eor x24, x24, x0 + bic V25.16B, V12.16B, V11.16B + eor x26, x26, x30 + bic V26.16B, V13.16B, V12.16B + bic V27.16B, V14.16B, V13.16B + bic V28.16B, V30.16B, V14.16B + bic V29.16B, V11.16B, V30.16B + eor V10.16B, V30.16B, V25.16B + eor V11.16B, V11.16B, V26.16B + eor V12.16B, V12.16B, V27.16B + eor V13.16B, V13.16B, V28.16B + eor V14.16B, V14.16B, V29.16B + bic V25.16B, V17.16B, V16.16B + bic V26.16B, V18.16B, V17.16B + bic V27.16B, V19.16B, V18.16B + bic V28.16B, V15.16B, V19.16B + bic V29.16B, V16.16B, V15.16B + eor V15.16B, V15.16B, V25.16B + eor V16.16B, V16.16B, V26.16B + eor V17.16B, V17.16B, V27.16B + eor V18.16B, V18.16B, V28.16B + eor V19.16B, V19.16B, V29.16B + bic V25.16B, V22.16B, V21.16B + bic V26.16B, V23.16B, V22.16B + bic V27.16B, V24.16B, V23.16B + bic V28.16B, V20.16B, V24.16B + bic V29.16B, V21.16B, V20.16B + eor V20.16B, V20.16B, V25.16B + eor V21.16B, V21.16B, V26.16B + eor V22.16B, V22.16B, V27.16B + eor V23.16B, V23.16B, V28.16B + eor V24.16B, V24.16B, V29.16B + ; Done transforming + ldp x27, x28, [x29, #48] + ldr x0, [x27], #8 + subs x28, x28, #1 + mov V30.D[0], x0 + mov V30.D[1], x0 + eor x1, x1, x0 + eor V0.16B, V0.16B, V30.16B + bne L_SHA3_transform_blocksx3_neon_begin + ldr x0, [x29, #40] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] + add x0, x0, #8 + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] + add x0, x0, #8 + stp x1, x2, [x0] + stp x3, x4, [x0, #16] + stp x5, x6, [x0, #32] + stp x7, x8, [x0, #48] + stp x9, x10, [x0, #64] + stp x11, x12, [x0, #80] + stp x13, x14, [x0, #96] + stp x15, x16, [x0, #112] + stp x17, x19, [x0, #128] + stp x20, x21, [x0, #144] + stp x22, x23, [x0, #160] + stp x24, x25, [x0, #176] + str x26, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT sha3_128_blocksx3_seed_neon +sha3_128_blocksx3_seed_neon PROC + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] + adrp x28, L_sha3_aarch64_r + add x28, x28, L_sha3_aarch64_r + str x0, [x29, #40] + add x0, x0, #32 + ld1 {V4.D}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {V4.D}[1], [x0] + ldp x4, x5, [x1], #16 + ldr x6, [x0, #200] + eor V5.16B, V5.16B, V5.16B + eor x7, x7, x7 + eor V6.16B, V6.16B, V6.16B + eor x8, x8, x8 + eor V7.16B, V7.16B, V7.16B + eor x9, x9, x9 + eor V8.16B, V8.16B, V8.16B + eor x10, x10, x10 + eor V9.16B, V9.16B, V9.16B + eor x11, x11, x11 + eor V10.16B, V10.16B, V10.16B + eor x12, x12, x12 + eor V11.16B, V11.16B, V11.16B + eor x13, x13, x13 + eor V12.16B, V12.16B, V12.16B + eor x14, x14, x14 + eor V13.16B, V13.16B, V13.16B + eor x15, x15, x15 + eor V14.16B, V14.16B, V14.16B + eor x16, x16, x16 + eor V15.16B, V15.16B, V15.16B + eor x17, x17, x17 + eor V16.16B, V16.16B, V16.16B + eor x19, x19, x19 + eor V17.16B, V17.16B, V17.16B + eor x20, x20, x20 + eor V18.16B, V18.16B, V18.16B + eor x21, x21, x21 + eor V19.16B, V19.16B, V19.16B + eor x22, x22, x22 + movz x23, #0x8000, lsl 48 + eor V21.16B, V21.16B, V21.16B + eor x24, x24, x24 + eor V22.16B, V22.16B, V22.16B + eor x25, x25, x25 + eor V23.16B, V23.16B, V23.16B + eor x26, x26, x26 + eor V24.16B, V24.16B, V24.16B + eor x27, x27, x27 + dup V0.2D, x2 + dup V1.2D, x3 + dup V2.2D, x4 + dup V3.2D, x5 + dup V20.2D, x23 + mov x1, #24 + ; Start of 24 rounds +L_SHA3_shake128_blocksx3_seed_neon_begin + stp x28, x1, [x29, #48] + ; Col Mix NEON + eor V30.16B, V4.16B, V9.16B + eor x0, x6, x11 + eor V27.16B, V1.16B, V6.16B + eor x30, x2, x7 + eor V30.16B, V30.16B, V14.16B + eor x28, x4, x9 + eor V27.16B, V27.16B, V11.16B + eor x0, x0, x16 + eor V30.16B, V30.16B, V19.16B + eor x30, x30, x12 + eor V27.16B, V27.16B, V16.16B + eor x28, x28, x14 + eor V30.16B, V30.16B, V24.16B + eor x0, x0, x22 + eor V27.16B, V27.16B, V21.16B + eor x30, x30, x17 + ushr V25.2D, V27.2D, #63 + eor x28, x28, x20 + sli V25.2D, V27.2D, #1 + eor x0, x0, x27 + eor V25.16B, V25.16B, V30.16B + eor x30, x30, x23 + eor V31.16B, V0.16B, V5.16B + eor x28, x28, x25 + eor V28.16B, V2.16B, V7.16B + str x0, [x29, #32] + eor V31.16B, V31.16B, V10.16B + str x28, [x29, #24] + eor V28.16B, V28.16B, V12.16B + eor x1, x3, x8 + eor V31.16B, V31.16B, V15.16B + eor x28, x5, x10 + eor V28.16B, V28.16B, V17.16B + eor x1, x1, x13 + eor V31.16B, V31.16B, V20.16B + eor x28, x28, x15 + eor V28.16B, V28.16B, V22.16B + eor x1, x1, x19 + ushr V29.2D, V30.2D, #63 + eor x28, x28, x21 + ushr V26.2D, V28.2D, #63 + eor x1, x1, x24 + sli V29.2D, V30.2D, #1 + eor x28, x28, x26 + sli V26.2D, V28.2D, #1 + eor x0, x0, x1, ror 63 + eor V28.16B, V28.16B, V29.16B + eor x1, x1, x28, ror 63 + eor V29.16B, V3.16B, V8.16B + eor x2, x2, x0 + eor V26.16B, V26.16B, V31.16B + eor x7, x7, x0 + eor V29.16B, V29.16B, V13.16B + eor x12, x12, x0 + eor V29.16B, V29.16B, V18.16B + eor x17, x17, x0 + eor V29.16B, V29.16B, V23.16B + eor x23, x23, x0 + ushr V30.2D, V29.2D, #63 + eor x4, x4, x1 + sli V30.2D, V29.2D, #1 + eor x9, x9, x1 + eor V27.16B, V27.16B, V30.16B + eor x14, x14, x1 + ushr V30.2D, V31.2D, #63 + eor x20, x20, x1 + sli V30.2D, V31.2D, #1 + eor x25, x25, x1 + eor V29.16B, V29.16B, V30.16B + ldr x0, [x29, #32] + ; Swap Rotate NEON + eor V0.16B, V0.16B, V25.16B + eor V31.16B, V1.16B, V26.16B + ldr x1, [x29, #24] + eor V6.16B, V6.16B, V26.16B + eor x28, x28, x30, ror 63 + ushr V30.2D, V31.2D, #63 + eor x30, x30, x1, ror 63 + ushr V1.2D, V6.2D, #20 + eor x1, x1, x0, ror 63 + sli V30.2D, V31.2D, #1 + eor x6, x6, x28 + sli V1.2D, V6.2D, #44 + eor x11, x11, x28 + eor V31.16B, V9.16B, V29.16B + eor x16, x16, x28 + eor V22.16B, V22.16B, V27.16B + eor x22, x22, x28 + ushr V6.2D, V31.2D, #44 + eor x27, x27, x28 + ushr V9.2D, V22.2D, #3 + eor x3, x3, x30 + sli V6.2D, V31.2D, #20 + eor x8, x8, x30 + sli V9.2D, V22.2D, #61 + eor x13, x13, x30 + eor V31.16B, V14.16B, V29.16B + eor x19, x19, x30 + eor V20.16B, V20.16B, V25.16B + eor x24, x24, x30 + ushr V22.2D, V31.2D, #25 + eor x5, x5, x1 + ushr V14.2D, V20.2D, #46 + eor x10, x10, x1 + sli V22.2D, V31.2D, #39 + eor x15, x15, x1 + sli V14.2D, V20.2D, #18 + eor x21, x21, x1 + eor V31.16B, V2.16B, V27.16B + eor x26, x26, x1 + ; Swap Rotate Base + eor V12.16B, V12.16B, V27.16B + ror x0, x3, #63 + ushr V20.2D, V31.2D, #2 + ror x3, x8, #20 + ushr V2.2D, V12.2D, #21 + ror x8, x11, #44 + sli V20.2D, V31.2D, #62 + ror x11, x25, #3 + sli V2.2D, V12.2D, #43 + ror x25, x16, #25 + eor V31.16B, V13.16B, V28.16B + ror x16, x23, #46 + eor V19.16B, V19.16B, V29.16B + ror x23, x4, #2 + ushr V12.2D, V31.2D, #39 + ror x4, x14, #21 + ushr V13.2D, V19.2D, #56 + ror x14, x15, #39 + sli V12.2D, V31.2D, #25 + ror x15, x22, #56 + sli V13.2D, V19.2D, #8 + ror x22, x26, #8 + eor V31.16B, V23.16B, V28.16B + ror x26, x17, #23 + eor V15.16B, V15.16B, V25.16B + ror x17, x6, #37 + ushr V19.2D, V31.2D, #8 + ror x6, x27, #50 + ushr V23.2D, V15.2D, #23 + ror x27, x24, #62 + sli V19.2D, V31.2D, #56 + ror x24, x10, #9 + sli V23.2D, V15.2D, #41 + ror x10, x19, #19 + eor V31.16B, V4.16B, V29.16B + ror x19, x7, #28 + eor V24.16B, V24.16B, V29.16B + ror x7, x5, #36 + ushr V15.2D, V31.2D, #37 + ror x5, x21, #43 + ushr V4.2D, V24.2D, #50 + ror x21, x20, #49 + sli V15.2D, V31.2D, #27 + ror x20, x13, #54 + sli V4.2D, V24.2D, #14 + ror x13, x9, #58 + eor V31.16B, V21.16B, V26.16B + ror x9, x12, #61 + ; Row Mix Base + eor V8.16B, V8.16B, V28.16B + bic x12, x4, x3 + ushr V24.2D, V31.2D, #62 + bic x1, x5, x4 + ushr V21.2D, V8.2D, #9 + bic x28, x2, x6 + sli V24.2D, V31.2D, #2 + bic x30, x3, x2 + sli V21.2D, V8.2D, #55 + eor x2, x2, x12 + eor V31.16B, V16.16B, V26.16B + eor x3, x3, x1 + eor V5.16B, V5.16B, V25.16B + bic x12, x6, x5 + ushr V8.2D, V31.2D, #19 + eor x5, x5, x28 + ushr V16.2D, V5.2D, #28 + eor x4, x4, x12 + sli V8.2D, V31.2D, #45 + eor x6, x6, x30 + sli V16.2D, V5.2D, #36 + bic x12, x9, x8 + eor V31.16B, V3.16B, V28.16B + bic x1, x10, x9 + eor V18.16B, V18.16B, V28.16B + bic x28, x7, x11 + ushr V5.2D, V31.2D, #36 + bic x30, x8, x7 + ushr V3.2D, V18.2D, #43 + eor x7, x7, x12 + sli V5.2D, V31.2D, #28 + eor x8, x8, x1 + sli V3.2D, V18.2D, #21 + bic x12, x11, x10 + eor V31.16B, V17.16B, V27.16B + eor x10, x10, x28 + eor V11.16B, V11.16B, V26.16B + eor x9, x9, x12 + ushr V18.2D, V31.2D, #49 + eor x11, x11, x30 + ushr V17.2D, V11.2D, #54 + bic x12, x14, x13 + sli V18.2D, V31.2D, #15 + bic x1, x15, x14 + sli V17.2D, V11.2D, #10 + bic x28, x0, x16 + eor V31.16B, V7.16B, V27.16B + bic x30, x13, x0 + eor V10.16B, V10.16B, V25.16B + eor x12, x0, x12 + ushr V11.2D, V31.2D, #58 + eor x13, x13, x1 + ushr V7.2D, V10.2D, #61 + bic x0, x16, x15 + sli V11.2D, V31.2D, #6 + eor x15, x15, x28 + sli V7.2D, V10.2D, #3 + eor x14, x14, x0 + ; Row Mix NEON + bic V25.16B, V2.16B, V1.16B + eor x16, x16, x30 + bic V26.16B, V3.16B, V2.16B + bic x0, x20, x19 + bic V27.16B, V4.16B, V3.16B + bic x1, x21, x20 + bic V28.16B, V0.16B, V4.16B + bic x28, x17, x22 + bic V29.16B, V1.16B, V0.16B + bic x30, x19, x17 + eor V0.16B, V0.16B, V25.16B + eor x17, x17, x0 + eor V1.16B, V1.16B, V26.16B + eor x19, x19, x1 + eor V2.16B, V2.16B, V27.16B + bic x0, x22, x21 + eor V3.16B, V3.16B, V28.16B + eor x21, x21, x28 + eor V4.16B, V4.16B, V29.16B + eor x20, x20, x0 + bic V25.16B, V7.16B, V6.16B + eor x22, x22, x30 + bic V26.16B, V8.16B, V7.16B + bic x0, x25, x24 + bic V27.16B, V9.16B, V8.16B + bic x1, x26, x25 + bic V28.16B, V5.16B, V9.16B + bic x28, x23, x27 + bic V29.16B, V6.16B, V5.16B + bic x30, x24, x23 + eor V5.16B, V5.16B, V25.16B + eor x23, x23, x0 + eor V6.16B, V6.16B, V26.16B + eor x24, x24, x1 + eor V7.16B, V7.16B, V27.16B + bic x0, x27, x26 + eor V8.16B, V8.16B, V28.16B + eor x26, x26, x28 + eor V9.16B, V9.16B, V29.16B + eor x25, x25, x0 + bic V25.16B, V12.16B, V11.16B + eor x27, x27, x30 + bic V26.16B, V13.16B, V12.16B + bic V27.16B, V14.16B, V13.16B + bic V28.16B, V30.16B, V14.16B + bic V29.16B, V11.16B, V30.16B + eor V10.16B, V30.16B, V25.16B + eor V11.16B, V11.16B, V26.16B + eor V12.16B, V12.16B, V27.16B + eor V13.16B, V13.16B, V28.16B + eor V14.16B, V14.16B, V29.16B + bic V25.16B, V17.16B, V16.16B + bic V26.16B, V18.16B, V17.16B + bic V27.16B, V19.16B, V18.16B + bic V28.16B, V15.16B, V19.16B + bic V29.16B, V16.16B, V15.16B + eor V15.16B, V15.16B, V25.16B + eor V16.16B, V16.16B, V26.16B + eor V17.16B, V17.16B, V27.16B + eor V18.16B, V18.16B, V28.16B + eor V19.16B, V19.16B, V29.16B + bic V25.16B, V22.16B, V21.16B + bic V26.16B, V23.16B, V22.16B + bic V27.16B, V24.16B, V23.16B + bic V28.16B, V20.16B, V24.16B + bic V29.16B, V21.16B, V20.16B + eor V20.16B, V20.16B, V25.16B + eor V21.16B, V21.16B, V26.16B + eor V22.16B, V22.16B, V27.16B + eor V23.16B, V23.16B, V28.16B + eor V24.16B, V24.16B, V29.16B + ; Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov V30.D[0], x0 + mov V30.D[1], x0 + eor x2, x2, x0 + eor V0.16B, V0.16B, V30.16B + bne L_SHA3_shake128_blocksx3_seed_neon_begin + ldr x0, [x29, #40] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] + add x0, x0, #8 + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT sha3_256_blocksx3_seed_neon +sha3_256_blocksx3_seed_neon PROC + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] + adrp x28, L_sha3_aarch64_r + add x28, x28, L_sha3_aarch64_r + str x0, [x29, #40] + add x0, x0, #32 + ld1 {V4.D}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {V4.D}[1], [x0] + ldp x4, x5, [x1], #16 + ldr x6, [x0, #200] + eor V5.16B, V5.16B, V5.16B + eor x7, x7, x7 + eor V6.16B, V6.16B, V6.16B + eor x8, x8, x8 + eor V7.16B, V7.16B, V7.16B + eor x9, x9, x9 + eor V8.16B, V8.16B, V8.16B + eor x10, x10, x10 + eor V9.16B, V9.16B, V9.16B + eor x11, x11, x11 + eor V10.16B, V10.16B, V10.16B + eor x12, x12, x12 + eor V11.16B, V11.16B, V11.16B + eor x13, x13, x13 + eor V12.16B, V12.16B, V12.16B + eor x14, x14, x14 + eor V13.16B, V13.16B, V13.16B + eor x15, x15, x15 + eor V14.16B, V14.16B, V14.16B + eor x16, x16, x16 + eor V15.16B, V15.16B, V15.16B + eor x17, x17, x17 + movz x19, #0x8000, lsl 48 + eor V17.16B, V17.16B, V17.16B + eor x20, x20, x20 + eor V18.16B, V18.16B, V18.16B + eor x21, x21, x21 + eor V19.16B, V19.16B, V19.16B + eor x22, x22, x22 + eor V20.16B, V20.16B, V20.16B + eor x23, x23, x23 + eor V21.16B, V21.16B, V21.16B + eor x24, x24, x24 + eor V22.16B, V22.16B, V22.16B + eor x25, x25, x25 + eor V23.16B, V23.16B, V23.16B + eor x26, x26, x26 + eor V24.16B, V24.16B, V24.16B + eor x27, x27, x27 + dup V0.2D, x2 + dup V1.2D, x3 + dup V2.2D, x4 + dup V3.2D, x5 + dup V16.2D, x19 + mov x1, #24 + ; Start of 24 rounds +L_SHA3_shake256_blocksx3_seed_neon_begin + stp x28, x1, [x29, #48] + ; Col Mix NEON + eor V30.16B, V4.16B, V9.16B + eor x0, x6, x11 + eor V27.16B, V1.16B, V6.16B + eor x30, x2, x7 + eor V30.16B, V30.16B, V14.16B + eor x28, x4, x9 + eor V27.16B, V27.16B, V11.16B + eor x0, x0, x16 + eor V30.16B, V30.16B, V19.16B + eor x30, x30, x12 + eor V27.16B, V27.16B, V16.16B + eor x28, x28, x14 + eor V30.16B, V30.16B, V24.16B + eor x0, x0, x22 + eor V27.16B, V27.16B, V21.16B + eor x30, x30, x17 + ushr V25.2D, V27.2D, #63 + eor x28, x28, x20 + sli V25.2D, V27.2D, #1 + eor x0, x0, x27 + eor V25.16B, V25.16B, V30.16B + eor x30, x30, x23 + eor V31.16B, V0.16B, V5.16B + eor x28, x28, x25 + eor V28.16B, V2.16B, V7.16B + str x0, [x29, #32] + eor V31.16B, V31.16B, V10.16B + str x28, [x29, #24] + eor V28.16B, V28.16B, V12.16B + eor x1, x3, x8 + eor V31.16B, V31.16B, V15.16B + eor x28, x5, x10 + eor V28.16B, V28.16B, V17.16B + eor x1, x1, x13 + eor V31.16B, V31.16B, V20.16B + eor x28, x28, x15 + eor V28.16B, V28.16B, V22.16B + eor x1, x1, x19 + ushr V29.2D, V30.2D, #63 + eor x28, x28, x21 + ushr V26.2D, V28.2D, #63 + eor x1, x1, x24 + sli V29.2D, V30.2D, #1 + eor x28, x28, x26 + sli V26.2D, V28.2D, #1 + eor x0, x0, x1, ror 63 + eor V28.16B, V28.16B, V29.16B + eor x1, x1, x28, ror 63 + eor V29.16B, V3.16B, V8.16B + eor x2, x2, x0 + eor V26.16B, V26.16B, V31.16B + eor x7, x7, x0 + eor V29.16B, V29.16B, V13.16B + eor x12, x12, x0 + eor V29.16B, V29.16B, V18.16B + eor x17, x17, x0 + eor V29.16B, V29.16B, V23.16B + eor x23, x23, x0 + ushr V30.2D, V29.2D, #63 + eor x4, x4, x1 + sli V30.2D, V29.2D, #1 + eor x9, x9, x1 + eor V27.16B, V27.16B, V30.16B + eor x14, x14, x1 + ushr V30.2D, V31.2D, #63 + eor x20, x20, x1 + sli V30.2D, V31.2D, #1 + eor x25, x25, x1 + eor V29.16B, V29.16B, V30.16B + ldr x0, [x29, #32] + ; Swap Rotate NEON + eor V0.16B, V0.16B, V25.16B + eor V31.16B, V1.16B, V26.16B + ldr x1, [x29, #24] + eor V6.16B, V6.16B, V26.16B + eor x28, x28, x30, ror 63 + ushr V30.2D, V31.2D, #63 + eor x30, x30, x1, ror 63 + ushr V1.2D, V6.2D, #20 + eor x1, x1, x0, ror 63 + sli V30.2D, V31.2D, #1 + eor x6, x6, x28 + sli V1.2D, V6.2D, #44 + eor x11, x11, x28 + eor V31.16B, V9.16B, V29.16B + eor x16, x16, x28 + eor V22.16B, V22.16B, V27.16B + eor x22, x22, x28 + ushr V6.2D, V31.2D, #44 + eor x27, x27, x28 + ushr V9.2D, V22.2D, #3 + eor x3, x3, x30 + sli V6.2D, V31.2D, #20 + eor x8, x8, x30 + sli V9.2D, V22.2D, #61 + eor x13, x13, x30 + eor V31.16B, V14.16B, V29.16B + eor x19, x19, x30 + eor V20.16B, V20.16B, V25.16B + eor x24, x24, x30 + ushr V22.2D, V31.2D, #25 + eor x5, x5, x1 + ushr V14.2D, V20.2D, #46 + eor x10, x10, x1 + sli V22.2D, V31.2D, #39 + eor x15, x15, x1 + sli V14.2D, V20.2D, #18 + eor x21, x21, x1 + eor V31.16B, V2.16B, V27.16B + eor x26, x26, x1 + ; Swap Rotate Base + eor V12.16B, V12.16B, V27.16B + ror x0, x3, #63 + ushr V20.2D, V31.2D, #2 + ror x3, x8, #20 + ushr V2.2D, V12.2D, #21 + ror x8, x11, #44 + sli V20.2D, V31.2D, #62 + ror x11, x25, #3 + sli V2.2D, V12.2D, #43 + ror x25, x16, #25 + eor V31.16B, V13.16B, V28.16B + ror x16, x23, #46 + eor V19.16B, V19.16B, V29.16B + ror x23, x4, #2 + ushr V12.2D, V31.2D, #39 + ror x4, x14, #21 + ushr V13.2D, V19.2D, #56 + ror x14, x15, #39 + sli V12.2D, V31.2D, #25 + ror x15, x22, #56 + sli V13.2D, V19.2D, #8 + ror x22, x26, #8 + eor V31.16B, V23.16B, V28.16B + ror x26, x17, #23 + eor V15.16B, V15.16B, V25.16B + ror x17, x6, #37 + ushr V19.2D, V31.2D, #8 + ror x6, x27, #50 + ushr V23.2D, V15.2D, #23 + ror x27, x24, #62 + sli V19.2D, V31.2D, #56 + ror x24, x10, #9 + sli V23.2D, V15.2D, #41 + ror x10, x19, #19 + eor V31.16B, V4.16B, V29.16B + ror x19, x7, #28 + eor V24.16B, V24.16B, V29.16B + ror x7, x5, #36 + ushr V15.2D, V31.2D, #37 + ror x5, x21, #43 + ushr V4.2D, V24.2D, #50 + ror x21, x20, #49 + sli V15.2D, V31.2D, #27 + ror x20, x13, #54 + sli V4.2D, V24.2D, #14 + ror x13, x9, #58 + eor V31.16B, V21.16B, V26.16B + ror x9, x12, #61 + ; Row Mix Base + eor V8.16B, V8.16B, V28.16B + bic x12, x4, x3 + ushr V24.2D, V31.2D, #62 + bic x1, x5, x4 + ushr V21.2D, V8.2D, #9 + bic x28, x2, x6 + sli V24.2D, V31.2D, #2 + bic x30, x3, x2 + sli V21.2D, V8.2D, #55 + eor x2, x2, x12 + eor V31.16B, V16.16B, V26.16B + eor x3, x3, x1 + eor V5.16B, V5.16B, V25.16B + bic x12, x6, x5 + ushr V8.2D, V31.2D, #19 + eor x5, x5, x28 + ushr V16.2D, V5.2D, #28 + eor x4, x4, x12 + sli V8.2D, V31.2D, #45 + eor x6, x6, x30 + sli V16.2D, V5.2D, #36 + bic x12, x9, x8 + eor V31.16B, V3.16B, V28.16B + bic x1, x10, x9 + eor V18.16B, V18.16B, V28.16B + bic x28, x7, x11 + ushr V5.2D, V31.2D, #36 + bic x30, x8, x7 + ushr V3.2D, V18.2D, #43 + eor x7, x7, x12 + sli V5.2D, V31.2D, #28 + eor x8, x8, x1 + sli V3.2D, V18.2D, #21 + bic x12, x11, x10 + eor V31.16B, V17.16B, V27.16B + eor x10, x10, x28 + eor V11.16B, V11.16B, V26.16B + eor x9, x9, x12 + ushr V18.2D, V31.2D, #49 + eor x11, x11, x30 + ushr V17.2D, V11.2D, #54 + bic x12, x14, x13 + sli V18.2D, V31.2D, #15 + bic x1, x15, x14 + sli V17.2D, V11.2D, #10 + bic x28, x0, x16 + eor V31.16B, V7.16B, V27.16B + bic x30, x13, x0 + eor V10.16B, V10.16B, V25.16B + eor x12, x0, x12 + ushr V11.2D, V31.2D, #58 + eor x13, x13, x1 + ushr V7.2D, V10.2D, #61 + bic x0, x16, x15 + sli V11.2D, V31.2D, #6 + eor x15, x15, x28 + sli V7.2D, V10.2D, #3 + eor x14, x14, x0 + ; Row Mix NEON + bic V25.16B, V2.16B, V1.16B + eor x16, x16, x30 + bic V26.16B, V3.16B, V2.16B + bic x0, x20, x19 + bic V27.16B, V4.16B, V3.16B + bic x1, x21, x20 + bic V28.16B, V0.16B, V4.16B + bic x28, x17, x22 + bic V29.16B, V1.16B, V0.16B + bic x30, x19, x17 + eor V0.16B, V0.16B, V25.16B + eor x17, x17, x0 + eor V1.16B, V1.16B, V26.16B + eor x19, x19, x1 + eor V2.16B, V2.16B, V27.16B + bic x0, x22, x21 + eor V3.16B, V3.16B, V28.16B + eor x21, x21, x28 + eor V4.16B, V4.16B, V29.16B + eor x20, x20, x0 + bic V25.16B, V7.16B, V6.16B + eor x22, x22, x30 + bic V26.16B, V8.16B, V7.16B + bic x0, x25, x24 + bic V27.16B, V9.16B, V8.16B + bic x1, x26, x25 + bic V28.16B, V5.16B, V9.16B + bic x28, x23, x27 + bic V29.16B, V6.16B, V5.16B + bic x30, x24, x23 + eor V5.16B, V5.16B, V25.16B + eor x23, x23, x0 + eor V6.16B, V6.16B, V26.16B + eor x24, x24, x1 + eor V7.16B, V7.16B, V27.16B + bic x0, x27, x26 + eor V8.16B, V8.16B, V28.16B + eor x26, x26, x28 + eor V9.16B, V9.16B, V29.16B + eor x25, x25, x0 + bic V25.16B, V12.16B, V11.16B + eor x27, x27, x30 + bic V26.16B, V13.16B, V12.16B + bic V27.16B, V14.16B, V13.16B + bic V28.16B, V30.16B, V14.16B + bic V29.16B, V11.16B, V30.16B + eor V10.16B, V30.16B, V25.16B + eor V11.16B, V11.16B, V26.16B + eor V12.16B, V12.16B, V27.16B + eor V13.16B, V13.16B, V28.16B + eor V14.16B, V14.16B, V29.16B + bic V25.16B, V17.16B, V16.16B + bic V26.16B, V18.16B, V17.16B + bic V27.16B, V19.16B, V18.16B + bic V28.16B, V15.16B, V19.16B + bic V29.16B, V16.16B, V15.16B + eor V15.16B, V15.16B, V25.16B + eor V16.16B, V16.16B, V26.16B + eor V17.16B, V17.16B, V27.16B + eor V18.16B, V18.16B, V28.16B + eor V19.16B, V19.16B, V29.16B + bic V25.16B, V22.16B, V21.16B + bic V26.16B, V23.16B, V22.16B + bic V27.16B, V24.16B, V23.16B + bic V28.16B, V20.16B, V24.16B + bic V29.16B, V21.16B, V20.16B + eor V20.16B, V20.16B, V25.16B + eor V21.16B, V21.16B, V26.16B + eor V22.16B, V22.16B, V27.16B + eor V23.16B, V23.16B, V28.16B + eor V24.16B, V24.16B, V29.16B + ; Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov V30.D[0], x0 + mov V30.D[1], x0 + eor x2, x2, x0 + eor V0.16B, V0.16B, V30.16B + bne L_SHA3_shake256_blocksx3_seed_neon_begin + ldr x0, [x29, #40] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] + add x0, x0, #8 + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret + ENDP + AREA |.text|, CODE, READONLY + ALIGN 4 + EXPORT sha3_256_blocksx3_seed_64_neon +sha3_256_blocksx3_seed_64_neon PROC + stp x29, x30, [sp, #-224]! + add x29, sp, #0 + stp x17, x19, [x29, #72] + stp x20, x21, [x29, #88] + stp x22, x23, [x29, #104] + stp x24, x25, [x29, #120] + stp x26, x27, [x29, #136] + str x28, [x29, #152] + stp D8, D9, [x29, #160] + stp D10, D11, [x29, #176] + stp D12, D13, [x29, #192] + stp D14, D15, [x29, #208] + adrp x28, L_sha3_aarch64_r + add x28, x28, L_sha3_aarch64_r + str x0, [x29, #40] + add x0, x0, #0x40 + ld1 {V8.D}[0], [x0] + ldp x2, x3, [x1], #16 + add x0, x0, #0xc8 + ld1 {V8.D}[1], [x0] + ldp x4, x5, [x1], #16 + ldp x6, x7, [x1], #16 + ldp x8, x9, [x1], #16 + ldr x10, [x0, #200] + eor V9.16B, V9.16B, V9.16B + eor x11, x11, x11 + eor V10.16B, V10.16B, V10.16B + eor x12, x12, x12 + eor V11.16B, V11.16B, V11.16B + eor x13, x13, x13 + eor V12.16B, V12.16B, V12.16B + eor x14, x14, x14 + eor V13.16B, V13.16B, V13.16B + eor x15, x15, x15 + eor V14.16B, V14.16B, V14.16B + eor x16, x16, x16 + eor V15.16B, V15.16B, V15.16B + eor x17, x17, x17 + movz x19, #0x8000, lsl 48 + eor V17.16B, V17.16B, V17.16B + eor x20, x20, x20 + eor V18.16B, V18.16B, V18.16B + eor x21, x21, x21 + eor V19.16B, V19.16B, V19.16B + eor x22, x22, x22 + eor V20.16B, V20.16B, V20.16B + eor x23, x23, x23 + eor V21.16B, V21.16B, V21.16B + eor x24, x24, x24 + eor V22.16B, V22.16B, V22.16B + eor x25, x25, x25 + eor V23.16B, V23.16B, V23.16B + eor x26, x26, x26 + eor V24.16B, V24.16B, V24.16B + eor x27, x27, x27 + dup V0.2D, x2 + dup V1.2D, x3 + dup V2.2D, x4 + dup V3.2D, x5 + dup V4.2D, x6 + dup V5.2D, x7 + dup V6.2D, x8 + dup V7.2D, x9 + dup V16.2D, x19 + mov x1, #24 + ; Start of 24 rounds +L_SHA3_shake256_blocksx3_seed_64_neon_begin + stp x28, x1, [x29, #48] + ; Col Mix NEON + eor V30.16B, V4.16B, V9.16B + eor x0, x6, x11 + eor V27.16B, V1.16B, V6.16B + eor x30, x2, x7 + eor V30.16B, V30.16B, V14.16B + eor x28, x4, x9 + eor V27.16B, V27.16B, V11.16B + eor x0, x0, x16 + eor V30.16B, V30.16B, V19.16B + eor x30, x30, x12 + eor V27.16B, V27.16B, V16.16B + eor x28, x28, x14 + eor V30.16B, V30.16B, V24.16B + eor x0, x0, x22 + eor V27.16B, V27.16B, V21.16B + eor x30, x30, x17 + ushr V25.2D, V27.2D, #63 + eor x28, x28, x20 + sli V25.2D, V27.2D, #1 + eor x0, x0, x27 + eor V25.16B, V25.16B, V30.16B + eor x30, x30, x23 + eor V31.16B, V0.16B, V5.16B + eor x28, x28, x25 + eor V28.16B, V2.16B, V7.16B + str x0, [x29, #32] + eor V31.16B, V31.16B, V10.16B + str x28, [x29, #24] + eor V28.16B, V28.16B, V12.16B + eor x1, x3, x8 + eor V31.16B, V31.16B, V15.16B + eor x28, x5, x10 + eor V28.16B, V28.16B, V17.16B + eor x1, x1, x13 + eor V31.16B, V31.16B, V20.16B + eor x28, x28, x15 + eor V28.16B, V28.16B, V22.16B + eor x1, x1, x19 + ushr V29.2D, V30.2D, #63 + eor x28, x28, x21 + ushr V26.2D, V28.2D, #63 + eor x1, x1, x24 + sli V29.2D, V30.2D, #1 + eor x28, x28, x26 + sli V26.2D, V28.2D, #1 + eor x0, x0, x1, ror 63 + eor V28.16B, V28.16B, V29.16B + eor x1, x1, x28, ror 63 + eor V29.16B, V3.16B, V8.16B + eor x2, x2, x0 + eor V26.16B, V26.16B, V31.16B + eor x7, x7, x0 + eor V29.16B, V29.16B, V13.16B + eor x12, x12, x0 + eor V29.16B, V29.16B, V18.16B + eor x17, x17, x0 + eor V29.16B, V29.16B, V23.16B + eor x23, x23, x0 + ushr V30.2D, V29.2D, #63 + eor x4, x4, x1 + sli V30.2D, V29.2D, #1 + eor x9, x9, x1 + eor V27.16B, V27.16B, V30.16B + eor x14, x14, x1 + ushr V30.2D, V31.2D, #63 + eor x20, x20, x1 + sli V30.2D, V31.2D, #1 + eor x25, x25, x1 + eor V29.16B, V29.16B, V30.16B + ldr x0, [x29, #32] + ; Swap Rotate NEON + eor V0.16B, V0.16B, V25.16B + eor V31.16B, V1.16B, V26.16B + ldr x1, [x29, #24] + eor V6.16B, V6.16B, V26.16B + eor x28, x28, x30, ror 63 + ushr V30.2D, V31.2D, #63 + eor x30, x30, x1, ror 63 + ushr V1.2D, V6.2D, #20 + eor x1, x1, x0, ror 63 + sli V30.2D, V31.2D, #1 + eor x6, x6, x28 + sli V1.2D, V6.2D, #44 + eor x11, x11, x28 + eor V31.16B, V9.16B, V29.16B + eor x16, x16, x28 + eor V22.16B, V22.16B, V27.16B + eor x22, x22, x28 + ushr V6.2D, V31.2D, #44 + eor x27, x27, x28 + ushr V9.2D, V22.2D, #3 + eor x3, x3, x30 + sli V6.2D, V31.2D, #20 + eor x8, x8, x30 + sli V9.2D, V22.2D, #61 + eor x13, x13, x30 + eor V31.16B, V14.16B, V29.16B + eor x19, x19, x30 + eor V20.16B, V20.16B, V25.16B + eor x24, x24, x30 + ushr V22.2D, V31.2D, #25 + eor x5, x5, x1 + ushr V14.2D, V20.2D, #46 + eor x10, x10, x1 + sli V22.2D, V31.2D, #39 + eor x15, x15, x1 + sli V14.2D, V20.2D, #18 + eor x21, x21, x1 + eor V31.16B, V2.16B, V27.16B + eor x26, x26, x1 + ; Swap Rotate Base + eor V12.16B, V12.16B, V27.16B + ror x0, x3, #63 + ushr V20.2D, V31.2D, #2 + ror x3, x8, #20 + ushr V2.2D, V12.2D, #21 + ror x8, x11, #44 + sli V20.2D, V31.2D, #62 + ror x11, x25, #3 + sli V2.2D, V12.2D, #43 + ror x25, x16, #25 + eor V31.16B, V13.16B, V28.16B + ror x16, x23, #46 + eor V19.16B, V19.16B, V29.16B + ror x23, x4, #2 + ushr V12.2D, V31.2D, #39 + ror x4, x14, #21 + ushr V13.2D, V19.2D, #56 + ror x14, x15, #39 + sli V12.2D, V31.2D, #25 + ror x15, x22, #56 + sli V13.2D, V19.2D, #8 + ror x22, x26, #8 + eor V31.16B, V23.16B, V28.16B + ror x26, x17, #23 + eor V15.16B, V15.16B, V25.16B + ror x17, x6, #37 + ushr V19.2D, V31.2D, #8 + ror x6, x27, #50 + ushr V23.2D, V15.2D, #23 + ror x27, x24, #62 + sli V19.2D, V31.2D, #56 + ror x24, x10, #9 + sli V23.2D, V15.2D, #41 + ror x10, x19, #19 + eor V31.16B, V4.16B, V29.16B + ror x19, x7, #28 + eor V24.16B, V24.16B, V29.16B + ror x7, x5, #36 + ushr V15.2D, V31.2D, #37 + ror x5, x21, #43 + ushr V4.2D, V24.2D, #50 + ror x21, x20, #49 + sli V15.2D, V31.2D, #27 + ror x20, x13, #54 + sli V4.2D, V24.2D, #14 + ror x13, x9, #58 + eor V31.16B, V21.16B, V26.16B + ror x9, x12, #61 + ; Row Mix Base + eor V8.16B, V8.16B, V28.16B + bic x12, x4, x3 + ushr V24.2D, V31.2D, #62 + bic x1, x5, x4 + ushr V21.2D, V8.2D, #9 + bic x28, x2, x6 + sli V24.2D, V31.2D, #2 + bic x30, x3, x2 + sli V21.2D, V8.2D, #55 + eor x2, x2, x12 + eor V31.16B, V16.16B, V26.16B + eor x3, x3, x1 + eor V5.16B, V5.16B, V25.16B + bic x12, x6, x5 + ushr V8.2D, V31.2D, #19 + eor x5, x5, x28 + ushr V16.2D, V5.2D, #28 + eor x4, x4, x12 + sli V8.2D, V31.2D, #45 + eor x6, x6, x30 + sli V16.2D, V5.2D, #36 + bic x12, x9, x8 + eor V31.16B, V3.16B, V28.16B + bic x1, x10, x9 + eor V18.16B, V18.16B, V28.16B + bic x28, x7, x11 + ushr V5.2D, V31.2D, #36 + bic x30, x8, x7 + ushr V3.2D, V18.2D, #43 + eor x7, x7, x12 + sli V5.2D, V31.2D, #28 + eor x8, x8, x1 + sli V3.2D, V18.2D, #21 + bic x12, x11, x10 + eor V31.16B, V17.16B, V27.16B + eor x10, x10, x28 + eor V11.16B, V11.16B, V26.16B + eor x9, x9, x12 + ushr V18.2D, V31.2D, #49 + eor x11, x11, x30 + ushr V17.2D, V11.2D, #54 + bic x12, x14, x13 + sli V18.2D, V31.2D, #15 + bic x1, x15, x14 + sli V17.2D, V11.2D, #10 + bic x28, x0, x16 + eor V31.16B, V7.16B, V27.16B + bic x30, x13, x0 + eor V10.16B, V10.16B, V25.16B + eor x12, x0, x12 + ushr V11.2D, V31.2D, #58 + eor x13, x13, x1 + ushr V7.2D, V10.2D, #61 + bic x0, x16, x15 + sli V11.2D, V31.2D, #6 + eor x15, x15, x28 + sli V7.2D, V10.2D, #3 + eor x14, x14, x0 + ; Row Mix NEON + bic V25.16B, V2.16B, V1.16B + eor x16, x16, x30 + bic V26.16B, V3.16B, V2.16B + bic x0, x20, x19 + bic V27.16B, V4.16B, V3.16B + bic x1, x21, x20 + bic V28.16B, V0.16B, V4.16B + bic x28, x17, x22 + bic V29.16B, V1.16B, V0.16B + bic x30, x19, x17 + eor V0.16B, V0.16B, V25.16B + eor x17, x17, x0 + eor V1.16B, V1.16B, V26.16B + eor x19, x19, x1 + eor V2.16B, V2.16B, V27.16B + bic x0, x22, x21 + eor V3.16B, V3.16B, V28.16B + eor x21, x21, x28 + eor V4.16B, V4.16B, V29.16B + eor x20, x20, x0 + bic V25.16B, V7.16B, V6.16B + eor x22, x22, x30 + bic V26.16B, V8.16B, V7.16B + bic x0, x25, x24 + bic V27.16B, V9.16B, V8.16B + bic x1, x26, x25 + bic V28.16B, V5.16B, V9.16B + bic x28, x23, x27 + bic V29.16B, V6.16B, V5.16B + bic x30, x24, x23 + eor V5.16B, V5.16B, V25.16B + eor x23, x23, x0 + eor V6.16B, V6.16B, V26.16B + eor x24, x24, x1 + eor V7.16B, V7.16B, V27.16B + bic x0, x27, x26 + eor V8.16B, V8.16B, V28.16B + eor x26, x26, x28 + eor V9.16B, V9.16B, V29.16B + eor x25, x25, x0 + bic V25.16B, V12.16B, V11.16B + eor x27, x27, x30 + bic V26.16B, V13.16B, V12.16B + bic V27.16B, V14.16B, V13.16B + bic V28.16B, V30.16B, V14.16B + bic V29.16B, V11.16B, V30.16B + eor V10.16B, V30.16B, V25.16B + eor V11.16B, V11.16B, V26.16B + eor V12.16B, V12.16B, V27.16B + eor V13.16B, V13.16B, V28.16B + eor V14.16B, V14.16B, V29.16B + bic V25.16B, V17.16B, V16.16B + bic V26.16B, V18.16B, V17.16B + bic V27.16B, V19.16B, V18.16B + bic V28.16B, V15.16B, V19.16B + bic V29.16B, V16.16B, V15.16B + eor V15.16B, V15.16B, V25.16B + eor V16.16B, V16.16B, V26.16B + eor V17.16B, V17.16B, V27.16B + eor V18.16B, V18.16B, V28.16B + eor V19.16B, V19.16B, V29.16B + bic V25.16B, V22.16B, V21.16B + bic V26.16B, V23.16B, V22.16B + bic V27.16B, V24.16B, V23.16B + bic V28.16B, V20.16B, V24.16B + bic V29.16B, V21.16B, V20.16B + eor V20.16B, V20.16B, V25.16B + eor V21.16B, V21.16B, V26.16B + eor V22.16B, V22.16B, V27.16B + eor V23.16B, V23.16B, V28.16B + eor V24.16B, V24.16B, V29.16B + ; Done transforming + ldp x28, x1, [x29, #48] + ldr x0, [x28], #8 + subs x1, x1, #1 + mov V30.D[0], x0 + mov V30.D[1], x0 + eor x2, x2, x0 + eor V0.16B, V0.16B, V30.16B + bne L_SHA3_shake256_blocksx3_seed_64_neon_begin + ldr x0, [x29, #40] + st4 {V0.D, V1.D, V2.D, V3.D}[0], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[0], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[0], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[0], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[0], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[0], [x0], #32 + st1 {V24.D}[0], [x0] + add x0, x0, #8 + st4 {V0.D, V1.D, V2.D, V3.D}[1], [x0], #32 + st4 {V4.D, V5.D, V6.D, V7.D}[1], [x0], #32 + st4 {V8.D, V9.D, V10.D, V11.D}[1], [x0], #32 + st4 {V12.D, V13.D, V14.D, V15.D}[1], [x0], #32 + st4 {V16.D, V17.D, V18.D, V19.D}[1], [x0], #32 + st4 {V20.D, V21.D, V22.D, V23.D}[1], [x0], #32 + st1 {V24.D}[1], [x0] + add x0, x0, #8 + stp x2, x3, [x0] + stp x4, x5, [x0, #16] + stp x6, x7, [x0, #32] + stp x8, x9, [x0, #48] + stp x10, x11, [x0, #64] + stp x12, x13, [x0, #80] + stp x14, x15, [x0, #96] + stp x16, x17, [x0, #112] + stp x19, x20, [x0, #128] + stp x21, x22, [x0, #144] + stp x23, x24, [x0, #160] + stp x25, x26, [x0, #176] + str x27, [x0, #192] + ldp x17, x19, [x29, #72] + ldp x20, x21, [x29, #88] + ldp x22, x23, [x29, #104] + ldp x24, x25, [x29, #120] + ldp x26, x27, [x29, #136] + ldr x28, [x29, #152] + ldp D8, D9, [x29, #160] + ldp D10, D11, [x29, #176] + ldp D12, D13, [x29, #192] + ldp D14, D15, [x29, #208] + ldp x29, x30, [sp], #0xe0 + ret + ENDP + ENDIF ENDIF END diff --git a/wolfcrypt/src/port/arm/armv8-sha3-asm_c.c b/wolfcrypt/src/port/arm/armv8-sha3-asm_c.c index 264be7b156e..4b427c75466 100644 --- a/wolfcrypt/src/port/arm/armv8-sha3-asm_c.c +++ b/wolfcrypt/src/port/arm/armv8-sha3-asm_c.c @@ -368,6 +368,2897 @@ void BlockSha3_base(word64* state) ); } +#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) +XALIGNED(16) static const word64 L_sha3_aarch64_r[] = { + 0x0000000000000001UL, 0x0000000000008082UL, + 0x800000000000808aUL, 0x8000000080008000UL, + 0x000000000000808bUL, 0x0000000080000001UL, + 0x8000000080008081UL, 0x8000000000008009UL, + 0x000000000000008aUL, 0x0000000000000088UL, + 0x0000000080008009UL, 0x000000008000000aUL, + 0x000000008000808bUL, 0x800000000000008bUL, + 0x8000000000008089UL, 0x8000000000008003UL, + 0x8000000000008002UL, 0x8000000000000080UL, + 0x000000000000800aUL, 0x800000008000000aUL, + 0x8000000080008081UL, 0x8000000000008080UL, + 0x0000000080000001UL, 0x8000000080008008UL, +}; + +#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 +void sha3_blocksx3_crypto(word64* state) +{ + const word64* r = L_sha3_aarch64_r; + __asm__ __volatile__ ( + "stp x29, x30, [sp, #-64]!\n\t" + "add x29, sp, #0\n\t" +#ifdef __APPLE__ + ".arch_extension sha3\n\t" +#endif /* __APPLE__ */ + "str %x[state], [x29, #40]\n\t" + "ld4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" + "ld4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" + "ld4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" + "ld4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" + "ld4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" + "ld4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" + "ld1 {v24.d}[0], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "ld4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" + "ld4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" + "ld4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" + "ld4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" + "ld4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" + "ld4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" + "ld1 {v24.d}[1], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "ldp x1, x2, [%x[state]]\n\t" + "ldp x3, x4, [%x[state], #16]\n\t" + "ldp x5, x6, [%x[state], #32]\n\t" + "ldp x7, x8, [%x[state], #48]\n\t" + "ldp x9, x10, [%x[state], #64]\n\t" + "ldp x11, x12, [%x[state], #80]\n\t" + "ldp x13, x14, [%x[state], #96]\n\t" + "ldp x15, x16, [%x[state], #112]\n\t" + "ldp x17, x19, [%x[state], #128]\n\t" + "ldp x20, x21, [%x[state], #144]\n\t" + "ldp x22, x23, [%x[state], #160]\n\t" + "ldp x24, x25, [%x[state], #176]\n\t" + "ldr x26, [%x[state], #192]\n\t" + "mov x28, #24\n\t" + /* Start of 24 rounds */ + "\n" + "L_SHA3_transform_blocksx3_crypto_begin_%=:\n\t" + "stp %[r], x28, [x29, #48]\n\t" + /* Col Mix */ + "eor3 v31.16b, v0.16b, v5.16b, v10.16b\n\t" + "eor %x[state], x5, x10\n\t" + "eor3 v27.16b, v1.16b, v6.16b, v11.16b\n\t" + "eor x30, x1, x6\n\t" + "eor3 v28.16b, v2.16b, v7.16b, v12.16b\n\t" + "eor x28, x3, x8\n\t" + "eor3 v29.16b, v3.16b, v8.16b, v13.16b\n\t" + "eor %x[state], %x[state], x15\n\t" + "eor3 v30.16b, v4.16b, v9.16b, v14.16b\n\t" + "eor x30, x30, x11\n\t" + "eor3 v31.16b, v31.16b, v15.16b, v20.16b\n\t" + "eor x28, x28, x13\n\t" + "eor3 v27.16b, v27.16b, v16.16b, v21.16b\n\t" + "eor %x[state], %x[state], x21\n\t" + "eor3 v28.16b, v28.16b, v17.16b, v22.16b\n\t" + "eor x30, x30, x16\n\t" + "eor3 v29.16b, v29.16b, v18.16b, v23.16b\n\t" + "eor x28, x28, x19\n\t" + "eor3 v30.16b, v30.16b, v19.16b, v24.16b\n\t" + "eor %x[state], %x[state], x26\n\t" + "rax1 v25.2d, v30.2d, v27.2d\n\t" + "eor x30, x30, x22\n\t" + "rax1 v26.2d, v31.2d, v28.2d\n\t" + "eor x28, x28, x24\n\t" + "rax1 v27.2d, v27.2d, v29.2d\n\t" + "str %x[state], [x29, #32]\n\t" + "rax1 v28.2d, v28.2d, v30.2d\n\t" + "str x28, [x29, #24]\n\t" + "rax1 v29.2d, v29.2d, v31.2d\n\t" + "eor %[r], x2, x7\n\t" + "eor v0.16b, v0.16b, v25.16b\n\t" + "xar v30.2d, v1.2d, v26.2d, #63\n\t" + "eor x28, x4, x9\n\t" + "xar v1.2d, v6.2d, v26.2d, #20\n\t" + "eor %[r], %[r], x12\n\t" + "xar v6.2d, v9.2d, v29.2d, #44\n\t" + "eor x28, x28, x14\n\t" + "xar v9.2d, v22.2d, v27.2d, #3\n\t" + "eor %[r], %[r], x17\n\t" + "xar v22.2d, v14.2d, v29.2d, #25\n\t" + "eor x28, x28, x20\n\t" + "xar v14.2d, v20.2d, v25.2d, #46\n\t" + "eor %[r], %[r], x23\n\t" + "xar v20.2d, v2.2d, v27.2d, #2\n\t" + "eor x28, x28, x25\n\t" + "xar v2.2d, v12.2d, v27.2d, #21\n\t" + "eor %x[state], %x[state], %[r], ror 63\n\t" + "xar v12.2d, v13.2d, v28.2d, #39\n\t" + "eor %[r], %[r], x28, ror 63\n\t" + "xar v13.2d, v19.2d, v29.2d, #56\n\t" + "eor x1, x1, %x[state]\n\t" + "xar v19.2d, v23.2d, v28.2d, #8\n\t" + "eor x6, x6, %x[state]\n\t" + "xar v23.2d, v15.2d, v25.2d, #23\n\t" + "eor x11, x11, %x[state]\n\t" + "xar v15.2d, v4.2d, v29.2d, #37\n\t" + "eor x16, x16, %x[state]\n\t" + "xar v4.2d, v24.2d, v29.2d, #50\n\t" + "eor x22, x22, %x[state]\n\t" + "xar v24.2d, v21.2d, v26.2d, #62\n\t" + "eor x3, x3, %[r]\n\t" + "xar v21.2d, v8.2d, v28.2d, #9\n\t" + "eor x8, x8, %[r]\n\t" + "xar v8.2d, v16.2d, v26.2d, #19\n\t" + "eor x13, x13, %[r]\n\t" + "xar v16.2d, v5.2d, v25.2d, #28\n\t" + "eor x19, x19, %[r]\n\t" + "xar v5.2d, v3.2d, v28.2d, #36\n\t" + "eor x24, x24, %[r]\n\t" + "xar v3.2d, v18.2d, v28.2d, #43\n\t" + "ldr %x[state], [x29, #32]\n\t" + "xar v18.2d, v17.2d, v27.2d, #49\n\t" + "ldr %[r], [x29, #24]\n\t" + "xar v17.2d, v11.2d, v26.2d, #54\n\t" + "eor x28, x28, x30, ror 63\n\t" + "xar v11.2d, v7.2d, v27.2d, #58\n\t" + "eor x30, x30, %[r], ror 63\n\t" + "xar v7.2d, v10.2d, v25.2d, #61\n\t" + "eor %[r], %[r], %x[state], ror 63\n\t" + /* Row Mix */ + "mov v25.16b, v0.16b\n\t" + "eor x5, x5, x28\n\t" + "mov v26.16b, v1.16b\n\t" + "eor x10, x10, x28\n\t" + "bcax v0.16b, v25.16b, v2.16b, v26.16b\n\t" + "eor x15, x15, x28\n\t" + "bcax v1.16b, v26.16b, v3.16b, v2.16b\n\t" + "eor x21, x21, x28\n\t" + "bcax v2.16b, v2.16b, v4.16b, v3.16b\n\t" + "eor x26, x26, x28\n\t" + "bcax v3.16b, v3.16b, v25.16b, v4.16b\n\t" + "eor x2, x2, x30\n\t" + "bcax v4.16b, v4.16b, v26.16b, v25.16b\n\t" + "eor x7, x7, x30\n\t" + "mov v25.16b, v5.16b\n\t" + "eor x12, x12, x30\n\t" + "mov v26.16b, v6.16b\n\t" + "eor x17, x17, x30\n\t" + "bcax v5.16b, v25.16b, v7.16b, v26.16b\n\t" + "eor x23, x23, x30\n\t" + "bcax v6.16b, v26.16b, v8.16b, v7.16b\n\t" + "eor x4, x4, %[r]\n\t" + "bcax v7.16b, v7.16b, v9.16b, v8.16b\n\t" + "eor x9, x9, %[r]\n\t" + "bcax v8.16b, v8.16b, v25.16b, v9.16b\n\t" + "eor x14, x14, %[r]\n\t" + "bcax v9.16b, v9.16b, v26.16b, v25.16b\n\t" + "eor x20, x20, %[r]\n\t" + "mov v26.16b, v11.16b\n\t" + "eor x25, x25, %[r]\n\t" + /* Swap Rotate Base */ + "bcax v10.16b, v30.16b, v12.16b, v26.16b\n\t" + "ror %x[state], x2, #63\n\t" + "bcax v11.16b, v26.16b, v13.16b, v12.16b\n\t" + "ror x2, x7, #20\n\t" + "bcax v12.16b, v12.16b, v14.16b, v13.16b\n\t" + "ror x7, x10, #44\n\t" + "bcax v13.16b, v13.16b, v30.16b, v14.16b\n\t" + "ror x10, x24, #3\n\t" + "bcax v14.16b, v14.16b, v26.16b, v30.16b\n\t" + "ror x24, x15, #25\n\t" + "mov v25.16b, v15.16b\n\t" + "ror x15, x22, #46\n\t" + "mov v26.16b, v16.16b\n\t" + "ror x22, x3, #2\n\t" + "bcax v15.16b, v25.16b, v17.16b, v26.16b\n\t" + "ror x3, x13, #21\n\t" + "bcax v16.16b, v26.16b, v18.16b, v17.16b\n\t" + "ror x13, x14, #39\n\t" + "bcax v17.16b, v17.16b, v19.16b, v18.16b\n\t" + "ror x14, x21, #56\n\t" + "bcax v18.16b, v18.16b, v25.16b, v19.16b\n\t" + "ror x21, x25, #8\n\t" + "bcax v19.16b, v19.16b, v26.16b, v25.16b\n\t" + "ror x25, x16, #23\n\t" + "mov v25.16b, v20.16b\n\t" + "ror x16, x5, #37\n\t" + "mov v26.16b, v21.16b\n\t" + "ror x5, x26, #50\n\t" + "bcax v20.16b, v25.16b, v22.16b, v26.16b\n\t" + "ror x26, x23, #62\n\t" + "bcax v21.16b, v26.16b, v23.16b, v22.16b\n\t" + "ror x23, x9, #9\n\t" + "bcax v22.16b, v22.16b, v24.16b, v23.16b\n\t" + "ror x9, x17, #19\n\t" + "bcax v23.16b, v23.16b, v25.16b, v24.16b\n\t" + "ror x17, x6, #28\n\t" + "bcax v24.16b, v24.16b, v26.16b, v25.16b\n\t" + "ror x6, x4, #36\n\t" + "ror x4, x20, #43\n\t" + "ror x20, x19, #49\n\t" + "ror x19, x12, #54\n\t" + "ror x12, x8, #58\n\t" + "ror x8, x11, #61\n\t" + /* Row Mix Base */ + "bic x11, x3, x2\n\t" + "bic %[r], x4, x3\n\t" + "bic x28, x1, x5\n\t" + "bic x30, x2, x1\n\t" + "eor x1, x1, x11\n\t" + "eor x2, x2, %[r]\n\t" + "bic x11, x5, x4\n\t" + "eor x4, x4, x28\n\t" + "eor x3, x3, x11\n\t" + "eor x5, x5, x30\n\t" + "bic x11, x8, x7\n\t" + "bic %[r], x9, x8\n\t" + "bic x28, x6, x10\n\t" + "bic x30, x7, x6\n\t" + "eor x6, x6, x11\n\t" + "eor x7, x7, %[r]\n\t" + "bic x11, x10, x9\n\t" + "eor x9, x9, x28\n\t" + "eor x8, x8, x11\n\t" + "eor x10, x10, x30\n\t" + "bic x11, x13, x12\n\t" + "bic %[r], x14, x13\n\t" + "bic x28, %x[state], x15\n\t" + "bic x30, x12, %x[state]\n\t" + "eor x11, %x[state], x11\n\t" + "eor x12, x12, %[r]\n\t" + "bic %x[state], x15, x14\n\t" + "eor x14, x14, x28\n\t" + "eor x13, x13, %x[state]\n\t" + "eor x15, x15, x30\n\t" + "bic %x[state], x19, x17\n\t" + "bic %[r], x20, x19\n\t" + "bic x28, x16, x21\n\t" + "bic x30, x17, x16\n\t" + "eor x16, x16, %x[state]\n\t" + "eor x17, x17, %[r]\n\t" + "bic %x[state], x21, x20\n\t" + "eor x20, x20, x28\n\t" + "eor x19, x19, %x[state]\n\t" + "eor x21, x21, x30\n\t" + "bic %x[state], x24, x23\n\t" + "bic %[r], x25, x24\n\t" + "bic x28, x22, x26\n\t" + "bic x30, x23, x22\n\t" + "eor x22, x22, %x[state]\n\t" + "eor x23, x23, %[r]\n\t" + "bic %x[state], x26, x25\n\t" + "eor x25, x25, x28\n\t" + "eor x24, x24, %x[state]\n\t" + "eor x26, x26, x30\n\t" + /* Done transforming */ + "ldp %[r], x28, [x29, #48]\n\t" + "ldr %x[state], [%[r]], #8\n\t" + "subs x28, x28, #1\n\t" + "mov v30.d[0], %x[state]\n\t" + "mov v30.d[1], %x[state]\n\t" + "eor x1, x1, %x[state]\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "b.ne L_SHA3_transform_blocksx3_crypto_begin_%=\n\t" + "ldr %x[state], [x29, #40]\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" + "st1 {v24.d}[0], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" + "st1 {v24.d}[1], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "stp x1, x2, [%x[state]]\n\t" + "stp x3, x4, [%x[state], #16]\n\t" + "stp x5, x6, [%x[state], #32]\n\t" + "stp x7, x8, [%x[state], #48]\n\t" + "stp x9, x10, [%x[state], #64]\n\t" + "stp x11, x12, [%x[state], #80]\n\t" + "stp x13, x14, [%x[state], #96]\n\t" + "stp x15, x16, [%x[state], #112]\n\t" + "stp x17, x19, [%x[state], #128]\n\t" + "stp x20, x21, [%x[state], #144]\n\t" + "stp x22, x23, [%x[state], #160]\n\t" + "stp x24, x25, [%x[state], #176]\n\t" + "str x26, [%x[state], #192]\n\t" + "ldp x29, x30, [sp], #0x40\n\t" + : [state] "+r" (state) + : [r] "r" (r) + : "memory", "cc", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", + "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", + "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x28", "v0", "v1", + "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", + "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31" + ); +} + +void sha3_128_blocksx3_seed_crypto(word64* state, byte* seed) +{ + const word64* r = L_sha3_aarch64_r; + __asm__ __volatile__ ( + "stp x29, x30, [sp, #-64]!\n\t" + "add x29, sp, #0\n\t" +#ifdef __APPLE__ + ".arch_extension sha3\n\t" +#endif /* __APPLE__ */ + "str %x[state], [x29, #40]\n\t" + "add %x[state], %x[state], #32\n\t" + "ld1 {v4.d}[0], [%x[state]]\n\t" + "ldp x2, x3, [%x[seed]], #16\n\t" + "add %x[state], %x[state], #0xc8\n\t" + "ld1 {v4.d}[1], [%x[state]]\n\t" + "ldp x4, x5, [%x[seed]], #16\n\t" + "ldr x6, [%x[state], #200]\n\t" + "eor v5.16b, v5.16b, v5.16b\n\t" + "eor x7, x7, x7\n\t" + "eor v6.16b, v6.16b, v6.16b\n\t" + "eor x8, x8, x8\n\t" + "eor v7.16b, v7.16b, v7.16b\n\t" + "eor x9, x9, x9\n\t" + "eor v8.16b, v8.16b, v8.16b\n\t" + "eor x10, x10, x10\n\t" + "eor v9.16b, v9.16b, v9.16b\n\t" + "eor x11, x11, x11\n\t" + "eor v10.16b, v10.16b, v10.16b\n\t" + "eor x12, x12, x12\n\t" + "eor v11.16b, v11.16b, v11.16b\n\t" + "eor x13, x13, x13\n\t" + "eor v12.16b, v12.16b, v12.16b\n\t" + "eor x14, x14, x14\n\t" + "eor v13.16b, v13.16b, v13.16b\n\t" + "eor x15, x15, x15\n\t" + "eor v14.16b, v14.16b, v14.16b\n\t" + "eor x16, x16, x16\n\t" + "eor v15.16b, v15.16b, v15.16b\n\t" + "eor x17, x17, x17\n\t" + "eor v16.16b, v16.16b, v16.16b\n\t" + "eor x19, x19, x19\n\t" + "eor v17.16b, v17.16b, v17.16b\n\t" + "eor x20, x20, x20\n\t" + "eor v18.16b, v18.16b, v18.16b\n\t" + "eor x21, x21, x21\n\t" + "eor v19.16b, v19.16b, v19.16b\n\t" + "eor x22, x22, x22\n\t" + "movz x23, #0x8000, lsl 48\n\t" + "eor v21.16b, v21.16b, v21.16b\n\t" + "eor x24, x24, x24\n\t" + "eor v22.16b, v22.16b, v22.16b\n\t" + "eor x25, x25, x25\n\t" + "eor v23.16b, v23.16b, v23.16b\n\t" + "eor x26, x26, x26\n\t" + "eor v24.16b, v24.16b, v24.16b\n\t" + "eor x27, x27, x27\n\t" + "dup v0.2d, x2\n\t" + "dup v1.2d, x3\n\t" + "dup v2.2d, x4\n\t" + "dup v3.2d, x5\n\t" + "dup v20.2d, x23\n\t" + "mov %x[seed], #24\n\t" + /* Start of 24 rounds */ + "\n" + "L_SHA3_shake128_blocksx3_seed_crypto_begin_%=:\n\t" + "stp %[r], %x[seed], [x29, #48]\n\t" + /* Col Mix */ + "eor3 v31.16b, v0.16b, v5.16b, v10.16b\n\t" + "eor %x[state], x6, x11\n\t" + "eor3 v27.16b, v1.16b, v6.16b, v11.16b\n\t" + "eor x30, x2, x7\n\t" + "eor3 v28.16b, v2.16b, v7.16b, v12.16b\n\t" + "eor %[r], x4, x9\n\t" + "eor3 v29.16b, v3.16b, v8.16b, v13.16b\n\t" + "eor %x[state], %x[state], x16\n\t" + "eor3 v30.16b, v4.16b, v9.16b, v14.16b\n\t" + "eor x30, x30, x12\n\t" + "eor3 v31.16b, v31.16b, v15.16b, v20.16b\n\t" + "eor %[r], %[r], x14\n\t" + "eor3 v27.16b, v27.16b, v16.16b, v21.16b\n\t" + "eor %x[state], %x[state], x22\n\t" + "eor3 v28.16b, v28.16b, v17.16b, v22.16b\n\t" + "eor x30, x30, x17\n\t" + "eor3 v29.16b, v29.16b, v18.16b, v23.16b\n\t" + "eor %[r], %[r], x20\n\t" + "eor3 v30.16b, v30.16b, v19.16b, v24.16b\n\t" + "eor %x[state], %x[state], x27\n\t" + "rax1 v25.2d, v30.2d, v27.2d\n\t" + "eor x30, x30, x23\n\t" + "rax1 v26.2d, v31.2d, v28.2d\n\t" + "eor %[r], %[r], x25\n\t" + "rax1 v27.2d, v27.2d, v29.2d\n\t" + "str %x[state], [x29, #32]\n\t" + "rax1 v28.2d, v28.2d, v30.2d\n\t" + "str %[r], [x29, #24]\n\t" + "rax1 v29.2d, v29.2d, v31.2d\n\t" + "eor %x[seed], x3, x8\n\t" + "eor v0.16b, v0.16b, v25.16b\n\t" + "xar v30.2d, v1.2d, v26.2d, #63\n\t" + "eor %[r], x5, x10\n\t" + "xar v1.2d, v6.2d, v26.2d, #20\n\t" + "eor %x[seed], %x[seed], x13\n\t" + "xar v6.2d, v9.2d, v29.2d, #44\n\t" + "eor %[r], %[r], x15\n\t" + "xar v9.2d, v22.2d, v27.2d, #3\n\t" + "eor %x[seed], %x[seed], x19\n\t" + "xar v22.2d, v14.2d, v29.2d, #25\n\t" + "eor %[r], %[r], x21\n\t" + "xar v14.2d, v20.2d, v25.2d, #46\n\t" + "eor %x[seed], %x[seed], x24\n\t" + "xar v20.2d, v2.2d, v27.2d, #2\n\t" + "eor %[r], %[r], x26\n\t" + "xar v2.2d, v12.2d, v27.2d, #21\n\t" + "eor %x[state], %x[state], %x[seed], ror 63\n\t" + "xar v12.2d, v13.2d, v28.2d, #39\n\t" + "eor %x[seed], %x[seed], %[r], ror 63\n\t" + "xar v13.2d, v19.2d, v29.2d, #56\n\t" + "eor x2, x2, %x[state]\n\t" + "xar v19.2d, v23.2d, v28.2d, #8\n\t" + "eor x7, x7, %x[state]\n\t" + "xar v23.2d, v15.2d, v25.2d, #23\n\t" + "eor x12, x12, %x[state]\n\t" + "xar v15.2d, v4.2d, v29.2d, #37\n\t" + "eor x17, x17, %x[state]\n\t" + "xar v4.2d, v24.2d, v29.2d, #50\n\t" + "eor x23, x23, %x[state]\n\t" + "xar v24.2d, v21.2d, v26.2d, #62\n\t" + "eor x4, x4, %x[seed]\n\t" + "xar v21.2d, v8.2d, v28.2d, #9\n\t" + "eor x9, x9, %x[seed]\n\t" + "xar v8.2d, v16.2d, v26.2d, #19\n\t" + "eor x14, x14, %x[seed]\n\t" + "xar v16.2d, v5.2d, v25.2d, #28\n\t" + "eor x20, x20, %x[seed]\n\t" + "xar v5.2d, v3.2d, v28.2d, #36\n\t" + "eor x25, x25, %x[seed]\n\t" + "xar v3.2d, v18.2d, v28.2d, #43\n\t" + "ldr %x[state], [x29, #32]\n\t" + "xar v18.2d, v17.2d, v27.2d, #49\n\t" + "ldr %x[seed], [x29, #24]\n\t" + "xar v17.2d, v11.2d, v26.2d, #54\n\t" + "eor %[r], %[r], x30, ror 63\n\t" + "xar v11.2d, v7.2d, v27.2d, #58\n\t" + "eor x30, x30, %x[seed], ror 63\n\t" + "xar v7.2d, v10.2d, v25.2d, #61\n\t" + "eor %x[seed], %x[seed], %x[state], ror 63\n\t" + /* Row Mix */ + "mov v25.16b, v0.16b\n\t" + "eor x6, x6, %[r]\n\t" + "mov v26.16b, v1.16b\n\t" + "eor x11, x11, %[r]\n\t" + "bcax v0.16b, v25.16b, v2.16b, v26.16b\n\t" + "eor x16, x16, %[r]\n\t" + "bcax v1.16b, v26.16b, v3.16b, v2.16b\n\t" + "eor x22, x22, %[r]\n\t" + "bcax v2.16b, v2.16b, v4.16b, v3.16b\n\t" + "eor x27, x27, %[r]\n\t" + "bcax v3.16b, v3.16b, v25.16b, v4.16b\n\t" + "eor x3, x3, x30\n\t" + "bcax v4.16b, v4.16b, v26.16b, v25.16b\n\t" + "eor x8, x8, x30\n\t" + "mov v25.16b, v5.16b\n\t" + "eor x13, x13, x30\n\t" + "mov v26.16b, v6.16b\n\t" + "eor x19, x19, x30\n\t" + "bcax v5.16b, v25.16b, v7.16b, v26.16b\n\t" + "eor x24, x24, x30\n\t" + "bcax v6.16b, v26.16b, v8.16b, v7.16b\n\t" + "eor x5, x5, %x[seed]\n\t" + "bcax v7.16b, v7.16b, v9.16b, v8.16b\n\t" + "eor x10, x10, %x[seed]\n\t" + "bcax v8.16b, v8.16b, v25.16b, v9.16b\n\t" + "eor x15, x15, %x[seed]\n\t" + "bcax v9.16b, v9.16b, v26.16b, v25.16b\n\t" + "eor x21, x21, %x[seed]\n\t" + "mov v26.16b, v11.16b\n\t" + "eor x26, x26, %x[seed]\n\t" + /* Swap Rotate Base */ + "bcax v10.16b, v30.16b, v12.16b, v26.16b\n\t" + "ror %x[state], x3, #63\n\t" + "bcax v11.16b, v26.16b, v13.16b, v12.16b\n\t" + "ror x3, x8, #20\n\t" + "bcax v12.16b, v12.16b, v14.16b, v13.16b\n\t" + "ror x8, x11, #44\n\t" + "bcax v13.16b, v13.16b, v30.16b, v14.16b\n\t" + "ror x11, x25, #3\n\t" + "bcax v14.16b, v14.16b, v26.16b, v30.16b\n\t" + "ror x25, x16, #25\n\t" + "mov v25.16b, v15.16b\n\t" + "ror x16, x23, #46\n\t" + "mov v26.16b, v16.16b\n\t" + "ror x23, x4, #2\n\t" + "bcax v15.16b, v25.16b, v17.16b, v26.16b\n\t" + "ror x4, x14, #21\n\t" + "bcax v16.16b, v26.16b, v18.16b, v17.16b\n\t" + "ror x14, x15, #39\n\t" + "bcax v17.16b, v17.16b, v19.16b, v18.16b\n\t" + "ror x15, x22, #56\n\t" + "bcax v18.16b, v18.16b, v25.16b, v19.16b\n\t" + "ror x22, x26, #8\n\t" + "bcax v19.16b, v19.16b, v26.16b, v25.16b\n\t" + "ror x26, x17, #23\n\t" + "mov v25.16b, v20.16b\n\t" + "ror x17, x6, #37\n\t" + "mov v26.16b, v21.16b\n\t" + "ror x6, x27, #50\n\t" + "bcax v20.16b, v25.16b, v22.16b, v26.16b\n\t" + "ror x27, x24, #62\n\t" + "bcax v21.16b, v26.16b, v23.16b, v22.16b\n\t" + "ror x24, x10, #9\n\t" + "bcax v22.16b, v22.16b, v24.16b, v23.16b\n\t" + "ror x10, x19, #19\n\t" + "bcax v23.16b, v23.16b, v25.16b, v24.16b\n\t" + "ror x19, x7, #28\n\t" + "bcax v24.16b, v24.16b, v26.16b, v25.16b\n\t" + "ror x7, x5, #36\n\t" + "ror x5, x21, #43\n\t" + "ror x21, x20, #49\n\t" + "ror x20, x13, #54\n\t" + "ror x13, x9, #58\n\t" + "ror x9, x12, #61\n\t" + /* Row Mix Base */ + "bic x12, x4, x3\n\t" + "bic %x[seed], x5, x4\n\t" + "bic %[r], x2, x6\n\t" + "bic x30, x3, x2\n\t" + "eor x2, x2, x12\n\t" + "eor x3, x3, %x[seed]\n\t" + "bic x12, x6, x5\n\t" + "eor x5, x5, %[r]\n\t" + "eor x4, x4, x12\n\t" + "eor x6, x6, x30\n\t" + "bic x12, x9, x8\n\t" + "bic %x[seed], x10, x9\n\t" + "bic %[r], x7, x11\n\t" + "bic x30, x8, x7\n\t" + "eor x7, x7, x12\n\t" + "eor x8, x8, %x[seed]\n\t" + "bic x12, x11, x10\n\t" + "eor x10, x10, %[r]\n\t" + "eor x9, x9, x12\n\t" + "eor x11, x11, x30\n\t" + "bic x12, x14, x13\n\t" + "bic %x[seed], x15, x14\n\t" + "bic %[r], %x[state], x16\n\t" + "bic x30, x13, %x[state]\n\t" + "eor x12, %x[state], x12\n\t" + "eor x13, x13, %x[seed]\n\t" + "bic %x[state], x16, x15\n\t" + "eor x15, x15, %[r]\n\t" + "eor x14, x14, %x[state]\n\t" + "eor x16, x16, x30\n\t" + "bic %x[state], x20, x19\n\t" + "bic %x[seed], x21, x20\n\t" + "bic %[r], x17, x22\n\t" + "bic x30, x19, x17\n\t" + "eor x17, x17, %x[state]\n\t" + "eor x19, x19, %x[seed]\n\t" + "bic %x[state], x22, x21\n\t" + "eor x21, x21, %[r]\n\t" + "eor x20, x20, %x[state]\n\t" + "eor x22, x22, x30\n\t" + "bic %x[state], x25, x24\n\t" + "bic %x[seed], x26, x25\n\t" + "bic %[r], x23, x27\n\t" + "bic x30, x24, x23\n\t" + "eor x23, x23, %x[state]\n\t" + "eor x24, x24, %x[seed]\n\t" + "bic %x[state], x27, x26\n\t" + "eor x26, x26, %[r]\n\t" + "eor x25, x25, %x[state]\n\t" + "eor x27, x27, x30\n\t" + /* Done transforming */ + "ldp %[r], %x[seed], [x29, #48]\n\t" + "ldr %x[state], [%[r]], #8\n\t" + "subs %x[seed], %x[seed], #1\n\t" + "mov v30.d[0], %x[state]\n\t" + "mov v30.d[1], %x[state]\n\t" + "eor x2, x2, %x[state]\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "b.ne L_SHA3_shake128_blocksx3_seed_crypto_begin_%=\n\t" + "ldr %x[state], [x29, #40]\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" + "st1 {v24.d}[0], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" + "st1 {v24.d}[1], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "stp x2, x3, [%x[state]]\n\t" + "stp x4, x5, [%x[state], #16]\n\t" + "stp x6, x7, [%x[state], #32]\n\t" + "stp x8, x9, [%x[state], #48]\n\t" + "stp x10, x11, [%x[state], #64]\n\t" + "stp x12, x13, [%x[state], #80]\n\t" + "stp x14, x15, [%x[state], #96]\n\t" + "stp x16, x17, [%x[state], #112]\n\t" + "stp x19, x20, [%x[state], #128]\n\t" + "stp x21, x22, [%x[state], #144]\n\t" + "stp x23, x24, [%x[state], #160]\n\t" + "stp x25, x26, [%x[state], #176]\n\t" + "str x27, [%x[state], #192]\n\t" + "ldp x29, x30, [sp], #0x40\n\t" + : [state] "+r" (state), [seed] "+r" (seed) + : [r] "r" (r) + : "memory", "cc", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", + "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", + "x21", "x22", "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", + "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", + "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31" + ); +} + +void sha3_256_blocksx3_seed_crypto(word64* state, byte* seed) +{ + const word64* r = L_sha3_aarch64_r; + __asm__ __volatile__ ( + "stp x29, x30, [sp, #-64]!\n\t" + "add x29, sp, #0\n\t" +#ifdef __APPLE__ + ".arch_extension sha3\n\t" +#endif /* __APPLE__ */ + "str %x[state], [x29, #40]\n\t" + "add %x[state], %x[state], #32\n\t" + "ld1 {v4.d}[0], [%x[state]]\n\t" + "ldp x2, x3, [%x[seed]], #16\n\t" + "add %x[state], %x[state], #0xc8\n\t" + "ld1 {v4.d}[1], [%x[state]]\n\t" + "ldp x4, x5, [%x[seed]], #16\n\t" + "ldr x6, [%x[state], #200]\n\t" + "eor v5.16b, v5.16b, v5.16b\n\t" + "eor x7, x7, x7\n\t" + "eor v6.16b, v6.16b, v6.16b\n\t" + "eor x8, x8, x8\n\t" + "eor v7.16b, v7.16b, v7.16b\n\t" + "eor x9, x9, x9\n\t" + "eor v8.16b, v8.16b, v8.16b\n\t" + "eor x10, x10, x10\n\t" + "eor v9.16b, v9.16b, v9.16b\n\t" + "eor x11, x11, x11\n\t" + "eor v10.16b, v10.16b, v10.16b\n\t" + "eor x12, x12, x12\n\t" + "eor v11.16b, v11.16b, v11.16b\n\t" + "eor x13, x13, x13\n\t" + "eor v12.16b, v12.16b, v12.16b\n\t" + "eor x14, x14, x14\n\t" + "eor v13.16b, v13.16b, v13.16b\n\t" + "eor x15, x15, x15\n\t" + "eor v14.16b, v14.16b, v14.16b\n\t" + "eor x16, x16, x16\n\t" + "eor v15.16b, v15.16b, v15.16b\n\t" + "eor x17, x17, x17\n\t" + "movz x19, #0x8000, lsl 48\n\t" + "eor v17.16b, v17.16b, v17.16b\n\t" + "eor x20, x20, x20\n\t" + "eor v18.16b, v18.16b, v18.16b\n\t" + "eor x21, x21, x21\n\t" + "eor v19.16b, v19.16b, v19.16b\n\t" + "eor x22, x22, x22\n\t" + "eor v20.16b, v20.16b, v20.16b\n\t" + "eor x23, x23, x23\n\t" + "eor v21.16b, v21.16b, v21.16b\n\t" + "eor x24, x24, x24\n\t" + "eor v22.16b, v22.16b, v22.16b\n\t" + "eor x25, x25, x25\n\t" + "eor v23.16b, v23.16b, v23.16b\n\t" + "eor x26, x26, x26\n\t" + "eor v24.16b, v24.16b, v24.16b\n\t" + "eor x27, x27, x27\n\t" + "dup v0.2d, x2\n\t" + "dup v1.2d, x3\n\t" + "dup v2.2d, x4\n\t" + "dup v3.2d, x5\n\t" + "dup v16.2d, x19\n\t" + "mov %x[seed], #24\n\t" + /* Start of 24 rounds */ + "\n" + "L_SHA3_shake256_blocksx3_seed_crypto_begin_%=:\n\t" + "stp %[r], %x[seed], [x29, #48]\n\t" + /* Col Mix */ + "eor3 v31.16b, v0.16b, v5.16b, v10.16b\n\t" + "eor %x[state], x6, x11\n\t" + "eor3 v27.16b, v1.16b, v6.16b, v11.16b\n\t" + "eor x30, x2, x7\n\t" + "eor3 v28.16b, v2.16b, v7.16b, v12.16b\n\t" + "eor %[r], x4, x9\n\t" + "eor3 v29.16b, v3.16b, v8.16b, v13.16b\n\t" + "eor %x[state], %x[state], x16\n\t" + "eor3 v30.16b, v4.16b, v9.16b, v14.16b\n\t" + "eor x30, x30, x12\n\t" + "eor3 v31.16b, v31.16b, v15.16b, v20.16b\n\t" + "eor %[r], %[r], x14\n\t" + "eor3 v27.16b, v27.16b, v16.16b, v21.16b\n\t" + "eor %x[state], %x[state], x22\n\t" + "eor3 v28.16b, v28.16b, v17.16b, v22.16b\n\t" + "eor x30, x30, x17\n\t" + "eor3 v29.16b, v29.16b, v18.16b, v23.16b\n\t" + "eor %[r], %[r], x20\n\t" + "eor3 v30.16b, v30.16b, v19.16b, v24.16b\n\t" + "eor %x[state], %x[state], x27\n\t" + "rax1 v25.2d, v30.2d, v27.2d\n\t" + "eor x30, x30, x23\n\t" + "rax1 v26.2d, v31.2d, v28.2d\n\t" + "eor %[r], %[r], x25\n\t" + "rax1 v27.2d, v27.2d, v29.2d\n\t" + "str %x[state], [x29, #32]\n\t" + "rax1 v28.2d, v28.2d, v30.2d\n\t" + "str %[r], [x29, #24]\n\t" + "rax1 v29.2d, v29.2d, v31.2d\n\t" + "eor %x[seed], x3, x8\n\t" + "eor v0.16b, v0.16b, v25.16b\n\t" + "xar v30.2d, v1.2d, v26.2d, #63\n\t" + "eor %[r], x5, x10\n\t" + "xar v1.2d, v6.2d, v26.2d, #20\n\t" + "eor %x[seed], %x[seed], x13\n\t" + "xar v6.2d, v9.2d, v29.2d, #44\n\t" + "eor %[r], %[r], x15\n\t" + "xar v9.2d, v22.2d, v27.2d, #3\n\t" + "eor %x[seed], %x[seed], x19\n\t" + "xar v22.2d, v14.2d, v29.2d, #25\n\t" + "eor %[r], %[r], x21\n\t" + "xar v14.2d, v20.2d, v25.2d, #46\n\t" + "eor %x[seed], %x[seed], x24\n\t" + "xar v20.2d, v2.2d, v27.2d, #2\n\t" + "eor %[r], %[r], x26\n\t" + "xar v2.2d, v12.2d, v27.2d, #21\n\t" + "eor %x[state], %x[state], %x[seed], ror 63\n\t" + "xar v12.2d, v13.2d, v28.2d, #39\n\t" + "eor %x[seed], %x[seed], %[r], ror 63\n\t" + "xar v13.2d, v19.2d, v29.2d, #56\n\t" + "eor x2, x2, %x[state]\n\t" + "xar v19.2d, v23.2d, v28.2d, #8\n\t" + "eor x7, x7, %x[state]\n\t" + "xar v23.2d, v15.2d, v25.2d, #23\n\t" + "eor x12, x12, %x[state]\n\t" + "xar v15.2d, v4.2d, v29.2d, #37\n\t" + "eor x17, x17, %x[state]\n\t" + "xar v4.2d, v24.2d, v29.2d, #50\n\t" + "eor x23, x23, %x[state]\n\t" + "xar v24.2d, v21.2d, v26.2d, #62\n\t" + "eor x4, x4, %x[seed]\n\t" + "xar v21.2d, v8.2d, v28.2d, #9\n\t" + "eor x9, x9, %x[seed]\n\t" + "xar v8.2d, v16.2d, v26.2d, #19\n\t" + "eor x14, x14, %x[seed]\n\t" + "xar v16.2d, v5.2d, v25.2d, #28\n\t" + "eor x20, x20, %x[seed]\n\t" + "xar v5.2d, v3.2d, v28.2d, #36\n\t" + "eor x25, x25, %x[seed]\n\t" + "xar v3.2d, v18.2d, v28.2d, #43\n\t" + "ldr %x[state], [x29, #32]\n\t" + "xar v18.2d, v17.2d, v27.2d, #49\n\t" + "ldr %x[seed], [x29, #24]\n\t" + "xar v17.2d, v11.2d, v26.2d, #54\n\t" + "eor %[r], %[r], x30, ror 63\n\t" + "xar v11.2d, v7.2d, v27.2d, #58\n\t" + "eor x30, x30, %x[seed], ror 63\n\t" + "xar v7.2d, v10.2d, v25.2d, #61\n\t" + "eor %x[seed], %x[seed], %x[state], ror 63\n\t" + /* Row Mix */ + "mov v25.16b, v0.16b\n\t" + "eor x6, x6, %[r]\n\t" + "mov v26.16b, v1.16b\n\t" + "eor x11, x11, %[r]\n\t" + "bcax v0.16b, v25.16b, v2.16b, v26.16b\n\t" + "eor x16, x16, %[r]\n\t" + "bcax v1.16b, v26.16b, v3.16b, v2.16b\n\t" + "eor x22, x22, %[r]\n\t" + "bcax v2.16b, v2.16b, v4.16b, v3.16b\n\t" + "eor x27, x27, %[r]\n\t" + "bcax v3.16b, v3.16b, v25.16b, v4.16b\n\t" + "eor x3, x3, x30\n\t" + "bcax v4.16b, v4.16b, v26.16b, v25.16b\n\t" + "eor x8, x8, x30\n\t" + "mov v25.16b, v5.16b\n\t" + "eor x13, x13, x30\n\t" + "mov v26.16b, v6.16b\n\t" + "eor x19, x19, x30\n\t" + "bcax v5.16b, v25.16b, v7.16b, v26.16b\n\t" + "eor x24, x24, x30\n\t" + "bcax v6.16b, v26.16b, v8.16b, v7.16b\n\t" + "eor x5, x5, %x[seed]\n\t" + "bcax v7.16b, v7.16b, v9.16b, v8.16b\n\t" + "eor x10, x10, %x[seed]\n\t" + "bcax v8.16b, v8.16b, v25.16b, v9.16b\n\t" + "eor x15, x15, %x[seed]\n\t" + "bcax v9.16b, v9.16b, v26.16b, v25.16b\n\t" + "eor x21, x21, %x[seed]\n\t" + "mov v26.16b, v11.16b\n\t" + "eor x26, x26, %x[seed]\n\t" + /* Swap Rotate Base */ + "bcax v10.16b, v30.16b, v12.16b, v26.16b\n\t" + "ror %x[state], x3, #63\n\t" + "bcax v11.16b, v26.16b, v13.16b, v12.16b\n\t" + "ror x3, x8, #20\n\t" + "bcax v12.16b, v12.16b, v14.16b, v13.16b\n\t" + "ror x8, x11, #44\n\t" + "bcax v13.16b, v13.16b, v30.16b, v14.16b\n\t" + "ror x11, x25, #3\n\t" + "bcax v14.16b, v14.16b, v26.16b, v30.16b\n\t" + "ror x25, x16, #25\n\t" + "mov v25.16b, v15.16b\n\t" + "ror x16, x23, #46\n\t" + "mov v26.16b, v16.16b\n\t" + "ror x23, x4, #2\n\t" + "bcax v15.16b, v25.16b, v17.16b, v26.16b\n\t" + "ror x4, x14, #21\n\t" + "bcax v16.16b, v26.16b, v18.16b, v17.16b\n\t" + "ror x14, x15, #39\n\t" + "bcax v17.16b, v17.16b, v19.16b, v18.16b\n\t" + "ror x15, x22, #56\n\t" + "bcax v18.16b, v18.16b, v25.16b, v19.16b\n\t" + "ror x22, x26, #8\n\t" + "bcax v19.16b, v19.16b, v26.16b, v25.16b\n\t" + "ror x26, x17, #23\n\t" + "mov v25.16b, v20.16b\n\t" + "ror x17, x6, #37\n\t" + "mov v26.16b, v21.16b\n\t" + "ror x6, x27, #50\n\t" + "bcax v20.16b, v25.16b, v22.16b, v26.16b\n\t" + "ror x27, x24, #62\n\t" + "bcax v21.16b, v26.16b, v23.16b, v22.16b\n\t" + "ror x24, x10, #9\n\t" + "bcax v22.16b, v22.16b, v24.16b, v23.16b\n\t" + "ror x10, x19, #19\n\t" + "bcax v23.16b, v23.16b, v25.16b, v24.16b\n\t" + "ror x19, x7, #28\n\t" + "bcax v24.16b, v24.16b, v26.16b, v25.16b\n\t" + "ror x7, x5, #36\n\t" + "ror x5, x21, #43\n\t" + "ror x21, x20, #49\n\t" + "ror x20, x13, #54\n\t" + "ror x13, x9, #58\n\t" + "ror x9, x12, #61\n\t" + /* Row Mix Base */ + "bic x12, x4, x3\n\t" + "bic %x[seed], x5, x4\n\t" + "bic %[r], x2, x6\n\t" + "bic x30, x3, x2\n\t" + "eor x2, x2, x12\n\t" + "eor x3, x3, %x[seed]\n\t" + "bic x12, x6, x5\n\t" + "eor x5, x5, %[r]\n\t" + "eor x4, x4, x12\n\t" + "eor x6, x6, x30\n\t" + "bic x12, x9, x8\n\t" + "bic %x[seed], x10, x9\n\t" + "bic %[r], x7, x11\n\t" + "bic x30, x8, x7\n\t" + "eor x7, x7, x12\n\t" + "eor x8, x8, %x[seed]\n\t" + "bic x12, x11, x10\n\t" + "eor x10, x10, %[r]\n\t" + "eor x9, x9, x12\n\t" + "eor x11, x11, x30\n\t" + "bic x12, x14, x13\n\t" + "bic %x[seed], x15, x14\n\t" + "bic %[r], %x[state], x16\n\t" + "bic x30, x13, %x[state]\n\t" + "eor x12, %x[state], x12\n\t" + "eor x13, x13, %x[seed]\n\t" + "bic %x[state], x16, x15\n\t" + "eor x15, x15, %[r]\n\t" + "eor x14, x14, %x[state]\n\t" + "eor x16, x16, x30\n\t" + "bic %x[state], x20, x19\n\t" + "bic %x[seed], x21, x20\n\t" + "bic %[r], x17, x22\n\t" + "bic x30, x19, x17\n\t" + "eor x17, x17, %x[state]\n\t" + "eor x19, x19, %x[seed]\n\t" + "bic %x[state], x22, x21\n\t" + "eor x21, x21, %[r]\n\t" + "eor x20, x20, %x[state]\n\t" + "eor x22, x22, x30\n\t" + "bic %x[state], x25, x24\n\t" + "bic %x[seed], x26, x25\n\t" + "bic %[r], x23, x27\n\t" + "bic x30, x24, x23\n\t" + "eor x23, x23, %x[state]\n\t" + "eor x24, x24, %x[seed]\n\t" + "bic %x[state], x27, x26\n\t" + "eor x26, x26, %[r]\n\t" + "eor x25, x25, %x[state]\n\t" + "eor x27, x27, x30\n\t" + /* Done transforming */ + "ldp %[r], %x[seed], [x29, #48]\n\t" + "ldr %x[state], [%[r]], #8\n\t" + "subs %x[seed], %x[seed], #1\n\t" + "mov v30.d[0], %x[state]\n\t" + "mov v30.d[1], %x[state]\n\t" + "eor x2, x2, %x[state]\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "b.ne L_SHA3_shake256_blocksx3_seed_crypto_begin_%=\n\t" + "ldr %x[state], [x29, #40]\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" + "st1 {v24.d}[0], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" + "st1 {v24.d}[1], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "stp x2, x3, [%x[state]]\n\t" + "stp x4, x5, [%x[state], #16]\n\t" + "stp x6, x7, [%x[state], #32]\n\t" + "stp x8, x9, [%x[state], #48]\n\t" + "stp x10, x11, [%x[state], #64]\n\t" + "stp x12, x13, [%x[state], #80]\n\t" + "stp x14, x15, [%x[state], #96]\n\t" + "stp x16, x17, [%x[state], #112]\n\t" + "stp x19, x20, [%x[state], #128]\n\t" + "stp x21, x22, [%x[state], #144]\n\t" + "stp x23, x24, [%x[state], #160]\n\t" + "stp x25, x26, [%x[state], #176]\n\t" + "str x27, [%x[state], #192]\n\t" + "ldp x29, x30, [sp], #0x40\n\t" + : [state] "+r" (state), [seed] "+r" (seed) + : [r] "r" (r) + : "memory", "cc", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", + "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", + "x21", "x22", "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", + "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", + "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31" + ); +} + +void sha3_256_blocksx3_seed_64_crypto(word64* state, byte* seed) +{ + const word64* r = L_sha3_aarch64_r; + __asm__ __volatile__ ( + "stp x29, x30, [sp, #-64]!\n\t" + "add x29, sp, #0\n\t" +#ifdef __APPLE__ + ".arch_extension sha3\n\t" +#endif /* __APPLE__ */ + "str %x[state], [x29, #40]\n\t" + "add %x[state], %x[state], #0x40\n\t" + "ld1 {v8.d}[0], [%x[state]]\n\t" + "ldp x2, x3, [%x[seed]], #16\n\t" + "add %x[state], %x[state], #0xc8\n\t" + "ld1 {v8.d}[1], [%x[state]]\n\t" + "ldp x4, x5, [%x[seed]], #16\n\t" + "ldp x6, x7, [%x[seed]], #16\n\t" + "ldp x8, x9, [%x[seed]], #16\n\t" + "ldr x10, [%x[state], #200]\n\t" + "eor v9.16b, v9.16b, v9.16b\n\t" + "eor x11, x11, x11\n\t" + "eor v10.16b, v10.16b, v10.16b\n\t" + "eor x12, x12, x12\n\t" + "eor v11.16b, v11.16b, v11.16b\n\t" + "eor x13, x13, x13\n\t" + "eor v12.16b, v12.16b, v12.16b\n\t" + "eor x14, x14, x14\n\t" + "eor v13.16b, v13.16b, v13.16b\n\t" + "eor x15, x15, x15\n\t" + "eor v14.16b, v14.16b, v14.16b\n\t" + "eor x16, x16, x16\n\t" + "eor v15.16b, v15.16b, v15.16b\n\t" + "eor x17, x17, x17\n\t" + "movz x19, #0x8000, lsl 48\n\t" + "eor v17.16b, v17.16b, v17.16b\n\t" + "eor x20, x20, x20\n\t" + "eor v18.16b, v18.16b, v18.16b\n\t" + "eor x21, x21, x21\n\t" + "eor v19.16b, v19.16b, v19.16b\n\t" + "eor x22, x22, x22\n\t" + "eor v20.16b, v20.16b, v20.16b\n\t" + "eor x23, x23, x23\n\t" + "eor v21.16b, v21.16b, v21.16b\n\t" + "eor x24, x24, x24\n\t" + "eor v22.16b, v22.16b, v22.16b\n\t" + "eor x25, x25, x25\n\t" + "eor v23.16b, v23.16b, v23.16b\n\t" + "eor x26, x26, x26\n\t" + "eor v24.16b, v24.16b, v24.16b\n\t" + "eor x27, x27, x27\n\t" + "dup v0.2d, x2\n\t" + "dup v1.2d, x3\n\t" + "dup v2.2d, x4\n\t" + "dup v3.2d, x5\n\t" + "dup v4.2d, x6\n\t" + "dup v5.2d, x7\n\t" + "dup v6.2d, x8\n\t" + "dup v7.2d, x9\n\t" + "dup v16.2d, x19\n\t" + "mov %x[seed], #24\n\t" + /* Start of 24 rounds */ + "\n" + "L_SHA3_shake256_blocksx3_seed_64_crypto_begin_%=:\n\t" + "stp %[r], %x[seed], [x29, #48]\n\t" + /* Col Mix */ + "eor3 v31.16b, v0.16b, v5.16b, v10.16b\n\t" + "eor %x[state], x6, x11\n\t" + "eor3 v27.16b, v1.16b, v6.16b, v11.16b\n\t" + "eor x30, x2, x7\n\t" + "eor3 v28.16b, v2.16b, v7.16b, v12.16b\n\t" + "eor %[r], x4, x9\n\t" + "eor3 v29.16b, v3.16b, v8.16b, v13.16b\n\t" + "eor %x[state], %x[state], x16\n\t" + "eor3 v30.16b, v4.16b, v9.16b, v14.16b\n\t" + "eor x30, x30, x12\n\t" + "eor3 v31.16b, v31.16b, v15.16b, v20.16b\n\t" + "eor %[r], %[r], x14\n\t" + "eor3 v27.16b, v27.16b, v16.16b, v21.16b\n\t" + "eor %x[state], %x[state], x22\n\t" + "eor3 v28.16b, v28.16b, v17.16b, v22.16b\n\t" + "eor x30, x30, x17\n\t" + "eor3 v29.16b, v29.16b, v18.16b, v23.16b\n\t" + "eor %[r], %[r], x20\n\t" + "eor3 v30.16b, v30.16b, v19.16b, v24.16b\n\t" + "eor %x[state], %x[state], x27\n\t" + "rax1 v25.2d, v30.2d, v27.2d\n\t" + "eor x30, x30, x23\n\t" + "rax1 v26.2d, v31.2d, v28.2d\n\t" + "eor %[r], %[r], x25\n\t" + "rax1 v27.2d, v27.2d, v29.2d\n\t" + "str %x[state], [x29, #32]\n\t" + "rax1 v28.2d, v28.2d, v30.2d\n\t" + "str %[r], [x29, #24]\n\t" + "rax1 v29.2d, v29.2d, v31.2d\n\t" + "eor %x[seed], x3, x8\n\t" + "eor v0.16b, v0.16b, v25.16b\n\t" + "xar v30.2d, v1.2d, v26.2d, #63\n\t" + "eor %[r], x5, x10\n\t" + "xar v1.2d, v6.2d, v26.2d, #20\n\t" + "eor %x[seed], %x[seed], x13\n\t" + "xar v6.2d, v9.2d, v29.2d, #44\n\t" + "eor %[r], %[r], x15\n\t" + "xar v9.2d, v22.2d, v27.2d, #3\n\t" + "eor %x[seed], %x[seed], x19\n\t" + "xar v22.2d, v14.2d, v29.2d, #25\n\t" + "eor %[r], %[r], x21\n\t" + "xar v14.2d, v20.2d, v25.2d, #46\n\t" + "eor %x[seed], %x[seed], x24\n\t" + "xar v20.2d, v2.2d, v27.2d, #2\n\t" + "eor %[r], %[r], x26\n\t" + "xar v2.2d, v12.2d, v27.2d, #21\n\t" + "eor %x[state], %x[state], %x[seed], ror 63\n\t" + "xar v12.2d, v13.2d, v28.2d, #39\n\t" + "eor %x[seed], %x[seed], %[r], ror 63\n\t" + "xar v13.2d, v19.2d, v29.2d, #56\n\t" + "eor x2, x2, %x[state]\n\t" + "xar v19.2d, v23.2d, v28.2d, #8\n\t" + "eor x7, x7, %x[state]\n\t" + "xar v23.2d, v15.2d, v25.2d, #23\n\t" + "eor x12, x12, %x[state]\n\t" + "xar v15.2d, v4.2d, v29.2d, #37\n\t" + "eor x17, x17, %x[state]\n\t" + "xar v4.2d, v24.2d, v29.2d, #50\n\t" + "eor x23, x23, %x[state]\n\t" + "xar v24.2d, v21.2d, v26.2d, #62\n\t" + "eor x4, x4, %x[seed]\n\t" + "xar v21.2d, v8.2d, v28.2d, #9\n\t" + "eor x9, x9, %x[seed]\n\t" + "xar v8.2d, v16.2d, v26.2d, #19\n\t" + "eor x14, x14, %x[seed]\n\t" + "xar v16.2d, v5.2d, v25.2d, #28\n\t" + "eor x20, x20, %x[seed]\n\t" + "xar v5.2d, v3.2d, v28.2d, #36\n\t" + "eor x25, x25, %x[seed]\n\t" + "xar v3.2d, v18.2d, v28.2d, #43\n\t" + "ldr %x[state], [x29, #32]\n\t" + "xar v18.2d, v17.2d, v27.2d, #49\n\t" + "ldr %x[seed], [x29, #24]\n\t" + "xar v17.2d, v11.2d, v26.2d, #54\n\t" + "eor %[r], %[r], x30, ror 63\n\t" + "xar v11.2d, v7.2d, v27.2d, #58\n\t" + "eor x30, x30, %x[seed], ror 63\n\t" + "xar v7.2d, v10.2d, v25.2d, #61\n\t" + "eor %x[seed], %x[seed], %x[state], ror 63\n\t" + /* Row Mix */ + "mov v25.16b, v0.16b\n\t" + "eor x6, x6, %[r]\n\t" + "mov v26.16b, v1.16b\n\t" + "eor x11, x11, %[r]\n\t" + "bcax v0.16b, v25.16b, v2.16b, v26.16b\n\t" + "eor x16, x16, %[r]\n\t" + "bcax v1.16b, v26.16b, v3.16b, v2.16b\n\t" + "eor x22, x22, %[r]\n\t" + "bcax v2.16b, v2.16b, v4.16b, v3.16b\n\t" + "eor x27, x27, %[r]\n\t" + "bcax v3.16b, v3.16b, v25.16b, v4.16b\n\t" + "eor x3, x3, x30\n\t" + "bcax v4.16b, v4.16b, v26.16b, v25.16b\n\t" + "eor x8, x8, x30\n\t" + "mov v25.16b, v5.16b\n\t" + "eor x13, x13, x30\n\t" + "mov v26.16b, v6.16b\n\t" + "eor x19, x19, x30\n\t" + "bcax v5.16b, v25.16b, v7.16b, v26.16b\n\t" + "eor x24, x24, x30\n\t" + "bcax v6.16b, v26.16b, v8.16b, v7.16b\n\t" + "eor x5, x5, %x[seed]\n\t" + "bcax v7.16b, v7.16b, v9.16b, v8.16b\n\t" + "eor x10, x10, %x[seed]\n\t" + "bcax v8.16b, v8.16b, v25.16b, v9.16b\n\t" + "eor x15, x15, %x[seed]\n\t" + "bcax v9.16b, v9.16b, v26.16b, v25.16b\n\t" + "eor x21, x21, %x[seed]\n\t" + "mov v26.16b, v11.16b\n\t" + "eor x26, x26, %x[seed]\n\t" + /* Swap Rotate Base */ + "bcax v10.16b, v30.16b, v12.16b, v26.16b\n\t" + "ror %x[state], x3, #63\n\t" + "bcax v11.16b, v26.16b, v13.16b, v12.16b\n\t" + "ror x3, x8, #20\n\t" + "bcax v12.16b, v12.16b, v14.16b, v13.16b\n\t" + "ror x8, x11, #44\n\t" + "bcax v13.16b, v13.16b, v30.16b, v14.16b\n\t" + "ror x11, x25, #3\n\t" + "bcax v14.16b, v14.16b, v26.16b, v30.16b\n\t" + "ror x25, x16, #25\n\t" + "mov v25.16b, v15.16b\n\t" + "ror x16, x23, #46\n\t" + "mov v26.16b, v16.16b\n\t" + "ror x23, x4, #2\n\t" + "bcax v15.16b, v25.16b, v17.16b, v26.16b\n\t" + "ror x4, x14, #21\n\t" + "bcax v16.16b, v26.16b, v18.16b, v17.16b\n\t" + "ror x14, x15, #39\n\t" + "bcax v17.16b, v17.16b, v19.16b, v18.16b\n\t" + "ror x15, x22, #56\n\t" + "bcax v18.16b, v18.16b, v25.16b, v19.16b\n\t" + "ror x22, x26, #8\n\t" + "bcax v19.16b, v19.16b, v26.16b, v25.16b\n\t" + "ror x26, x17, #23\n\t" + "mov v25.16b, v20.16b\n\t" + "ror x17, x6, #37\n\t" + "mov v26.16b, v21.16b\n\t" + "ror x6, x27, #50\n\t" + "bcax v20.16b, v25.16b, v22.16b, v26.16b\n\t" + "ror x27, x24, #62\n\t" + "bcax v21.16b, v26.16b, v23.16b, v22.16b\n\t" + "ror x24, x10, #9\n\t" + "bcax v22.16b, v22.16b, v24.16b, v23.16b\n\t" + "ror x10, x19, #19\n\t" + "bcax v23.16b, v23.16b, v25.16b, v24.16b\n\t" + "ror x19, x7, #28\n\t" + "bcax v24.16b, v24.16b, v26.16b, v25.16b\n\t" + "ror x7, x5, #36\n\t" + "ror x5, x21, #43\n\t" + "ror x21, x20, #49\n\t" + "ror x20, x13, #54\n\t" + "ror x13, x9, #58\n\t" + "ror x9, x12, #61\n\t" + /* Row Mix Base */ + "bic x12, x4, x3\n\t" + "bic %x[seed], x5, x4\n\t" + "bic %[r], x2, x6\n\t" + "bic x30, x3, x2\n\t" + "eor x2, x2, x12\n\t" + "eor x3, x3, %x[seed]\n\t" + "bic x12, x6, x5\n\t" + "eor x5, x5, %[r]\n\t" + "eor x4, x4, x12\n\t" + "eor x6, x6, x30\n\t" + "bic x12, x9, x8\n\t" + "bic %x[seed], x10, x9\n\t" + "bic %[r], x7, x11\n\t" + "bic x30, x8, x7\n\t" + "eor x7, x7, x12\n\t" + "eor x8, x8, %x[seed]\n\t" + "bic x12, x11, x10\n\t" + "eor x10, x10, %[r]\n\t" + "eor x9, x9, x12\n\t" + "eor x11, x11, x30\n\t" + "bic x12, x14, x13\n\t" + "bic %x[seed], x15, x14\n\t" + "bic %[r], %x[state], x16\n\t" + "bic x30, x13, %x[state]\n\t" + "eor x12, %x[state], x12\n\t" + "eor x13, x13, %x[seed]\n\t" + "bic %x[state], x16, x15\n\t" + "eor x15, x15, %[r]\n\t" + "eor x14, x14, %x[state]\n\t" + "eor x16, x16, x30\n\t" + "bic %x[state], x20, x19\n\t" + "bic %x[seed], x21, x20\n\t" + "bic %[r], x17, x22\n\t" + "bic x30, x19, x17\n\t" + "eor x17, x17, %x[state]\n\t" + "eor x19, x19, %x[seed]\n\t" + "bic %x[state], x22, x21\n\t" + "eor x21, x21, %[r]\n\t" + "eor x20, x20, %x[state]\n\t" + "eor x22, x22, x30\n\t" + "bic %x[state], x25, x24\n\t" + "bic %x[seed], x26, x25\n\t" + "bic %[r], x23, x27\n\t" + "bic x30, x24, x23\n\t" + "eor x23, x23, %x[state]\n\t" + "eor x24, x24, %x[seed]\n\t" + "bic %x[state], x27, x26\n\t" + "eor x26, x26, %[r]\n\t" + "eor x25, x25, %x[state]\n\t" + "eor x27, x27, x30\n\t" + /* Done transforming */ + "ldp %[r], %x[seed], [x29, #48]\n\t" + "ldr %x[state], [%[r]], #8\n\t" + "subs %x[seed], %x[seed], #1\n\t" + "mov v30.d[0], %x[state]\n\t" + "mov v30.d[1], %x[state]\n\t" + "eor x2, x2, %x[state]\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "b.ne L_SHA3_shake256_blocksx3_seed_64_crypto_begin_%=\n\t" + "ldr %x[state], [x29, #40]\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" + "st1 {v24.d}[0], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" + "st1 {v24.d}[1], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "stp x2, x3, [%x[state]]\n\t" + "stp x4, x5, [%x[state], #16]\n\t" + "stp x6, x7, [%x[state], #32]\n\t" + "stp x8, x9, [%x[state], #48]\n\t" + "stp x10, x11, [%x[state], #64]\n\t" + "stp x12, x13, [%x[state], #80]\n\t" + "stp x14, x15, [%x[state], #96]\n\t" + "stp x16, x17, [%x[state], #112]\n\t" + "stp x19, x20, [%x[state], #128]\n\t" + "stp x21, x22, [%x[state], #144]\n\t" + "stp x23, x24, [%x[state], #160]\n\t" + "stp x25, x26, [%x[state], #176]\n\t" + "str x27, [%x[state], #192]\n\t" + "ldp x29, x30, [sp], #0x40\n\t" + : [state] "+r" (state), [seed] "+r" (seed) + : [r] "r" (r) + : "memory", "cc", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", + "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", + "x21", "x22", "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", + "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", + "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31" + ); +} + +#endif /* WOLFSSL_ARMASM_CRYPTO_SHA3 */ +void sha3_blocksx3_neon(word64* state) +{ + const word64* r = L_sha3_aarch64_r; + __asm__ __volatile__ ( + "stp x29, x30, [sp, #-64]!\n\t" + "add x29, sp, #0\n\t" + "str %x[state], [x29, #40]\n\t" + "ld4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" + "ld4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" + "ld4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" + "ld4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" + "ld4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" + "ld4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" + "ld1 {v24.d}[0], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "ld4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" + "ld4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" + "ld4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" + "ld4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" + "ld4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" + "ld4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" + "ld1 {v24.d}[1], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "ldp x1, x2, [%x[state]]\n\t" + "ldp x3, x4, [%x[state], #16]\n\t" + "ldp x5, x6, [%x[state], #32]\n\t" + "ldp x7, x8, [%x[state], #48]\n\t" + "ldp x9, x10, [%x[state], #64]\n\t" + "ldp x11, x12, [%x[state], #80]\n\t" + "ldp x13, x14, [%x[state], #96]\n\t" + "ldp x15, x16, [%x[state], #112]\n\t" + "ldp x17, x19, [%x[state], #128]\n\t" + "ldp x20, x21, [%x[state], #144]\n\t" + "ldp x22, x23, [%x[state], #160]\n\t" + "ldp x24, x25, [%x[state], #176]\n\t" + "ldr x26, [%x[state], #192]\n\t" + "mov x28, #24\n\t" + /* Start of 24 rounds */ + "\n" + "L_SHA3_transform_blocksx3_neon_begin_%=:\n\t" + "stp %[r], x28, [x29, #48]\n\t" + /* Col Mix NEON */ + "eor v30.16b, v4.16b, v9.16b\n\t" + "eor %x[state], x5, x10\n\t" + "eor v27.16b, v1.16b, v6.16b\n\t" + "eor x30, x1, x6\n\t" + "eor v30.16b, v30.16b, v14.16b\n\t" + "eor x28, x3, x8\n\t" + "eor v27.16b, v27.16b, v11.16b\n\t" + "eor %x[state], %x[state], x15\n\t" + "eor v30.16b, v30.16b, v19.16b\n\t" + "eor x30, x30, x11\n\t" + "eor v27.16b, v27.16b, v16.16b\n\t" + "eor x28, x28, x13\n\t" + "eor v30.16b, v30.16b, v24.16b\n\t" + "eor %x[state], %x[state], x21\n\t" + "eor v27.16b, v27.16b, v21.16b\n\t" + "eor x30, x30, x16\n\t" + "ushr v25.2d, v27.2d, #63\n\t" + "eor x28, x28, x19\n\t" + "sli v25.2d, v27.2d, #1\n\t" + "eor %x[state], %x[state], x26\n\t" + "eor v25.16b, v25.16b, v30.16b\n\t" + "eor x30, x30, x22\n\t" + "eor v31.16b, v0.16b, v5.16b\n\t" + "eor x28, x28, x24\n\t" + "eor v28.16b, v2.16b, v7.16b\n\t" + "str %x[state], [x29, #32]\n\t" + "eor v31.16b, v31.16b, v10.16b\n\t" + "str x28, [x29, #24]\n\t" + "eor v28.16b, v28.16b, v12.16b\n\t" + "eor %[r], x2, x7\n\t" + "eor v31.16b, v31.16b, v15.16b\n\t" + "eor x28, x4, x9\n\t" + "eor v28.16b, v28.16b, v17.16b\n\t" + "eor %[r], %[r], x12\n\t" + "eor v31.16b, v31.16b, v20.16b\n\t" + "eor x28, x28, x14\n\t" + "eor v28.16b, v28.16b, v22.16b\n\t" + "eor %[r], %[r], x17\n\t" + "ushr v29.2d, v30.2d, #63\n\t" + "eor x28, x28, x20\n\t" + "ushr v26.2d, v28.2d, #63\n\t" + "eor %[r], %[r], x23\n\t" + "sli v29.2d, v30.2d, #1\n\t" + "eor x28, x28, x25\n\t" + "sli v26.2d, v28.2d, #1\n\t" + "eor %x[state], %x[state], %[r], ror 63\n\t" + "eor v28.16b, v28.16b, v29.16b\n\t" + "eor %[r], %[r], x28, ror 63\n\t" + "eor v29.16b, v3.16b, v8.16b\n\t" + "eor x1, x1, %x[state]\n\t" + "eor v26.16b, v26.16b, v31.16b\n\t" + "eor x6, x6, %x[state]\n\t" + "eor v29.16b, v29.16b, v13.16b\n\t" + "eor x11, x11, %x[state]\n\t" + "eor v29.16b, v29.16b, v18.16b\n\t" + "eor x16, x16, %x[state]\n\t" + "eor v29.16b, v29.16b, v23.16b\n\t" + "eor x22, x22, %x[state]\n\t" + "ushr v30.2d, v29.2d, #63\n\t" + "eor x3, x3, %[r]\n\t" + "sli v30.2d, v29.2d, #1\n\t" + "eor x8, x8, %[r]\n\t" + "eor v27.16b, v27.16b, v30.16b\n\t" + "eor x13, x13, %[r]\n\t" + "ushr v30.2d, v31.2d, #63\n\t" + "eor x19, x19, %[r]\n\t" + "sli v30.2d, v31.2d, #1\n\t" + "eor x24, x24, %[r]\n\t" + "eor v29.16b, v29.16b, v30.16b\n\t" + "ldr %x[state], [x29, #32]\n\t" + /* Swap Rotate NEON */ + "eor v0.16b, v0.16b, v25.16b\n\t" + "eor v31.16b, v1.16b, v26.16b\n\t" + "ldr %[r], [x29, #24]\n\t" + "eor v6.16b, v6.16b, v26.16b\n\t" + "eor x28, x28, x30, ror 63\n\t" + "ushr v30.2d, v31.2d, #63\n\t" + "eor x30, x30, %[r], ror 63\n\t" + "ushr v1.2d, v6.2d, #20\n\t" + "eor %[r], %[r], %x[state], ror 63\n\t" + "sli v30.2d, v31.2d, #1\n\t" + "eor x5, x5, x28\n\t" + "sli v1.2d, v6.2d, #44\n\t" + "eor x10, x10, x28\n\t" + "eor v31.16b, v9.16b, v29.16b\n\t" + "eor x15, x15, x28\n\t" + "eor v22.16b, v22.16b, v27.16b\n\t" + "eor x21, x21, x28\n\t" + "ushr v6.2d, v31.2d, #44\n\t" + "eor x26, x26, x28\n\t" + "ushr v9.2d, v22.2d, #3\n\t" + "eor x2, x2, x30\n\t" + "sli v6.2d, v31.2d, #20\n\t" + "eor x7, x7, x30\n\t" + "sli v9.2d, v22.2d, #61\n\t" + "eor x12, x12, x30\n\t" + "eor v31.16b, v14.16b, v29.16b\n\t" + "eor x17, x17, x30\n\t" + "eor v20.16b, v20.16b, v25.16b\n\t" + "eor x23, x23, x30\n\t" + "ushr v22.2d, v31.2d, #25\n\t" + "eor x4, x4, %[r]\n\t" + "ushr v14.2d, v20.2d, #46\n\t" + "eor x9, x9, %[r]\n\t" + "sli v22.2d, v31.2d, #39\n\t" + "eor x14, x14, %[r]\n\t" + "sli v14.2d, v20.2d, #18\n\t" + "eor x20, x20, %[r]\n\t" + "eor v31.16b, v2.16b, v27.16b\n\t" + "eor x25, x25, %[r]\n\t" + /* Swap Rotate Base */ + "eor v12.16b, v12.16b, v27.16b\n\t" + "ror %x[state], x2, #63\n\t" + "ushr v20.2d, v31.2d, #2\n\t" + "ror x2, x7, #20\n\t" + "ushr v2.2d, v12.2d, #21\n\t" + "ror x7, x10, #44\n\t" + "sli v20.2d, v31.2d, #62\n\t" + "ror x10, x24, #3\n\t" + "sli v2.2d, v12.2d, #43\n\t" + "ror x24, x15, #25\n\t" + "eor v31.16b, v13.16b, v28.16b\n\t" + "ror x15, x22, #46\n\t" + "eor v19.16b, v19.16b, v29.16b\n\t" + "ror x22, x3, #2\n\t" + "ushr v12.2d, v31.2d, #39\n\t" + "ror x3, x13, #21\n\t" + "ushr v13.2d, v19.2d, #56\n\t" + "ror x13, x14, #39\n\t" + "sli v12.2d, v31.2d, #25\n\t" + "ror x14, x21, #56\n\t" + "sli v13.2d, v19.2d, #8\n\t" + "ror x21, x25, #8\n\t" + "eor v31.16b, v23.16b, v28.16b\n\t" + "ror x25, x16, #23\n\t" + "eor v15.16b, v15.16b, v25.16b\n\t" + "ror x16, x5, #37\n\t" + "ushr v19.2d, v31.2d, #8\n\t" + "ror x5, x26, #50\n\t" + "ushr v23.2d, v15.2d, #23\n\t" + "ror x26, x23, #62\n\t" + "sli v19.2d, v31.2d, #56\n\t" + "ror x23, x9, #9\n\t" + "sli v23.2d, v15.2d, #41\n\t" + "ror x9, x17, #19\n\t" + "eor v31.16b, v4.16b, v29.16b\n\t" + "ror x17, x6, #28\n\t" + "eor v24.16b, v24.16b, v29.16b\n\t" + "ror x6, x4, #36\n\t" + "ushr v15.2d, v31.2d, #37\n\t" + "ror x4, x20, #43\n\t" + "ushr v4.2d, v24.2d, #50\n\t" + "ror x20, x19, #49\n\t" + "sli v15.2d, v31.2d, #27\n\t" + "ror x19, x12, #54\n\t" + "sli v4.2d, v24.2d, #14\n\t" + "ror x12, x8, #58\n\t" + "eor v31.16b, v21.16b, v26.16b\n\t" + "ror x8, x11, #61\n\t" + /* Row Mix Base */ + "eor v8.16b, v8.16b, v28.16b\n\t" + "bic x11, x3, x2\n\t" + "ushr v24.2d, v31.2d, #62\n\t" + "bic %[r], x4, x3\n\t" + "ushr v21.2d, v8.2d, #9\n\t" + "bic x28, x1, x5\n\t" + "sli v24.2d, v31.2d, #2\n\t" + "bic x30, x2, x1\n\t" + "sli v21.2d, v8.2d, #55\n\t" + "eor x1, x1, x11\n\t" + "eor v31.16b, v16.16b, v26.16b\n\t" + "eor x2, x2, %[r]\n\t" + "eor v5.16b, v5.16b, v25.16b\n\t" + "bic x11, x5, x4\n\t" + "ushr v8.2d, v31.2d, #19\n\t" + "eor x4, x4, x28\n\t" + "ushr v16.2d, v5.2d, #28\n\t" + "eor x3, x3, x11\n\t" + "sli v8.2d, v31.2d, #45\n\t" + "eor x5, x5, x30\n\t" + "sli v16.2d, v5.2d, #36\n\t" + "bic x11, x8, x7\n\t" + "eor v31.16b, v3.16b, v28.16b\n\t" + "bic %[r], x9, x8\n\t" + "eor v18.16b, v18.16b, v28.16b\n\t" + "bic x28, x6, x10\n\t" + "ushr v5.2d, v31.2d, #36\n\t" + "bic x30, x7, x6\n\t" + "ushr v3.2d, v18.2d, #43\n\t" + "eor x6, x6, x11\n\t" + "sli v5.2d, v31.2d, #28\n\t" + "eor x7, x7, %[r]\n\t" + "sli v3.2d, v18.2d, #21\n\t" + "bic x11, x10, x9\n\t" + "eor v31.16b, v17.16b, v27.16b\n\t" + "eor x9, x9, x28\n\t" + "eor v11.16b, v11.16b, v26.16b\n\t" + "eor x8, x8, x11\n\t" + "ushr v18.2d, v31.2d, #49\n\t" + "eor x10, x10, x30\n\t" + "ushr v17.2d, v11.2d, #54\n\t" + "bic x11, x13, x12\n\t" + "sli v18.2d, v31.2d, #15\n\t" + "bic %[r], x14, x13\n\t" + "sli v17.2d, v11.2d, #10\n\t" + "bic x28, %x[state], x15\n\t" + "eor v31.16b, v7.16b, v27.16b\n\t" + "bic x30, x12, %x[state]\n\t" + "eor v10.16b, v10.16b, v25.16b\n\t" + "eor x11, %x[state], x11\n\t" + "ushr v11.2d, v31.2d, #58\n\t" + "eor x12, x12, %[r]\n\t" + "ushr v7.2d, v10.2d, #61\n\t" + "bic %x[state], x15, x14\n\t" + "sli v11.2d, v31.2d, #6\n\t" + "eor x14, x14, x28\n\t" + "sli v7.2d, v10.2d, #3\n\t" + "eor x13, x13, %x[state]\n\t" + /* Row Mix NEON */ + "bic v25.16b, v2.16b, v1.16b\n\t" + "eor x15, x15, x30\n\t" + "bic v26.16b, v3.16b, v2.16b\n\t" + "bic %x[state], x19, x17\n\t" + "bic v27.16b, v4.16b, v3.16b\n\t" + "bic %[r], x20, x19\n\t" + "bic v28.16b, v0.16b, v4.16b\n\t" + "bic x28, x16, x21\n\t" + "bic v29.16b, v1.16b, v0.16b\n\t" + "bic x30, x17, x16\n\t" + "eor v0.16b, v0.16b, v25.16b\n\t" + "eor x16, x16, %x[state]\n\t" + "eor v1.16b, v1.16b, v26.16b\n\t" + "eor x17, x17, %[r]\n\t" + "eor v2.16b, v2.16b, v27.16b\n\t" + "bic %x[state], x21, x20\n\t" + "eor v3.16b, v3.16b, v28.16b\n\t" + "eor x20, x20, x28\n\t" + "eor v4.16b, v4.16b, v29.16b\n\t" + "eor x19, x19, %x[state]\n\t" + "bic v25.16b, v7.16b, v6.16b\n\t" + "eor x21, x21, x30\n\t" + "bic v26.16b, v8.16b, v7.16b\n\t" + "bic %x[state], x24, x23\n\t" + "bic v27.16b, v9.16b, v8.16b\n\t" + "bic %[r], x25, x24\n\t" + "bic v28.16b, v5.16b, v9.16b\n\t" + "bic x28, x22, x26\n\t" + "bic v29.16b, v6.16b, v5.16b\n\t" + "bic x30, x23, x22\n\t" + "eor v5.16b, v5.16b, v25.16b\n\t" + "eor x22, x22, %x[state]\n\t" + "eor v6.16b, v6.16b, v26.16b\n\t" + "eor x23, x23, %[r]\n\t" + "eor v7.16b, v7.16b, v27.16b\n\t" + "bic %x[state], x26, x25\n\t" + "eor v8.16b, v8.16b, v28.16b\n\t" + "eor x25, x25, x28\n\t" + "eor v9.16b, v9.16b, v29.16b\n\t" + "eor x24, x24, %x[state]\n\t" + "bic v25.16b, v12.16b, v11.16b\n\t" + "eor x26, x26, x30\n\t" + "bic v26.16b, v13.16b, v12.16b\n\t" + "bic v27.16b, v14.16b, v13.16b\n\t" + "bic v28.16b, v30.16b, v14.16b\n\t" + "bic v29.16b, v11.16b, v30.16b\n\t" + "eor v10.16b, v30.16b, v25.16b\n\t" + "eor v11.16b, v11.16b, v26.16b\n\t" + "eor v12.16b, v12.16b, v27.16b\n\t" + "eor v13.16b, v13.16b, v28.16b\n\t" + "eor v14.16b, v14.16b, v29.16b\n\t" + "bic v25.16b, v17.16b, v16.16b\n\t" + "bic v26.16b, v18.16b, v17.16b\n\t" + "bic v27.16b, v19.16b, v18.16b\n\t" + "bic v28.16b, v15.16b, v19.16b\n\t" + "bic v29.16b, v16.16b, v15.16b\n\t" + "eor v15.16b, v15.16b, v25.16b\n\t" + "eor v16.16b, v16.16b, v26.16b\n\t" + "eor v17.16b, v17.16b, v27.16b\n\t" + "eor v18.16b, v18.16b, v28.16b\n\t" + "eor v19.16b, v19.16b, v29.16b\n\t" + "bic v25.16b, v22.16b, v21.16b\n\t" + "bic v26.16b, v23.16b, v22.16b\n\t" + "bic v27.16b, v24.16b, v23.16b\n\t" + "bic v28.16b, v20.16b, v24.16b\n\t" + "bic v29.16b, v21.16b, v20.16b\n\t" + "eor v20.16b, v20.16b, v25.16b\n\t" + "eor v21.16b, v21.16b, v26.16b\n\t" + "eor v22.16b, v22.16b, v27.16b\n\t" + "eor v23.16b, v23.16b, v28.16b\n\t" + "eor v24.16b, v24.16b, v29.16b\n\t" + /* Done transforming */ + "ldp %[r], x28, [x29, #48]\n\t" + "ldr %x[state], [%[r]], #8\n\t" + "subs x28, x28, #1\n\t" + "mov v30.d[0], %x[state]\n\t" + "mov v30.d[1], %x[state]\n\t" + "eor x1, x1, %x[state]\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "b.ne L_SHA3_transform_blocksx3_neon_begin_%=\n\t" + "ldr %x[state], [x29, #40]\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" + "st1 {v24.d}[0], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" + "st1 {v24.d}[1], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "stp x1, x2, [%x[state]]\n\t" + "stp x3, x4, [%x[state], #16]\n\t" + "stp x5, x6, [%x[state], #32]\n\t" + "stp x7, x8, [%x[state], #48]\n\t" + "stp x9, x10, [%x[state], #64]\n\t" + "stp x11, x12, [%x[state], #80]\n\t" + "stp x13, x14, [%x[state], #96]\n\t" + "stp x15, x16, [%x[state], #112]\n\t" + "stp x17, x19, [%x[state], #128]\n\t" + "stp x20, x21, [%x[state], #144]\n\t" + "stp x22, x23, [%x[state], #160]\n\t" + "stp x24, x25, [%x[state], #176]\n\t" + "str x26, [%x[state], #192]\n\t" + "ldp x29, x30, [sp], #0x40\n\t" + : [state] "+r" (state) + : [r] "r" (r) + : "memory", "cc", "x1", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", + "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", + "x20", "x21", "x22", "x23", "x24", "x25", "x26", "x28", "v0", "v1", + "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", + "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31" + ); +} + +void sha3_128_blocksx3_seed_neon(word64* state, byte* seed) +{ + const word64* r = L_sha3_aarch64_r; + __asm__ __volatile__ ( + "stp x29, x30, [sp, #-64]!\n\t" + "add x29, sp, #0\n\t" + "str %x[state], [x29, #40]\n\t" + "add %x[state], %x[state], #32\n\t" + "ld1 {v4.d}[0], [%x[state]]\n\t" + "ldp x2, x3, [%x[seed]], #16\n\t" + "add %x[state], %x[state], #0xc8\n\t" + "ld1 {v4.d}[1], [%x[state]]\n\t" + "ldp x4, x5, [%x[seed]], #16\n\t" + "ldr x6, [%x[state], #200]\n\t" + "eor v5.16b, v5.16b, v5.16b\n\t" + "eor x7, x7, x7\n\t" + "eor v6.16b, v6.16b, v6.16b\n\t" + "eor x8, x8, x8\n\t" + "eor v7.16b, v7.16b, v7.16b\n\t" + "eor x9, x9, x9\n\t" + "eor v8.16b, v8.16b, v8.16b\n\t" + "eor x10, x10, x10\n\t" + "eor v9.16b, v9.16b, v9.16b\n\t" + "eor x11, x11, x11\n\t" + "eor v10.16b, v10.16b, v10.16b\n\t" + "eor x12, x12, x12\n\t" + "eor v11.16b, v11.16b, v11.16b\n\t" + "eor x13, x13, x13\n\t" + "eor v12.16b, v12.16b, v12.16b\n\t" + "eor x14, x14, x14\n\t" + "eor v13.16b, v13.16b, v13.16b\n\t" + "eor x15, x15, x15\n\t" + "eor v14.16b, v14.16b, v14.16b\n\t" + "eor x16, x16, x16\n\t" + "eor v15.16b, v15.16b, v15.16b\n\t" + "eor x17, x17, x17\n\t" + "eor v16.16b, v16.16b, v16.16b\n\t" + "eor x19, x19, x19\n\t" + "eor v17.16b, v17.16b, v17.16b\n\t" + "eor x20, x20, x20\n\t" + "eor v18.16b, v18.16b, v18.16b\n\t" + "eor x21, x21, x21\n\t" + "eor v19.16b, v19.16b, v19.16b\n\t" + "eor x22, x22, x22\n\t" + "movz x23, #0x8000, lsl 48\n\t" + "eor v21.16b, v21.16b, v21.16b\n\t" + "eor x24, x24, x24\n\t" + "eor v22.16b, v22.16b, v22.16b\n\t" + "eor x25, x25, x25\n\t" + "eor v23.16b, v23.16b, v23.16b\n\t" + "eor x26, x26, x26\n\t" + "eor v24.16b, v24.16b, v24.16b\n\t" + "eor x27, x27, x27\n\t" + "dup v0.2d, x2\n\t" + "dup v1.2d, x3\n\t" + "dup v2.2d, x4\n\t" + "dup v3.2d, x5\n\t" + "dup v20.2d, x23\n\t" + "mov %x[seed], #24\n\t" + /* Start of 24 rounds */ + "\n" + "L_SHA3_shake128_blocksx3_seed_neon_begin_%=:\n\t" + "stp %[r], %x[seed], [x29, #48]\n\t" + /* Col Mix NEON */ + "eor v30.16b, v4.16b, v9.16b\n\t" + "eor %x[state], x6, x11\n\t" + "eor v27.16b, v1.16b, v6.16b\n\t" + "eor x30, x2, x7\n\t" + "eor v30.16b, v30.16b, v14.16b\n\t" + "eor %[r], x4, x9\n\t" + "eor v27.16b, v27.16b, v11.16b\n\t" + "eor %x[state], %x[state], x16\n\t" + "eor v30.16b, v30.16b, v19.16b\n\t" + "eor x30, x30, x12\n\t" + "eor v27.16b, v27.16b, v16.16b\n\t" + "eor %[r], %[r], x14\n\t" + "eor v30.16b, v30.16b, v24.16b\n\t" + "eor %x[state], %x[state], x22\n\t" + "eor v27.16b, v27.16b, v21.16b\n\t" + "eor x30, x30, x17\n\t" + "ushr v25.2d, v27.2d, #63\n\t" + "eor %[r], %[r], x20\n\t" + "sli v25.2d, v27.2d, #1\n\t" + "eor %x[state], %x[state], x27\n\t" + "eor v25.16b, v25.16b, v30.16b\n\t" + "eor x30, x30, x23\n\t" + "eor v31.16b, v0.16b, v5.16b\n\t" + "eor %[r], %[r], x25\n\t" + "eor v28.16b, v2.16b, v7.16b\n\t" + "str %x[state], [x29, #32]\n\t" + "eor v31.16b, v31.16b, v10.16b\n\t" + "str %[r], [x29, #24]\n\t" + "eor v28.16b, v28.16b, v12.16b\n\t" + "eor %x[seed], x3, x8\n\t" + "eor v31.16b, v31.16b, v15.16b\n\t" + "eor %[r], x5, x10\n\t" + "eor v28.16b, v28.16b, v17.16b\n\t" + "eor %x[seed], %x[seed], x13\n\t" + "eor v31.16b, v31.16b, v20.16b\n\t" + "eor %[r], %[r], x15\n\t" + "eor v28.16b, v28.16b, v22.16b\n\t" + "eor %x[seed], %x[seed], x19\n\t" + "ushr v29.2d, v30.2d, #63\n\t" + "eor %[r], %[r], x21\n\t" + "ushr v26.2d, v28.2d, #63\n\t" + "eor %x[seed], %x[seed], x24\n\t" + "sli v29.2d, v30.2d, #1\n\t" + "eor %[r], %[r], x26\n\t" + "sli v26.2d, v28.2d, #1\n\t" + "eor %x[state], %x[state], %x[seed], ror 63\n\t" + "eor v28.16b, v28.16b, v29.16b\n\t" + "eor %x[seed], %x[seed], %[r], ror 63\n\t" + "eor v29.16b, v3.16b, v8.16b\n\t" + "eor x2, x2, %x[state]\n\t" + "eor v26.16b, v26.16b, v31.16b\n\t" + "eor x7, x7, %x[state]\n\t" + "eor v29.16b, v29.16b, v13.16b\n\t" + "eor x12, x12, %x[state]\n\t" + "eor v29.16b, v29.16b, v18.16b\n\t" + "eor x17, x17, %x[state]\n\t" + "eor v29.16b, v29.16b, v23.16b\n\t" + "eor x23, x23, %x[state]\n\t" + "ushr v30.2d, v29.2d, #63\n\t" + "eor x4, x4, %x[seed]\n\t" + "sli v30.2d, v29.2d, #1\n\t" + "eor x9, x9, %x[seed]\n\t" + "eor v27.16b, v27.16b, v30.16b\n\t" + "eor x14, x14, %x[seed]\n\t" + "ushr v30.2d, v31.2d, #63\n\t" + "eor x20, x20, %x[seed]\n\t" + "sli v30.2d, v31.2d, #1\n\t" + "eor x25, x25, %x[seed]\n\t" + "eor v29.16b, v29.16b, v30.16b\n\t" + "ldr %x[state], [x29, #32]\n\t" + /* Swap Rotate NEON */ + "eor v0.16b, v0.16b, v25.16b\n\t" + "eor v31.16b, v1.16b, v26.16b\n\t" + "ldr %x[seed], [x29, #24]\n\t" + "eor v6.16b, v6.16b, v26.16b\n\t" + "eor %[r], %[r], x30, ror 63\n\t" + "ushr v30.2d, v31.2d, #63\n\t" + "eor x30, x30, %x[seed], ror 63\n\t" + "ushr v1.2d, v6.2d, #20\n\t" + "eor %x[seed], %x[seed], %x[state], ror 63\n\t" + "sli v30.2d, v31.2d, #1\n\t" + "eor x6, x6, %[r]\n\t" + "sli v1.2d, v6.2d, #44\n\t" + "eor x11, x11, %[r]\n\t" + "eor v31.16b, v9.16b, v29.16b\n\t" + "eor x16, x16, %[r]\n\t" + "eor v22.16b, v22.16b, v27.16b\n\t" + "eor x22, x22, %[r]\n\t" + "ushr v6.2d, v31.2d, #44\n\t" + "eor x27, x27, %[r]\n\t" + "ushr v9.2d, v22.2d, #3\n\t" + "eor x3, x3, x30\n\t" + "sli v6.2d, v31.2d, #20\n\t" + "eor x8, x8, x30\n\t" + "sli v9.2d, v22.2d, #61\n\t" + "eor x13, x13, x30\n\t" + "eor v31.16b, v14.16b, v29.16b\n\t" + "eor x19, x19, x30\n\t" + "eor v20.16b, v20.16b, v25.16b\n\t" + "eor x24, x24, x30\n\t" + "ushr v22.2d, v31.2d, #25\n\t" + "eor x5, x5, %x[seed]\n\t" + "ushr v14.2d, v20.2d, #46\n\t" + "eor x10, x10, %x[seed]\n\t" + "sli v22.2d, v31.2d, #39\n\t" + "eor x15, x15, %x[seed]\n\t" + "sli v14.2d, v20.2d, #18\n\t" + "eor x21, x21, %x[seed]\n\t" + "eor v31.16b, v2.16b, v27.16b\n\t" + "eor x26, x26, %x[seed]\n\t" + /* Swap Rotate Base */ + "eor v12.16b, v12.16b, v27.16b\n\t" + "ror %x[state], x3, #63\n\t" + "ushr v20.2d, v31.2d, #2\n\t" + "ror x3, x8, #20\n\t" + "ushr v2.2d, v12.2d, #21\n\t" + "ror x8, x11, #44\n\t" + "sli v20.2d, v31.2d, #62\n\t" + "ror x11, x25, #3\n\t" + "sli v2.2d, v12.2d, #43\n\t" + "ror x25, x16, #25\n\t" + "eor v31.16b, v13.16b, v28.16b\n\t" + "ror x16, x23, #46\n\t" + "eor v19.16b, v19.16b, v29.16b\n\t" + "ror x23, x4, #2\n\t" + "ushr v12.2d, v31.2d, #39\n\t" + "ror x4, x14, #21\n\t" + "ushr v13.2d, v19.2d, #56\n\t" + "ror x14, x15, #39\n\t" + "sli v12.2d, v31.2d, #25\n\t" + "ror x15, x22, #56\n\t" + "sli v13.2d, v19.2d, #8\n\t" + "ror x22, x26, #8\n\t" + "eor v31.16b, v23.16b, v28.16b\n\t" + "ror x26, x17, #23\n\t" + "eor v15.16b, v15.16b, v25.16b\n\t" + "ror x17, x6, #37\n\t" + "ushr v19.2d, v31.2d, #8\n\t" + "ror x6, x27, #50\n\t" + "ushr v23.2d, v15.2d, #23\n\t" + "ror x27, x24, #62\n\t" + "sli v19.2d, v31.2d, #56\n\t" + "ror x24, x10, #9\n\t" + "sli v23.2d, v15.2d, #41\n\t" + "ror x10, x19, #19\n\t" + "eor v31.16b, v4.16b, v29.16b\n\t" + "ror x19, x7, #28\n\t" + "eor v24.16b, v24.16b, v29.16b\n\t" + "ror x7, x5, #36\n\t" + "ushr v15.2d, v31.2d, #37\n\t" + "ror x5, x21, #43\n\t" + "ushr v4.2d, v24.2d, #50\n\t" + "ror x21, x20, #49\n\t" + "sli v15.2d, v31.2d, #27\n\t" + "ror x20, x13, #54\n\t" + "sli v4.2d, v24.2d, #14\n\t" + "ror x13, x9, #58\n\t" + "eor v31.16b, v21.16b, v26.16b\n\t" + "ror x9, x12, #61\n\t" + /* Row Mix Base */ + "eor v8.16b, v8.16b, v28.16b\n\t" + "bic x12, x4, x3\n\t" + "ushr v24.2d, v31.2d, #62\n\t" + "bic %x[seed], x5, x4\n\t" + "ushr v21.2d, v8.2d, #9\n\t" + "bic %[r], x2, x6\n\t" + "sli v24.2d, v31.2d, #2\n\t" + "bic x30, x3, x2\n\t" + "sli v21.2d, v8.2d, #55\n\t" + "eor x2, x2, x12\n\t" + "eor v31.16b, v16.16b, v26.16b\n\t" + "eor x3, x3, %x[seed]\n\t" + "eor v5.16b, v5.16b, v25.16b\n\t" + "bic x12, x6, x5\n\t" + "ushr v8.2d, v31.2d, #19\n\t" + "eor x5, x5, %[r]\n\t" + "ushr v16.2d, v5.2d, #28\n\t" + "eor x4, x4, x12\n\t" + "sli v8.2d, v31.2d, #45\n\t" + "eor x6, x6, x30\n\t" + "sli v16.2d, v5.2d, #36\n\t" + "bic x12, x9, x8\n\t" + "eor v31.16b, v3.16b, v28.16b\n\t" + "bic %x[seed], x10, x9\n\t" + "eor v18.16b, v18.16b, v28.16b\n\t" + "bic %[r], x7, x11\n\t" + "ushr v5.2d, v31.2d, #36\n\t" + "bic x30, x8, x7\n\t" + "ushr v3.2d, v18.2d, #43\n\t" + "eor x7, x7, x12\n\t" + "sli v5.2d, v31.2d, #28\n\t" + "eor x8, x8, %x[seed]\n\t" + "sli v3.2d, v18.2d, #21\n\t" + "bic x12, x11, x10\n\t" + "eor v31.16b, v17.16b, v27.16b\n\t" + "eor x10, x10, %[r]\n\t" + "eor v11.16b, v11.16b, v26.16b\n\t" + "eor x9, x9, x12\n\t" + "ushr v18.2d, v31.2d, #49\n\t" + "eor x11, x11, x30\n\t" + "ushr v17.2d, v11.2d, #54\n\t" + "bic x12, x14, x13\n\t" + "sli v18.2d, v31.2d, #15\n\t" + "bic %x[seed], x15, x14\n\t" + "sli v17.2d, v11.2d, #10\n\t" + "bic %[r], %x[state], x16\n\t" + "eor v31.16b, v7.16b, v27.16b\n\t" + "bic x30, x13, %x[state]\n\t" + "eor v10.16b, v10.16b, v25.16b\n\t" + "eor x12, %x[state], x12\n\t" + "ushr v11.2d, v31.2d, #58\n\t" + "eor x13, x13, %x[seed]\n\t" + "ushr v7.2d, v10.2d, #61\n\t" + "bic %x[state], x16, x15\n\t" + "sli v11.2d, v31.2d, #6\n\t" + "eor x15, x15, %[r]\n\t" + "sli v7.2d, v10.2d, #3\n\t" + "eor x14, x14, %x[state]\n\t" + /* Row Mix NEON */ + "bic v25.16b, v2.16b, v1.16b\n\t" + "eor x16, x16, x30\n\t" + "bic v26.16b, v3.16b, v2.16b\n\t" + "bic %x[state], x20, x19\n\t" + "bic v27.16b, v4.16b, v3.16b\n\t" + "bic %x[seed], x21, x20\n\t" + "bic v28.16b, v0.16b, v4.16b\n\t" + "bic %[r], x17, x22\n\t" + "bic v29.16b, v1.16b, v0.16b\n\t" + "bic x30, x19, x17\n\t" + "eor v0.16b, v0.16b, v25.16b\n\t" + "eor x17, x17, %x[state]\n\t" + "eor v1.16b, v1.16b, v26.16b\n\t" + "eor x19, x19, %x[seed]\n\t" + "eor v2.16b, v2.16b, v27.16b\n\t" + "bic %x[state], x22, x21\n\t" + "eor v3.16b, v3.16b, v28.16b\n\t" + "eor x21, x21, %[r]\n\t" + "eor v4.16b, v4.16b, v29.16b\n\t" + "eor x20, x20, %x[state]\n\t" + "bic v25.16b, v7.16b, v6.16b\n\t" + "eor x22, x22, x30\n\t" + "bic v26.16b, v8.16b, v7.16b\n\t" + "bic %x[state], x25, x24\n\t" + "bic v27.16b, v9.16b, v8.16b\n\t" + "bic %x[seed], x26, x25\n\t" + "bic v28.16b, v5.16b, v9.16b\n\t" + "bic %[r], x23, x27\n\t" + "bic v29.16b, v6.16b, v5.16b\n\t" + "bic x30, x24, x23\n\t" + "eor v5.16b, v5.16b, v25.16b\n\t" + "eor x23, x23, %x[state]\n\t" + "eor v6.16b, v6.16b, v26.16b\n\t" + "eor x24, x24, %x[seed]\n\t" + "eor v7.16b, v7.16b, v27.16b\n\t" + "bic %x[state], x27, x26\n\t" + "eor v8.16b, v8.16b, v28.16b\n\t" + "eor x26, x26, %[r]\n\t" + "eor v9.16b, v9.16b, v29.16b\n\t" + "eor x25, x25, %x[state]\n\t" + "bic v25.16b, v12.16b, v11.16b\n\t" + "eor x27, x27, x30\n\t" + "bic v26.16b, v13.16b, v12.16b\n\t" + "bic v27.16b, v14.16b, v13.16b\n\t" + "bic v28.16b, v30.16b, v14.16b\n\t" + "bic v29.16b, v11.16b, v30.16b\n\t" + "eor v10.16b, v30.16b, v25.16b\n\t" + "eor v11.16b, v11.16b, v26.16b\n\t" + "eor v12.16b, v12.16b, v27.16b\n\t" + "eor v13.16b, v13.16b, v28.16b\n\t" + "eor v14.16b, v14.16b, v29.16b\n\t" + "bic v25.16b, v17.16b, v16.16b\n\t" + "bic v26.16b, v18.16b, v17.16b\n\t" + "bic v27.16b, v19.16b, v18.16b\n\t" + "bic v28.16b, v15.16b, v19.16b\n\t" + "bic v29.16b, v16.16b, v15.16b\n\t" + "eor v15.16b, v15.16b, v25.16b\n\t" + "eor v16.16b, v16.16b, v26.16b\n\t" + "eor v17.16b, v17.16b, v27.16b\n\t" + "eor v18.16b, v18.16b, v28.16b\n\t" + "eor v19.16b, v19.16b, v29.16b\n\t" + "bic v25.16b, v22.16b, v21.16b\n\t" + "bic v26.16b, v23.16b, v22.16b\n\t" + "bic v27.16b, v24.16b, v23.16b\n\t" + "bic v28.16b, v20.16b, v24.16b\n\t" + "bic v29.16b, v21.16b, v20.16b\n\t" + "eor v20.16b, v20.16b, v25.16b\n\t" + "eor v21.16b, v21.16b, v26.16b\n\t" + "eor v22.16b, v22.16b, v27.16b\n\t" + "eor v23.16b, v23.16b, v28.16b\n\t" + "eor v24.16b, v24.16b, v29.16b\n\t" + /* Done transforming */ + "ldp %[r], %x[seed], [x29, #48]\n\t" + "ldr %x[state], [%[r]], #8\n\t" + "subs %x[seed], %x[seed], #1\n\t" + "mov v30.d[0], %x[state]\n\t" + "mov v30.d[1], %x[state]\n\t" + "eor x2, x2, %x[state]\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "b.ne L_SHA3_shake128_blocksx3_seed_neon_begin_%=\n\t" + "ldr %x[state], [x29, #40]\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" + "st1 {v24.d}[0], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" + "st1 {v24.d}[1], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "stp x2, x3, [%x[state]]\n\t" + "stp x4, x5, [%x[state], #16]\n\t" + "stp x6, x7, [%x[state], #32]\n\t" + "stp x8, x9, [%x[state], #48]\n\t" + "stp x10, x11, [%x[state], #64]\n\t" + "stp x12, x13, [%x[state], #80]\n\t" + "stp x14, x15, [%x[state], #96]\n\t" + "stp x16, x17, [%x[state], #112]\n\t" + "stp x19, x20, [%x[state], #128]\n\t" + "stp x21, x22, [%x[state], #144]\n\t" + "stp x23, x24, [%x[state], #160]\n\t" + "stp x25, x26, [%x[state], #176]\n\t" + "str x27, [%x[state], #192]\n\t" + "ldp x29, x30, [sp], #0x40\n\t" + : [state] "+r" (state), [seed] "+r" (seed) + : [r] "r" (r) + : "memory", "cc", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", + "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", + "x21", "x22", "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", + "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", + "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31" + ); +} + +void sha3_256_blocksx3_seed_neon(word64* state, byte* seed) +{ + const word64* r = L_sha3_aarch64_r; + __asm__ __volatile__ ( + "stp x29, x30, [sp, #-64]!\n\t" + "add x29, sp, #0\n\t" + "str %x[state], [x29, #40]\n\t" + "add %x[state], %x[state], #32\n\t" + "ld1 {v4.d}[0], [%x[state]]\n\t" + "ldp x2, x3, [%x[seed]], #16\n\t" + "add %x[state], %x[state], #0xc8\n\t" + "ld1 {v4.d}[1], [%x[state]]\n\t" + "ldp x4, x5, [%x[seed]], #16\n\t" + "ldr x6, [%x[state], #200]\n\t" + "eor v5.16b, v5.16b, v5.16b\n\t" + "eor x7, x7, x7\n\t" + "eor v6.16b, v6.16b, v6.16b\n\t" + "eor x8, x8, x8\n\t" + "eor v7.16b, v7.16b, v7.16b\n\t" + "eor x9, x9, x9\n\t" + "eor v8.16b, v8.16b, v8.16b\n\t" + "eor x10, x10, x10\n\t" + "eor v9.16b, v9.16b, v9.16b\n\t" + "eor x11, x11, x11\n\t" + "eor v10.16b, v10.16b, v10.16b\n\t" + "eor x12, x12, x12\n\t" + "eor v11.16b, v11.16b, v11.16b\n\t" + "eor x13, x13, x13\n\t" + "eor v12.16b, v12.16b, v12.16b\n\t" + "eor x14, x14, x14\n\t" + "eor v13.16b, v13.16b, v13.16b\n\t" + "eor x15, x15, x15\n\t" + "eor v14.16b, v14.16b, v14.16b\n\t" + "eor x16, x16, x16\n\t" + "eor v15.16b, v15.16b, v15.16b\n\t" + "eor x17, x17, x17\n\t" + "movz x19, #0x8000, lsl 48\n\t" + "eor v17.16b, v17.16b, v17.16b\n\t" + "eor x20, x20, x20\n\t" + "eor v18.16b, v18.16b, v18.16b\n\t" + "eor x21, x21, x21\n\t" + "eor v19.16b, v19.16b, v19.16b\n\t" + "eor x22, x22, x22\n\t" + "eor v20.16b, v20.16b, v20.16b\n\t" + "eor x23, x23, x23\n\t" + "eor v21.16b, v21.16b, v21.16b\n\t" + "eor x24, x24, x24\n\t" + "eor v22.16b, v22.16b, v22.16b\n\t" + "eor x25, x25, x25\n\t" + "eor v23.16b, v23.16b, v23.16b\n\t" + "eor x26, x26, x26\n\t" + "eor v24.16b, v24.16b, v24.16b\n\t" + "eor x27, x27, x27\n\t" + "dup v0.2d, x2\n\t" + "dup v1.2d, x3\n\t" + "dup v2.2d, x4\n\t" + "dup v3.2d, x5\n\t" + "dup v16.2d, x19\n\t" + "mov %x[seed], #24\n\t" + /* Start of 24 rounds */ + "\n" + "L_SHA3_shake256_blocksx3_seed_neon_begin_%=:\n\t" + "stp %[r], %x[seed], [x29, #48]\n\t" + /* Col Mix NEON */ + "eor v30.16b, v4.16b, v9.16b\n\t" + "eor %x[state], x6, x11\n\t" + "eor v27.16b, v1.16b, v6.16b\n\t" + "eor x30, x2, x7\n\t" + "eor v30.16b, v30.16b, v14.16b\n\t" + "eor %[r], x4, x9\n\t" + "eor v27.16b, v27.16b, v11.16b\n\t" + "eor %x[state], %x[state], x16\n\t" + "eor v30.16b, v30.16b, v19.16b\n\t" + "eor x30, x30, x12\n\t" + "eor v27.16b, v27.16b, v16.16b\n\t" + "eor %[r], %[r], x14\n\t" + "eor v30.16b, v30.16b, v24.16b\n\t" + "eor %x[state], %x[state], x22\n\t" + "eor v27.16b, v27.16b, v21.16b\n\t" + "eor x30, x30, x17\n\t" + "ushr v25.2d, v27.2d, #63\n\t" + "eor %[r], %[r], x20\n\t" + "sli v25.2d, v27.2d, #1\n\t" + "eor %x[state], %x[state], x27\n\t" + "eor v25.16b, v25.16b, v30.16b\n\t" + "eor x30, x30, x23\n\t" + "eor v31.16b, v0.16b, v5.16b\n\t" + "eor %[r], %[r], x25\n\t" + "eor v28.16b, v2.16b, v7.16b\n\t" + "str %x[state], [x29, #32]\n\t" + "eor v31.16b, v31.16b, v10.16b\n\t" + "str %[r], [x29, #24]\n\t" + "eor v28.16b, v28.16b, v12.16b\n\t" + "eor %x[seed], x3, x8\n\t" + "eor v31.16b, v31.16b, v15.16b\n\t" + "eor %[r], x5, x10\n\t" + "eor v28.16b, v28.16b, v17.16b\n\t" + "eor %x[seed], %x[seed], x13\n\t" + "eor v31.16b, v31.16b, v20.16b\n\t" + "eor %[r], %[r], x15\n\t" + "eor v28.16b, v28.16b, v22.16b\n\t" + "eor %x[seed], %x[seed], x19\n\t" + "ushr v29.2d, v30.2d, #63\n\t" + "eor %[r], %[r], x21\n\t" + "ushr v26.2d, v28.2d, #63\n\t" + "eor %x[seed], %x[seed], x24\n\t" + "sli v29.2d, v30.2d, #1\n\t" + "eor %[r], %[r], x26\n\t" + "sli v26.2d, v28.2d, #1\n\t" + "eor %x[state], %x[state], %x[seed], ror 63\n\t" + "eor v28.16b, v28.16b, v29.16b\n\t" + "eor %x[seed], %x[seed], %[r], ror 63\n\t" + "eor v29.16b, v3.16b, v8.16b\n\t" + "eor x2, x2, %x[state]\n\t" + "eor v26.16b, v26.16b, v31.16b\n\t" + "eor x7, x7, %x[state]\n\t" + "eor v29.16b, v29.16b, v13.16b\n\t" + "eor x12, x12, %x[state]\n\t" + "eor v29.16b, v29.16b, v18.16b\n\t" + "eor x17, x17, %x[state]\n\t" + "eor v29.16b, v29.16b, v23.16b\n\t" + "eor x23, x23, %x[state]\n\t" + "ushr v30.2d, v29.2d, #63\n\t" + "eor x4, x4, %x[seed]\n\t" + "sli v30.2d, v29.2d, #1\n\t" + "eor x9, x9, %x[seed]\n\t" + "eor v27.16b, v27.16b, v30.16b\n\t" + "eor x14, x14, %x[seed]\n\t" + "ushr v30.2d, v31.2d, #63\n\t" + "eor x20, x20, %x[seed]\n\t" + "sli v30.2d, v31.2d, #1\n\t" + "eor x25, x25, %x[seed]\n\t" + "eor v29.16b, v29.16b, v30.16b\n\t" + "ldr %x[state], [x29, #32]\n\t" + /* Swap Rotate NEON */ + "eor v0.16b, v0.16b, v25.16b\n\t" + "eor v31.16b, v1.16b, v26.16b\n\t" + "ldr %x[seed], [x29, #24]\n\t" + "eor v6.16b, v6.16b, v26.16b\n\t" + "eor %[r], %[r], x30, ror 63\n\t" + "ushr v30.2d, v31.2d, #63\n\t" + "eor x30, x30, %x[seed], ror 63\n\t" + "ushr v1.2d, v6.2d, #20\n\t" + "eor %x[seed], %x[seed], %x[state], ror 63\n\t" + "sli v30.2d, v31.2d, #1\n\t" + "eor x6, x6, %[r]\n\t" + "sli v1.2d, v6.2d, #44\n\t" + "eor x11, x11, %[r]\n\t" + "eor v31.16b, v9.16b, v29.16b\n\t" + "eor x16, x16, %[r]\n\t" + "eor v22.16b, v22.16b, v27.16b\n\t" + "eor x22, x22, %[r]\n\t" + "ushr v6.2d, v31.2d, #44\n\t" + "eor x27, x27, %[r]\n\t" + "ushr v9.2d, v22.2d, #3\n\t" + "eor x3, x3, x30\n\t" + "sli v6.2d, v31.2d, #20\n\t" + "eor x8, x8, x30\n\t" + "sli v9.2d, v22.2d, #61\n\t" + "eor x13, x13, x30\n\t" + "eor v31.16b, v14.16b, v29.16b\n\t" + "eor x19, x19, x30\n\t" + "eor v20.16b, v20.16b, v25.16b\n\t" + "eor x24, x24, x30\n\t" + "ushr v22.2d, v31.2d, #25\n\t" + "eor x5, x5, %x[seed]\n\t" + "ushr v14.2d, v20.2d, #46\n\t" + "eor x10, x10, %x[seed]\n\t" + "sli v22.2d, v31.2d, #39\n\t" + "eor x15, x15, %x[seed]\n\t" + "sli v14.2d, v20.2d, #18\n\t" + "eor x21, x21, %x[seed]\n\t" + "eor v31.16b, v2.16b, v27.16b\n\t" + "eor x26, x26, %x[seed]\n\t" + /* Swap Rotate Base */ + "eor v12.16b, v12.16b, v27.16b\n\t" + "ror %x[state], x3, #63\n\t" + "ushr v20.2d, v31.2d, #2\n\t" + "ror x3, x8, #20\n\t" + "ushr v2.2d, v12.2d, #21\n\t" + "ror x8, x11, #44\n\t" + "sli v20.2d, v31.2d, #62\n\t" + "ror x11, x25, #3\n\t" + "sli v2.2d, v12.2d, #43\n\t" + "ror x25, x16, #25\n\t" + "eor v31.16b, v13.16b, v28.16b\n\t" + "ror x16, x23, #46\n\t" + "eor v19.16b, v19.16b, v29.16b\n\t" + "ror x23, x4, #2\n\t" + "ushr v12.2d, v31.2d, #39\n\t" + "ror x4, x14, #21\n\t" + "ushr v13.2d, v19.2d, #56\n\t" + "ror x14, x15, #39\n\t" + "sli v12.2d, v31.2d, #25\n\t" + "ror x15, x22, #56\n\t" + "sli v13.2d, v19.2d, #8\n\t" + "ror x22, x26, #8\n\t" + "eor v31.16b, v23.16b, v28.16b\n\t" + "ror x26, x17, #23\n\t" + "eor v15.16b, v15.16b, v25.16b\n\t" + "ror x17, x6, #37\n\t" + "ushr v19.2d, v31.2d, #8\n\t" + "ror x6, x27, #50\n\t" + "ushr v23.2d, v15.2d, #23\n\t" + "ror x27, x24, #62\n\t" + "sli v19.2d, v31.2d, #56\n\t" + "ror x24, x10, #9\n\t" + "sli v23.2d, v15.2d, #41\n\t" + "ror x10, x19, #19\n\t" + "eor v31.16b, v4.16b, v29.16b\n\t" + "ror x19, x7, #28\n\t" + "eor v24.16b, v24.16b, v29.16b\n\t" + "ror x7, x5, #36\n\t" + "ushr v15.2d, v31.2d, #37\n\t" + "ror x5, x21, #43\n\t" + "ushr v4.2d, v24.2d, #50\n\t" + "ror x21, x20, #49\n\t" + "sli v15.2d, v31.2d, #27\n\t" + "ror x20, x13, #54\n\t" + "sli v4.2d, v24.2d, #14\n\t" + "ror x13, x9, #58\n\t" + "eor v31.16b, v21.16b, v26.16b\n\t" + "ror x9, x12, #61\n\t" + /* Row Mix Base */ + "eor v8.16b, v8.16b, v28.16b\n\t" + "bic x12, x4, x3\n\t" + "ushr v24.2d, v31.2d, #62\n\t" + "bic %x[seed], x5, x4\n\t" + "ushr v21.2d, v8.2d, #9\n\t" + "bic %[r], x2, x6\n\t" + "sli v24.2d, v31.2d, #2\n\t" + "bic x30, x3, x2\n\t" + "sli v21.2d, v8.2d, #55\n\t" + "eor x2, x2, x12\n\t" + "eor v31.16b, v16.16b, v26.16b\n\t" + "eor x3, x3, %x[seed]\n\t" + "eor v5.16b, v5.16b, v25.16b\n\t" + "bic x12, x6, x5\n\t" + "ushr v8.2d, v31.2d, #19\n\t" + "eor x5, x5, %[r]\n\t" + "ushr v16.2d, v5.2d, #28\n\t" + "eor x4, x4, x12\n\t" + "sli v8.2d, v31.2d, #45\n\t" + "eor x6, x6, x30\n\t" + "sli v16.2d, v5.2d, #36\n\t" + "bic x12, x9, x8\n\t" + "eor v31.16b, v3.16b, v28.16b\n\t" + "bic %x[seed], x10, x9\n\t" + "eor v18.16b, v18.16b, v28.16b\n\t" + "bic %[r], x7, x11\n\t" + "ushr v5.2d, v31.2d, #36\n\t" + "bic x30, x8, x7\n\t" + "ushr v3.2d, v18.2d, #43\n\t" + "eor x7, x7, x12\n\t" + "sli v5.2d, v31.2d, #28\n\t" + "eor x8, x8, %x[seed]\n\t" + "sli v3.2d, v18.2d, #21\n\t" + "bic x12, x11, x10\n\t" + "eor v31.16b, v17.16b, v27.16b\n\t" + "eor x10, x10, %[r]\n\t" + "eor v11.16b, v11.16b, v26.16b\n\t" + "eor x9, x9, x12\n\t" + "ushr v18.2d, v31.2d, #49\n\t" + "eor x11, x11, x30\n\t" + "ushr v17.2d, v11.2d, #54\n\t" + "bic x12, x14, x13\n\t" + "sli v18.2d, v31.2d, #15\n\t" + "bic %x[seed], x15, x14\n\t" + "sli v17.2d, v11.2d, #10\n\t" + "bic %[r], %x[state], x16\n\t" + "eor v31.16b, v7.16b, v27.16b\n\t" + "bic x30, x13, %x[state]\n\t" + "eor v10.16b, v10.16b, v25.16b\n\t" + "eor x12, %x[state], x12\n\t" + "ushr v11.2d, v31.2d, #58\n\t" + "eor x13, x13, %x[seed]\n\t" + "ushr v7.2d, v10.2d, #61\n\t" + "bic %x[state], x16, x15\n\t" + "sli v11.2d, v31.2d, #6\n\t" + "eor x15, x15, %[r]\n\t" + "sli v7.2d, v10.2d, #3\n\t" + "eor x14, x14, %x[state]\n\t" + /* Row Mix NEON */ + "bic v25.16b, v2.16b, v1.16b\n\t" + "eor x16, x16, x30\n\t" + "bic v26.16b, v3.16b, v2.16b\n\t" + "bic %x[state], x20, x19\n\t" + "bic v27.16b, v4.16b, v3.16b\n\t" + "bic %x[seed], x21, x20\n\t" + "bic v28.16b, v0.16b, v4.16b\n\t" + "bic %[r], x17, x22\n\t" + "bic v29.16b, v1.16b, v0.16b\n\t" + "bic x30, x19, x17\n\t" + "eor v0.16b, v0.16b, v25.16b\n\t" + "eor x17, x17, %x[state]\n\t" + "eor v1.16b, v1.16b, v26.16b\n\t" + "eor x19, x19, %x[seed]\n\t" + "eor v2.16b, v2.16b, v27.16b\n\t" + "bic %x[state], x22, x21\n\t" + "eor v3.16b, v3.16b, v28.16b\n\t" + "eor x21, x21, %[r]\n\t" + "eor v4.16b, v4.16b, v29.16b\n\t" + "eor x20, x20, %x[state]\n\t" + "bic v25.16b, v7.16b, v6.16b\n\t" + "eor x22, x22, x30\n\t" + "bic v26.16b, v8.16b, v7.16b\n\t" + "bic %x[state], x25, x24\n\t" + "bic v27.16b, v9.16b, v8.16b\n\t" + "bic %x[seed], x26, x25\n\t" + "bic v28.16b, v5.16b, v9.16b\n\t" + "bic %[r], x23, x27\n\t" + "bic v29.16b, v6.16b, v5.16b\n\t" + "bic x30, x24, x23\n\t" + "eor v5.16b, v5.16b, v25.16b\n\t" + "eor x23, x23, %x[state]\n\t" + "eor v6.16b, v6.16b, v26.16b\n\t" + "eor x24, x24, %x[seed]\n\t" + "eor v7.16b, v7.16b, v27.16b\n\t" + "bic %x[state], x27, x26\n\t" + "eor v8.16b, v8.16b, v28.16b\n\t" + "eor x26, x26, %[r]\n\t" + "eor v9.16b, v9.16b, v29.16b\n\t" + "eor x25, x25, %x[state]\n\t" + "bic v25.16b, v12.16b, v11.16b\n\t" + "eor x27, x27, x30\n\t" + "bic v26.16b, v13.16b, v12.16b\n\t" + "bic v27.16b, v14.16b, v13.16b\n\t" + "bic v28.16b, v30.16b, v14.16b\n\t" + "bic v29.16b, v11.16b, v30.16b\n\t" + "eor v10.16b, v30.16b, v25.16b\n\t" + "eor v11.16b, v11.16b, v26.16b\n\t" + "eor v12.16b, v12.16b, v27.16b\n\t" + "eor v13.16b, v13.16b, v28.16b\n\t" + "eor v14.16b, v14.16b, v29.16b\n\t" + "bic v25.16b, v17.16b, v16.16b\n\t" + "bic v26.16b, v18.16b, v17.16b\n\t" + "bic v27.16b, v19.16b, v18.16b\n\t" + "bic v28.16b, v15.16b, v19.16b\n\t" + "bic v29.16b, v16.16b, v15.16b\n\t" + "eor v15.16b, v15.16b, v25.16b\n\t" + "eor v16.16b, v16.16b, v26.16b\n\t" + "eor v17.16b, v17.16b, v27.16b\n\t" + "eor v18.16b, v18.16b, v28.16b\n\t" + "eor v19.16b, v19.16b, v29.16b\n\t" + "bic v25.16b, v22.16b, v21.16b\n\t" + "bic v26.16b, v23.16b, v22.16b\n\t" + "bic v27.16b, v24.16b, v23.16b\n\t" + "bic v28.16b, v20.16b, v24.16b\n\t" + "bic v29.16b, v21.16b, v20.16b\n\t" + "eor v20.16b, v20.16b, v25.16b\n\t" + "eor v21.16b, v21.16b, v26.16b\n\t" + "eor v22.16b, v22.16b, v27.16b\n\t" + "eor v23.16b, v23.16b, v28.16b\n\t" + "eor v24.16b, v24.16b, v29.16b\n\t" + /* Done transforming */ + "ldp %[r], %x[seed], [x29, #48]\n\t" + "ldr %x[state], [%[r]], #8\n\t" + "subs %x[seed], %x[seed], #1\n\t" + "mov v30.d[0], %x[state]\n\t" + "mov v30.d[1], %x[state]\n\t" + "eor x2, x2, %x[state]\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "b.ne L_SHA3_shake256_blocksx3_seed_neon_begin_%=\n\t" + "ldr %x[state], [x29, #40]\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" + "st1 {v24.d}[0], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" + "st1 {v24.d}[1], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "stp x2, x3, [%x[state]]\n\t" + "stp x4, x5, [%x[state], #16]\n\t" + "stp x6, x7, [%x[state], #32]\n\t" + "stp x8, x9, [%x[state], #48]\n\t" + "stp x10, x11, [%x[state], #64]\n\t" + "stp x12, x13, [%x[state], #80]\n\t" + "stp x14, x15, [%x[state], #96]\n\t" + "stp x16, x17, [%x[state], #112]\n\t" + "stp x19, x20, [%x[state], #128]\n\t" + "stp x21, x22, [%x[state], #144]\n\t" + "stp x23, x24, [%x[state], #160]\n\t" + "stp x25, x26, [%x[state], #176]\n\t" + "str x27, [%x[state], #192]\n\t" + "ldp x29, x30, [sp], #0x40\n\t" + : [state] "+r" (state), [seed] "+r" (seed) + : [r] "r" (r) + : "memory", "cc", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", + "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", + "x21", "x22", "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", + "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", + "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31" + ); +} + +void sha3_256_blocksx3_seed_64_neon(word64* state, byte* seed) +{ + const word64* r = L_sha3_aarch64_r; + __asm__ __volatile__ ( + "stp x29, x30, [sp, #-64]!\n\t" + "add x29, sp, #0\n\t" + "str %x[state], [x29, #40]\n\t" + "add %x[state], %x[state], #0x40\n\t" + "ld1 {v8.d}[0], [%x[state]]\n\t" + "ldp x2, x3, [%x[seed]], #16\n\t" + "add %x[state], %x[state], #0xc8\n\t" + "ld1 {v8.d}[1], [%x[state]]\n\t" + "ldp x4, x5, [%x[seed]], #16\n\t" + "ldp x6, x7, [%x[seed]], #16\n\t" + "ldp x8, x9, [%x[seed]], #16\n\t" + "ldr x10, [%x[state], #200]\n\t" + "eor v9.16b, v9.16b, v9.16b\n\t" + "eor x11, x11, x11\n\t" + "eor v10.16b, v10.16b, v10.16b\n\t" + "eor x12, x12, x12\n\t" + "eor v11.16b, v11.16b, v11.16b\n\t" + "eor x13, x13, x13\n\t" + "eor v12.16b, v12.16b, v12.16b\n\t" + "eor x14, x14, x14\n\t" + "eor v13.16b, v13.16b, v13.16b\n\t" + "eor x15, x15, x15\n\t" + "eor v14.16b, v14.16b, v14.16b\n\t" + "eor x16, x16, x16\n\t" + "eor v15.16b, v15.16b, v15.16b\n\t" + "eor x17, x17, x17\n\t" + "movz x19, #0x8000, lsl 48\n\t" + "eor v17.16b, v17.16b, v17.16b\n\t" + "eor x20, x20, x20\n\t" + "eor v18.16b, v18.16b, v18.16b\n\t" + "eor x21, x21, x21\n\t" + "eor v19.16b, v19.16b, v19.16b\n\t" + "eor x22, x22, x22\n\t" + "eor v20.16b, v20.16b, v20.16b\n\t" + "eor x23, x23, x23\n\t" + "eor v21.16b, v21.16b, v21.16b\n\t" + "eor x24, x24, x24\n\t" + "eor v22.16b, v22.16b, v22.16b\n\t" + "eor x25, x25, x25\n\t" + "eor v23.16b, v23.16b, v23.16b\n\t" + "eor x26, x26, x26\n\t" + "eor v24.16b, v24.16b, v24.16b\n\t" + "eor x27, x27, x27\n\t" + "dup v0.2d, x2\n\t" + "dup v1.2d, x3\n\t" + "dup v2.2d, x4\n\t" + "dup v3.2d, x5\n\t" + "dup v4.2d, x6\n\t" + "dup v5.2d, x7\n\t" + "dup v6.2d, x8\n\t" + "dup v7.2d, x9\n\t" + "dup v16.2d, x19\n\t" + "mov %x[seed], #24\n\t" + /* Start of 24 rounds */ + "\n" + "L_SHA3_shake256_blocksx3_seed_64_neon_begin_%=:\n\t" + "stp %[r], %x[seed], [x29, #48]\n\t" + /* Col Mix NEON */ + "eor v30.16b, v4.16b, v9.16b\n\t" + "eor %x[state], x6, x11\n\t" + "eor v27.16b, v1.16b, v6.16b\n\t" + "eor x30, x2, x7\n\t" + "eor v30.16b, v30.16b, v14.16b\n\t" + "eor %[r], x4, x9\n\t" + "eor v27.16b, v27.16b, v11.16b\n\t" + "eor %x[state], %x[state], x16\n\t" + "eor v30.16b, v30.16b, v19.16b\n\t" + "eor x30, x30, x12\n\t" + "eor v27.16b, v27.16b, v16.16b\n\t" + "eor %[r], %[r], x14\n\t" + "eor v30.16b, v30.16b, v24.16b\n\t" + "eor %x[state], %x[state], x22\n\t" + "eor v27.16b, v27.16b, v21.16b\n\t" + "eor x30, x30, x17\n\t" + "ushr v25.2d, v27.2d, #63\n\t" + "eor %[r], %[r], x20\n\t" + "sli v25.2d, v27.2d, #1\n\t" + "eor %x[state], %x[state], x27\n\t" + "eor v25.16b, v25.16b, v30.16b\n\t" + "eor x30, x30, x23\n\t" + "eor v31.16b, v0.16b, v5.16b\n\t" + "eor %[r], %[r], x25\n\t" + "eor v28.16b, v2.16b, v7.16b\n\t" + "str %x[state], [x29, #32]\n\t" + "eor v31.16b, v31.16b, v10.16b\n\t" + "str %[r], [x29, #24]\n\t" + "eor v28.16b, v28.16b, v12.16b\n\t" + "eor %x[seed], x3, x8\n\t" + "eor v31.16b, v31.16b, v15.16b\n\t" + "eor %[r], x5, x10\n\t" + "eor v28.16b, v28.16b, v17.16b\n\t" + "eor %x[seed], %x[seed], x13\n\t" + "eor v31.16b, v31.16b, v20.16b\n\t" + "eor %[r], %[r], x15\n\t" + "eor v28.16b, v28.16b, v22.16b\n\t" + "eor %x[seed], %x[seed], x19\n\t" + "ushr v29.2d, v30.2d, #63\n\t" + "eor %[r], %[r], x21\n\t" + "ushr v26.2d, v28.2d, #63\n\t" + "eor %x[seed], %x[seed], x24\n\t" + "sli v29.2d, v30.2d, #1\n\t" + "eor %[r], %[r], x26\n\t" + "sli v26.2d, v28.2d, #1\n\t" + "eor %x[state], %x[state], %x[seed], ror 63\n\t" + "eor v28.16b, v28.16b, v29.16b\n\t" + "eor %x[seed], %x[seed], %[r], ror 63\n\t" + "eor v29.16b, v3.16b, v8.16b\n\t" + "eor x2, x2, %x[state]\n\t" + "eor v26.16b, v26.16b, v31.16b\n\t" + "eor x7, x7, %x[state]\n\t" + "eor v29.16b, v29.16b, v13.16b\n\t" + "eor x12, x12, %x[state]\n\t" + "eor v29.16b, v29.16b, v18.16b\n\t" + "eor x17, x17, %x[state]\n\t" + "eor v29.16b, v29.16b, v23.16b\n\t" + "eor x23, x23, %x[state]\n\t" + "ushr v30.2d, v29.2d, #63\n\t" + "eor x4, x4, %x[seed]\n\t" + "sli v30.2d, v29.2d, #1\n\t" + "eor x9, x9, %x[seed]\n\t" + "eor v27.16b, v27.16b, v30.16b\n\t" + "eor x14, x14, %x[seed]\n\t" + "ushr v30.2d, v31.2d, #63\n\t" + "eor x20, x20, %x[seed]\n\t" + "sli v30.2d, v31.2d, #1\n\t" + "eor x25, x25, %x[seed]\n\t" + "eor v29.16b, v29.16b, v30.16b\n\t" + "ldr %x[state], [x29, #32]\n\t" + /* Swap Rotate NEON */ + "eor v0.16b, v0.16b, v25.16b\n\t" + "eor v31.16b, v1.16b, v26.16b\n\t" + "ldr %x[seed], [x29, #24]\n\t" + "eor v6.16b, v6.16b, v26.16b\n\t" + "eor %[r], %[r], x30, ror 63\n\t" + "ushr v30.2d, v31.2d, #63\n\t" + "eor x30, x30, %x[seed], ror 63\n\t" + "ushr v1.2d, v6.2d, #20\n\t" + "eor %x[seed], %x[seed], %x[state], ror 63\n\t" + "sli v30.2d, v31.2d, #1\n\t" + "eor x6, x6, %[r]\n\t" + "sli v1.2d, v6.2d, #44\n\t" + "eor x11, x11, %[r]\n\t" + "eor v31.16b, v9.16b, v29.16b\n\t" + "eor x16, x16, %[r]\n\t" + "eor v22.16b, v22.16b, v27.16b\n\t" + "eor x22, x22, %[r]\n\t" + "ushr v6.2d, v31.2d, #44\n\t" + "eor x27, x27, %[r]\n\t" + "ushr v9.2d, v22.2d, #3\n\t" + "eor x3, x3, x30\n\t" + "sli v6.2d, v31.2d, #20\n\t" + "eor x8, x8, x30\n\t" + "sli v9.2d, v22.2d, #61\n\t" + "eor x13, x13, x30\n\t" + "eor v31.16b, v14.16b, v29.16b\n\t" + "eor x19, x19, x30\n\t" + "eor v20.16b, v20.16b, v25.16b\n\t" + "eor x24, x24, x30\n\t" + "ushr v22.2d, v31.2d, #25\n\t" + "eor x5, x5, %x[seed]\n\t" + "ushr v14.2d, v20.2d, #46\n\t" + "eor x10, x10, %x[seed]\n\t" + "sli v22.2d, v31.2d, #39\n\t" + "eor x15, x15, %x[seed]\n\t" + "sli v14.2d, v20.2d, #18\n\t" + "eor x21, x21, %x[seed]\n\t" + "eor v31.16b, v2.16b, v27.16b\n\t" + "eor x26, x26, %x[seed]\n\t" + /* Swap Rotate Base */ + "eor v12.16b, v12.16b, v27.16b\n\t" + "ror %x[state], x3, #63\n\t" + "ushr v20.2d, v31.2d, #2\n\t" + "ror x3, x8, #20\n\t" + "ushr v2.2d, v12.2d, #21\n\t" + "ror x8, x11, #44\n\t" + "sli v20.2d, v31.2d, #62\n\t" + "ror x11, x25, #3\n\t" + "sli v2.2d, v12.2d, #43\n\t" + "ror x25, x16, #25\n\t" + "eor v31.16b, v13.16b, v28.16b\n\t" + "ror x16, x23, #46\n\t" + "eor v19.16b, v19.16b, v29.16b\n\t" + "ror x23, x4, #2\n\t" + "ushr v12.2d, v31.2d, #39\n\t" + "ror x4, x14, #21\n\t" + "ushr v13.2d, v19.2d, #56\n\t" + "ror x14, x15, #39\n\t" + "sli v12.2d, v31.2d, #25\n\t" + "ror x15, x22, #56\n\t" + "sli v13.2d, v19.2d, #8\n\t" + "ror x22, x26, #8\n\t" + "eor v31.16b, v23.16b, v28.16b\n\t" + "ror x26, x17, #23\n\t" + "eor v15.16b, v15.16b, v25.16b\n\t" + "ror x17, x6, #37\n\t" + "ushr v19.2d, v31.2d, #8\n\t" + "ror x6, x27, #50\n\t" + "ushr v23.2d, v15.2d, #23\n\t" + "ror x27, x24, #62\n\t" + "sli v19.2d, v31.2d, #56\n\t" + "ror x24, x10, #9\n\t" + "sli v23.2d, v15.2d, #41\n\t" + "ror x10, x19, #19\n\t" + "eor v31.16b, v4.16b, v29.16b\n\t" + "ror x19, x7, #28\n\t" + "eor v24.16b, v24.16b, v29.16b\n\t" + "ror x7, x5, #36\n\t" + "ushr v15.2d, v31.2d, #37\n\t" + "ror x5, x21, #43\n\t" + "ushr v4.2d, v24.2d, #50\n\t" + "ror x21, x20, #49\n\t" + "sli v15.2d, v31.2d, #27\n\t" + "ror x20, x13, #54\n\t" + "sli v4.2d, v24.2d, #14\n\t" + "ror x13, x9, #58\n\t" + "eor v31.16b, v21.16b, v26.16b\n\t" + "ror x9, x12, #61\n\t" + /* Row Mix Base */ + "eor v8.16b, v8.16b, v28.16b\n\t" + "bic x12, x4, x3\n\t" + "ushr v24.2d, v31.2d, #62\n\t" + "bic %x[seed], x5, x4\n\t" + "ushr v21.2d, v8.2d, #9\n\t" + "bic %[r], x2, x6\n\t" + "sli v24.2d, v31.2d, #2\n\t" + "bic x30, x3, x2\n\t" + "sli v21.2d, v8.2d, #55\n\t" + "eor x2, x2, x12\n\t" + "eor v31.16b, v16.16b, v26.16b\n\t" + "eor x3, x3, %x[seed]\n\t" + "eor v5.16b, v5.16b, v25.16b\n\t" + "bic x12, x6, x5\n\t" + "ushr v8.2d, v31.2d, #19\n\t" + "eor x5, x5, %[r]\n\t" + "ushr v16.2d, v5.2d, #28\n\t" + "eor x4, x4, x12\n\t" + "sli v8.2d, v31.2d, #45\n\t" + "eor x6, x6, x30\n\t" + "sli v16.2d, v5.2d, #36\n\t" + "bic x12, x9, x8\n\t" + "eor v31.16b, v3.16b, v28.16b\n\t" + "bic %x[seed], x10, x9\n\t" + "eor v18.16b, v18.16b, v28.16b\n\t" + "bic %[r], x7, x11\n\t" + "ushr v5.2d, v31.2d, #36\n\t" + "bic x30, x8, x7\n\t" + "ushr v3.2d, v18.2d, #43\n\t" + "eor x7, x7, x12\n\t" + "sli v5.2d, v31.2d, #28\n\t" + "eor x8, x8, %x[seed]\n\t" + "sli v3.2d, v18.2d, #21\n\t" + "bic x12, x11, x10\n\t" + "eor v31.16b, v17.16b, v27.16b\n\t" + "eor x10, x10, %[r]\n\t" + "eor v11.16b, v11.16b, v26.16b\n\t" + "eor x9, x9, x12\n\t" + "ushr v18.2d, v31.2d, #49\n\t" + "eor x11, x11, x30\n\t" + "ushr v17.2d, v11.2d, #54\n\t" + "bic x12, x14, x13\n\t" + "sli v18.2d, v31.2d, #15\n\t" + "bic %x[seed], x15, x14\n\t" + "sli v17.2d, v11.2d, #10\n\t" + "bic %[r], %x[state], x16\n\t" + "eor v31.16b, v7.16b, v27.16b\n\t" + "bic x30, x13, %x[state]\n\t" + "eor v10.16b, v10.16b, v25.16b\n\t" + "eor x12, %x[state], x12\n\t" + "ushr v11.2d, v31.2d, #58\n\t" + "eor x13, x13, %x[seed]\n\t" + "ushr v7.2d, v10.2d, #61\n\t" + "bic %x[state], x16, x15\n\t" + "sli v11.2d, v31.2d, #6\n\t" + "eor x15, x15, %[r]\n\t" + "sli v7.2d, v10.2d, #3\n\t" + "eor x14, x14, %x[state]\n\t" + /* Row Mix NEON */ + "bic v25.16b, v2.16b, v1.16b\n\t" + "eor x16, x16, x30\n\t" + "bic v26.16b, v3.16b, v2.16b\n\t" + "bic %x[state], x20, x19\n\t" + "bic v27.16b, v4.16b, v3.16b\n\t" + "bic %x[seed], x21, x20\n\t" + "bic v28.16b, v0.16b, v4.16b\n\t" + "bic %[r], x17, x22\n\t" + "bic v29.16b, v1.16b, v0.16b\n\t" + "bic x30, x19, x17\n\t" + "eor v0.16b, v0.16b, v25.16b\n\t" + "eor x17, x17, %x[state]\n\t" + "eor v1.16b, v1.16b, v26.16b\n\t" + "eor x19, x19, %x[seed]\n\t" + "eor v2.16b, v2.16b, v27.16b\n\t" + "bic %x[state], x22, x21\n\t" + "eor v3.16b, v3.16b, v28.16b\n\t" + "eor x21, x21, %[r]\n\t" + "eor v4.16b, v4.16b, v29.16b\n\t" + "eor x20, x20, %x[state]\n\t" + "bic v25.16b, v7.16b, v6.16b\n\t" + "eor x22, x22, x30\n\t" + "bic v26.16b, v8.16b, v7.16b\n\t" + "bic %x[state], x25, x24\n\t" + "bic v27.16b, v9.16b, v8.16b\n\t" + "bic %x[seed], x26, x25\n\t" + "bic v28.16b, v5.16b, v9.16b\n\t" + "bic %[r], x23, x27\n\t" + "bic v29.16b, v6.16b, v5.16b\n\t" + "bic x30, x24, x23\n\t" + "eor v5.16b, v5.16b, v25.16b\n\t" + "eor x23, x23, %x[state]\n\t" + "eor v6.16b, v6.16b, v26.16b\n\t" + "eor x24, x24, %x[seed]\n\t" + "eor v7.16b, v7.16b, v27.16b\n\t" + "bic %x[state], x27, x26\n\t" + "eor v8.16b, v8.16b, v28.16b\n\t" + "eor x26, x26, %[r]\n\t" + "eor v9.16b, v9.16b, v29.16b\n\t" + "eor x25, x25, %x[state]\n\t" + "bic v25.16b, v12.16b, v11.16b\n\t" + "eor x27, x27, x30\n\t" + "bic v26.16b, v13.16b, v12.16b\n\t" + "bic v27.16b, v14.16b, v13.16b\n\t" + "bic v28.16b, v30.16b, v14.16b\n\t" + "bic v29.16b, v11.16b, v30.16b\n\t" + "eor v10.16b, v30.16b, v25.16b\n\t" + "eor v11.16b, v11.16b, v26.16b\n\t" + "eor v12.16b, v12.16b, v27.16b\n\t" + "eor v13.16b, v13.16b, v28.16b\n\t" + "eor v14.16b, v14.16b, v29.16b\n\t" + "bic v25.16b, v17.16b, v16.16b\n\t" + "bic v26.16b, v18.16b, v17.16b\n\t" + "bic v27.16b, v19.16b, v18.16b\n\t" + "bic v28.16b, v15.16b, v19.16b\n\t" + "bic v29.16b, v16.16b, v15.16b\n\t" + "eor v15.16b, v15.16b, v25.16b\n\t" + "eor v16.16b, v16.16b, v26.16b\n\t" + "eor v17.16b, v17.16b, v27.16b\n\t" + "eor v18.16b, v18.16b, v28.16b\n\t" + "eor v19.16b, v19.16b, v29.16b\n\t" + "bic v25.16b, v22.16b, v21.16b\n\t" + "bic v26.16b, v23.16b, v22.16b\n\t" + "bic v27.16b, v24.16b, v23.16b\n\t" + "bic v28.16b, v20.16b, v24.16b\n\t" + "bic v29.16b, v21.16b, v20.16b\n\t" + "eor v20.16b, v20.16b, v25.16b\n\t" + "eor v21.16b, v21.16b, v26.16b\n\t" + "eor v22.16b, v22.16b, v27.16b\n\t" + "eor v23.16b, v23.16b, v28.16b\n\t" + "eor v24.16b, v24.16b, v29.16b\n\t" + /* Done transforming */ + "ldp %[r], %x[seed], [x29, #48]\n\t" + "ldr %x[state], [%[r]], #8\n\t" + "subs %x[seed], %x[seed], #1\n\t" + "mov v30.d[0], %x[state]\n\t" + "mov v30.d[1], %x[state]\n\t" + "eor x2, x2, %x[state]\n\t" + "eor v0.16b, v0.16b, v30.16b\n\t" + "b.ne L_SHA3_shake256_blocksx3_seed_64_neon_begin_%=\n\t" + "ldr %x[state], [x29, #40]\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[0], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[0], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[0], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[0], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[0], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[0], [%x[state]], #32\n\t" + "st1 {v24.d}[0], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "st4 {v0.d, v1.d, v2.d, v3.d}[1], [%x[state]], #32\n\t" + "st4 {v4.d, v5.d, v6.d, v7.d}[1], [%x[state]], #32\n\t" + "st4 {v8.d, v9.d, v10.d, v11.d}[1], [%x[state]], #32\n\t" + "st4 {v12.d, v13.d, v14.d, v15.d}[1], [%x[state]], #32\n\t" + "st4 {v16.d, v17.d, v18.d, v19.d}[1], [%x[state]], #32\n\t" + "st4 {v20.d, v21.d, v22.d, v23.d}[1], [%x[state]], #32\n\t" + "st1 {v24.d}[1], [%x[state]]\n\t" + "add %x[state], %x[state], #8\n\t" + "stp x2, x3, [%x[state]]\n\t" + "stp x4, x5, [%x[state], #16]\n\t" + "stp x6, x7, [%x[state], #32]\n\t" + "stp x8, x9, [%x[state], #48]\n\t" + "stp x10, x11, [%x[state], #64]\n\t" + "stp x12, x13, [%x[state], #80]\n\t" + "stp x14, x15, [%x[state], #96]\n\t" + "stp x16, x17, [%x[state], #112]\n\t" + "stp x19, x20, [%x[state], #128]\n\t" + "stp x21, x22, [%x[state], #144]\n\t" + "stp x23, x24, [%x[state], #160]\n\t" + "stp x25, x26, [%x[state], #176]\n\t" + "str x27, [%x[state], #192]\n\t" + "ldp x29, x30, [sp], #0x40\n\t" + : [state] "+r" (state), [seed] "+r" (seed) + : [r] "r" (r) + : "memory", "cc", "x2", "x3", "x4", "x5", "x6", "x7", "x8", "x9", "x10", + "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x19", "x20", + "x21", "x22", "x23", "x24", "x25", "x26", "x27", "v0", "v1", "v2", + "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", + "v13", "v14", "v15", "v16", "v17", "v18", "v19", "v20", "v21", + "v22", "v23", "v24", "v25", "v26", "v27", "v28", "v29", "v30", + "v31" + ); +} + +#endif /* defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) */ #endif /* WOLFSSL_SHA3 */ #endif /* __aarch64__ */ #endif /* WOLFSSL_ARMASM */ diff --git a/wolfcrypt/src/wc_mldsa.c b/wolfcrypt/src/wc_mldsa.c index 353ef26288f..dc0e9c3a273 100644 --- a/wolfcrypt/src/wc_mldsa.c +++ b/wolfcrypt/src/wc_mldsa.c @@ -187,8 +187,12 @@ #endif #endif -#if defined(USE_INTEL_SPEEDUP) +#if defined(USE_INTEL_SPEEDUP) || (defined(__aarch64__) && \ + defined(WOLFSSL_ARMASM)) static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER; +#endif + +#if defined(USE_INTEL_SPEEDUP) /* AVX2 NTT/invNTT flavor selection: the non-full AVX2 NTT/invNTT keep the * NTT-domain coefficients in a permuted (lane-interleaved) order that only @@ -220,6 +224,132 @@ static cpuid_flags_t cpuid_flags = WC_CPUID_INITIALIZER; #endif #endif +#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) +/* NEON NTT/invNTT flavor selection, the same split as AVX2 has: the non-full + * transforms leave the coefficients lane-permuted (each block of 8 held as + * 0 2 4 6 1 3 5 7), which saves the un-permute and is understood by the + * positionwise multiplies and by the matching non-full inverse transform. + * The _full variants use the standard order, which is what the naturally + * ordered matrix A and t1 have to be multiplied against. */ +#ifndef WOLFSSL_AARCH64_NO_SQRDMLSH +#define MLDSA_NEON_PICK(fn, ...) \ + do { \ + if (IS_AARCH64_RDM(cpuid_flags)) { \ + fn##_sqrdmlsh_neon(__VA_ARGS__); \ + } \ + else { \ + fn##_neon(__VA_ARGS__); \ + } \ + } \ + while (0) +#else +#define MLDSA_NEON_PICK(fn, ...) fn##_neon(__VA_ARGS__) +#endif + +#define MLDSA_NTT_NEON(r) MLDSA_NEON_PICK(mldsa_ntt, r) +#define MLDSA_NTT_FULL_NEON(r) MLDSA_NEON_PICK(mldsa_ntt_full, r) +#define MLDSA_INVNTT_NEON(r) MLDSA_NEON_PICK(mldsa_invntt, r) +#define MLDSA_INVNTT_FULL_NEON(r) MLDSA_NEON_PICK(mldsa_invntt_full, r) +#define MLDSA_MUL_NEON(r, a, b) MLDSA_NEON_PICK(mldsa_mul, r, a, b) + +/* These come from armv8-sha3-asm, which is built whenever SHA-3 is, so they + * are available to ML-DSA whether or not ML-KEM is enabled. + * + * ML-DSA shares ML-KEM's 3-way-parallel Keccak. Those have a NEON version and + * a version built on the optional ARMv8.2 SHA-3 instructions (EOR3/RAX1/XAR/ + * BCAX), which is roughly 1.7x the speed. Pick between them at run time from + * the CPU ID flags, exactly as wc_mlkem_poly.c does -- selecting at build time + * would SIGILL on an aarch64 CPU without FEAT_SHA3 (e.g. Cortex-A55). */ +#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 + +static void mlkem_sha3_blocksx3(word64* state) +{ + if (IS_AARCH64_SHA3(cpuid_flags)) { + sha3_blocksx3_crypto(state); + } + else { + sha3_blocksx3_neon(state); + } +} + +static void mlkem_shake128_blocksx3_seed(word64* state, byte* seed) +{ + if (IS_AARCH64_SHA3(cpuid_flags)) { + sha3_128_blocksx3_seed_crypto(state, seed); + } + else { + sha3_128_blocksx3_seed_neon(state, seed); + } +} + +static void mlkem_shake256_blocksx3_seed_64(word64* state, byte* seed) +{ + if (IS_AARCH64_SHA3(cpuid_flags)) { + sha3_256_blocksx3_seed_64_crypto(state, seed); + } + else { + sha3_256_blocksx3_seed_64_neon(state, seed); + } +} + +#else + +#define mlkem_sha3_blocksx3 sha3_blocksx3_neon +#define mlkem_shake128_blocksx3_seed sha3_128_blocksx3_seed_neon +#define mlkem_shake256_blocksx3_seed_64 sha3_256_blocksx3_seed_64_neon + +#endif /* WOLFSSL_ARMASM_CRYPTO_SHA3 */ +#endif + +#if defined(WOLFSSL_ARMASM) && !defined(__aarch64__) && \ + !defined(WOLFSSL_ARMASM_THUMB2) && !defined(WOLFSSL_ARMASM_NO_NEON) && \ + !defined(WC_SHA3_NO_ASM) + +/* AArch32 shares the same batched Keccak but has no SHA-3 instruction + * extension, so there is nothing to select at run time. */ +#define mlkem_sha3_blocksx3 sha3_blocksx3_neon +#define mlkem_shake128_blocksx3_seed sha3_128_blocksx3_seed_neon +#define mlkem_shake256_blocksx3_seed_64 sha3_256_blocksx3_seed_64_neon + +#endif + +/* Targets carrying the three-way batched Keccak in assembly. gen_s and gen_y + * need only that; gen_matrix additionally needs a 32-bit rejection sampler, + * which every target carrying the batched Keccak now has. */ +#if defined(WOLFSSL_ARMASM) && !defined(WC_SHA3_NO_ASM) && \ + (defined(__aarch64__) || (!defined(WOLFSSL_ARMASM_THUMB2) && \ + !defined(WOLFSSL_ARMASM_NO_NEON))) + #define MLDSA_ASM_BLOCKSX3 +#endif + +/* Targets with the forward NTT in assembly. */ +#if defined(WOLFSSL_ARMASM) && (defined(__aarch64__) || \ + (!defined(WOLFSSL_ARMASM_THUMB2) && !defined(WOLFSSL_ARMASM_NO_NEON))) + #define MLDSA_ASM_NTT + #define MLDSA_ASM_INVNTT + #define MLDSA_ASM_DECOMPOSE + #define MLDSA_ASM_MUL + #define MLDSA_ASM_POLY_OPS + #define MLDSA_ASM_DECODE + #define MLDSA_ASM_ENCODE + #define MLDSA_ASM_EXTRACT + #define MLDSA_ASM_MAKE_HINT +#endif + +/* Targets that drive the Keccak state directly instead of going through the + * SHAKE-256 API. There is no run-time instruction-set choice to make on Arm, + * so the absorb can be written straight into the state and BlockSha3() called + * on it - which also lets sample_in_ball refill its block in place. Thumb-2 is + * left on the generic path. */ +#if defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_ARMASM_THUMB2) +#define MLDSA_SHAKE_DIRECT +#endif + +#if defined(MLDSA_ASM_BLOCKSX3) +#define MLDSA_REJ_UNIFORM_ASM mldsa_rej_uniform_neon +#endif + + #ifdef DEBUG_MLDSA void print_polys(const char* name, const sword32* a, int d1, int d2); void print_polys(const char* name, const sword32* a, int d1, int d2) @@ -304,6 +434,9 @@ void print_data(const char* name, const byte* d, int len) #define MLDSA_GEN_S_BYTES \ (MLDSA_GEN_S_NBLOCKS * MLDSA_GEN_S_BLOCK_BYTES) +/* Number of bytes generated by SHAKE256. */ +#define SHA3_256_BYTES (WC_SHA3_256_COUNT * 8) + /* Length of the hash OID to include in pre-hash message. */ #define MLDSA_HASH_OID_LEN 11 @@ -579,6 +712,43 @@ static int mldsa_shake256(wc_Shake* shake256, const byte* data, XMEMCPY(hash, shake256->s, hashLen); } ret = 0; +#elif defined(MLDSA_SHAKE_DIRECT) + word64* state = shake256->s; + word8 *state8 = (word8*)state; + + if (dataLen >= WC_SHA3_256_COUNT * 8) { + XMEMCPY(state, data, WC_SHA3_256_COUNT * 8); + XMEMSET(state + WC_SHA3_256_COUNT, 0, + sizeof(shake256->s) - WC_SHA3_256_COUNT * 8); + dataLen -= WC_SHA3_256_COUNT * 8; + data += WC_SHA3_256_COUNT * 8; + BlockSha3(state); + if (dataLen >= WC_SHA3_256_COUNT * 8) { + while (dataLen >= WC_SHA3_256_COUNT * 8) { + xorbuf(state, data, WC_SHA3_256_COUNT * 8); + dataLen -= WC_SHA3_256_COUNT * 8; + data += WC_SHA3_256_COUNT * 8; + BlockSha3(state); + } + } + if (dataLen > 0) { + xorbuf(state, data, dataLen); + } + state8[dataLen] ^= 0x1f; + state8[WC_SHA3_256_COUNT * 8 - 1] ^= 0x80; + } + else { + XMEMCPY(state, data, dataLen); + state8[dataLen] = 0x1f; + XMEMSET(state8 + dataLen + 1, 0, sizeof(shake256->s) - (dataLen + 1)); + state8[WC_SHA3_256_COUNT * 8 - 1] ^= 0x80; + } + + BlockSha3(state); + if (hash != (byte*)shake256->s) { + XMEMCPY(hash, shake256->s, hashLen); + } + ret = 0; #else /* Initialize SHAKE-256 operation. */ ret = wc_InitShake256(shake256, NULL, INVALID_DEVID); @@ -688,7 +858,7 @@ static int mldsa_hash256(wc_Shake* shake256, const byte* data1, XMEMCPY(state8 + data1Len, data2, data2Len); state8[dataLen] = 0x1f; XMEMSET(state8 + dataLen + 1, 0, sizeof(shake256->s) - (dataLen + 1)); - state8[WC_SHA3_256_COUNT * 8 - 1] = 0x80; + state8[WC_SHA3_256_COUNT * 8 - 1] ^= 0x80; } #ifndef WC_SHA3_NO_ASM @@ -706,6 +876,50 @@ static int mldsa_hash256(wc_Shake* shake256, const byte* data1, } XMEMCPY(hash, shake256->s, hashLen); ret = 0; +#elif defined(MLDSA_SHAKE_DIRECT) + word64* state = shake256->s; + word8 *state8 = (word8*)state; + + if (data2Len > (WOLFSSL_MAX_32BIT - data1Len)) { + return BAD_FUNC_ARG; + } + if (data1Len + data2Len >= WC_SHA3_256_COUNT * 8) { + XMEMCPY(state8, data1, data1Len); + XMEMCPY(state8 + data1Len, data2, WC_SHA3_256_COUNT * 8 - data1Len); + XMEMSET(state + WC_SHA3_256_COUNT, 0, + sizeof(shake256->s) - WC_SHA3_256_COUNT * 8); + data2Len -= WC_SHA3_256_COUNT * 8 - data1Len; + data2 += WC_SHA3_256_COUNT * 8 - data1Len; + BlockSha3(state); + + if (data2Len >= WC_SHA3_256_COUNT * 8) { + while (data2Len >= WC_SHA3_256_COUNT * 8) { + xorbuf(state, data2, WC_SHA3_256_COUNT * 8); + data2Len -= WC_SHA3_256_COUNT * 8; + data2 += WC_SHA3_256_COUNT * 8; + BlockSha3(state); + } + } + + if (data2Len > 0) { + xorbuf(state, data2, data2Len); + } + state8[data2Len] ^= 0x1f; + state8[WC_SHA3_256_COUNT * 8 - 1] ^= 0x80; + } + else { + word32 dataLen = data1Len + data2Len; + + XMEMCPY(state8, data1, data1Len); + XMEMCPY(state8 + data1Len, data2, data2Len); + state8[dataLen] = 0x1f; + XMEMSET(state8 + dataLen + 1, 0, sizeof(shake256->s) - (dataLen + 1)); + state8[WC_SHA3_256_COUNT * 8 - 1] ^= 0x80; + } + + BlockSha3(state); + XMEMCPY(hash, shake256->s, hashLen); + ret = 0; #else /* Initialize SHAKE-256 operation. */ ret = wc_InitShake256(shake256, NULL, INVALID_DEVID); @@ -1007,6 +1221,21 @@ static int mldsa_squeeze256(wc_Shake* shake256, const byte* in, out += WC_SHA3_256_COUNT * 8; } ret = 0; +#elif defined(MLDSA_SHAKE_DIRECT) + word64* state = shake256->s; + word8* state8 = (word8*)state; + + XMEMCPY(state, in, inLen); + state8[inLen] = 0x1f; + XMEMSET(state8 + inLen + 1, 0, sizeof(shake256->s) - (inLen + 1)); + state8[WC_SHA3_256_COUNT * 8 - 1] ^= 0x80; + + for (; outBlocks > 0; outBlocks--) { + BlockSha3(state); + XMEMCPY(out, shake256->s, WC_SHA3_256_COUNT * 8); + out += WC_SHA3_256_COUNT * 8; + } + ret = 0; #else /* Initialize SHAKE-256 operation. */ ret = wc_InitShake256(shake256, NULL, INVALID_DEVID); @@ -1220,6 +1449,8 @@ static void mldsa_vec_encode_eta_bits(const sword32* s, byte d, byte eta, * @param [in] p Buffer of data to decode. * @param [in] s Vector of decoded polynomials. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_DECODE) static void mldsa_decode_eta_2_bits_c(const byte* p, sword32* s) { unsigned int j; @@ -1242,6 +1473,7 @@ static void mldsa_decode_eta_2_bits_c(const byte* p, sword32* s) p += MLDSA_ETA_2_BITS; } } +#endif /* !MLDSA_ASM_DECODE */ /* Decode polynomial with range -2..2. * @@ -1250,6 +1482,9 @@ static void mldsa_decode_eta_2_bits_c(const byte* p, sword32* s) */ static void mldsa_decode_eta_2_bits(const byte* p, sword32* s) { +#if defined(MLDSA_ASM_DECODE) + mldsa_decode_eta_2_neon(p, s); +#else #ifdef USE_INTEL_SPEEDUP if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { wc_mldsa_decode_eta_2_avx2(p, s); @@ -1260,6 +1495,7 @@ static void mldsa_decode_eta_2_bits(const byte* p, sword32* s) { mldsa_decode_eta_2_bits_c(p, s); } +#endif /* MLDSA_ASM_DECODE */ } #endif #ifndef WOLFSSL_NO_ML_DSA_65 @@ -1282,6 +1518,8 @@ static void mldsa_decode_eta_2_bits(const byte* p, sword32* s) * @param [in] p Buffer of data to decode. * @param [in] s Vector of decoded polynomials. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_DECODE) static void mldsa_decode_eta_4_bits_c(const byte* p, sword32* s) { unsigned int j; @@ -1312,6 +1550,7 @@ static void mldsa_decode_eta_4_bits_c(const byte* p, sword32* s) } #endif /* WOLFSSL_MLDSA_SMALL */ } +#endif /* !MLDSA_ASM_DECODE */ /* Decode polynomial with range -4..4. * @@ -1320,6 +1559,9 @@ static void mldsa_decode_eta_4_bits_c(const byte* p, sword32* s) */ static void mldsa_decode_eta_4_bits(const byte* p, sword32* s) { +#if defined(MLDSA_ASM_DECODE) + mldsa_decode_eta_4_neon(p, s); +#else #ifdef USE_INTEL_SPEEDUP if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { wc_mldsa_decode_eta_4_avx2(p, s); @@ -1330,6 +1572,7 @@ static void mldsa_decode_eta_4_bits(const byte* p, sword32* s) { mldsa_decode_eta_4_bits_c(p, s); } +#endif /* MLDSA_ASM_DECODE */ } #endif @@ -1615,6 +1858,8 @@ static void mldsa_vec_encode_t0_t1(const sword32* t, byte d, byte* t0, * @param [in] t0 Encoded values of t0. * @param [out] t Vector of polynomials. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_DECODE) static void mldsa_decode_t0_c(const byte* t0, sword32* t) { unsigned int j; @@ -1681,6 +1926,7 @@ static void mldsa_decode_t0_c(const byte* t0, sword32* t) t0 += MLDSA_D; } } +#endif /* !MLDSA_ASM_DECODE */ /* Decode bottom D bits of t as t0. * @@ -1689,6 +1935,9 @@ static void mldsa_decode_t0_c(const byte* t0, sword32* t) */ static void mldsa_decode_t0(const byte* t0, sword32* t) { +#if defined(MLDSA_ASM_DECODE) + mldsa_decode_t0_neon(t0, t); +#else #ifdef USE_INTEL_SPEEDUP if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { wc_mldsa_decode_t0_avx2(t0, t); @@ -1699,6 +1948,7 @@ static void mldsa_decode_t0(const byte* t0, sword32* t) { mldsa_decode_t0_c(t0, t); } +#endif /* MLDSA_ASM_DECODE */ } #if defined(WOLFSSL_MLDSA_CHECK_KEY) || \ @@ -1758,6 +2008,8 @@ static void mldsa_vec_decode_t0(const byte* t0, byte d, sword32* t) * @param [in] t1 Encoded values of t1. * @param [out] t Polynomials. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_DECODE) static void mldsa_decode_t1_c(const byte* t1, sword32* t) { unsigned int j; @@ -1820,6 +2072,7 @@ static void mldsa_decode_t1_c(const byte* t1, sword32* t) t1 += MLDSA_U; } } +#endif /* !MLDSA_ASM_DECODE */ /* Decode top bits of t as t1. * @@ -1833,6 +2086,9 @@ static void mldsa_decode_t1_c(const byte* t1, sword32* t) */ static void mldsa_decode_t1(const byte* t1, sword32* t) { +#if defined(MLDSA_ASM_DECODE) + mldsa_decode_t1_neon(t1, t); +#else #ifdef USE_INTEL_SPEEDUP if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { wc_mldsa_decode_t1_avx2(t1, t); @@ -1843,6 +2099,7 @@ static void mldsa_decode_t1(const byte* t1, sword32* t) { mldsa_decode_t1_c(t1, t); } +#endif /* MLDSA_ASM_DECODE */ } #endif @@ -1902,6 +2159,8 @@ static void mldsa_vec_decode_t1(const byte* t1, byte d, sword32* t) * @param [in] z Polynomial to encode. * @param [out] s Buffer to encode into. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_ENCODE) static void mldsa_encode_gamma1_17_bits_c(const sword32* z, byte* s) { unsigned int j; @@ -1938,6 +2197,7 @@ static void mldsa_encode_gamma1_17_bits_c(const sword32* z, byte* s) s += MLDSA_GAMMA1_17_ENC_BITS / 2; } } +#endif /* !MLDSA_ASM_ENCODE */ /* Encode z with range of -(GAMMA1-1)...GAMMA1 * @@ -1946,6 +2206,9 @@ static void mldsa_encode_gamma1_17_bits_c(const sword32* z, byte* s) */ static void mldsa_encode_gamma1_17_bits(const sword32* z, byte* s) { +#if defined(MLDSA_ASM_ENCODE) + mldsa_encode_gamma1_17_neon(z, s); +#else #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { wc_mldsa_encode_gamma1_17_avx512(z, s); @@ -1963,6 +2226,7 @@ static void mldsa_encode_gamma1_17_bits(const sword32* z, byte* s) { mldsa_encode_gamma1_17_bits_c(z, s); } +#endif /* MLDSA_ASM_ENCODE */ } #endif #if !defined(WOLFSSL_NO_ML_DSA_65) || !defined(WOLFSSL_NO_ML_DSA_87) @@ -1976,6 +2240,8 @@ static void mldsa_encode_gamma1_17_bits(const sword32* z, byte* s) * @param [in] z Polynomial to encode. * @param [out] s Buffer to encode into. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_ENCODE) static void mldsa_encode_gamma1_19_bits_c(const sword32* z, byte* s) { unsigned int j; @@ -2014,6 +2280,7 @@ static void mldsa_encode_gamma1_19_bits_c(const sword32* z, byte* s) s += MLDSA_GAMMA1_19_ENC_BITS / 2; } } +#endif /* !MLDSA_ASM_ENCODE */ /* Encode z with range of -(GAMMA1-1)...GAMMA1 * @@ -2022,6 +2289,9 @@ static void mldsa_encode_gamma1_19_bits_c(const sword32* z, byte* s) */ static void mldsa_encode_gamma1_19_bits(const sword32* z, byte* s) { +#if defined(MLDSA_ASM_ENCODE) + mldsa_encode_gamma1_19_neon(z, s); +#else #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { wc_mldsa_encode_gamma1_19_avx512(z, s); @@ -2039,6 +2309,7 @@ static void mldsa_encode_gamma1_19_bits(const sword32* z, byte* s) { mldsa_encode_gamma1_19_bits_c(z, s); } +#endif /* MLDSA_ASM_ENCODE */ } #endif @@ -2109,6 +2380,8 @@ static void mldsa_vec_encode_gamma1(const sword32* z, byte l, int bits, * @param [in] bits Number of bits used in encoding - GAMMA1 bits. * @param [out] z Polynomial to fill. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_DECODE) static void mldsa_decode_gamma1_c(const byte* s, int bits, sword32* z) { unsigned int i; @@ -2370,6 +2643,7 @@ static void mldsa_decode_gamma1_c(const byte* s, int bits, sword32* z) } #endif } +#endif /* !MLDSA_ASM_DECODE */ /* Decode polynomial with range -(GAMMA1-1)..GAMMA1. * @@ -2379,6 +2653,14 @@ static void mldsa_decode_gamma1_c(const byte* s, int bits, sword32* z) */ static void mldsa_decode_gamma1(const byte* s, int bits, sword32* z) { +#if defined(MLDSA_ASM_DECODE) + if (bits == MLDSA_GAMMA1_BITS_17) { + mldsa_decode_gamma1_17_neon(s, z); + } + else { + mldsa_decode_gamma1_19_neon(s, z); + } +#else #ifdef USE_INTEL_SPEEDUP if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { if (bits == MLDSA_GAMMA1_BITS_17) { @@ -2394,6 +2676,7 @@ static void mldsa_decode_gamma1(const byte* s, int bits, sword32* z) { mldsa_decode_gamma1_c(s, bits, z); } +#endif /* MLDSA_ASM_DECODE */ } #endif @@ -2464,6 +2747,8 @@ static void mldsa_vec_decode_gamma1(const byte* x, byte l, int bits, * @param [in] w1 Vector of polynomials to encode. * @param [out] w1e Buffer to encode into. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_ENCODE) static void mldsa_encode_w1_88_c(const sword32* w1, byte* w1e) { unsigned int j; @@ -2509,6 +2794,7 @@ static void mldsa_encode_w1_88_c(const sword32* w1, byte* w1e) w1e += MLDSA_Q_HI_88_ENC_BITS * 2; } } +#endif /* !MLDSA_ASM_ENCODE */ /* Encode w1 with range of 0..((q-1)/(2*GAMMA2)-1). * @@ -2517,6 +2803,9 @@ static void mldsa_encode_w1_88_c(const sword32* w1, byte* w1e) */ static void mldsa_encode_w1_88(const sword32* w1, byte* w1e) { +#if defined(MLDSA_ASM_ENCODE) + mldsa_encode_w1_88_neon(w1, w1e); +#else #ifdef USE_INTEL_SPEEDUP if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { wc_mldsa_encode_w1_88_avx2(w1, w1e); @@ -2527,6 +2816,7 @@ static void mldsa_encode_w1_88(const sword32* w1, byte* w1e) { mldsa_encode_w1_88_c(w1, w1e); } +#endif /* MLDSA_ASM_ENCODE */ } WOLFSSL_TEST_VIS void wc_mldsa_encode_w1_88(const sword32* w1, byte* w1e) @@ -2547,6 +2837,8 @@ WOLFSSL_TEST_VIS void wc_mldsa_encode_w1_88(const sword32* w1, byte* w1e) * @param [in] w1 Vector of polynomials to encode. * @param [out] w1e Buffer to encode into. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_ENCODE) static void mldsa_encode_w1_32_c(const sword32* w1, byte* w1e) { unsigned int j; @@ -2586,6 +2878,7 @@ static void mldsa_encode_w1_32_c(const sword32* w1, byte* w1e) w1e += MLDSA_Q_HI_32_ENC_BITS * 2; } } +#endif /* !MLDSA_ASM_ENCODE */ /* Encode w1 with range of 0..((q-1)/(2*GAMMA2)-1). * @@ -2594,6 +2887,9 @@ static void mldsa_encode_w1_32_c(const sword32* w1, byte* w1e) */ static void mldsa_encode_w1_32(const sword32* w1, byte* w1e) { +#if defined(MLDSA_ASM_ENCODE) + mldsa_encode_w1_32_neon(w1, w1e); +#else #ifdef USE_INTEL_SPEEDUP if (IS_INTEL_AVX2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { wc_mldsa_encode_w1_32_avx2(w1, w1e); @@ -2604,6 +2900,7 @@ static void mldsa_encode_w1_32(const sword32* w1, byte* w1e) { mldsa_encode_w1_32_c(w1, w1e); } +#endif /* MLDSA_ASM_ENCODE */ } WOLFSSL_TEST_VIS void wc_mldsa_encode_w1_32(const sword32* w1, byte* w1e) @@ -3012,7 +3309,8 @@ static int mldsa_rej_ntt_poly(wc_Shake* shake128, byte* seed, sword32* a, (!defined(WOLFSSL_MLDSA_NO_SIGN) && \ (!defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM) || \ defined(WC_MLDSA_CACHE_MATRIX_A))) -#if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) +#if (defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM)) || \ + defined(MLDSA_ASM_BLOCKSX3) #define SHA3_128_BYTES (WC_SHA3_128_COUNT * 8) /* Number of blocks to generate for matrix. */ @@ -3028,6 +3326,9 @@ static int mldsa_rej_ntt_poly(wc_Shake* shake128, byte* seed, sword32* a, * the end of the buffer. */ wc_static_assert(GEN_MATRIX_SIZE >= MLDSA_N * 3); +#endif + +#if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifndef WOLFSSL_NO_ML_DSA_44 /* Deterministically generate a matrix (or transpose) of uniform integers mod q. * @@ -3507,6 +3808,234 @@ static int wc_mldsa_gen_matrix_avx512(sword32* a, byte* seed, byte k, } #endif /* WOLFSSL_MLDSA_HAVE_INTEL_AVX512 */ +#elif defined(MLDSA_ASM_BLOCKSX3) +#ifndef WOLFSSL_NO_ML_DSA_44 +/* Deterministically generate a matrix (or transpose) of uniform integers mod q. + * + * Seed used with XOF to generate random bytes. + * + * @param [out] a Matrix of uniform integers. + * @param [in] seed Bytes to seed XOF generation. + * @return 0 on success. + * @return MEMORY_E when dynamic memory allocation fails. Only possible when + * WOLFSSL_SMALL_STACK is defined. + */ +static int wc_mldsa_gen_matrix_4x4_arm(sword32* a, byte* seed) +{ + int j; + unsigned int ctr0; + unsigned int ctr1; + unsigned int ctr2; + byte* p; + WC_DECLARE_VAR(state, word64, 3 * 25, NULL); + word64* st; + + WC_ALLOC_VAR_EX(state, word64, 3 * 25, NULL, DYNAMIC_TYPE_MLDSA, + return MEMORY_E); + st = (word64*)state; + + /* 0-0 -> 3-2 */ + for (j = 0; j < 15; j += 3) { + state[0*25 + 4] = 0x1f0000 + (((j + 0) / 4) << 8) + ((j + 0) % 4); + state[1*25 + 4] = 0x1f0000 + (((j + 1) / 4) << 8) + ((j + 1) % 4); + state[2*25 + 4] = 0x1f0000 + (((j + 2) / 4) << 8) + ((j + 2) % 4); + + mlkem_shake128_blocksx3_seed(state, seed); + /* Sample random bytes to create a polynomial. */ + p = (byte*)st; + ctr0 = MLDSA_REJ_UNIFORM_ASM(a + 0 * MLDSA_N, MLDSA_N, p, + SHA3_128_BYTES); + p += 25 * 8; + ctr1 = MLDSA_REJ_UNIFORM_ASM(a + 1 * MLDSA_N, MLDSA_N, p, + SHA3_128_BYTES); + p += 25 * 8; + ctr2 = MLDSA_REJ_UNIFORM_ASM(a + 2 * MLDSA_N, MLDSA_N, p, + SHA3_128_BYTES); + while ((ctr0 < MLDSA_N) || (ctr1 < MLDSA_N) || (ctr2 < MLDSA_N)) { + mlkem_sha3_blocksx3(st); + + p = (byte*)st; + ctr0 += MLDSA_REJ_UNIFORM_ASM(a + 0 * MLDSA_N + ctr0, + MLDSA_N - ctr0, p, SHA3_128_BYTES); + p += 25 * 8; + ctr1 += MLDSA_REJ_UNIFORM_ASM(a + 1 * MLDSA_N + ctr1, + MLDSA_N - ctr1, p, SHA3_128_BYTES); + p += 25 * 8; + ctr2 += MLDSA_REJ_UNIFORM_ASM(a + 2 * MLDSA_N + ctr2, + MLDSA_N - ctr2, p, SHA3_128_BYTES); + } + + a += 3 * MLDSA_N; + } + + /* Last one: 3-3 */ + readUnalignedWords64(state, seed, 4); + state[4] = 0x1f0000 + (3 << 8) + 3; + XMEMSET(state + 5, 0, sizeof(*state) * (25 - 5)); + state[20] = W64LIT(0x8000000000000000); + BlockSha3(state); + p = (byte*)st; + ctr0 = MLDSA_REJ_UNIFORM_ASM(a, MLDSA_N, p, SHA3_128_BYTES); + while (ctr0 < MLDSA_N) { + BlockSha3(state); + ctr0 += MLDSA_REJ_UNIFORM_ASM(a + ctr0, MLDSA_N - ctr0, p, + SHA3_128_BYTES); + } + + WC_FREE_VAR_EX(state, NULL, DYNAMIC_TYPE_MLDSA); + + return 0; +} +#endif /* WOLFSSL_NO_ML_DSA_44 */ +#ifndef WOLFSSL_NO_ML_DSA_65 +/* Deterministically generate a matrix (or transpose) of uniform integers mod q. + * + * Seed used with XOF to generate random bytes. + * + * @param [out] a Matrix of uniform integers. + * @param [in] seed Bytes to seed XOF generation. + * @return 0 on success. + * @return MEMORY_E when dynamic memory allocation fails. Only possible when + * WOLFSSL_SMALL_STACK is defined. + */ +static int wc_mldsa_gen_matrix_6x5_arm(sword32* a, byte* seed) +{ + int j; + unsigned int ctr0; + unsigned int ctr1; + unsigned int ctr2; + byte* p; + WC_DECLARE_VAR(state, word64, 3 * 25, NULL); + word64* st; + + WC_ALLOC_VAR_EX(state, word64, 3 * 25, NULL, DYNAMIC_TYPE_MLDSA, + return MEMORY_E); + st = (word64*)state; + + /* 0-0 -> 5-4 */ + for (j = 0; j < 30; j += 3) { + state[0*25 + 4] = 0x1f0000 + (((j + 0) / 5) << 8) + ((j + 0) % 5); + state[1*25 + 4] = 0x1f0000 + (((j + 1) / 5) << 8) + ((j + 1) % 5); + state[2*25 + 4] = 0x1f0000 + (((j + 2) / 5) << 8) + ((j + 2) % 5); + + mlkem_shake128_blocksx3_seed(state, seed); + /* Sample random bytes to create a polynomial. */ + p = (byte*)st; + ctr0 = MLDSA_REJ_UNIFORM_ASM(a + 0 * MLDSA_N, MLDSA_N, p, + SHA3_128_BYTES); + p += 25 * 8; + ctr1 = MLDSA_REJ_UNIFORM_ASM(a + 1 * MLDSA_N, MLDSA_N, p, + SHA3_128_BYTES); + p += 25 * 8; + ctr2 = MLDSA_REJ_UNIFORM_ASM(a + 2 * MLDSA_N, MLDSA_N, p, + SHA3_128_BYTES); + while ((ctr0 < MLDSA_N) || (ctr1 < MLDSA_N) || (ctr2 < MLDSA_N)) { + mlkem_sha3_blocksx3(st); + + p = (byte*)st; + ctr0 += MLDSA_REJ_UNIFORM_ASM(a + 0 * MLDSA_N + ctr0, + MLDSA_N - ctr0, p, SHA3_128_BYTES); + p += 25 * 8; + ctr1 += MLDSA_REJ_UNIFORM_ASM(a + 1 * MLDSA_N + ctr1, + MLDSA_N - ctr1, p, SHA3_128_BYTES); + p += 25 * 8; + ctr2 += MLDSA_REJ_UNIFORM_ASM(a + 2 * MLDSA_N + ctr2, + MLDSA_N - ctr2, p, SHA3_128_BYTES); + } + + a += 3 * MLDSA_N; + } + + WC_FREE_VAR_EX(state, NULL, DYNAMIC_TYPE_MLDSA); + + return 0; +} +#endif /* WOLFSSL_NO_ML_DSA_65 */ +#ifndef WOLFSSL_NO_ML_DSA_87 +/* Deterministically generate a matrix (or transpose) of uniform integers mod q. + * + * Seed used with XOF to generate random bytes. + * + * @param [out] a Matrix of uniform integers. + * @param [in] seed Bytes to seed XOF generation. + * @return 0 on success. + * @return MEMORY_E when dynamic memory allocation fails. Only possible when + * WOLFSSL_SMALL_STACK is defined. + */ +static int wc_mldsa_gen_matrix_8x7_arm(sword32* a, byte* seed) +{ + int j; + unsigned int ctr0; + unsigned int ctr1; + unsigned int ctr2; + byte* p; + WC_DECLARE_VAR(state, word64, 3 * 25, NULL); + word64* st; + + WC_ALLOC_VAR_EX(state, word64, 3 * 25, NULL, DYNAMIC_TYPE_MLDSA, + return MEMORY_E); + st = (word64*)state; + + /* 0-0 -> 7-4 */ + for (j = 0; j < 54; j += 3) { + state[0*25 + 4] = 0x1f0000 + (((j + 0) / 7) << 8) + ((j + 0) % 7); + state[1*25 + 4] = 0x1f0000 + (((j + 1) / 7) << 8) + ((j + 1) % 7); + state[2*25 + 4] = 0x1f0000 + (((j + 2) / 7) << 8) + ((j + 2) % 7); + + mlkem_shake128_blocksx3_seed(state, seed); + /* Sample random bytes to create a polynomial. */ + p = (byte*)st; + ctr0 = MLDSA_REJ_UNIFORM_ASM(a + 0 * MLDSA_N, MLDSA_N, p, + SHA3_128_BYTES); + p += 25 * 8; + ctr1 = MLDSA_REJ_UNIFORM_ASM(a + 1 * MLDSA_N, MLDSA_N, p, + SHA3_128_BYTES); + p += 25 * 8; + ctr2 = MLDSA_REJ_UNIFORM_ASM(a + 2 * MLDSA_N, MLDSA_N, p, + SHA3_128_BYTES); + while ((ctr0 < MLDSA_N) || (ctr1 < MLDSA_N) || (ctr2 < MLDSA_N)) { + mlkem_sha3_blocksx3(st); + + p = (byte*)st; + ctr0 += MLDSA_REJ_UNIFORM_ASM(a + 0 * MLDSA_N + ctr0, + MLDSA_N - ctr0, p, SHA3_128_BYTES); + p += 25 * 8; + ctr1 += MLDSA_REJ_UNIFORM_ASM(a + 1 * MLDSA_N + ctr1, + MLDSA_N - ctr1, p, SHA3_128_BYTES); + p += 25 * 8; + ctr2 += MLDSA_REJ_UNIFORM_ASM(a + 2 * MLDSA_N + ctr2, + MLDSA_N - ctr2, p, SHA3_128_BYTES); + } + + a += 3 * MLDSA_N; + } + + /* Last 2: 7-5, 7-6 */ + state[0*25 + 4] = 0x1f0000 + (7 << 8) + 5; + state[1*25 + 4] = 0x1f0000 + (7 << 8) + 6; + + mlkem_shake128_blocksx3_seed(state, seed); + /* Sample random bytes to create a polynomial. */ + p = (byte*)st; + ctr0 = MLDSA_REJ_UNIFORM_ASM(a + 0 * MLDSA_N, MLDSA_N, p, SHA3_128_BYTES); + p += 25 * 8; + ctr1 = MLDSA_REJ_UNIFORM_ASM(a + 1 * MLDSA_N, MLDSA_N, p, SHA3_128_BYTES); + while ((ctr0 < MLDSA_N) || (ctr1 < MLDSA_N)) { + mlkem_sha3_blocksx3(st); + + p = (byte*)st; + ctr0 += MLDSA_REJ_UNIFORM_ASM(a + 0 * MLDSA_N + ctr0, MLDSA_N - ctr0, + p, SHA3_128_BYTES); + p += 25 * 8; + ctr1 += MLDSA_REJ_UNIFORM_ASM(a + 1 * MLDSA_N + ctr1, MLDSA_N - ctr1, + p, SHA3_128_BYTES); + } + + WC_FREE_VAR_EX(state, NULL, DYNAMIC_TYPE_MLDSA); + + return 0; +} +#endif /* WOLFSSL_NO_ML_DSA_87 */ #endif /* Expand the seed to create matrix a. @@ -3581,14 +4110,37 @@ static int mldsa_expand_a(wc_Shake* shake128, const byte* pub_seed, byte k, byte l, sword32* a, void* heap) { int ret; -#if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) +#if (defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM)) || \ + defined(MLDSA_ASM_BLOCKSX3) byte seed[MLDSA_GEN_A_SEED_SZ]; #endif -#if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) -#ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 - /* Eight SHAKE-128 instances per permutation instead of four. Handles any - * k x l, so it comes before the fixed-size AVX2 implementations. */ +#if defined(MLDSA_ASM_BLOCKSX3) +#ifndef WOLFSSL_NO_ML_DSA_44 + if ((k == 4) && (l == 4)) { + XMEMCPY(seed, pub_seed, MLDSA_PUB_SEED_SZ); + ret = wc_mldsa_gen_matrix_4x4_arm(a, seed); + } + else +#endif +#ifndef WOLFSSL_NO_ML_DSA_65 + if ((k == 6) && (l == 5)) { + XMEMCPY(seed, pub_seed, MLDSA_PUB_SEED_SZ); + ret = wc_mldsa_gen_matrix_6x5_arm(a, seed); + } + else +#endif +#ifndef WOLFSSL_NO_ML_DSA_87 + if ((k == 8) && (l == 7)) { + XMEMCPY(seed, pub_seed, MLDSA_PUB_SEED_SZ); + ret = wc_mldsa_gen_matrix_8x7_arm(a, seed); + } + else +#endif +#elif defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) +#ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 + /* Eight SHAKE-128 instances per permutation instead of four. Handles any + * k x l, so it comes before the fixed-size AVX2 implementations. */ if (USE_INTEL_AVX512(cpuid_flags) && IS_INTEL_BMI2(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { XMEMCPY(seed, pub_seed, MLDSA_PUB_SEED_SZ); @@ -3649,11 +4201,14 @@ static int mldsa_expand_a(wc_Shake* shake128, const byte* pub_seed, #define MLDSA_COEFF_S_VALID_ETA2(b) \ ((b) < MLDSA_ETA_2_MOD) +/* Unreferenced when the assembly extractor is in use. */ +#if !defined(MLDSA_ASM_EXTRACT) static const signed char mldsa_coeff_eta2[] = { 2, 1, 0, -1, -2, 2, 1, 0, -1, -2, 2, 1, 0, -1, -2 }; +#endif /* !MLDSA_ASM_EXTRACT */ /* Convert random value 0..15 to a value in range of -2..2. * * FIPS 204 Section 7.1, Algorithm 15 CoeffFromHalfByte(b) @@ -3855,6 +4410,14 @@ static const signed char mldsa_coeff_eta2[] = { static void mldsa_extract_coeffs(const byte* z, unsigned int zLen, byte eta, sword32* s, unsigned int* cnt) { +#if defined(MLDSA_ASM_EXTRACT) + if (eta == MLDSA_ETA_2) { + mldsa_extract_coeffs_eta2_neon(z, zLen, s, cnt); + } + else { + mldsa_extract_coeffs_eta4_neon(z, zLen, s, cnt); + } +#else #ifdef WOLFSSL_MLDSA_NO_LARGE_CODE unsigned int j = *cnt; unsigned int c; @@ -3945,6 +4508,7 @@ static void mldsa_extract_coeffs(const byte* z, unsigned int zLen, *cnt = j; #endif +#endif /* MLDSA_ASM_EXTRACT */ } /* Create polynomial from hashing the seed with bounded values. @@ -4584,6 +5148,312 @@ static int wc_mldsa_gen_s_avx512(sword32* s1, byte s1Len, sword32* s2, return 0; } #endif /* WOLFSSL_MLDSA_HAVE_INTEL_AVX512 */ +#elif defined(MLDSA_ASM_BLOCKSX3) +#ifndef WOLFSSL_NO_ML_DSA_44 +/* Deterministically generate a matrix (or transpose) of uniform integers mod q. + * + * Seed used with XOF to generate random bytes. + * + * @param [out] a Vectos of uniform integers. + * @param [in] seed Bytes to seed XOF generation. + * @return 0 on success. + * @return MEMORY_E when dynamic memory allocation fails. Only possible when + * WOLFSSL_SMALL_STACK is defined. + */ +static int wc_mldsa_gen_s_4_4_arm(sword32* s[2], byte* seed) +{ + int l; +#ifdef WOLFSSL_SMALL_STACK + word64 *state = NULL; +#else + word64 state[25 * 3]; +#endif + unsigned int ctr0; + unsigned int ctr1; + unsigned int ctr2; + byte* p; + +#ifdef WOLFSSL_SMALL_STACK + state = (word64*)XMALLOC(sizeof(word64) * 25 * 3, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (state == NULL) { + XFREE(state, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return MEMORY_E; + } +#endif + + /* 4xN arrays for s1 + 2xN arrays for s2. */ + for (l = 0; l < 6; l += 3) { + state[0*25 + 8] = 0x1f0000 + l + 0; + state[1*25 + 8] = 0x1f0000 + l + 1; + state[2*25 + 8] = 0x1f0000 + l + 2; + + mlkem_shake256_blocksx3_seed_64(state, seed); + + ctr0 = 0; + ctr1 = 0; + ctr2 = 0; + + do { + p = (byte*)state; + + if (ctr0 < MLDSA_N) { + mldsa_extract_coeffs(p, MLDSA_GEN_S_BLOCK_BYTES, + MLDSA_ETA_2, + s[(l + 0) / 4] + ((l + 0) % 4) * MLDSA_N, &ctr0); + } + p += 25 * 8; + if (ctr1 < MLDSA_N) { + mldsa_extract_coeffs(p, MLDSA_GEN_S_BLOCK_BYTES, + MLDSA_ETA_2, + s[(l + 1) / 4] + ((l + 1) % 4) * MLDSA_N, &ctr1); + } + p += 25 * 8; + if (ctr2 < MLDSA_N) { + mldsa_extract_coeffs(p, MLDSA_GEN_S_BLOCK_BYTES, + MLDSA_ETA_2, + s[(l + 2) / 4] + ((l + 2) % 4) * MLDSA_N, &ctr2); + } + + if ((ctr0 < MLDSA_N) || (ctr1 < MLDSA_N) || (ctr2 < MLDSA_N)) { + mlkem_sha3_blocksx3(state); + } + } + /* Create more blocks if too many rejected. */ + while ((ctr0 < MLDSA_N) || (ctr1 < MLDSA_N) || (ctr2 < MLDSA_N)); + } + + /* 2xN arrays for s2 */ + state[0*25 + 8] = 0x1f0000 + 6; + state[1*25 + 8] = 0x1f0000 + 7; + + mlkem_shake256_blocksx3_seed_64(state, seed); + + ctr0 = 0; + ctr1 = 0; + + do { + p = (byte*)state; + + if (ctr0 < MLDSA_N) { + mldsa_extract_coeffs(p, MLDSA_GEN_S_BLOCK_BYTES, + MLDSA_ETA_2, s[1] + 2 * MLDSA_N, &ctr0); + } + p += 25 * 8; + if (ctr1 < MLDSA_N) { + mldsa_extract_coeffs(p, MLDSA_GEN_S_BLOCK_BYTES, + MLDSA_ETA_2, s[1] + 3 * MLDSA_N, &ctr1); + } + + if ((ctr0 < MLDSA_N) || (ctr1 < MLDSA_N)) { + mlkem_sha3_blocksx3(state); + } + } + /* Create more blocks if too many rejected. */ + while ((ctr0 < MLDSA_N) || (ctr1 < MLDSA_N)); + + /* state holds secret s-vector material. */ + ForceZero(state, sizeof(word64) * 25 * 3); + WC_FREE_VAR_EX(state, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return 0; +} +#endif +#ifndef WOLFSSL_NO_ML_DSA_65 +/* Deterministically generate a matrix (or transpose) of uniform integers mod q. + * + * Seed used with XOF to generate random bytes. + * + * @param [out] a Vectos of uniform integers. + * @param [in] seed Bytes to seed XOF generation. + * @return 0 on success. + * @return MEMORY_E when dynamic memory allocation fails. Only possible when + * WOLFSSL_SMALL_STACK is defined. + */ +static int wc_mldsa_gen_s_5_6_arm(sword32* s[2], byte* seed) +{ + int l; +#ifdef WOLFSSL_SMALL_STACK + word64 *state = NULL; +#else + word64 state[25 * 3]; +#endif + unsigned int ctr0; + unsigned int ctr1; + unsigned int ctr2; + byte* p; + +#ifdef WOLFSSL_SMALL_STACK + state = (word64*)XMALLOC(sizeof(word64) * 25 * 3, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (state == NULL) { + XFREE(state, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return MEMORY_E; + } +#endif + + /* 5xN arrays for s1 + 4xN arrays for s2. */ + for (l = 0; l < 9; l += 3) { + state[0*25 + 8] = 0x1f0000 + l + 0; + state[1*25 + 8] = 0x1f0000 + l + 1; + state[2*25 + 8] = 0x1f0000 + l + 2; + + mlkem_shake256_blocksx3_seed_64(state, seed); + + ctr0 = 0; + ctr1 = 0; + ctr2 = 0; + + do { + p = (byte*)state; + + if (ctr0 < MLDSA_N) { + mldsa_extract_coeffs(p, MLDSA_GEN_S_BLOCK_BYTES, + MLDSA_ETA_4, + s[(l + 0) / 5] + ((l + 0) % 5) * MLDSA_N, &ctr0); + } + p += 25 * 8; + if (ctr1 < MLDSA_N) { + mldsa_extract_coeffs(p, MLDSA_GEN_S_BLOCK_BYTES, + MLDSA_ETA_4, + s[(l + 1) / 5] + ((l + 1) % 5) * MLDSA_N, &ctr1); + } + p += 25 * 8; + if (ctr2 < MLDSA_N) { + mldsa_extract_coeffs(p, MLDSA_GEN_S_BLOCK_BYTES, + MLDSA_ETA_4, + s[(l + 2) / 5] + ((l + 2) % 5) * MLDSA_N, &ctr2); + } + + if ((ctr0 < MLDSA_N) || (ctr1 < MLDSA_N) || (ctr2 < MLDSA_N)) { + mlkem_sha3_blocksx3(state); + } + } + /* Create more blocks if too many rejected. */ + while ((ctr0 < MLDSA_N) || (ctr1 < MLDSA_N) || (ctr2 < MLDSA_N)); + } + + /* 2xN arrays for s2 */ + state[0*25 + 8] = 0x1f0000 + 9; + state[1*25 + 8] = 0x1f0000 + 10; + + mlkem_shake256_blocksx3_seed_64(state, seed); + + ctr0 = 0; + ctr1 = 0; + + do { + p = (byte*)state; + + if (ctr0 < MLDSA_N) { + mldsa_extract_coeffs(p, MLDSA_GEN_S_BLOCK_BYTES, + MLDSA_ETA_4, s[1] + 4 * MLDSA_N, &ctr0); + } + p += 25 * 8; + if (ctr1 < MLDSA_N) { + mldsa_extract_coeffs(p, MLDSA_GEN_S_BLOCK_BYTES, + MLDSA_ETA_4, s[1] + 5 * MLDSA_N, &ctr1); + } + + if ((ctr0 < MLDSA_N) || (ctr1 < MLDSA_N)) { + mlkem_sha3_blocksx3(state); + } + } + /* Create more blocks if too many rejected. */ + while ((ctr0 < MLDSA_N) || (ctr1 < MLDSA_N)); + + /* state holds secret s-vector material. */ + ForceZero(state, sizeof(word64) * 25 * 3); + WC_FREE_VAR_EX(state, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return 0; +} +#endif +#ifndef WOLFSSL_NO_ML_DSA_87 +/* Deterministically generate a matrix (or transpose) of uniform integers mod q. + * + * Seed used with XOF to generate random bytes. + * + * @param [out] a Vectos of uniform integers. + * @param [in] seed Bytes to seed XOF generation. + * @return 0 on success. + * @return MEMORY_E when dynamic memory allocation fails. Only possible when + * WOLFSSL_SMALL_STACK is defined. + */ +static int wc_mldsa_gen_s_7_8_arm(sword32* s[2], byte* seed) +{ + int l; +#ifdef WOLFSSL_SMALL_STACK + word64 *state = NULL; +#else + word64 state[25 * 3]; +#endif + unsigned int ctr0; + unsigned int ctr1; + unsigned int ctr2; + byte* p; + sword32* sa[5][3] = { + { &s[0][0 * MLDSA_N], &s[0][1 * MLDSA_N], &s[0][2 * MLDSA_N] }, + { &s[0][3 * MLDSA_N], &s[0][4 * MLDSA_N], &s[0][5 * MLDSA_N] }, + { &s[0][6 * MLDSA_N], &s[1][0 * MLDSA_N], &s[1][1 * MLDSA_N] }, + { &s[1][2 * MLDSA_N], &s[1][3 * MLDSA_N], &s[1][4 * MLDSA_N] }, + { &s[1][5 * MLDSA_N], &s[1][6 * MLDSA_N], &s[1][7 * MLDSA_N] }, + }; + +#ifdef WOLFSSL_SMALL_STACK + state = (word64*)XMALLOC(sizeof(word64) * 25 * 3, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if (state == NULL) { + XFREE(state, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return MEMORY_E; + } +#endif + + /* 7xN arrays for s1 + 8xN arrays for s2. */ + for (l = 0; l < 15; l += 3) { + state[0*25 + 8] = 0x1f0000 + l + 0; + state[1*25 + 8] = 0x1f0000 + l + 1; + state[2*25 + 8] = 0x1f0000 + l + 2; + + mlkem_shake256_blocksx3_seed_64(state, seed); + + ctr0 = 0; + ctr1 = 0; + ctr2 = 0; + + do { + p = (byte*)state; + + if (ctr0 < MLDSA_N) { + mldsa_extract_coeffs(p, MLDSA_GEN_S_BLOCK_BYTES, + MLDSA_ETA_2, sa[l/3][0] , &ctr0); + } + p += 25 * 8; + if (ctr1 < MLDSA_N) { + mldsa_extract_coeffs(p, MLDSA_GEN_S_BLOCK_BYTES, + MLDSA_ETA_2, sa[l/3][1] , &ctr1); + } + p += 25 * 8; + if (ctr2 < MLDSA_N) { + mldsa_extract_coeffs(p, MLDSA_GEN_S_BLOCK_BYTES, + MLDSA_ETA_2, sa[l/3][2] , &ctr2); + } + + if ((ctr0 < MLDSA_N) || (ctr1 < MLDSA_N) || (ctr2 < MLDSA_N)) { + mlkem_sha3_blocksx3(state); + } + } + /* Create more blocks if too many rejected. */ + while ((ctr0 < MLDSA_N) || (ctr1 < MLDSA_N) || (ctr2 < MLDSA_N)); + } + + /* state holds secret s-vector material. */ + ForceZero(state, sizeof(word64) * 25 * 3); + WC_FREE_VAR_EX(state, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return 0; +} +#endif #endif /* Expand private seed into vectors s1 and s2. @@ -4673,7 +5543,29 @@ static int mldsa_expand_s(wc_Shake* shake256, byte* priv_seed, byte eta, (void)heap; -#if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) +#if defined(MLDSA_ASM_BLOCKSX3) + #ifndef WOLFSSL_NO_ML_DSA_44 + if (s1Len == 4) { + sword32* s[2] = { s1, s2 }; + ret = wc_mldsa_gen_s_4_4_arm(s, priv_seed); + } + else + #endif + #ifndef WOLFSSL_NO_ML_DSA_65 + if (s1Len == 5) { + sword32* s[2] = { s1, s2 }; + ret = wc_mldsa_gen_s_5_6_arm(s, priv_seed); + } + else + #endif + #ifndef WOLFSSL_NO_ML_DSA_87 + if (s1Len == 7) { + sword32* s[2] = { s1, s2 }; + ret = wc_mldsa_gen_s_7_8_arm(s, priv_seed); + } + else + #endif +#elif defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 /* Eight SHAKE-256 instances per permutation instead of four. Handles any * vector dimensions, so it comes before the fixed-size AVX2 code. */ @@ -4714,7 +5606,7 @@ static int mldsa_expand_s(wc_Shake* shake256, byte* priv_seed, byte eta, } else #endif -#endif /* USE_INTEL_SPEEDUP && !WC_SHA3_NO_ASM */ +#endif { ret = mldsa_expand_s_c(shake256, priv_seed, eta, s1, s1Len, s2, s2Len); @@ -4726,7 +5618,6 @@ static int mldsa_expand_s(wc_Shake* shake256, byte* priv_seed, byte eta, #ifndef WOLFSSL_MLDSA_NO_SIGN #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) -#define SHA3_256_BYTES (WC_SHA3_256_COUNT * 8) #ifndef WOLFSSL_NO_ML_DSA_44 /* Deterministically generate a matrix (or transpose) of uniform integers mod q. @@ -4983,6 +5874,7 @@ static int wc_mldsa_gen_y_7_avx2(sword32* y, byte* seed, word16 kappa) } #endif + #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 /* Deterministically expand the private random seed into the vector y. * @@ -5074,6 +5966,267 @@ static int wc_mldsa_gen_y_avx512(sword32* y, byte* seed, word16 kappa, return 0; } #endif /* WOLFSSL_MLDSA_HAVE_INTEL_AVX512 */ +#elif defined(MLDSA_ASM_BLOCKSX3) +#ifndef WOLFSSL_NO_ML_DSA_44 +/* Deterministically generate a matrix (or transpose) of uniform integers mod q. + * + * Seed used with XOF to generate random bytes. + * + * @param [out] a Vectos of uniform integers. + * @param [in] seed Bytes to seed XOF generation. + * @return 0 on success. + * @return MEMORY_E when dynamic memory allocation fails. Only possible when + * WOLFSSL_SMALL_STACK is defined. + */ +static int wc_mldsa_gen_y_4_arm(sword32* s, byte* seed, word16 kappa) +{ + int l; +#ifdef WOLFSSL_SMALL_STACK + byte *rand = NULL; + word64 *state = NULL; +#else + byte rand[3 * MLDSA_MAX_V]; + word64 state[25 * 3]; +#endif + byte* p; + +#ifdef WOLFSSL_SMALL_STACK + rand = (byte*)XMALLOC(3 * MLDSA_MAX_V, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + state = (word64*)XMALLOC(sizeof(word64) * 25 * 3, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if ((rand == NULL) || (state == NULL)) { + XFREE(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(state, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return MEMORY_E; + } +#endif + + state[0*25 + 8] = 0x1f0000 + kappa + 0; + state[1*25 + 8] = 0x1f0000 + kappa + 1; + state[2*25 + 8] = 0x1f0000 + kappa + 2; + + mlkem_shake256_blocksx3_seed_64(state, seed); + XMEMCPY(rand + 0 * MLDSA_MAX_V, state + 0 * 25, SHA3_256_BYTES); + XMEMCPY(rand + 1 * MLDSA_MAX_V, state + 1 * 25, SHA3_256_BYTES); + XMEMCPY(rand + 2 * MLDSA_MAX_V, state + 2 * 25, SHA3_256_BYTES); + for (l = 1; l < MLDSA_MAX_V_BLOCKS; l++) { + mlkem_sha3_blocksx3(state); + XMEMCPY(rand + 0 * MLDSA_MAX_V + l * SHA3_256_BYTES, state + 0 * 25, + SHA3_256_BYTES); + XMEMCPY(rand + 1 * MLDSA_MAX_V + l * SHA3_256_BYTES, state + 1 * 25, + SHA3_256_BYTES); + XMEMCPY(rand + 2 * MLDSA_MAX_V + l * SHA3_256_BYTES, state + 2 * 25, + SHA3_256_BYTES); + } + + p = (byte*)rand; + mldsa_decode_gamma1(p, PARAMS_ML_DSA_44_GAMMA1_BITS, s + 0 * MLDSA_N); + p += MLDSA_MAX_V; + mldsa_decode_gamma1(p, PARAMS_ML_DSA_44_GAMMA1_BITS, s + 1 * MLDSA_N); + p += MLDSA_MAX_V; + mldsa_decode_gamma1(p, PARAMS_ML_DSA_44_GAMMA1_BITS, s + 2 * MLDSA_N); + + readUnalignedWords64(state, seed, 8); + state[8] = 0x1f0000 + kappa + 3; + XMEMSET(state + 9, 0, sizeof(*state) * (25 - 9)); + state[16] = W64LIT(0x8000000000000000); + BlockSha3(state); + XMEMCPY(rand, state, SHA3_256_BYTES); + for (l = 1; l < MLDSA_MAX_V_BLOCKS; l++) { + BlockSha3(state); + XMEMCPY(rand + l * SHA3_256_BYTES, state, SHA3_256_BYTES); + } + mldsa_decode_gamma1(rand, PARAMS_ML_DSA_44_GAMMA1_BITS, + s + 3 * MLDSA_N); + + /* rand/state hold the secret mask y. */ + ForceZero(rand, 3 * MLDSA_MAX_V); + ForceZero(state, sizeof(word64) * 25 * 3); + WC_FREE_VAR_EX(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER); + WC_FREE_VAR_EX(state, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return 0; +} +#endif + +#ifndef WOLFSSL_NO_ML_DSA_65 +/* Deterministically generate a matrix (or transpose) of uniform integers mod q. + * + * Seed used with XOF to generate random bytes. + * + * @param [out] a Vectos of uniform integers. + * @param [in] seed Bytes to seed XOF generation. + * @return 0 on success. + * @return MEMORY_E when dynamic memory allocation fails. Only possible when + * WOLFSSL_SMALL_STACK is defined. + */ +static int wc_mldsa_gen_y_5_arm(sword32* s, byte* seed, word16 kappa) +{ + int l; +#ifdef WOLFSSL_SMALL_STACK + byte *rand = NULL; + word64 *state = NULL; +#else + byte rand[3 * MLDSA_MAX_V]; + word64 state[25 * 3]; +#endif + byte* p; + +#ifdef WOLFSSL_SMALL_STACK + rand = (byte*)XMALLOC(3 * MLDSA_MAX_V, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + state = (word64*)XMALLOC(sizeof(word64) * 25 * 3, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if ((rand == NULL) || (state == NULL)) { + XFREE(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(state, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return MEMORY_E; + } +#endif + + state[0*25 + 8] = 0x1f0000 + kappa + 0; + state[1*25 + 8] = 0x1f0000 + kappa + 1; + state[2*25 + 8] = 0x1f0000 + kappa + 2; + + mlkem_shake256_blocksx3_seed_64(state, seed); + XMEMCPY(rand + 0 * MLDSA_MAX_V, state + 0 * 25, SHA3_256_BYTES); + XMEMCPY(rand + 1 * MLDSA_MAX_V, state + 1 * 25, SHA3_256_BYTES); + XMEMCPY(rand + 2 * MLDSA_MAX_V, state + 2 * 25, SHA3_256_BYTES); + for (l = 1; l < MLDSA_MAX_V_BLOCKS; l++) { + mlkem_sha3_blocksx3(state); + XMEMCPY(rand + 0 * MLDSA_MAX_V + l * SHA3_256_BYTES, state + 0 * 25, + SHA3_256_BYTES); + XMEMCPY(rand + 1 * MLDSA_MAX_V + l * SHA3_256_BYTES, state + 1 * 25, + SHA3_256_BYTES); + XMEMCPY(rand + 2 * MLDSA_MAX_V + l * SHA3_256_BYTES, state + 2 * 25, + SHA3_256_BYTES); + } + + p = (byte*)rand; + mldsa_decode_gamma1(p, PARAMS_ML_DSA_65_GAMMA1_BITS, s + 0 * MLDSA_N); + p += MLDSA_MAX_V; + mldsa_decode_gamma1(p, PARAMS_ML_DSA_65_GAMMA1_BITS, s + 1 * MLDSA_N); + p += MLDSA_MAX_V; + mldsa_decode_gamma1(p, PARAMS_ML_DSA_65_GAMMA1_BITS, s + 2 * MLDSA_N); + + state[0*25 + 8] = 0x1f0000 + kappa + 3; + state[1*25 + 8] = 0x1f0000 + kappa + 4; + + mlkem_shake256_blocksx3_seed_64(state, seed); + XMEMCPY(rand + 0 * MLDSA_MAX_V, state + 0 * 25, SHA3_256_BYTES); + XMEMCPY(rand + 1 * MLDSA_MAX_V, state + 1 * 25, SHA3_256_BYTES); + for (l = 1; l < MLDSA_MAX_V_BLOCKS; l++) { + mlkem_sha3_blocksx3(state); + XMEMCPY(rand + 0 * MLDSA_MAX_V + l * SHA3_256_BYTES, state + 0 * 25, + SHA3_256_BYTES); + XMEMCPY(rand + 1 * MLDSA_MAX_V + l * SHA3_256_BYTES, state + 1 * 25, + SHA3_256_BYTES); + } + + p = (byte*)rand; + mldsa_decode_gamma1(p, PARAMS_ML_DSA_65_GAMMA1_BITS, s + 3 * MLDSA_N); + p += MLDSA_MAX_V; + mldsa_decode_gamma1(p, PARAMS_ML_DSA_65_GAMMA1_BITS, s + 4 * MLDSA_N); + + /* rand/state hold the secret mask y. */ + ForceZero(rand, 3 * MLDSA_MAX_V); + ForceZero(state, sizeof(word64) * 25 * 3); + WC_FREE_VAR_EX(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER); + WC_FREE_VAR_EX(state, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return 0; +} +#endif + +#ifndef WOLFSSL_NO_ML_DSA_87 +/* Deterministically generate a matrix (or transpose) of uniform integers mod q. + * + * Seed used with XOF to generate random bytes. + * + * @param [out] a Vectos of uniform integers. + * @param [in] seed Bytes to seed XOF generation. + * @return 0 on success. + * @return MEMORY_E when dynamic memory allocation fails. Only possible when + * WOLFSSL_SMALL_STACK is defined. + */ +static int wc_mldsa_gen_y_7_arm(sword32* s, byte* seed, word16 kappa) +{ + int i; + int l; +#ifdef WOLFSSL_SMALL_STACK + byte *rand = NULL; + word64 *state = NULL; +#else + byte rand[3 * MLDSA_MAX_V]; + word64 state[25 * 3]; +#endif + byte* p; + +#ifdef WOLFSSL_SMALL_STACK + rand = (byte*)XMALLOC(3 * MLDSA_MAX_V, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + state = (word64*)XMALLOC(sizeof(word64) * 25 * 3, NULL, + DYNAMIC_TYPE_TMP_BUFFER); + if ((rand == NULL) || (state == NULL)) { + XFREE(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER); + XFREE(state, NULL, DYNAMIC_TYPE_TMP_BUFFER); + return MEMORY_E; + } +#endif + + for (i = 0; i < 6; i += 3) { + state[0*25 + 8] = 0x1f0000 + kappa + i + 0; + state[1*25 + 8] = 0x1f0000 + kappa + i + 1; + state[2*25 + 8] = 0x1f0000 + kappa + i + 2; + + mlkem_shake256_blocksx3_seed_64(state, seed); + XMEMCPY(rand + 0 * MLDSA_MAX_V, state + 0 * 25, SHA3_256_BYTES); + XMEMCPY(rand + 1 * MLDSA_MAX_V, state + 1 * 25, SHA3_256_BYTES); + XMEMCPY(rand + 2 * MLDSA_MAX_V, state + 2 * 25, SHA3_256_BYTES); + for (l = 1; l < MLDSA_MAX_V_BLOCKS; l++) { + mlkem_sha3_blocksx3(state); + XMEMCPY(rand + 0 * MLDSA_MAX_V + l * SHA3_256_BYTES, + state + 0 * 25, SHA3_256_BYTES); + XMEMCPY(rand + 1 * MLDSA_MAX_V + l * SHA3_256_BYTES, + state + 1 * 25, SHA3_256_BYTES); + XMEMCPY(rand + 2 * MLDSA_MAX_V + l * SHA3_256_BYTES, + state + 2 * 25, SHA3_256_BYTES); + } + + p = (byte*)rand; + mldsa_decode_gamma1(p, PARAMS_ML_DSA_87_GAMMA1_BITS, + s + (i + 0) * MLDSA_N); + p += MLDSA_MAX_V; + mldsa_decode_gamma1(p, PARAMS_ML_DSA_87_GAMMA1_BITS, + s + (i + 1) * MLDSA_N); + p += MLDSA_MAX_V; + mldsa_decode_gamma1(p, PARAMS_ML_DSA_87_GAMMA1_BITS, + s + (i + 2) * MLDSA_N); + } + + readUnalignedWords64(state, seed, 8); + state[8] = 0x1f0000 + kappa + 6; + XMEMSET(state + 9, 0, sizeof(*state) * (25 - 9)); + state[16] = W64LIT(0x8000000000000000); + BlockSha3(state); + XMEMCPY(rand, state, SHA3_256_BYTES); + for (l = 1; l < MLDSA_MAX_V_BLOCKS; l++) { + BlockSha3(state); + XMEMCPY(rand + l * SHA3_256_BYTES, state, SHA3_256_BYTES); + } + mldsa_decode_gamma1(rand, PARAMS_ML_DSA_87_GAMMA1_BITS, + s + 6 * MLDSA_N); + + /* rand/state hold the secret mask y. */ + ForceZero(rand, 3 * MLDSA_MAX_V); + ForceZero(state, sizeof(word64) * 25 * 3); + WC_FREE_VAR_EX(rand, NULL, DYNAMIC_TYPE_TMP_BUFFER); + WC_FREE_VAR_EX(state, NULL, DYNAMIC_TYPE_TMP_BUFFER); + + return 0; +} +#endif /* WOLFSSL_NO_ML_DSA_87 */ #endif /* Expand the private random seed into vector y. @@ -5152,7 +6305,26 @@ static int mldsa_vec_expand_mask(wc_Shake* shake256, byte* seed, (void)heap; -#if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) +#if defined(MLDSA_ASM_BLOCKSX3) + #ifndef WOLFSSL_NO_ML_DSA_44 + if (l == 4) { + ret = wc_mldsa_gen_y_4_arm(y, seed, kappa); + } + else + #endif + #ifndef WOLFSSL_NO_ML_DSA_65 + if (l == 5) { + ret = wc_mldsa_gen_y_5_arm(y, seed, kappa); + } + else + #endif + #ifndef WOLFSSL_NO_ML_DSA_87 + if (l == 7) { + ret = wc_mldsa_gen_y_7_arm(y, seed, kappa); + } + else + #endif +#elif defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 /* Whole vector in one eight-way run, whatever the dimension. */ if (USE_INTEL_AVX512(cpuid_flags) && IS_INTEL_BMI2(cpuid_flags) && @@ -5210,82 +6382,19 @@ static int mldsa_vec_expand_mask(wc_Shake* shake256, byte* seed, * 12: return c * * @param [in] level Security level of the parameter set. - * @param [in] shake256 SHAKE-256 object. - * @param [in] seed Buffer containing seed to expand. - * @param [in] seedLen Length of seed in bytes. - * @param [in] tau Number of +/- 1s in polynomial. - * @param [out] c Commit polynomial. - * @param [in] block Memory to use for block from key. - * @return 0 on success. - * @return Negative on hash error. - */ -static int mldsa_sample_in_ball_ex(int level, wc_Shake* shake256, - const byte* seed, word32 seedLen, byte tau, sword32* c, byte* block) -{ -#ifndef USE_INTEL_SPEEDUP - int ret = 0; - byte signs[MLDSA_SIGN_BYTES]; - unsigned int i; - /* Step 1: Initialize sign bit index. */ - unsigned int s = 0; - /* Step 2: First 8 bytes are used for sign. */ - unsigned int k = MLDSA_SIGN_BYTES; - - /* Set polynomial to all zeros. */ - XMEMSET(c, 0, MLDSA_POLY_SIZE); - - /* Generate a block of data from seed. */ -#ifdef WOLFSSL_MLDSA_FIPS204_DRAFT - if (level >= WC_ML_DSA_DRAFT) { - ret = mldsa_shake256(shake256, seed, MLDSA_SEED_SZ, block, - MLDSA_GEN_C_BLOCK_BYTES); - } - else -#endif - { - (void)level; - ret = mldsa_shake256(shake256, seed, seedLen, block, - MLDSA_GEN_C_BLOCK_BYTES); - } - if (ret == 0) { - /* Copy first 8 bytes of first hash block as random sign bits. */ - XMEMCPY(signs, block, MLDSA_SIGN_BYTES); - } - - /* Step 3: Put in TAU +/- 1s. */ - for (i = (unsigned int)MLDSA_N - tau; (ret == 0) && (i < MLDSA_N); i++) { - unsigned int j; - do { - /* Check whether block is exhausted. */ - if (k == MLDSA_GEN_C_BLOCK_BYTES) { - /* Generate a new block. */ - ret = wc_Shake256_SqueezeBlocks(shake256, block, 1); - - /* Restart hash block index. */ - k = 0; - } - /* Step 7: Get random byte from block as index. - * Step 5 and 10: Increment hash block index. - */ - j = block[k++]; - } - /* Step 4: Get another random if random index is a future swap index. */ - while ((ret == 0) && (j > i)); - - /* Step 8: Move value from random index to current index. */ - c[i] = c[j]; - /* Step 9: Set value at random index to +/- 1. - * Cast to sword32 before the subtract: where a byte is as wide as int, - * signs[] promotes to unsigned and -1 would widen as 0x0000ffff. - * Matches the USE_INTEL_SPEEDUP path. */ - c[j] = (sword32)1 - - (sword32)((((signs[s >> 3]) >> (s & 0x7)) & 0x1) << 1); - /* Next sign bit index. */ - s++; - } - - return ret; -#else + * @param [in] shake256 SHAKE-256 object. + * @param [in] seed Buffer containing seed to expand. + * @param [in] seedLen Length of seed in bytes. + * @param [in] tau Number of +/- 1s in polynomial. + * @param [out] c Commit polynomial. + * @param [in] block Memory to use for block from key. + * @return 0 on success. + * @return Negative on hash error. + */ +static int mldsa_sample_in_ball_ex(int level, wc_Shake* shake256, + const byte* seed, word32 seedLen, byte tau, sword32* c, byte* block) +{ +#ifdef USE_INTEL_SPEEDUP int ret = 0; /* SHA-3 state. */ word64* state = shake256->s; @@ -5360,6 +6469,132 @@ static int mldsa_sample_in_ball_ex(int level, wc_Shake* shake256, } } + return ret; +#elif defined(MLDSA_SHAKE_DIRECT) + int ret = 0; + /* SHA-3 state. */ + word64* state = shake256->s; + word64 signs; + unsigned int i; + /* Step 2: First 8 bytes are used for sign. */ + unsigned int k = MLDSA_SIGN_BYTES; + + block = (byte*)state; + + /* Set polynomial to all zeros. */ + XMEMSET(c, 0, MLDSA_POLY_SIZE); + + /* Generate a block of data from seed. */ +#ifdef WOLFSSL_MLDSA_FIPS204_DRAFT + if (level >= WC_ML_DSA_DRAFT) { + ret = mldsa_shake256(shake256, seed, MLDSA_SEED_SZ, block, + MLDSA_GEN_C_BLOCK_BYTES); + } + else +#endif + { + (void)level; + ret = mldsa_shake256(shake256, seed, seedLen, block, + MLDSA_GEN_C_BLOCK_BYTES); + } + if (ret == 0) { + /* Step 1: Initialize sign bit index. */ + /* Copy first 8 bytes of first hash block as random sign bits. */ + signs = *(word64*)block; + + /* Step 3: Put in TAU +/- 1s. */ + for (i = MLDSA_N - tau; i < MLDSA_N; i++) { + unsigned int j; + do { + /* Check whether block is exhausted. */ + if (k == MLDSA_GEN_C_BLOCK_BYTES) { + /* Generate a new block. */ + BlockSha3(state); + + /* Restart hash block index. */ + k = 0; + } + /* Step 7: Get random byte from block as index. + * Step 5 and 10: Increment hash block index. + */ + j = block[k++]; + } + /* Step 4: Get another random if random index is a future swap + * index. */ + while (j > i); + + /* Step 8: Move value from random index to current index. */ + c[i] = c[j]; + /* Step 9: Set value at random index to +/- 1. */ + c[j] = (sword32)1 - (sword32)((sword32)(signs & 0x1) << 1); + /* Next sign bit index. */ + signs >>= 1; + } + } + + return ret; +#else + int ret = 0; + byte signs[MLDSA_SIGN_BYTES]; + unsigned int i; + /* Step 1: Initialize sign bit index. */ + unsigned int s = 0; + /* Step 2: First 8 bytes are used for sign. */ + unsigned int k = MLDSA_SIGN_BYTES; + + /* Set polynomial to all zeros. */ + XMEMSET(c, 0, MLDSA_POLY_SIZE); + + /* Generate a block of data from seed. */ +#ifdef WOLFSSL_MLDSA_FIPS204_DRAFT + if (level >= WC_ML_DSA_DRAFT) { + ret = mldsa_shake256(shake256, seed, MLDSA_SEED_SZ, block, + MLDSA_GEN_C_BLOCK_BYTES); + } + else +#endif + { + (void)level; + ret = mldsa_shake256(shake256, seed, seedLen, block, + MLDSA_GEN_C_BLOCK_BYTES); + } + if (ret == 0) { + /* Copy first 8 bytes of first hash block as random sign bits. */ + XMEMCPY(signs, block, MLDSA_SIGN_BYTES); + } + + /* Step 3: Put in TAU +/- 1s. */ + for (i = (unsigned int)MLDSA_N - tau; (ret == 0) && (i < MLDSA_N); i++) { + unsigned int j; + do { + /* Check whether block is exhausted. */ + if (k == MLDSA_GEN_C_BLOCK_BYTES) { + /* Generate a new block. */ + ret = wc_Shake256_SqueezeBlocks(shake256, block, 1); + + /* Restart hash block index. */ + k = 0; + } + /* Step 7: Get random byte from block as index. + * Step 5 and 10: Increment hash block index. + */ + j = block[k++]; + } + /* Step 4: Get another random if random index is a future swap index. */ + while ((ret == 0) && (j > i)); + + /* Step 8: Move value from random index to current index. */ + c[i] = c[j]; + /* Step 9: Set value at random index to +/- 1. + * Cast to sword32 before the subtract: where a byte is as wide as int, + * signs[] promotes to unsigned and -1 would widen as 0x0000ffff. + * Matches the USE_INTEL_SPEEDUP path. */ + c[j] = (sword32)1 - + (sword32)((((signs[s >> 3]) >> (s & 0x7)) & 0x1) << 1); + /* Next sign bit index. */ + s++; + } + return ret; #endif } @@ -5419,7 +6654,14 @@ static int mldsa_sample_in_ball(int level, wc_Shake* shake256, * Decompose operations ******************************************************************************/ -#if !defined(WOLFSSL_MLDSA_NO_SIGN) || !defined(WOLFSSL_MLDSA_NO_VERIFY) +/* Callers of the per-value decompose: mldsa_vec_decompose_c() when signing + * without the assembly, the small-memory sign path, and use_hint on + * every target. With the assembly in use, a non-small-memory sign-only build + * has none of those, so they must not be defined there. */ +#if !defined(WOLFSSL_MLDSA_NO_VERIFY) || \ + (!defined(WOLFSSL_MLDSA_NO_SIGN) && \ + (!defined(MLDSA_ASM_DECOMPOSE) || \ + defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM))) #ifndef WOLFSSL_NO_ML_DSA_44 /* Decompose value into high and low based on GAMMA2 being ((q-1) / 88). * @@ -5548,6 +6790,10 @@ static void mldsa_decompose_q32(sword32 r, sword32* r0, sword32* r1) * @param [out] r0 Low parts in vector of polynomials. * @param [out] r1 High parts in vector of polynomials. */ +/* Unreferenced when the assembly is in use. The per-value + * mldsa_decompose_q88/q32 above stay: use_hint and the small-memory sign + * path call them. */ +#if !defined(MLDSA_ASM_DECOMPOSE) static void mldsa_vec_decompose_c(const sword32* r, byte k, sword32 gamma2, sword32* r0, sword32* r1) { @@ -5587,6 +6833,7 @@ static void mldsa_vec_decompose_c(const sword32* r, byte k, sword32 gamma2, } #endif } +#endif /* !MLDSA_ASM_DECOMPOSE */ /* Decompose vector of polynomials into high and low based on GAMMA2. * @@ -5599,6 +6846,26 @@ static void mldsa_vec_decompose_c(const sword32* r, byte k, sword32 gamma2, static void mldsa_vec_decompose(const sword32* r, byte k, sword32 gamma2, sword32* r0, sword32* r1) { +#if defined(MLDSA_ASM_DECOMPOSE) + { + unsigned int i; + for (i = 0; i < k; i++) { + #ifndef WOLFSSL_NO_ML_DSA_44 + if (gamma2 == MLDSA_Q_LOW_88) { + mldsa_decompose_q88_neon(r, r0, r1); + } + #endif + #if !defined(WOLFSSL_NO_ML_DSA_65) || !defined(WOLFSSL_NO_ML_DSA_87) + if (gamma2 == MLDSA_Q_LOW_32) { + mldsa_decompose_q32_neon(r, r0, r1); + } + #endif + r += MLDSA_N; + r0 += MLDSA_N; + r1 += MLDSA_N; + } + } +#else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { @@ -5634,6 +6901,7 @@ static void mldsa_vec_decompose(const sword32* r, byte k, sword32 gamma2, { mldsa_vec_decompose_c(r, k, gamma2, r0, r1); } +#endif /* MLDSA_ASM_DECOMPOSE */ } #endif @@ -5652,7 +6920,7 @@ static void mldsa_vec_decompose(const sword32* r, byte k, sword32 gamma2, * @param [in] a Polynomial. * @param [in] hi Largest value in range. */ -static int mldsa_check_low(const sword32* a, sword32 hi) +WC_MAYBE_UNUSED static int mldsa_check_low(const sword32* a, sword32 hi) { int ret = 1; unsigned int j; @@ -5684,6 +6952,8 @@ static int mldsa_check_low(const sword32* a, sword32 hi) * @param [in] l Dimension of vector. * @param [in] hi Largest value in range. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_POLY_OPS) static int mldsa_vec_check_low_c(const sword32* a, byte l, sword32 hi) { int ret = 1; @@ -5701,6 +6971,7 @@ static int mldsa_vec_check_low_c(const sword32* a, byte l, sword32 hi) return ret; } +#endif /* !MLDSA_ASM_POLY_OPS */ #endif /* Check that the values of the vector are in range. @@ -5711,6 +6982,9 @@ static int mldsa_vec_check_low_c(const sword32* a, byte l, sword32 hi) */ static int mldsa_vec_check_low(const sword32* a, byte l, sword32 hi) { +#if defined(MLDSA_ASM_POLY_OPS) + return mldsa_vec_check_low_neon(a, l, hi); +#else int ret; #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 @@ -5731,6 +7005,7 @@ static int mldsa_vec_check_low(const sword32* a, byte l, sword32 hi) } return ret; +#endif /* MLDSA_ASM_POLY_OPS */ } #endif @@ -5782,6 +7057,17 @@ static int mldsa_vec_check_low(const sword32* a, byte l, sword32 hi) static int mldsa_make_hint_88(const sword32* s, const sword32* w1, byte* h, byte *idxp) { +#if defined(MLDSA_ASM_MAKE_HINT) + { + int n = mldsa_make_hint_neon(s, w1, h + *idxp, (int)*idxp, + PARAMS_ML_DSA_44_OMEGA, (sword32)MLDSA_Q_LOW_88); + if (n < 0) { + return -1; + } + *idxp = (byte)n; + return 0; + } +#else unsigned int j; byte idx; @@ -5820,6 +7106,7 @@ static int mldsa_make_hint_88(const sword32* s, const sword32* w1, byte* h, *idxp = idx; return 0; +#endif /* MLDSA_ASM_MAKE_HINT */ } #endif #if !defined(WOLFSSL_NO_ML_DSA_65) || !defined(WOLFSSL_NO_ML_DSA_87) @@ -5865,6 +7152,17 @@ static int mldsa_make_hint_88(const sword32* s, const sword32* w1, byte* h, static int mldsa_make_hint_32(const sword32* s, const sword32* w1, byte omega, byte* h, byte *idxp) { +#if defined(MLDSA_ASM_MAKE_HINT) + { + int n = mldsa_make_hint_neon(s, w1, h + *idxp, (int)*idxp, + (int)omega, (sword32)MLDSA_Q_LOW_32); + if (n < 0) { + return -1; + } + *idxp = (byte)n; + return 0; + } +#else unsigned int j; byte idx; @@ -5904,6 +7202,7 @@ static int mldsa_make_hint_32(const sword32* s, const sword32* w1, *idxp = idx; return 0; +#endif /* MLDSA_ASM_MAKE_HINT */ } #endif @@ -6080,6 +7379,8 @@ static int mldsa_check_hint(const byte* h, byte k, byte omega) * @param [in] i Dimension index. * @param [in, out] op Pointer to current offset into hints. */ +/* Replaced by mldsa_use_hint_neon() when the assembly is in use. */ +#if !defined(MLDSA_ASM_DECOMPOSE) static void mldsa_use_hint_88(sword32* w1, const byte* h, unsigned int i, byte* op) { @@ -6135,6 +7436,7 @@ static void mldsa_use_hint_88(sword32* w1, const byte* h, unsigned int i, } *op = o; } +#endif /* !MLDSA_ASM_DECOMPOSE */ #endif /* !WOLFSSL_NO_ML_DSA_44 */ #if !defined(WOLFSSL_NO_ML_DSA_65) || !defined(WOLFSSL_NO_ML_DSA_87) @@ -6153,6 +7455,8 @@ static void mldsa_use_hint_88(sword32* w1, const byte* h, unsigned int i, * @param [in] i Dimension index. * @param [in, out] op Pointer to current offset into hints. */ +/* Replaced by mldsa_use_hint_neon() when the assembly is in use. */ +#if !defined(MLDSA_ASM_DECOMPOSE) static void mldsa_use_hint_32(sword32* w1, const byte* h, byte omega, unsigned int i, byte* op) { @@ -6201,6 +7505,100 @@ static void mldsa_use_hint_32(sword32* w1, const byte* h, byte omega, } *op = o; } +#endif /* !MLDSA_ASM_DECOMPOSE */ +#endif + +/* Outside the WOLFSSL_MLDSA_VERIFY_SMALL_MEM guard below: the small-memory + * verify path applies hints a polynomial at a time and calls this directly, + * so it is needed in both modes. Only mldsa_vec_use_hint(), which walks the + * whole vector, is specific to the non-small-memory path. */ +#if defined(MLDSA_ASM_DECOMPOSE) + +/* Largest OMEGA over the parameter sets built here, which bounds the number + * of hints in any one polynomial. mldsa_check_hint() has already rejected a + * signature whose per-polynomial counts exceed OMEGA. */ +#ifndef WOLFSSL_NO_ML_DSA_44 +#define MLDSA_USE_HINT_MAX_H PARAMS_ML_DSA_44_OMEGA +#elif !defined(WOLFSSL_NO_ML_DSA_87) +#define MLDSA_USE_HINT_MAX_H PARAMS_ML_DSA_87_OMEGA +#else +#define MLDSA_USE_HINT_MAX_H PARAMS_ML_DSA_65_OMEGA +#endif + +/* Apply hints to one polynomial with the assembly. + * + * The assembly does every coefficient as if no hint applied - convert to the + * positive range, decompose, keep the high part - which is the whole loop bar + * the at most OMEGA coefficients a hint touches. Those need the low part as + * well, so they are computed from the original values first and written back + * over the assembly's result. The high part is 0..43 (0..15 for q32), so a + * byte holds it. + * + * @param [in, out] w1 Polynomial to apply hints to. + * @param [in] h Hints to apply, in signature encoding. + * @param [in] end Index one past this polynomial's last hint. + * @param [in] q88 Whether GAMMA2 is (q-1)/88. + * @param [in, out] op Pointer to current offset into hints. + */ +static void mldsa_use_hint_neon(sword32* w1, const byte* h, byte end, int q88, + byte* op) +{ + byte o = *op; + byte n = 0; + byte idx[MLDSA_USE_HINT_MAX_H]; + byte val[MLDSA_USE_HINT_MAX_H]; + + /* Work out the hinted coefficients before the bulk pass overwrites them. */ + while ((o + n < end) && (n < (byte)MLDSA_USE_HINT_MAX_H)) { + sword32 r0; + sword32 r1; + sword32 w = w1[h[o + n]]; + sword32 r = w + (sword32)((0 - (((word32)w) >> 31)) & MLDSA_Q); + + #ifndef WOLFSSL_NO_ML_DSA_44 + if (q88) { + mldsa_decompose_q88(r, &r0, &r1); + /* r0 > 0 adds, r0 <= 0 subtracts (FIPS 204). */ + r1 += (sword32)(1U - (2U * (((word32)(r0 - 1)) >> 31))); + /* 44 is too large and wraps to 0; -1 is really 43. */ + r1 &= (sword32)(0U - (((word32)r1 - 44U) >> 31)); + r1 += (sword32)((0U - (((word32)r1) >> 31)) & 44U); + } + #endif + #if !defined(WOLFSSL_NO_ML_DSA_65) || !defined(WOLFSSL_NO_ML_DSA_87) + if (!q88) { + mldsa_decompose_q32(r, &r0, &r1); + r1 += (sword32)(1 - (2 * (((word32)(r0 - 1)) >> 31))); + /* Masking maps 16 to 0 and -1 to 15. */ + r1 &= 0xf; + } + #endif + idx[n] = h[o + n]; + val[n] = (byte)r1; + n++; + } + + /* Every coefficient no hint touches. */ +#ifndef WOLFSSL_NO_ML_DSA_44 + if (q88) { + mldsa_use_hint_q88_neon(w1); + } +#endif +#if !defined(WOLFSSL_NO_ML_DSA_65) || !defined(WOLFSSL_NO_ML_DSA_87) + if (!q88) { + mldsa_use_hint_q32_neon(w1); + } +#endif + + { + byte t; + for (t = 0; t < n; t++) { + w1[idx[t]] = (sword32)val[t]; + } + } + + *op = (byte)(o + n); +} #endif #ifndef WOLFSSL_MLDSA_VERIFY_SMALL_MEM @@ -6219,6 +7617,7 @@ static void mldsa_use_hint_32(sword32* w1, const byte* h, byte omega, * @param [in] omega Max number of hints. Hint counts after this index. * @param [in] h Hints to apply. In signature encoding. */ + static void mldsa_vec_use_hint(sword32* w1, byte k, sword32 gamma2, byte omega, const byte* h) { @@ -6248,7 +7647,12 @@ static void mldsa_vec_use_hint(sword32* w1, byte k, sword32 gamma2, { /* For each polynomial of vector. */ for (i = 0; i < PARAMS_ML_DSA_44_K; i++) { +#if defined(MLDSA_ASM_DECOMPOSE) + mldsa_use_hint_neon(w1, h, + h[PARAMS_ML_DSA_44_OMEGA + i], 1, &o); +#else mldsa_use_hint_88(w1, h, i, &o); +#endif w1 += MLDSA_N; } } @@ -6274,7 +7678,11 @@ static void mldsa_vec_use_hint(sword32* w1, byte k, sword32 gamma2, { /* For each polynomial of vector. */ for (i = 0; i < k; i++) { +#if defined(MLDSA_ASM_DECOMPOSE) + mldsa_use_hint_neon(w1, h, h[omega + i], 0, &o); +#else mldsa_use_hint_32(w1, h, omega, i, &o); +#endif w1 += MLDSA_N; } } @@ -6292,11 +7700,15 @@ static void mldsa_vec_use_hint(sword32* w1, byte k, sword32 gamma2, #define MLDSA_QINV 58728449 /* Montgomery reduce a. + * + * Whether this is referenced depends on the build: where the transforms and + * multiplies are all assembly, only the small-memory paths are left as + * callers, and those are themselves optional. * * @param [in] a 64-bit value to be reduced. * @return Montgomery reduction result. */ -static sword32 mldsa_mont_red(sword64 a) +WC_MAYBE_UNUSED static sword32 mldsa_mont_red(sword64 a) { #ifndef MLDSA_MUL_QINV_SLOW #ifdef MLDSA_MUL_QINV_WIDE64 @@ -6314,7 +7726,7 @@ static sword32 mldsa_mont_red(sword64 a) ((word32)a << 23) + ((word32)a << 26)); #endif #ifndef MLDSA_MUL_Q_SLOW - return (sword32)((a - ((sword32)t * (sword64)MLDSA_Q)) >> 32); + return (sword32)((a - (t * (sword32)MLDSA_Q)) >> 32); #else /* The whole expression is computed in word64 with a single conversion * back: shifts on word64 avoid the signed-shift UB, |t << 23| < 2^54 @@ -6339,7 +7751,7 @@ static sword32 mldsa_mont_red(sword64 a) * @param [in] a 32-bit value to be reduced to range of q. * @return Modulo result. */ -static sword32 mldsa_red(sword32 a) +WC_MAYBE_UNUSED static sword32 mldsa_red(sword32 a) { sword32 t = (sword32)((a + ((sword32)1 << 22)) >> 23); #ifndef MLDSA_MUL_Q_SLOW @@ -6353,6 +7765,9 @@ static sword32 mldsa_red(sword32 a) } #endif +/* Only mldsa_ntt_c() and mldsa_ntt_small_c() read these, so they go with them + * when the assembly replaces them. */ +#if !defined(MLDSA_ASM_NTT) /* Zetas for NTT. */ static const sword32 zetas[MLDSA_N] = { -41978, 25847, -2608894, -518909, 237124, -777960, -876248, 466468, @@ -6388,9 +7803,11 @@ static const sword32 zetas[MLDSA_N] = { -2939036, -2235985, -420899, -2286327, 183443, -976891, 1612842, -3545687, -554416, 3919660, -48306, -1362209, 3937738, 1400424, -846154, 1976782 }; +#endif /* !MLDSA_ASM_NTT */ -#ifndef WOLFSSL_MLDSA_SMALL -/* Zetas for inverse NTT. */ +/* Zetas for inverse NTT. Only mldsa_invntt_c() reads these, so they go with + * it when the assembly replaces it. */ +#if !defined(WOLFSSL_MLDSA_SMALL) && !defined(MLDSA_ASM_INVNTT) static const sword32 zetas_inv[MLDSA_N] = { -1976782, 846154, -1400424, -3937738, 1362209, 48306, -3919660, 554416, 3545687, -1612842, 976891, -183443, 2286327, 420899, 2235985, 2939036, @@ -6429,7 +7846,8 @@ static const sword32 zetas_inv[MLDSA_N] = { #if !defined(WOLFSSL_MLDSA_NO_SIGN) || \ !defined(WOLFSSL_MLDSA_NO_VERIFY) || \ - (!defined(WOLFSSL_MLDSA_NO_MAKE) && defined(WOLFSSL_MLDSA_SMALL)) + (!defined(WOLFSSL_MLDSA_NO_MAKE) && \ + (defined(WOLFSSL_MLDSA_SMALL) || defined(MLDSA_ASM_NTT))) /* One iteration of Number-Theoretic Transform. * @@ -6451,9 +7869,13 @@ do { \ while (0) /* Number-Theoretic Transform. + * + * Not built when the assembly is in use: both entry points go to the + * NEON transform there, leaving this unreferenced. * * @param [in, out] r Polynomial to transform. */ +#if !defined(MLDSA_ASM_NTT) static void mldsa_ntt_c(sword32* r) { #ifdef WOLFSSL_MLDSA_SMALL @@ -6819,6 +8241,7 @@ static void mldsa_ntt_c(sword32* r) } #endif } +#endif /* !MLDSA_ASM_NTT */ #if !defined(WOLFSSL_MLDSA_NO_SIGN) || \ defined(WC_MLDSA_CACHE_PRIV_VECTORS) @@ -6828,6 +8251,15 @@ static void mldsa_ntt_c(sword32* r) */ static void mldsa_ntt(sword32* r) { +#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) + MLDSA_NTT_NEON(r); +#elif defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_ARMASM_THUMB2) && \ + !defined(WOLFSSL_ARMASM_NO_NEON) + /* AArch32 transforms in place and leaves the coefficients in their natural + * order, which is what the C inverse transform expects. It therefore + * serves both the plain and the "full" entry points. */ + mldsa_ntt_neon(r); +#else #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { MLDSA_NTT_AVX512(r); @@ -6846,6 +8278,7 @@ static void mldsa_ntt(sword32* r) { mldsa_ntt_c(r); } +#endif /* __aarch64__ && WOLFSSL_ARMASM */ } #endif @@ -6853,7 +8286,7 @@ static void mldsa_ntt(sword32* r) (!defined(WOLFSSL_MLDSA_NO_SIGN) && \ (!defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM) || \ defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC))) || \ - (defined(WOLFSSL_MLDSA_SMALL) && \ + ((defined(WOLFSSL_MLDSA_SMALL) || defined(MLDSA_ASM_NTT)) && \ (!defined(WOLFSSL_MLDSA_NO_MAKE_KEY) || \ defined(WOLFSSL_MLDSA_CHECK_KEY))) /* Number-Theoretic Transform. @@ -6862,6 +8295,15 @@ static void mldsa_ntt(sword32* r) */ static void mldsa_ntt_full(sword32* r) { +#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) + MLDSA_NTT_FULL_NEON(r); +#elif defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_ARMASM_THUMB2) && \ + !defined(WOLFSSL_ARMASM_NO_NEON) + /* AArch32 transforms in place and leaves the coefficients in their natural + * order, which is what the C inverse transform expects. It therefore + * serves both the plain and the "full" entry points. */ + mldsa_ntt_neon(r); +#else #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { wc_mldsa_ntt_full_1p_avx512(r); @@ -6879,6 +8321,7 @@ static void mldsa_ntt_full(sword32* r) { mldsa_ntt_c(r); } +#endif /* __aarch64__ && WOLFSSL_ARMASM */ } #endif @@ -6920,7 +8363,7 @@ static void mldsa_vec_ntt(sword32* r, byte l) (!defined(WOLFSSL_MLDSA_NO_SIGN) && \ (!defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM) || \ defined(WOLFSSL_MLDSA_SIGN_SMALL_MEM_PRECALC)))) || \ - (defined(WOLFSSL_MLDSA_SMALL) && \ + ((defined(WOLFSSL_MLDSA_SMALL) || defined(MLDSA_ASM_NTT)) && \ (!defined(WOLFSSL_MLDSA_NO_MAKE_KEY) || \ defined(WOLFSSL_MLDSA_CHECK_KEY))) /* Number-Theoretic Transform. @@ -6939,7 +8382,11 @@ static void mldsa_vec_ntt_full(sword32* r, byte l) } #endif -#ifndef WOLFSSL_MLDSA_SMALL +/* The specialised small-coefficient transform only pays off against the C + * general one. An assembly transform is faster than either and, as the comment + * below records, produces identical output for the -26..26 inputs this is + * called with, so those targets take the same aliases as a small build. */ +#if !defined(WOLFSSL_MLDSA_SMALL) && !defined(MLDSA_ASM_NTT) /* Zeta index value 1 not in montgomery form. */ #define MLDSA_NTT_ZETA_1 ((sword32)-3572223) @@ -7429,7 +8876,7 @@ static void mldsa_vec_ntt_small_full(sword32* r, byte l) #define mldsa_vec_ntt_small_full mldsa_vec_ntt_full -#endif /* WOLFSSL_MLDSA_SMALL */ +#endif /* !WOLFSSL_MLDSA_SMALL && !MLDSA_ASM_NTT */ /* One iteration of Inverse Number-Theoretic Transform. @@ -7456,6 +8903,9 @@ while (0) * * @param [in, out] r Polynomial to transform. */ +/* Not built when the assembly is in use - both entry points + * go to the NEON transform there, leaving this unreferenced. */ +#if !defined(MLDSA_ASM_INVNTT) static void mldsa_invntt_c(sword32* r) { #ifdef WOLFSSL_MLDSA_SMALL @@ -7849,6 +9299,7 @@ static void mldsa_invntt_c(sword32* r) } #endif } +#endif /* !MLDSA_ASM_INVNTT */ #if !defined(WOLFSSL_MLDSA_NO_SIGN) /* Inverse Number-Theoretic Transform. @@ -7857,6 +9308,13 @@ static void mldsa_invntt_c(sword32* r) */ static void mldsa_invntt(sword32* r) { +#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) + MLDSA_INVNTT_NEON(r); +#elif defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_ARMASM_THUMB2) && \ + !defined(WOLFSSL_ARMASM_NO_NEON) + /* AArch32 takes and leaves natural order, so it serves both entry points. */ + mldsa_invntt_neon(r); +#else #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { /* MLDSA_INVNTT_AVX512: see the flavor-selection note by its @@ -7877,6 +9335,7 @@ static void mldsa_invntt(sword32* r) { mldsa_invntt_c(r); } +#endif /* __aarch64__ && WOLFSSL_ARMASM */ } #endif @@ -7886,6 +9345,13 @@ static void mldsa_invntt(sword32* r) */ static void mldsa_invntt_full(sword32* r) { +#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) + MLDSA_INVNTT_FULL_NEON(r); +#elif defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_ARMASM_THUMB2) && \ + !defined(WOLFSSL_ARMASM_NO_NEON) + /* AArch32 takes and leaves natural order, so it serves both entry points. */ + mldsa_invntt_neon(r); +#else #if defined(USE_INTEL_SPEEDUP) && defined(WOLFSSL_MLDSA_HAVE_INTEL_AVX512) if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { wc_mldsa_invntt_full_1p_avx512(r); @@ -7903,6 +9369,7 @@ static void mldsa_invntt_full(sword32* r) { mldsa_invntt_c(r); } +#endif /* __aarch64__ && WOLFSSL_ARMASM */ } #if !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) || \ @@ -7952,6 +9419,8 @@ static void mldsa_vec_invntt_full(sword32* r, byte l) * @param [in] k First dimension of matrix and dimension of result. * @param [in] l Second dimension of matrix and dimension of v. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_MUL) static void mldsa_matrix_mul_c(sword32* r, const sword32* m, const sword32* v, byte k, byte l) { @@ -8105,6 +9574,7 @@ static void mldsa_matrix_mul_c(sword32* r, const sword32* m, r += MLDSA_N; } } +#endif /* !MLDSA_ASM_MUL */ /* Matrix multiplication. * @@ -8117,6 +9587,24 @@ static void mldsa_matrix_mul_c(sword32* r, const sword32* m, static void mldsa_matrix_mul(sword32* r, const sword32* m, const sword32* v, byte k, byte l) { +#if defined(MLDSA_ASM_MUL) + { + int i; + for (i = 0; i < (int)k; i++) { + if (l == 4) { + mldsa_mul_vec_4_neon(r, m, v); + } + else if (l == 5) { + mldsa_mul_vec_5_neon(r, m, v); + } + else { + mldsa_mul_vec_7_neon(r, m, v); + } + m += (unsigned int)l * MLDSA_N; + r += MLDSA_N; + } + } +#else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { @@ -8176,6 +9664,7 @@ static void mldsa_matrix_mul(sword32* r, const sword32* m, const sword32* v, { mldsa_matrix_mul_c(r, m, v, k, l); } +#endif /* MLDSA_ASM_MUL */ } #endif @@ -8188,6 +9677,8 @@ static void mldsa_matrix_mul(sword32* r, const sword32* m, const sword32* v, * @param [in] a Polynomial * @param [in] b Polynomial. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_MUL) static void mldsa_mul_c(sword32* r, sword32* a, sword32* b) { unsigned int e; @@ -8227,6 +9718,7 @@ static void mldsa_mul_c(sword32* r, sword32* a, sword32* b) } #endif } +#endif /* !MLDSA_ASM_MUL */ #if !defined(WOLFSSL_MLDSA_NO_SIGN) /* Polynomial multiplication. @@ -8237,6 +9729,13 @@ static void mldsa_mul_c(sword32* r, sword32* a, sword32* b) */ static void mldsa_mul(sword32* r, sword32* a, sword32* b) { +#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) + MLDSA_MUL_NEON(r, a, b); +#elif defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_ARMASM_THUMB2) && \ + !defined(WOLFSSL_ARMASM_NO_NEON) + /* AArch32 has one implementation - no run-time choice to make. */ + mldsa_mul_neon(r, a, b); +#else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { @@ -8254,6 +9753,7 @@ static void mldsa_mul(sword32* r, sword32* a, sword32* b) { mldsa_mul_c(r, a, b); } +#endif /* __aarch64__ && WOLFSSL_ARMASM */ } #ifndef WOLFSSL_MLDSA_SIGN_SMALL_MEM @@ -8297,6 +9797,28 @@ static void mldsa_mul_invntt(sword32* r, sword32* c, sword32* v) */ static void mldsa_vec_mul(sword32* r, sword32* a, sword32* b, byte l) { +#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) + { + int i; + for (i = 0; i < l; i++) { + MLDSA_MUL_NEON(r, a, b); + r += MLDSA_N; + b += MLDSA_N; + } + } +#elif defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_ARMASM_THUMB2) && \ + !defined(WOLFSSL_ARMASM_NO_NEON) + { + /* These are l independent products, so the single-polynomial + * assembly serves here too. */ + int i; + for (i = 0; i < l; i++) { + mldsa_mul_neon(r, a, b); + r += MLDSA_N; + b += MLDSA_N; + } + } +#else byte i; #ifdef USE_INTEL_SPEEDUP @@ -8328,6 +9850,7 @@ static void mldsa_vec_mul(sword32* r, sword32* a, sword32* b, byte l) b += MLDSA_N; } } +#endif /* __aarch64__ && WOLFSSL_ARMASM */ } #endif #endif @@ -8342,6 +9865,8 @@ static void mldsa_vec_mul(sword32* r, sword32* a, sword32* b, byte l) * * @param [in, out] a Polynomial. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_POLY_OPS) static void mldsa_poly_red_c(sword32* a) { unsigned int j; @@ -8362,6 +9887,7 @@ static void mldsa_poly_red_c(sword32* a) } #endif } +#endif /* !MLDSA_ASM_POLY_OPS */ /* Modulo reduce values in polynomial. Range (-2^31)..(2^31-1). * @@ -8369,6 +9895,9 @@ static void mldsa_poly_red_c(sword32* a) */ static void mldsa_poly_red(sword32* a) { +#if defined(MLDSA_ASM_POLY_OPS) + mldsa_poly_red_neon(a); +#else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { @@ -8386,6 +9915,7 @@ static void mldsa_poly_red(sword32* a) { mldsa_poly_red_c(a); } +#endif /* MLDSA_ASM_POLY_OPS */ } #if (defined(WOLFSSL_MLDSA_SMALL) && \ @@ -8421,6 +9951,8 @@ static void mldsa_vec_red(sword32* a, byte l) * @param [out] r Polynomial to subtract from. * @param [in] a Polynomial to subtract. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_POLY_OPS) static void mldsa_sub_c(sword32* r, const sword32* a) { unsigned int j; @@ -8441,6 +9973,7 @@ static void mldsa_sub_c(sword32* r, const sword32* a) } #endif } +#endif /* !MLDSA_ASM_POLY_OPS */ /* Subtract polynomials a from r. r -= a. * @@ -8449,6 +9982,9 @@ static void mldsa_sub_c(sword32* r, const sword32* a) */ static void mldsa_sub(sword32* r, const sword32* a) { +#if defined(MLDSA_ASM_POLY_OPS) + mldsa_poly_sub_neon(r, a); +#else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { @@ -8466,6 +10002,7 @@ static void mldsa_sub(sword32* r, const sword32* a) { mldsa_sub_c(r, a); } +#endif /* MLDSA_ASM_POLY_OPS */ } #if defined(WOLFSSL_MLDSA_CHECK_KEY) || \ @@ -8496,6 +10033,8 @@ static void mldsa_vec_sub(sword32* r, const sword32* a, byte l) * @param [out] r Polynomial to add to. * @param [in] a Polynomial to add. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_POLY_OPS) static void mldsa_add_c(sword32* r, const sword32* a) { unsigned int j; @@ -8516,6 +10055,7 @@ static void mldsa_add_c(sword32* r, const sword32* a) } #endif } +#endif /* !MLDSA_ASM_POLY_OPS */ /* Add polynomials a to r. r += a. * @@ -8524,6 +10064,9 @@ static void mldsa_add_c(sword32* r, const sword32* a) */ static void mldsa_add(sword32* r, const sword32* a) { +#if defined(MLDSA_ASM_POLY_OPS) + mldsa_poly_add_neon(r, a); +#else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { @@ -8541,6 +10084,7 @@ static void mldsa_add(sword32* r, const sword32* a) { mldsa_add_c(r, a); } +#endif /* MLDSA_ASM_POLY_OPS */ } #if !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) || \ @@ -8575,6 +10119,8 @@ static void mldsa_vec_add(sword32* r, const sword32* a, byte l) * * @param [in, out] a Polynomial. */ +/* Unreferenced when the assembly is in use. */ +#if !defined(MLDSA_ASM_POLY_OPS) static void mldsa_make_pos_c(sword32* a) { unsigned int j; @@ -8595,6 +10141,7 @@ static void mldsa_make_pos_c(sword32* a) } #endif } +#endif /* !MLDSA_ASM_POLY_OPS */ /* Make values in polynomial be in positive range. * @@ -8602,6 +10149,9 @@ static void mldsa_make_pos_c(sword32* a) */ static void mldsa_make_pos(sword32* a) { +#if defined(MLDSA_ASM_POLY_OPS) + mldsa_poly_make_pos_neon(a); +#else #ifdef USE_INTEL_SPEEDUP #ifdef WOLFSSL_MLDSA_HAVE_INTEL_AVX512 if (USE_INTEL_AVX512(cpuid_flags) && (SAVE_VECTOR_REGISTERS2() == 0)) { @@ -8619,6 +10169,7 @@ static void mldsa_make_pos(sword32* a) { mldsa_make_pos_c(a); } +#endif /* MLDSA_ASM_POLY_OPS */ } #if !defined(WOLFSSL_MLDSA_NO_MAKE_KEY) || \ @@ -10999,7 +12550,11 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, #ifndef WOLFSSL_NO_ML_DSA_44 if (params->gamma2 == MLDSA_Q_LOW_88) { /* Step 11: Use hint to give full w1. */ +#if defined(MLDSA_ASM_DECOMPOSE) + mldsa_use_hint_neon(w, h, h[params->omega + r], 1, &o); +#else mldsa_use_hint_88(w, h, r, &o); +#endif /* Step 12: Encode w1. */ mldsa_encode_w1_88(w, encW1); encW1 += MLDSA_Q_HI_88_ENC_BITS * 2 * MLDSA_N / 16; @@ -11009,7 +12564,11 @@ static int mldsa_verify_with_mu(wc_MlDsaKey* key, const byte* mu, #if !defined(WOLFSSL_NO_ML_DSA_65) || !defined(WOLFSSL_NO_ML_DSA_87) if (params->gamma2 == MLDSA_Q_LOW_32) { /* Step 11: Use hint to give full w1. */ +#if defined(MLDSA_ASM_DECOMPOSE) + mldsa_use_hint_neon(w, h, h[params->omega + r], 0, &o); +#else mldsa_use_hint_32(w, h, params->omega, r, &o); +#endif /* Step 12: Encode w1. */ mldsa_encode_w1_32(w, encW1); encW1 += MLDSA_Q_HI_32_ENC_BITS * 2 * MLDSA_N / 16; @@ -11927,9 +13486,25 @@ int wc_MlDsaKey_Init(wc_MlDsaKey* key, void* heap, int devId) key->heap = heap; } -#if defined(USE_INTEL_SPEEDUP) +#if defined(USE_INTEL_SPEEDUP) || (defined(__aarch64__) && \ + defined(WOLFSSL_ARMASM)) + /* AArch64 needs these for the FEAT_RDM check that picks the NTT flavor and + * the FEAT_SHA3 check that picks the Keccak one. This has to stay ahead of + * anything that can reach those dispatchers: cpuid_flags starts out as + * WC_CPUID_INITIALIZER, which is all ones, so a read before this call + * reports every feature present and would send a CPU without FEAT_SHA3 + * into the SHA-3 instruction path and SIGILL. */ cpuid_get_flags_ex(&cpuid_flags); #endif +#if defined(MLDSA_SHAKE_DIRECT) + /* The direct-state paths never take the object through wc_InitShake*(), so + * do it here: the heap hint and device id have to reach it, and freeing a + * SHAKE object that was never initialized is not safe under + * WOLFSSL_ASYNC_CRYPT. */ + if (ret == 0) { + ret = wc_InitShake128(&key->shake, heap, devId); + } +#endif return ret; } diff --git a/wolfcrypt/src/wc_mlkem_poly.c b/wolfcrypt/src/wc_mlkem_poly.c index c587c9f660c..45066c88593 100644 --- a/wolfcrypt/src/wc_mlkem_poly.c +++ b/wolfcrypt/src/wc_mlkem_poly.c @@ -1272,38 +1272,38 @@ void mlkem_init(void) static void mlkem_sha3_blocksx3(word64* state) { if (IS_AARCH64_SHA3(cpuid_flags)) { - mlkem_sha3_blocksx3_crypto(state); + sha3_blocksx3_crypto(state); } else { - mlkem_sha3_blocksx3_neon(state); + sha3_blocksx3_neon(state); } } static void mlkem_shake128_blocksx3_seed(word64* state, byte* seed) { if (IS_AARCH64_SHA3(cpuid_flags)) { - mlkem_shake128_blocksx3_seed_crypto(state, seed); + sha3_128_blocksx3_seed_crypto(state, seed); } else { - mlkem_shake128_blocksx3_seed_neon(state, seed); + sha3_128_blocksx3_seed_neon(state, seed); } } static void mlkem_shake256_blocksx3_seed(word64* state, byte* seed) { if (IS_AARCH64_SHA3(cpuid_flags)) { - mlkem_shake256_blocksx3_seed_crypto(state, seed); + sha3_256_blocksx3_seed_crypto(state, seed); } else { - mlkem_shake256_blocksx3_seed_neon(state, seed); + sha3_256_blocksx3_seed_neon(state, seed); } } #else -#define mlkem_sha3_blocksx3 mlkem_sha3_blocksx3_neon -#define mlkem_shake128_blocksx3_seed mlkem_shake128_blocksx3_seed_neon -#define mlkem_shake256_blocksx3_seed mlkem_shake256_blocksx3_seed_neon +#define mlkem_sha3_blocksx3 sha3_blocksx3_neon +#define mlkem_shake128_blocksx3_seed sha3_128_blocksx3_seed_neon +#define mlkem_shake256_blocksx3_seed sha3_256_blocksx3_seed_neon #endif /* WOLFSSL_ARMASM_CRYPTO_SHA3 */ @@ -2356,6 +2356,31 @@ void mlkem_decapsulate(const sword16* s, sword16* w, sword16* u, #endif /* !WOLFSSL_MLKEM_NO_DECAPSULATE */ #endif +#if defined(WOLFSSL_ARMASM) && !defined(__aarch64__) && \ + !defined(WOLFSSL_ARMASM_THUMB2) && !defined(WOLFSSL_ARMASM_NO_NEON) && \ + !defined(WC_SHA3_NO_ASM) + +/* AArch32 has the same three-way batched Keccak, but no SHA-3 instruction + * extension, so there is only the NEON implementation and nothing to select + * at run time. */ +#define mlkem_sha3_blocksx3 sha3_blocksx3_neon +#define mlkem_shake128_blocksx3_seed sha3_128_blocksx3_seed_neon +#define mlkem_shake256_blocksx3_seed sha3_256_blocksx3_seed_neon + +#endif + +/* Targets carrying the three-way batched Keccak in assembly, with the + * rejection sampler that goes alongside it. */ +#if defined(WOLFSSL_ARMASM) && defined(__aarch64__) + #define MLKEM_ASM_BLOCKSX3 + #define MLKEM_REJ_UNIFORM_ASM mlkem_rej_uniform_neon +#elif defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_ARMASM_THUMB2) && \ + !defined(WOLFSSL_ARMASM_NO_NEON) && !defined(WC_SHA3_NO_ASM) + #define MLKEM_ASM_BLOCKSX3 + #define MLKEM_REJ_UNIFORM_ASM mlkem_arm32_rej_uniform +#endif + + /******************************************************************************/ #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) @@ -3158,7 +3183,7 @@ static int mlkem_gen_matrix_k4_avx512(sword16* a, byte* seed, int transposed) } #endif /* WOLFSSL_MLKEM_HAVE_INTEL_AVX512 */ #endif /* WOLFSSL_KYBER1024 || WOLFSSL_WC_ML_KEM_1024 */ -#elif defined(WOLFSSL_ARMASM) && defined(__aarch64__) +#elif defined(MLKEM_ASM_BLOCKSX3) #if defined(WOLFSSL_KYBER512) || defined(WOLFSSL_WC_ML_KEM_512) /* Deterministically generate a matrix (or transpose) of uniform integers mod q. * @@ -3169,7 +3194,7 @@ static int mlkem_gen_matrix_k4_avx512(sword16* a, byte* seed, int transposed) * @param [in] transposed Whether A or A^T is generated. * @return 0 on success. */ -static int mlkem_gen_matrix_k2_aarch64(sword16* a, byte* seed, int transposed) +static int mlkem_gen_matrix_k2_arm(sword16* a, byte* seed, int transposed) { word64 state[3 * 25]; word64* st = (word64*)state; @@ -3192,22 +3217,22 @@ static int mlkem_gen_matrix_k2_aarch64(sword16* a, byte* seed, int transposed) mlkem_shake128_blocksx3_seed(state, seed); /* Sample random bytes to create a polynomial. */ p = (byte*)st; - ctr0 = mlkem_rej_uniform_neon(a + 0 * MLKEM_N, MLKEM_N, p, XOF_BLOCK_SIZE); + ctr0 = MLKEM_REJ_UNIFORM_ASM(a + 0 * MLKEM_N, MLKEM_N, p, XOF_BLOCK_SIZE); p += 25 * 8; - ctr1 = mlkem_rej_uniform_neon(a + 1 * MLKEM_N, MLKEM_N, p, XOF_BLOCK_SIZE); + ctr1 = MLKEM_REJ_UNIFORM_ASM(a + 1 * MLKEM_N, MLKEM_N, p, XOF_BLOCK_SIZE); p += 25 * 8; - ctr2 = mlkem_rej_uniform_neon(a + 2 * MLKEM_N, MLKEM_N, p, XOF_BLOCK_SIZE); + ctr2 = MLKEM_REJ_UNIFORM_ASM(a + 2 * MLKEM_N, MLKEM_N, p, XOF_BLOCK_SIZE); while ((ctr0 < MLKEM_N) || (ctr1 < MLKEM_N) || (ctr2 < MLKEM_N)) { mlkem_sha3_blocksx3(st); p = (byte*)st; - ctr0 += mlkem_rej_uniform_neon(a + 0 * MLKEM_N + ctr0, MLKEM_N - ctr0, + ctr0 += MLKEM_REJ_UNIFORM_ASM(a + 0 * MLKEM_N + ctr0, MLKEM_N - ctr0, p, XOF_BLOCK_SIZE); p += 25 * 8; - ctr1 += mlkem_rej_uniform_neon(a + 1 * MLKEM_N + ctr1, MLKEM_N - ctr1, + ctr1 += MLKEM_REJ_UNIFORM_ASM(a + 1 * MLKEM_N + ctr1, MLKEM_N - ctr1, p, XOF_BLOCK_SIZE); p += 25 * 8; - ctr2 += mlkem_rej_uniform_neon(a + 2 * MLKEM_N + ctr2, MLKEM_N - ctr2, + ctr2 += MLKEM_REJ_UNIFORM_ASM(a + 2 * MLKEM_N + ctr2, MLKEM_N - ctr2, p, XOF_BLOCK_SIZE); } @@ -3220,10 +3245,10 @@ static int mlkem_gen_matrix_k2_aarch64(sword16* a, byte* seed, int transposed) state[20] = W64LIT(0x8000000000000000); BlockSha3(state); p = (byte*)state; - ctr0 = mlkem_rej_uniform_neon(a, MLKEM_N, p, XOF_BLOCK_SIZE); + ctr0 = MLKEM_REJ_UNIFORM_ASM(a, MLKEM_N, p, XOF_BLOCK_SIZE); while (ctr0 < MLKEM_N) { BlockSha3(state); - ctr0 += mlkem_rej_uniform_neon(a + ctr0, MLKEM_N - ctr0, p, + ctr0 += MLKEM_REJ_UNIFORM_ASM(a + ctr0, MLKEM_N - ctr0, p, XOF_BLOCK_SIZE); } @@ -3241,7 +3266,7 @@ static int mlkem_gen_matrix_k2_aarch64(sword16* a, byte* seed, int transposed) * @param [in] transposed Whether A or A^T is generated. * @return 0 on success. */ -static int mlkem_gen_matrix_k3_aarch64(sword16* a, byte* seed, int transposed) +static int mlkem_gen_matrix_k3_arm(sword16* a, byte* seed, int transposed) { int i; int k; @@ -3265,26 +3290,26 @@ static int mlkem_gen_matrix_k3_aarch64(sword16* a, byte* seed, int transposed) mlkem_shake128_blocksx3_seed(state, seed); /* Sample random bytes to create a polynomial. */ p = (byte*)st; - ctr0 = mlkem_rej_uniform_neon(a + 0 * MLKEM_N, MLKEM_N, p, + ctr0 = MLKEM_REJ_UNIFORM_ASM(a + 0 * MLKEM_N, MLKEM_N, p, XOF_BLOCK_SIZE); p += 25 * 8; - ctr1 = mlkem_rej_uniform_neon(a + 1 * MLKEM_N, MLKEM_N, p, + ctr1 = MLKEM_REJ_UNIFORM_ASM(a + 1 * MLKEM_N, MLKEM_N, p, XOF_BLOCK_SIZE); p += 25 * 8; - ctr2 = mlkem_rej_uniform_neon(a + 2 * MLKEM_N, MLKEM_N, p, + ctr2 = MLKEM_REJ_UNIFORM_ASM(a + 2 * MLKEM_N, MLKEM_N, p, XOF_BLOCK_SIZE); /* Create more blocks if too many rejected. */ while ((ctr0 < MLKEM_N) || (ctr1 < MLKEM_N) || (ctr2 < MLKEM_N)) { mlkem_sha3_blocksx3(st); p = (byte*)st; - ctr0 += mlkem_rej_uniform_neon(a + 0 * MLKEM_N + ctr0, + ctr0 += MLKEM_REJ_UNIFORM_ASM(a + 0 * MLKEM_N + ctr0, MLKEM_N - ctr0, p, XOF_BLOCK_SIZE); p += 25 * 8; - ctr1 += mlkem_rej_uniform_neon(a + 1 * MLKEM_N + ctr1, + ctr1 += MLKEM_REJ_UNIFORM_ASM(a + 1 * MLKEM_N + ctr1, MLKEM_N - ctr1, p, XOF_BLOCK_SIZE); p += 25 * 8; - ctr2 += mlkem_rej_uniform_neon(a + 2 * MLKEM_N + ctr2, + ctr2 += MLKEM_REJ_UNIFORM_ASM(a + 2 * MLKEM_N + ctr2, MLKEM_N - ctr2, p, XOF_BLOCK_SIZE); } @@ -3305,7 +3330,7 @@ static int mlkem_gen_matrix_k3_aarch64(sword16* a, byte* seed, int transposed) * @param [in] transposed Whether A or A^T is generated. * @return 0 on success. */ -static int mlkem_gen_matrix_k4_aarch64(sword16* a, byte* seed, int transposed) +static int mlkem_gen_matrix_k4_arm(sword16* a, byte* seed, int transposed) { int i; int k; @@ -3331,26 +3356,26 @@ static int mlkem_gen_matrix_k4_aarch64(sword16* a, byte* seed, int transposed) mlkem_shake128_blocksx3_seed(state, seed); /* Sample random bytes to create a polynomial. */ p = (byte*)st; - ctr0 = mlkem_rej_uniform_neon(a + 0 * MLKEM_N, MLKEM_N, p, + ctr0 = MLKEM_REJ_UNIFORM_ASM(a + 0 * MLKEM_N, MLKEM_N, p, XOF_BLOCK_SIZE); p += 25 * 8; - ctr1 = mlkem_rej_uniform_neon(a + 1 * MLKEM_N, MLKEM_N, p, + ctr1 = MLKEM_REJ_UNIFORM_ASM(a + 1 * MLKEM_N, MLKEM_N, p, XOF_BLOCK_SIZE); p += 25 * 8; - ctr2 = mlkem_rej_uniform_neon(a + 2 * MLKEM_N, MLKEM_N, p, + ctr2 = MLKEM_REJ_UNIFORM_ASM(a + 2 * MLKEM_N, MLKEM_N, p, XOF_BLOCK_SIZE); /* Create more blocks if too many rejected. */ while ((ctr0 < MLKEM_N) || (ctr1 < MLKEM_N) || (ctr2 < MLKEM_N)) { mlkem_sha3_blocksx3(st); p = (byte*)st; - ctr0 += mlkem_rej_uniform_neon(a + 0 * MLKEM_N + ctr0, + ctr0 += MLKEM_REJ_UNIFORM_ASM(a + 0 * MLKEM_N + ctr0, MLKEM_N - ctr0, p, XOF_BLOCK_SIZE); p += 25 * 8; - ctr1 += mlkem_rej_uniform_neon(a + 1 * MLKEM_N + ctr1, + ctr1 += MLKEM_REJ_UNIFORM_ASM(a + 1 * MLKEM_N + ctr1, MLKEM_N - ctr1, p, XOF_BLOCK_SIZE); p += 25 * 8; - ctr2 += mlkem_rej_uniform_neon(a + 2 * MLKEM_N + ctr2, + ctr2 += MLKEM_REJ_UNIFORM_ASM(a + 2 * MLKEM_N + ctr2, MLKEM_N - ctr2, p, XOF_BLOCK_SIZE); } @@ -3364,10 +3389,10 @@ static int mlkem_gen_matrix_k4_aarch64(sword16* a, byte* seed, int transposed) state[20] = W64LIT(0x8000000000000000); BlockSha3(state); p = (byte*)state; - ctr0 = mlkem_rej_uniform_neon(a, MLKEM_N, p, XOF_BLOCK_SIZE); + ctr0 = MLKEM_REJ_UNIFORM_ASM(a, MLKEM_N, p, XOF_BLOCK_SIZE); while (ctr0 < MLKEM_N) { BlockSha3(state); - ctr0 += mlkem_rej_uniform_neon(a + ctr0, MLKEM_N - ctr0, p, + ctr0 += MLKEM_REJ_UNIFORM_ASM(a + ctr0, MLKEM_N - ctr0, p, XOF_BLOCK_SIZE); } @@ -3376,7 +3401,7 @@ static int mlkem_gen_matrix_k4_aarch64(sword16* a, byte* seed, int transposed) #endif #endif /* USE_INTEL_SPEEDUP */ -#if !(defined(WOLFSSL_ARMASM) && defined(__aarch64__)) +#if !defined(MLKEM_ASM_BLOCKSX3) /* Absorb the seed data for squeezing out pseudo-random data. * * FIPS 203, Section 4.1: @@ -3537,7 +3562,7 @@ void mlkem_prf_free(wc_Shake* prf) wc_Shake256_Free(prf); } -#if !(defined(WOLFSSL_ARMASM) && defined(__aarch64__)) +#if !defined(MLKEM_ASM_BLOCKSX3) /* Create pseudo-random data from the key using SHAKE-256. * * FIPS 203, Section 4.1, 4.3: @@ -3930,7 +3955,7 @@ static unsigned int mlkem_rej_uniform_c(sword16* p, unsigned int len, #if !defined(WOLFSSL_MLKEM_MAKEKEY_SMALL_MEM) || \ !defined(WOLFSSL_MLKEM_ENCAPSULATE_SMALL_MEM) -#if !(defined(WOLFSSL_ARMASM) && defined(__aarch64__)) +#if !defined(MLKEM_ASM_BLOCKSX3) /* Deterministically generate a matrix (or transpose) of uniform integers mod q. * * Seed used with XOF to generate random bytes. @@ -4082,8 +4107,8 @@ int mlkem_gen_matrix(MLKEM_PRF_T* prf, sword16* a, int k, byte* seed, #if defined(WOLFSSL_KYBER512) || defined(WOLFSSL_WC_ML_KEM_512) if (k == WC_ML_KEM_512_K) { -#if defined(WOLFSSL_ARMASM) && defined(__aarch64__) - ret = mlkem_gen_matrix_k2_aarch64(a, seed, transposed); +#if defined(MLKEM_ASM_BLOCKSX3) + ret = mlkem_gen_matrix_k2_arm(a, seed, transposed); #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 @@ -4108,8 +4133,8 @@ int mlkem_gen_matrix(MLKEM_PRF_T* prf, sword16* a, int k, byte* seed, #endif #if defined(WOLFSSL_KYBER768) || defined(WOLFSSL_WC_ML_KEM_768) if (k == WC_ML_KEM_768_K) { -#if defined(WOLFSSL_ARMASM) && defined(__aarch64__) - ret = mlkem_gen_matrix_k3_aarch64(a, seed, transposed); +#if defined(MLKEM_ASM_BLOCKSX3) + ret = mlkem_gen_matrix_k3_arm(a, seed, transposed); #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 @@ -4134,8 +4159,8 @@ int mlkem_gen_matrix(MLKEM_PRF_T* prf, sword16* a, int k, byte* seed, #endif #if defined(WOLFSSL_KYBER1024) || defined(WOLFSSL_WC_ML_KEM_1024) if (k == WC_ML_KEM_1024_K) { -#if defined(WOLFSSL_ARMASM) && defined(__aarch64__) - ret = mlkem_gen_matrix_k4_aarch64(a, seed, transposed); +#if defined(MLKEM_ASM_BLOCKSX3) + ret = mlkem_gen_matrix_k4_arm(a, seed, transposed); #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 @@ -4563,7 +4588,7 @@ static void mlkem_cbd_eta3(sword16* p, const byte* r) } #endif -#if !(defined(__aarch64__) && defined(WOLFSSL_ARMASM)) +#if !defined(MLKEM_ASM_BLOCKSX3) /* Get noise/error by calculating random bytes and sampling to a binomial * distribution. @@ -5163,7 +5188,7 @@ static int mlkem_get_noise_k4_avx512(MLKEM_PRF_T* prf, sword16* vec1, #endif #endif /* USE_INTEL_SPEEDUP */ -#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) +#if defined(MLKEM_ASM_BLOCKSX3) #define PRF_RAND_SZ (2 * SHA3_256_BYTES) @@ -5181,7 +5206,7 @@ static int mlkem_get_noise_k4_avx512(MLKEM_PRF_T* prf, sword16* vec1, * @param [in] seed Seed to generate random from. * @param [in] o Offset of seed count. */ -static void mlkem_get_noise_x3_eta2_aarch64(word64* rand, byte* seed, byte o) +static void mlkem_get_noise_x3_eta2_arm(word64* rand, byte* seed, byte o) { /* Only rand[i*25 + 4] is set here - the rest of the state is zeroed in * registers by the assembly. */ @@ -5210,7 +5235,7 @@ static void mlkem_get_noise_x3_eta2_aarch64(word64* rand, byte* seed, byte o) * @param [in] seed Seed to generate random from. * @param [in] o Offset of seed count. */ -static void mlkem_get_noise_x3_eta3_aarch64(byte* rand, byte* seed, byte o) +static void mlkem_get_noise_x3_eta3_arm(byte* rand, byte* seed, byte o) { /* Only state[i*25 + 4] is read by the assembly - the rest of the state is * zeroed in registers there. */ @@ -5254,7 +5279,7 @@ static void mlkem_get_noise_x3_eta3_aarch64(byte* rand, byte* seed, byte o) * @param [in] seed Seed to generate random from. * @param [in] o Offset of seed count. */ -static void mlkem_get_noise_eta3_aarch64(byte* rand, byte* seed, byte o) +static void mlkem_get_noise_eta3_arm(byte* rand, byte* seed, byte o) { /* ETA3_RAND_SIZE is larger than the SHAKE-256 rate - two squeezes are * needed, so the state cannot be squeezed in place over the output. */ @@ -5288,22 +5313,22 @@ static void mlkem_get_noise_eta3_aarch64(byte* rand, byte* seed, byte o) * @param [in] seed Seed to use when calculating random. * @return 0 on success. */ -static int mlkem_get_noise_k2_aarch64(sword16* vec1, sword16* vec2, +static int mlkem_get_noise_k2_arm(sword16* vec1, sword16* vec2, sword16* poly, byte* seed) { int ret = 0; word64 rand[3 * 25]; - mlkem_get_noise_x3_eta3_aarch64((byte*)rand, seed, 0); + mlkem_get_noise_x3_eta3_arm((byte*)rand, seed, 0); mlkem_cbd_eta3(vec1 , (byte*)rand + 0 * ETA3_RAND_SIZE); mlkem_cbd_eta3(vec1 + MLKEM_N, (byte*)rand + 1 * ETA3_RAND_SIZE); if (poly == NULL) { mlkem_cbd_eta3(vec2 , (byte*)rand + 2 * ETA3_RAND_SIZE); - mlkem_get_noise_eta3_aarch64((byte*)rand, seed, 3); + mlkem_get_noise_eta3_arm((byte*)rand, seed, 3); mlkem_cbd_eta3(vec2 + MLKEM_N, (byte*)rand ); } else { - mlkem_get_noise_x3_eta2_aarch64(rand, seed, 2); + mlkem_get_noise_x3_eta2_arm(rand, seed, 2); mlkem_cbd_eta2(vec2 , (byte*)rand + 0 * 25 * 8); mlkem_cbd_eta2(vec2 + MLKEM_N, (byte*)rand + 1 * 25 * 8); mlkem_cbd_eta2(poly , (byte*)rand + 2 * 25 * 8); @@ -5335,7 +5360,7 @@ static int mlkem_get_noise_k2_aarch64(sword16* vec1, sword16* vec2, * @param [in] seed Seed to generate random from. * @param [in] o Offset of seed count. */ -static void mlkem_get_noise_eta2_aarch64(word64* rand, byte* seed, byte o) +static void mlkem_get_noise_eta2_arm(word64* rand, byte* seed, byte o) { readUnalignedWords64(rand, seed, 4); /* Transposed value same as not. */ @@ -5354,21 +5379,21 @@ static void mlkem_get_noise_eta2_aarch64(word64* rand, byte* seed, byte o) * @param [in] seed Seed to use when calculating random. * @return 0 on success. */ -static int mlkem_get_noise_k3_aarch64(sword16* vec1, sword16* vec2, +static int mlkem_get_noise_k3_arm(sword16* vec1, sword16* vec2, sword16* poly, byte* seed) { word64 rand[3 * 25]; - mlkem_get_noise_x3_eta2_aarch64(rand, seed, 0); + mlkem_get_noise_x3_eta2_arm(rand, seed, 0); mlkem_cbd_eta2(vec1 , (byte*)rand + 0 * 25 * 8); mlkem_cbd_eta2(vec1 + 1 * MLKEM_N, (byte*)rand + 1 * 25 * 8); mlkem_cbd_eta2(vec1 + 2 * MLKEM_N, (byte*)rand + 2 * 25 * 8); - mlkem_get_noise_x3_eta2_aarch64(rand, seed, 3); + mlkem_get_noise_x3_eta2_arm(rand, seed, 3); mlkem_cbd_eta2(vec2 , (byte*)rand + 0 * 25 * 8); mlkem_cbd_eta2(vec2 + 1 * MLKEM_N, (byte*)rand + 1 * 25 * 8); mlkem_cbd_eta2(vec2 + 2 * MLKEM_N, (byte*)rand + 2 * 25 * 8); if (poly != NULL) { - mlkem_get_noise_eta2_aarch64(rand, seed, 6); + mlkem_get_noise_eta2_arm(rand, seed, 6); mlkem_cbd_eta2(poly , (byte*)rand + 0 * 25 * 8); } @@ -5394,21 +5419,21 @@ static int mlkem_get_noise_k3_aarch64(sword16* vec1, sword16* vec2, * @param [in] seed Seed to use when calculating random. * @return 0 on success. */ -static int mlkem_get_noise_k4_aarch64(sword16* vec1, sword16* vec2, +static int mlkem_get_noise_k4_arm(sword16* vec1, sword16* vec2, sword16* poly, byte* seed) { int ret = 0; word64 rand[3 * 25]; - mlkem_get_noise_x3_eta2_aarch64(rand, seed, 0); + mlkem_get_noise_x3_eta2_arm(rand, seed, 0); mlkem_cbd_eta2(vec1 , (byte*)rand + 0 * 25 * 8); mlkem_cbd_eta2(vec1 + 1 * MLKEM_N, (byte*)rand + 1 * 25 * 8); mlkem_cbd_eta2(vec1 + 2 * MLKEM_N, (byte*)rand + 2 * 25 * 8); - mlkem_get_noise_x3_eta2_aarch64(rand, seed, 3); + mlkem_get_noise_x3_eta2_arm(rand, seed, 3); mlkem_cbd_eta2(vec1 + 3 * MLKEM_N, (byte*)rand + 0 * 25 * 8); mlkem_cbd_eta2(vec2 , (byte*)rand + 1 * 25 * 8); mlkem_cbd_eta2(vec2 + 1 * MLKEM_N, (byte*)rand + 2 * 25 * 8); - mlkem_get_noise_x3_eta2_aarch64(rand, seed, 6); + mlkem_get_noise_x3_eta2_arm(rand, seed, 6); mlkem_cbd_eta2(vec2 + 2 * MLKEM_N, (byte*)rand + 0 * 25 * 8); mlkem_cbd_eta2(vec2 + 3 * MLKEM_N, (byte*)rand + 1 * 25 * 8); if (poly != NULL) { @@ -5428,7 +5453,7 @@ static int mlkem_get_noise_k4_aarch64(sword16* vec1, sword16* vec2, #endif #endif /* __aarch64__ && WOLFSSL_ARMASM */ -#if !(defined(__aarch64__) && defined(WOLFSSL_ARMASM)) +#if !defined(MLKEM_ASM_BLOCKSX3) /* Get the noise/error by calculating random bytes and sampling to a binomial * distribution. @@ -5499,8 +5524,8 @@ int mlkem_get_noise(MLKEM_PRF_T* prf, int k, sword16* vec1, sword16* vec2, #if defined(WOLFSSL_KYBER512) || defined(WOLFSSL_WC_ML_KEM_512) if (k == WC_ML_KEM_512_K) { -#if defined(WOLFSSL_ARMASM) && defined(__aarch64__) - ret = mlkem_get_noise_k2_aarch64(vec1, vec2, poly, seed); +#if defined(MLKEM_ASM_BLOCKSX3) + ret = mlkem_get_noise_k2_arm(vec1, vec2, poly, seed); #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 @@ -5530,8 +5555,8 @@ int mlkem_get_noise(MLKEM_PRF_T* prf, int k, sword16* vec1, sword16* vec2, #endif #if defined(WOLFSSL_KYBER768) || defined(WOLFSSL_WC_ML_KEM_768) if (k == WC_ML_KEM_768_K) { -#if defined(WOLFSSL_ARMASM) && defined(__aarch64__) - ret = mlkem_get_noise_k3_aarch64(vec1, vec2, poly, seed); +#if defined(MLKEM_ASM_BLOCKSX3) + ret = mlkem_get_noise_k3_arm(vec1, vec2, poly, seed); #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 @@ -5557,8 +5582,8 @@ int mlkem_get_noise(MLKEM_PRF_T* prf, int k, sword16* vec1, sword16* vec2, #endif #if defined(WOLFSSL_KYBER1024) || defined(WOLFSSL_WC_ML_KEM_1024) if (k == WC_ML_KEM_1024_K) { -#if defined(WOLFSSL_ARMASM) && defined(__aarch64__) - ret = mlkem_get_noise_k4_aarch64(vec1, vec2, poly, seed); +#if defined(MLKEM_ASM_BLOCKSX3) + ret = mlkem_get_noise_k4_arm(vec1, vec2, poly, seed); #else #if defined(USE_INTEL_SPEEDUP) && !defined(WC_SHA3_NO_ASM) #ifdef WOLFSSL_MLKEM_HAVE_INTEL_AVX512 diff --git a/wolfssl-VS2022.vcxproj b/wolfssl-VS2022.vcxproj index 9e7f78151f7..33f9e5148dd 100644 --- a/wolfssl-VS2022.vcxproj +++ b/wolfssl-VS2022.vcxproj @@ -76,7 +76,7 @@ the WOLFSSL_ARMASM* defines below); a symbol that is left undefined here simply drops the matching assembly (safe - the C caller then uses a C fallback), while a symbol enabled here but not in C is dead code. --> - -PreDefine "HAVE_AES_CBC SETL {TRUE}" -PreDefine "HAVE_AESCCM SETL {TRUE}" -PreDefine "HAVE_AESGCM SETL {TRUE}" -PreDefine "HAVE_AES_ECB SETL {TRUE}" -PreDefine "HAVE_AES_DECRYPT SETL {TRUE}" -PreDefine "WOLFSSL_AES_COUNTER SETL {TRUE}" -PreDefine "WOLFSSL_AES_DIRECT SETL {TRUE}" -PreDefine "WOLFSSL_AES_XTS SETL {TRUE}" -PreDefine "WOLFSSL_AESGCM_STREAM SETL {TRUE}" -PreDefine "WOLFSSL_SHA224 SETL {TRUE}" -PreDefine "WOLFSSL_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_SHA384 SETL {TRUE}" -PreDefine "WOLFSSL_SHA512 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA512 SETL {TRUE}" -PreDefine "HAVE_CHACHA SETL {TRUE}" -PreDefine "HAVE_CURVE25519 SETL {TRUE}" -PreDefine "HAVE_ED25519 SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_MLKEM SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_FRODOKEM SETL {TRUE}" + -PreDefine "HAVE_AES_CBC SETL {TRUE}" -PreDefine "HAVE_AESCCM SETL {TRUE}" -PreDefine "HAVE_AESGCM SETL {TRUE}" -PreDefine "HAVE_AES_ECB SETL {TRUE}" -PreDefine "HAVE_AES_DECRYPT SETL {TRUE}" -PreDefine "WOLFSSL_AES_COUNTER SETL {TRUE}" -PreDefine "WOLFSSL_AES_DIRECT SETL {TRUE}" -PreDefine "WOLFSSL_AES_XTS SETL {TRUE}" -PreDefine "WOLFSSL_AESGCM_STREAM SETL {TRUE}" -PreDefine "WOLFSSL_SHA224 SETL {TRUE}" -PreDefine "WOLFSSL_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_SHA384 SETL {TRUE}" -PreDefine "WOLFSSL_SHA512 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA512 SETL {TRUE}" -PreDefine "HAVE_CHACHA SETL {TRUE}" -PreDefine "HAVE_CURVE25519 SETL {TRUE}" -PreDefine "HAVE_ED25519 SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_MLKEM SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_MLDSA SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_FRODOKEM SETL {TRUE}" StaticLibrary @@ -672,6 +672,12 @@ armasm64.exe -o "$(IntDir)%(Filename).obj" $(WolfSSLArmAsmDefs) %(Identity) $(IntDir)%(Filename).obj + + true + false + armasm64.exe -o "$(IntDir)%(Filename).obj" $(WolfSSLArmAsmDefs) %(Identity) + $(IntDir)%(Filename).obj + true false diff --git a/wolfssl.vcxproj b/wolfssl.vcxproj index 30c290811a5..f52f0f5fd17 100644 --- a/wolfssl.vcxproj +++ b/wolfssl.vcxproj @@ -75,7 +75,7 @@ the WOLFSSL_ARMASM* defines below); a symbol that is left undefined here simply drops the matching assembly (safe - the C caller then uses a C fallback), while a symbol enabled here but not in C is dead code. --> - -PreDefine "HAVE_AES_CBC SETL {TRUE}" -PreDefine "HAVE_AESCCM SETL {TRUE}" -PreDefine "HAVE_AESGCM SETL {TRUE}" -PreDefine "HAVE_AES_ECB SETL {TRUE}" -PreDefine "HAVE_AES_DECRYPT SETL {TRUE}" -PreDefine "WOLFSSL_AES_COUNTER SETL {TRUE}" -PreDefine "WOLFSSL_AES_DIRECT SETL {TRUE}" -PreDefine "WOLFSSL_AES_XTS SETL {TRUE}" -PreDefine "WOLFSSL_AESGCM_STREAM SETL {TRUE}" -PreDefine "WOLFSSL_SHA224 SETL {TRUE}" -PreDefine "WOLFSSL_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_SHA384 SETL {TRUE}" -PreDefine "WOLFSSL_SHA512 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA512 SETL {TRUE}" -PreDefine "HAVE_CHACHA SETL {TRUE}" -PreDefine "HAVE_CURVE25519 SETL {TRUE}" -PreDefine "HAVE_ED25519 SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_MLKEM SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_FRODOKEM SETL {TRUE}" + -PreDefine "HAVE_AES_CBC SETL {TRUE}" -PreDefine "HAVE_AESCCM SETL {TRUE}" -PreDefine "HAVE_AESGCM SETL {TRUE}" -PreDefine "HAVE_AES_ECB SETL {TRUE}" -PreDefine "HAVE_AES_DECRYPT SETL {TRUE}" -PreDefine "WOLFSSL_AES_COUNTER SETL {TRUE}" -PreDefine "WOLFSSL_AES_DIRECT SETL {TRUE}" -PreDefine "WOLFSSL_AES_XTS SETL {TRUE}" -PreDefine "WOLFSSL_AESGCM_STREAM SETL {TRUE}" -PreDefine "WOLFSSL_SHA224 SETL {TRUE}" -PreDefine "WOLFSSL_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA3 SETL {TRUE}" -PreDefine "WOLFSSL_SHA384 SETL {TRUE}" -PreDefine "WOLFSSL_SHA512 SETL {TRUE}" -PreDefine "WOLFSSL_ARMASM_CRYPTO_SHA512 SETL {TRUE}" -PreDefine "HAVE_CHACHA SETL {TRUE}" -PreDefine "HAVE_CURVE25519 SETL {TRUE}" -PreDefine "HAVE_ED25519 SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_MLKEM SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_MLDSA SETL {TRUE}" -PreDefine "WOLFSSL_HAVE_FRODOKEM SETL {TRUE}" StaticLibrary @@ -672,6 +672,12 @@ armasm64.exe -o "$(IntDir)%(Filename).obj" $(WolfSSLArmAsmDefs) %(Identity) $(IntDir)%(Filename).obj + + true + false + armasm64.exe -o "$(IntDir)%(Filename).obj" $(WolfSSLArmAsmDefs) %(Identity) + $(IntDir)%(Filename).obj + true false diff --git a/wolfssl/wolfcrypt/sha3.h b/wolfssl/wolfcrypt/sha3.h index d0d1aa86a3f..a9865668a43 100644 --- a/wolfssl/wolfcrypt/sha3.h +++ b/wolfssl/wolfcrypt/sha3.h @@ -451,10 +451,37 @@ WOLFSSL_LOCAL void BlockSha3(word64 *s); WOLFSSL_LOCAL void BlockSha3_base(word64 *s); #endif + +/* Three-way batched Keccak for ML-KEM and ML-DSA. Declared outside the + * WC_SHA3_NO_ASM selection above: AArch64 keeps these even under it, because + * armv8-sha3-asm is not suppressed by WC_SHA3_NO_ASM. The AArch32 NEON file + * is suppressed, so that branch follows it. */ +#if defined(WOLFSSL_HAVE_MLKEM) || defined(WOLFSSL_HAVE_MLDSA) +#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) + WOLFSSL_LOCAL void sha3_blocksx3_neon(word64* s); + WOLFSSL_LOCAL void sha3_128_blocksx3_seed_neon(word64* s, byte* seed); + WOLFSSL_LOCAL void sha3_256_blocksx3_seed_neon(word64* s, byte* seed); + WOLFSSL_LOCAL void sha3_256_blocksx3_seed_64_neon(word64* s, byte* seed); + #ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 + WOLFSSL_LOCAL void sha3_blocksx3_crypto(word64* s); + WOLFSSL_LOCAL void sha3_128_blocksx3_seed_crypto(word64* s, byte* seed); + WOLFSSL_LOCAL void sha3_256_blocksx3_seed_crypto(word64* s, byte* seed); + WOLFSSL_LOCAL void sha3_256_blocksx3_seed_64_crypto(word64* s, + byte* seed); + #endif +#elif defined(WOLFSSL_ARMASM) && !defined(WOLFSSL_ARMASM_THUMB2) && \ + !defined(WOLFSSL_ARMASM_NO_NEON) && !defined(WC_SHA3_NO_ASM) + WOLFSSL_LOCAL void sha3_blocksx3_neon(word64* s); + WOLFSSL_LOCAL void sha3_128_blocksx3_seed_neon(word64* s, byte* seed); + WOLFSSL_LOCAL void sha3_256_blocksx3_seed_neon(word64* s, byte* seed); + WOLFSSL_LOCAL void sha3_256_blocksx3_seed_64_neon(word64* s, byte* seed); +#endif +#endif + + #ifdef __cplusplus } /* extern "C" */ #endif - #endif /* WOLFSSL_SHA3 */ #endif /* WOLF_CRYPT_SHA3_H */ diff --git a/wolfssl/wolfcrypt/wc_mldsa.h b/wolfssl/wolfcrypt/wc_mldsa.h index 64e0472ebc7..2961c26019c 100644 --- a/wolfssl/wolfcrypt/wc_mldsa.h +++ b/wolfssl/wolfcrypt/wc_mldsa.h @@ -1052,6 +1052,146 @@ WOLFSSL_LOCAL void wc_mldsa_encode_t0_t1_avx512_vbmi(const sword32* t, #endif /* WOLFSSL_MLDSA_HAVE_INTEL_AVX512_VBMI */ #endif +/* AArch32 NEON. Every ML-DSA operation the AArch64 assembly covers is here + * too; only the FEAT_RDM (sqrdmlsh) flavours have no AArch32 counterpart, and + * the transforms leave coefficients in natural order so one entry point + * serves where AArch64 needs both a plain and a "full" form. */ +#if defined(WOLFSSL_ARMASM) && !defined(__aarch64__) && \ + !defined(WOLFSSL_ARMASM_THUMB2) && !defined(WOLFSSL_ARMASM_NO_NEON) +WOLFSSL_LOCAL void mldsa_mul_neon(sword32* r, sword32* a, sword32* b); +/* Transforms in place, leaving coefficients in natural order. */ +WOLFSSL_LOCAL void mldsa_ntt_neon(sword32* r); +WOLFSSL_LOCAL void mldsa_invntt_neon(sword32* r); +WOLFSSL_LOCAL void mldsa_decompose_q88_neon(const sword32* r, sword32* r0, + sword32* r1); +WOLFSSL_LOCAL void mldsa_decompose_q32_neon(const sword32* r, sword32* r0, + sword32* r1); +WOLFSSL_LOCAL void mldsa_use_hint_q88_neon(sword32* w1); +WOLFSSL_LOCAL void mldsa_use_hint_q32_neon(sword32* w1); +WOLFSSL_LOCAL void mldsa_poly_add_neon(sword32* r, const sword32* a); +WOLFSSL_LOCAL void mldsa_poly_sub_neon(sword32* r, const sword32* a); +WOLFSSL_LOCAL void mldsa_poly_make_pos_neon(sword32* a); +WOLFSSL_LOCAL void mldsa_poly_red_neon(sword32* a); +WOLFSSL_LOCAL int mldsa_vec_check_low_neon(const sword32* a, byte l, + sword32 hi); +WOLFSSL_LOCAL void mldsa_decode_eta_2_neon(const byte* p, sword32* z); +WOLFSSL_LOCAL void mldsa_decode_eta_4_neon(const byte* p, sword32* z); +WOLFSSL_LOCAL void mldsa_decode_t1_neon(const byte* p, sword32* z); +WOLFSSL_LOCAL void mldsa_decode_t0_neon(const byte* p, sword32* z); +WOLFSSL_LOCAL void mldsa_decode_gamma1_17_neon(const byte* p, sword32* z); +WOLFSSL_LOCAL void mldsa_decode_gamma1_19_neon(const byte* p, sword32* z); +WOLFSSL_LOCAL void mldsa_encode_w1_88_neon(const sword32* w1, byte* w1e); +WOLFSSL_LOCAL void mldsa_encode_w1_32_neon(const sword32* w1, byte* w1e); +WOLFSSL_LOCAL void mldsa_encode_gamma1_17_neon(const sword32* z, byte* s); +WOLFSSL_LOCAL void mldsa_encode_gamma1_19_neon(const sword32* z, byte* s); +WOLFSSL_LOCAL void mldsa_mul_vec_4_neon(sword32* r, const sword32* m, + const sword32* v); +WOLFSSL_LOCAL void mldsa_mul_vec_5_neon(sword32* r, const sword32* m, + const sword32* v); +WOLFSSL_LOCAL void mldsa_mul_vec_7_neon(sword32* r, const sword32* m, + const sword32* v); +WOLFSSL_LOCAL void mldsa_extract_coeffs_eta2_neon(const byte* z, + unsigned int zLen, sword32* s, unsigned int* cnt); +WOLFSSL_LOCAL void mldsa_extract_coeffs_eta4_neon(const byte* z, + unsigned int zLen, sword32* s, unsigned int* cnt); + +/* MakeHint over one polynomial. h is the write position, idx the running + * count. Returns the new count, or -1 when it passed omega. */ +WOLFSSL_LOCAL int mldsa_make_hint_neon(const sword32* s, const sword32* w1, + byte* h, int idx, int omega, sword32 gamma2); + +/* Rejection sample 23-bit values below q. Returns the number written. */ +WOLFSSL_LOCAL unsigned int mldsa_rej_uniform_neon(sword32* p, unsigned int len, + const byte* r, unsigned int rLen); +#endif + +#if defined(__aarch64__) && defined(WOLFSSL_ARMASM) +#ifndef WC_SHA3_NO_ASM +WOLFSSL_LOCAL unsigned int mldsa_rej_uniform_neon(sword32* p, unsigned int len, + const byte* r, unsigned int rLen); +#endif + +/* The plain transforms leave the coefficients lane-permuted; the _full ones + * use the standard order. See the flavor-selection note in wc_mldsa.c. */ +WOLFSSL_LOCAL void mldsa_ntt_neon(sword32* r); +WOLFSSL_LOCAL void mldsa_ntt_full_neon(sword32* r); +WOLFSSL_LOCAL void mldsa_invntt_neon(sword32* r); +WOLFSSL_LOCAL void mldsa_invntt_full_neon(sword32* r); +#ifndef WOLFSSL_AARCH64_NO_SQRDMLSH +/* Same transforms, using the ARMv8.1 rounding-doubling multiply-subtract to + * fold the Montgomery reduction into one instruction. Chosen at run time on + * CPUs reporting FEAT_RDM. */ +WOLFSSL_LOCAL void mldsa_ntt_sqrdmlsh_neon(sword32* r); +WOLFSSL_LOCAL void mldsa_ntt_full_sqrdmlsh_neon(sword32* r); +WOLFSSL_LOCAL void mldsa_invntt_sqrdmlsh_neon(sword32* r); +WOLFSSL_LOCAL void mldsa_invntt_full_sqrdmlsh_neon(sword32* r); +#endif + +WOLFSSL_LOCAL void mldsa_mul_neon(sword32* r, sword32* a, sword32* b); +#ifndef WOLFSSL_AARCH64_NO_SQRDMLSH +WOLFSSL_LOCAL void mldsa_mul_sqrdmlsh_neon(sword32* r, sword32* a, sword32* b); +#endif + +/* One row of the matrix-vector product: the l products are accumulated at 64 + * bits and Montgomery reduced once, matching mldsa_matrix_mul_c(). */ +WOLFSSL_LOCAL void mldsa_mul_vec_4_neon(sword32* r, const sword32* m, + const sword32* v); +WOLFSSL_LOCAL void mldsa_mul_vec_5_neon(sword32* r, const sword32* m, + const sword32* v); +WOLFSSL_LOCAL void mldsa_mul_vec_7_neon(sword32* r, const sword32* m, + const sword32* v); + +/* Decompose one polynomial. GAMMA2 picks the variant, as it does in the C. */ +WOLFSSL_LOCAL void mldsa_decompose_q88_neon(const sword32* r, sword32* r0, + sword32* r1); +WOLFSSL_LOCAL void mldsa_decompose_q32_neon(const sword32* r, sword32* r0, + sword32* r1); + +/* UseHint for every coefficient no hint touches: convert to the positive + * range, decompose and keep the high part, in place. */ +WOLFSSL_LOCAL void mldsa_use_hint_q88_neon(sword32* w1); +WOLFSSL_LOCAL void mldsa_use_hint_q32_neon(sword32* w1); + +/* Unpack MLDSA_N bit-packed fields into 32-bit coefficients. */ +WOLFSSL_LOCAL void mldsa_decode_eta_2_neon(const byte* p, sword32* z); +WOLFSSL_LOCAL void mldsa_decode_eta_4_neon(const byte* p, sword32* z); +WOLFSSL_LOCAL void mldsa_decode_t1_neon(const byte* p, sword32* z); +WOLFSSL_LOCAL void mldsa_decode_t0_neon(const byte* p, sword32* z); +WOLFSSL_LOCAL void mldsa_decode_gamma1_17_neon(const byte* p, sword32* z); +WOLFSSL_LOCAL void mldsa_decode_gamma1_19_neon(const byte* p, sword32* z); + +/* Pack MLDSA_N high-part values into the w1 encoding. */ +WOLFSSL_LOCAL void mldsa_encode_w1_88_neon(const sword32* w1, byte* w1e); +WOLFSSL_LOCAL void mldsa_encode_w1_32_neon(const sword32* w1, byte* w1e); + +/* Pack MLDSA_N values into the gamma1 encoding. */ +WOLFSSL_LOCAL void mldsa_encode_gamma1_17_neon(const sword32* z, byte* s); +WOLFSSL_LOCAL void mldsa_encode_gamma1_19_neon(const sword32* z, byte* s); + +/* 1 when every value of every polynomial is inside -(hi-1)..(hi-1). */ +WOLFSSL_LOCAL int mldsa_vec_check_low_neon(const sword32* a, byte l, + sword32 hi); + +/* Coefficient-wise add, subtract, and add q to the negative values. */ +WOLFSSL_LOCAL void mldsa_poly_add_neon(sword32* r, const sword32* a); +WOLFSSL_LOCAL void mldsa_poly_sub_neon(sword32* r, const sword32* a); +WOLFSSL_LOCAL void mldsa_poly_make_pos_neon(sword32* a); + +/* Barrett reduce every coefficient of a polynomial. */ +WOLFSSL_LOCAL void mldsa_poly_red_neon(sword32* a); + +/* Rejection sample nibbles into -eta..eta, appending from *cnt. */ +WOLFSSL_LOCAL void mldsa_extract_coeffs_eta2_neon(const byte* z, + unsigned int zLen, sword32* s, unsigned int* cnt); +WOLFSSL_LOCAL void mldsa_extract_coeffs_eta4_neon(const byte* z, + unsigned int zLen, sword32* s, unsigned int* cnt); + +/* MakeHint over one polynomial. h is the write position, idx the running + * count. Returns the new count, or -1 when it passed omega. */ +WOLFSSL_LOCAL int mldsa_make_hint_neon(const sword32* s, const sword32* w1, + byte* h, int idx, int omega, sword32 gamma2); +#endif + #define WC_ML_DSA_DRAFT 10 diff --git a/wolfssl/wolfcrypt/wc_mlkem.h b/wolfssl/wolfcrypt/wc_mlkem.h index 739bd4a7937..0ea6356e934 100644 --- a/wolfssl/wolfcrypt/wc_mlkem.h +++ b/wolfssl/wolfcrypt/wc_mlkem.h @@ -811,16 +811,6 @@ WOLFSSL_LOCAL void mlkem_add3_reduce(sword16* r, const sword16* a, WOLFSSL_LOCAL void mlkem_rsub_reduce(sword16* r, const sword16* a); WOLFSSL_LOCAL void mlkem_to_mont(sword16* p); WOLFSSL_LOCAL void mlkem_to_mont_sqrdmlsh(sword16* p); -WOLFSSL_LOCAL void mlkem_sha3_blocksx3_neon(word64* state); -WOLFSSL_LOCAL void mlkem_shake128_blocksx3_seed_neon(word64* state, byte* seed); -WOLFSSL_LOCAL void mlkem_shake256_blocksx3_seed_neon(word64* state, byte* seed); -#ifdef WOLFSSL_ARMASM_CRYPTO_SHA3 -WOLFSSL_LOCAL void mlkem_sha3_blocksx3_crypto(word64* state); -WOLFSSL_LOCAL void mlkem_shake128_blocksx3_seed_crypto(word64* state, - byte* seed); -WOLFSSL_LOCAL void mlkem_shake256_blocksx3_seed_crypto(word64* state, - byte* seed); -#endif WOLFSSL_LOCAL unsigned int mlkem_rej_uniform_neon(sword16* p, unsigned int len, const byte* r, unsigned int rLen); WOLFSSL_LOCAL int mlkem_cmp_neon(const byte* a, const byte* b, int sz);