ntt_n256_l8_s32_bar.s

///
/// Copyright (c) 2021 Arm Limited
/// Copyright (c) 2022 Hanno Becker
/// Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer
/// SPDX-License-Identifier: MIT
///
/// Permission is hereby granted, free of charge, to any person obtaining a copy
/// of this software and associated documentation files (the "Software"), to deal
/// in the Software without restriction, including without limitation the rights
/// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
/// copies of the Software, and to permit persons to whom the Software is
/// furnished to do so, subject to the following conditions:
///
/// The above copyright notice and this permission notice shall be included in all
/// copies or substantial portions of the Software.
///
/// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
/// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
/// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
/// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
/// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
/// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
/// SOFTWARE.
///

.data
roots:
#include "ntt_n256_l8_s32_twiddles.s"
.text

// Barrett multiplication
.macro mulmod dst, src, const, const_twisted
        vmul.s32       \dst,  \src, \const
        vqrdmulh.s32   \src,  \src, \const_twisted
        vmla.s32       \dst,  \src, modulus
.endm

.macro ct_butterfly a, b, root, root_twisted
        mulmod tmp, \b, \root, \root_twisted
        vsub.u32       \b,    \a, tmp
        vadd.u32       \a,    \a, tmp
.endm

.align 4
roots_addr: .word roots
.syntax unified
.type ntt_n256_u32_33556993_28678040_complete_manual, %function
.global ntt_n256_u32_33556993_28678040_complete_manual
ntt_n256_u32_33556993_28678040_complete_manual:

        push {r4-r11,lr}
        // Save MVE vector registers
        vpush {d8-d15}

        modulus  .req r12
        root_ptr .req r11

        .equ modulus_const, -33556993
        movw modulus, #:lower16:modulus_const
        movt modulus, #:upper16:modulus_const
        ldr  root_ptr, roots_addr

        in_low       .req r0
        in_high      .req r1

        add in_high, in_low, #(4*128)

        root0         .req r2
        root0_twisted .req r3
        root1         .req r4
        root1_twisted .req r5
        root2         .req r6
        root2_twisted .req r7

        data0 .req q0
        data1 .req q1
        data2 .req q2
        data3 .req q3

        tmp .req q4

        /* Layers 1-2 */

        ldrd root0, root0_twisted, [root_ptr], #+8
        ldrd root1, root1_twisted, [root_ptr], #+8
        ldrd root2, root2_twisted, [root_ptr], #+8

        mov lr, #16
        vldrw.u32 q2, [in_high, #256]              // *.....
        vmul.s32 q0, q2, root0                     // .*....
        vldrw.u32 q5, [in_low, #256]               // ...*..
        vqrdmulh.s32 q2, q2, root0_twisted         // ..*...
        nop                                        // .....*
        vmla.s32 q0, q2, modulus                   // ....*.
        
        // original source code
        // vldrw.u32 q6, [in_high, #256]           // *.....
        // vmul.s32 q0, q6, root0                  // .*....
        // vqrdmulh.s32 q4, q6, root0_twisted      // ...*..
        // vldrw.u32 q5, [in_low, #256]            // ..*...
        // vmla.s32 q0, q4, modulus                // .....*
        // nop                                     // ....*.
        
        sub lr, lr, #1
layer12_loop:
        vsub.u32 q1, q5, q0                        // ............*...............
        vqrdmulh.s32 q4, q1, root2_twisted         // ....................*.......
        vldrw.u32 q7, [in_high]                    // ..*.........................
        vmul.s32 q2, q7, root0                     // ....*.......................
        vldrw.u32 q6, [in_high, #272]              // ...e........................
        vqrdmulh.s32 q7, q7, root0_twisted         // .....*......................
        vadd.u32 q0, q5, q0                        // .............*..............
        vmla.s32 q2, q7, modulus                   // ......*.....................
        vldrw.u32 q3, [in_low]                     // *...........................
        vmul.s32 q5, q1, root2                     // ...................*........
        vsub.u32 q7, q3, q2                        // .......*....................
        vmla.s32 q5, q4, modulus                   // .....................*......
        vadd.u32 q4, q3, q2                        // ........*...................
        vmul.s32 q3, q0, root1                     // ..............*.............
        vadd.u32 q2, q7, q5                        // .......................*....
        vqrdmulh.s32 q1, q0, root1_twisted         // ...............*............
        vstrw.u32 q2, [in_high] , #16              // ..........................*.
        vmla.s32 q3, q1, modulus                   // ................*...........
        vsub.u32 q2, q7, q5                        // ......................*.....
        vstrw.u32 q2, [in_high, #240]              // ...........................*
        vsub.u32 q5, q4, q3                        // .................*..........
        vmul.s32 q0, q6, root0                     // .........e..................
        vstrw.u32 q5, [in_low, #256]               // .........................*..
        vadd.u32 q1, q4, q3                        // ..................*.........
        vqrdmulh.s32 q4, q6, root0_twisted         // ..........e.................
        vldrw.u32 q5, [in_low, #272]               // .e..........................
        vmla.s32 q0, q4, modulus                   // ...........e................
        vstrw.u32 q1, [in_low] , #16               // ........................*...
        
        // original source code
        // vldrw.u32 data0, [in_low]                     // ................................*...................
        // vldrw.u32 data1, [in_low, #256]               // .....................e..............................
        // vldrw.u32 data2, [in_high]                    // ..........................*.........................
        // vldrw.u32 data3, [in_high, #256]              // e...................................................
        // vmul.s32 tmp, data2, root0                    // ...........................*........................
        // vqrdmulh.s32 data2, data2, root0_twisted      // .............................*......................
        // vmla.s32 tmp, data2, modulus                  // ...............................*....................
        // vsub.u32 data2, data0, tmp                    // ..................................*.................
        // vadd.u32 data0, data0, tmp                    // ....................................*...............
        // vmul.s32 tmp, data3, root0                    // .................e..................................
        // vqrdmulh.s32 data3, data3, root0_twisted      // ....................e...............................
        // vmla.s32 tmp, data3, modulus                  // ......................e.............................
        // vsub.u32 data3, data1, tmp                    // ........................*...........................
        // vadd.u32 data1, data1, tmp                    // ..............................*.....................
        // vmul.s32 tmp, data1, root1                    // .....................................*..............
        // vqrdmulh.s32 data1, data1, root1_twisted      // .......................................*............
        // vmla.s32 tmp, data1, modulus                  // .........................................*..........
        // vsub.u32 data1, data0, tmp                    // ............................................*.......
        // vadd.u32 data0, data0, tmp                    // ...............................................*....
        // vmul.s32 tmp, data3, root2                    // .................................*..................
        // vqrdmulh.s32 data3, data3, root2_twisted      // .........................*..........................
        // vmla.s32 tmp, data3, modulus                  // ...................................*................
        // vsub.u32 data3, data2, tmp                    // ..........................................*.........
        // vadd.u32 data2, data2, tmp                    // ......................................*.............
        // vstrw.u32 data0, [in_low] , #16               // ...................................................*
        // vstrw.u32 data1, [in_low, #240]               // ..............................................*.....
        // vstrw.u32 data2, [in_high] , #16              // ........................................*...........
        // vstrw.u32 data3, [in_high, #240]              // ...........................................*........
        
        le lr, layer12_loop
        vsub.u32 q1, q5, q0                        // *.......................
        vqrdmulh.s32 q7, q1, root2_twisted         // .*......................
        vldrw.u32 q3, [in_high]                    // ..*.....................
        vqrdmulh.s32 q6, q3, root0_twisted         // ....*...................
        vadd.u32 q0, q5, q0                        // .....*..................
        vmul.s32 q2, q3, root0                     // ...*....................
        vldrw.u32 q3, [in_low]                     // .......*................
        vmla.s32 q2, q6, modulus                   // ......*.................
        nop                                        // .......................*
        vmul.s32 q5, q1, root2                     // ........*...............
        vsub.u32 q1, q3, q2                        // .........*..............
        vmla.s32 q5, q7, modulus                   // ..........*.............
        vadd.u32 q2, q3, q2                        // ...........*............
        vmul.s32 q3, q0, root1                     // ............*...........
        vadd.u32 q6, q1, q5                        // .............*..........
        vstrw.u32 q6, [in_high] , #16              // ...............*........
        vqrdmulh.s32 q4, q0, root1_twisted         // ..............*.........
        vsub.u32 q7, q1, q5                        // .................*......
        vmla.s32 q3, q4, modulus                   // ................*.......
        vstrw.u32 q7, [in_high, #240]              // ..................*.....
        vsub.u32 q5, q2, q3                        // ...................*....
        vstrw.u32 q5, [in_low, #256]               // ....................*...
        vadd.u32 q2, q2, q3                        // .....................*..
        vstrw.u32 q2, [in_low] , #16               // ......................*.
        
        // original source code
        // vsub.u32 q1, q5, q0                     // *.......................
        // vqrdmulh.s32 q4, q1, root2_twisted      // .*......................
        // vldrw.u32 q7, [in_high]                 // ..*.....................
        // vmul.s32 q2, q7, root0                  // .....*..................
        // vqrdmulh.s32 q7, q7, root0_twisted      // ...*....................
        // vadd.u32 q0, q5, q0                     // ....*...................
        // vmla.s32 q2, q7, modulus                // .......*................
        // vldrw.u32 q3, [in_low]                  // ......*.................
        // vmul.s32 q5, q1, root2                  // .........*..............
        // vsub.u32 q7, q3, q2                     // ..........*.............
        // vmla.s32 q5, q4, modulus                // ...........*............
        // vadd.u32 q4, q3, q2                     // ............*...........
        // vmul.s32 q3, q0, root1                  // .............*..........
        // vadd.u32 q2, q7, q5                     // ..............*.........
        // vqrdmulh.s32 q1, q0, root1_twisted      // ................*.......
        // vstrw.u32 q2, [in_high] , #16           // ...............*........
        // vmla.s32 q3, q1, modulus                // ..................*.....
        // vsub.u32 q2, q7, q5                     // .................*......
        // vstrw.u32 q2, [in_high, #240]           // ...................*....
        // vsub.u32 q5, q4, q3                     // ....................*...
        // vstrw.u32 q5, [in_low, #256]            // .....................*..
        // vadd.u32 q1, q4, q3                     // ......................*.
        // vstrw.u32 q1, [in_low] , #16            // .......................*
        // nop                                     // ........*...............
        
layer12_loop_end:
        .unreq in_high
        .unreq in_low

        in .req r0
        sub in, in, #(64*4)

        /* Layers 3,4 */

        // 4 butterfly blocks per root config, 4 root configs
        // loop over root configs

        count .req r1
        mov count, #4

out_start:
        ldrd root0, root0_twisted, [root_ptr], #+8
        ldrd root1, root1_twisted, [root_ptr], #+8
        ldrd root2, root2_twisted, [root_ptr], #+8

        mov lr, #4
        vldrw.u32 q2, [in, #192]                   // *.....
        vmul.s32 q0, q2, root0                     // .*....
        vldrw.u32 q5, [in, #64]                    // ...*..
        vqrdmulh.s32 q2, q2, root0_twisted         // ..*...
        nop                                        // .....*
        vmla.s32 q0, q2, modulus                   // ....*.
        
        // original source code
        // vldrw.u32 q6, [in, #192]                // *.....
        // vmul.s32 q0, q6, root0                  // .*....
        // vqrdmulh.s32 q4, q6, root0_twisted      // ...*..
        // vldrw.u32 q5, [in, #64]                 // ..*...
        // vmla.s32 q0, q4, modulus                // .....*
        // nop                                     // ....*.
        
        sub lr, lr, #1
layer34_loop:
        vsub.u32 q1, q5, q0                        // ............*...............
        vqrdmulh.s32 q4, q1, root2_twisted         // ....................*.......
        vldrw.u32 q7, [in, #128]                   // ..*.........................
        vmul.s32 q2, q7, root0                     // ....*.......................
        vldrw.u32 q6, [in, #208]                   // ...e........................
        vqrdmulh.s32 q7, q7, root0_twisted         // .....*......................
        vadd.u32 q0, q5, q0                        // .............*..............
        vmla.s32 q2, q7, modulus                   // ......*.....................
        vldrw.u32 q3, [in]                         // *...........................
        vmul.s32 q5, q1, root2                     // ...................*........
        vsub.u32 q7, q3, q2                        // .......*....................
        vmla.s32 q5, q4, modulus                   // .....................*......
        vadd.u32 q4, q3, q2                        // ........*...................
        vmul.s32 q3, q0, root1                     // ..............*.............
        vadd.u32 q2, q7, q5                        // .......................*....
        vqrdmulh.s32 q1, q0, root1_twisted         // ...............*............
        vstrw.u32 q2, [in, #128]                   // ..........................*.
        vmla.s32 q3, q1, modulus                   // ................*...........
        vsub.u32 q2, q7, q5                        // ......................*.....
        vstrw.u32 q2, [in, #192]                   // ...........................*
        vsub.u32 q5, q4, q3                        // .................*..........
        vmul.s32 q0, q6, root0                     // .........e..................
        vstrw.u32 q5, [in, #64]                    // .........................*..
        vadd.u32 q1, q4, q3                        // ..................*.........
        vqrdmulh.s32 q4, q6, root0_twisted         // ..........e.................
        vldrw.u32 q5, [in, #80]                    // .e..........................
        vmla.s32 q0, q4, modulus                   // ...........e................
        vstrw.u32 q1, [in] , #16                   // ........................*...
        
        // original source code
        // vldrw.u32 data0, [in]                         // ................................*...................
        // vldrw.u32 data1, [in, #64]                    // .....................e..............................
        // vldrw.u32 data2, [in, #128]                   // ..........................*.........................
        // vldrw.u32 data3, [in, #192]                   // e...................................................
        // vmul.s32 tmp, data2, root0                    // ...........................*........................
        // vqrdmulh.s32 data2, data2, root0_twisted      // .............................*......................
        // vmla.s32 tmp, data2, modulus                  // ...............................*....................
        // vsub.u32 data2, data0, tmp                    // ..................................*.................
        // vadd.u32 data0, data0, tmp                    // ....................................*...............
        // vmul.s32 tmp, data3, root0                    // .................e..................................
        // vqrdmulh.s32 data3, data3, root0_twisted      // ....................e...............................
        // vmla.s32 tmp, data3, modulus                  // ......................e.............................
        // vsub.u32 data3, data1, tmp                    // ........................*...........................
        // vadd.u32 data1, data1, tmp                    // ..............................*.....................
        // vmul.s32 tmp, data1, root1                    // .....................................*..............
        // vqrdmulh.s32 data1, data1, root1_twisted      // .......................................*............
        // vmla.s32 tmp, data1, modulus                  // .........................................*..........
        // vsub.u32 data1, data0, tmp                    // ............................................*.......
        // vadd.u32 data0, data0, tmp                    // ...............................................*....
        // vmul.s32 tmp, data3, root2                    // .................................*..................
        // vqrdmulh.s32 data3, data3, root2_twisted      // .........................*..........................
        // vmla.s32 tmp, data3, modulus                  // ...................................*................
        // vsub.u32 data3, data2, tmp                    // ..........................................*.........
        // vadd.u32 data2, data2, tmp                    // ......................................*.............
        // vstrw.u32 data0, [in] , #16                   // ...................................................*
        // vstrw.u32 data1, [in, #48]                    // ..............................................*.....
        // vstrw.u32 data2, [in, #112]                   // ........................................*...........
        // vstrw.u32 data3, [in, #176]                   // ...........................................*........
        
        le lr, layer34_loop
        vsub.u32 q1, q5, q0                        // *.......................
        vqrdmulh.s32 q7, q1, root2_twisted         // .*......................
        vldrw.u32 q3, [in, #128]                   // ..*.....................
        vqrdmulh.s32 q6, q3, root0_twisted         // ....*...................
        vadd.u32 q0, q5, q0                        // .....*..................
        vmul.s32 q2, q3, root0                     // ...*....................
        vldrw.u32 q3, [in]                         // .......*................
        vmla.s32 q2, q6, modulus                   // ......*.................
        nop                                        // .......................*
        vmul.s32 q5, q1, root2                     // ........*...............
        vsub.u32 q1, q3, q2                        // .........*..............
        vmla.s32 q5, q7, modulus                   // ..........*.............
        vadd.u32 q2, q3, q2                        // ...........*............
        vmul.s32 q3, q0, root1                     // ............*...........
        vadd.u32 q6, q1, q5                        // .............*..........
        vstrw.u32 q6, [in, #128]                   // ...............*........
        vqrdmulh.s32 q4, q0, root1_twisted         // ..............*.........
        vsub.u32 q7, q1, q5                        // .................*......
        vmla.s32 q3, q4, modulus                   // ................*.......
        vstrw.u32 q7, [in, #192]                   // ..................*.....
        vsub.u32 q5, q2, q3                        // ...................*....
        vstrw.u32 q5, [in, #64]                    // ....................*...
        vadd.u32 q2, q2, q3                        // .....................*..
        vstrw.u32 q2, [in] , #16                   // ......................*.
        
        // original source code
        // vsub.u32 q1, q5, q0                     // *.......................
        // vqrdmulh.s32 q4, q1, root2_twisted      // .*......................
        // vldrw.u32 q7, [in, #128]                // ..*.....................
        // vmul.s32 q2, q7, root0                  // .....*..................
        // vqrdmulh.s32 q7, q7, root0_twisted      // ...*....................
        // vadd.u32 q0, q5, q0                     // ....*...................
        // vmla.s32 q2, q7, modulus                // .......*................
        // vldrw.u32 q3, [in]                      // ......*.................
        // vmul.s32 q5, q1, root2                  // .........*..............
        // vsub.u32 q7, q3, q2                     // ..........*.............
        // vmla.s32 q5, q4, modulus                // ...........*............
        // vadd.u32 q4, q3, q2                     // ............*...........
        // vmul.s32 q3, q0, root1                  // .............*..........
        // vadd.u32 q2, q7, q5                     // ..............*.........
        // vqrdmulh.s32 q1, q0, root1_twisted      // ................*.......
        // vstrw.u32 q2, [in, #128]                // ...............*........
        // vmla.s32 q3, q1, modulus                // ..................*.....
        // vsub.u32 q2, q7, q5                     // .................*......
        // vstrw.u32 q2, [in, #192]                // ...................*....
        // vsub.u32 q5, q4, q3                     // ....................*...
        // vstrw.u32 q5, [in, #64]                 // .....................*..
        // vadd.u32 q1, q4, q3                     // ......................*.
        // vstrw.u32 q1, [in] , #16                // .......................*
        // nop                                     // ........*...............
        
layer34_loop_end:

        add in, in, #(4*64 - 4*16)
        subs count, count, #1
        bne out_start

        sub in, in, #(4*256)

        /* Layers 5,6 */

        // 1 butterfly blocks per root config, 16 root configs
        // loop over root configs

        mov lr, #16
        ldrd r9, r7, [root_ptr] , #24         // ...*...
        vldrw.u32 q5, [in, #48]               // *......
        vqrdmulh.s32 q2, q5, r7               // ....*..
        vldrw.u32 q6, [in, #16]               // .*.....
        vmul.s32 q5, q5, r9                   // .....*.
        ldrd r2, r1, [root_ptr, #-8]          // ..*....
        vmla.s32 q5, q2, modulus              // ......*
        
        // original source code
        // vldrw.u32 q5, [in, #48]            // .*.....
        // vldrw.u32 q6, [in, #16]            // ...*...
        // ldrd r2, r1, [root_ptr, #16]       // .....*.
        // ldrd r9, r7, [root_ptr] , #24      // *......
        // vqrdmulh.s32 q0, q5, r7            // ..*....
        // vmul.s32 q5, q5, r9                // ....*..
        // vmla.s32 q5, q0, modulus           // ......*
        
        sub lr, lr, #1
layer56_loop:
        vsub.u32 q1, q6, q5                    // ...............*...............
        vqrdmulh.s32 q4, q1, r1                // .......................*.......
        vldrw.u32 q0, [in, #32]                // .....*.........................
        vmul.s32 q2, q0, r9                    // .......*.......................
        ldrd r9, r8, [root_ptr, #-16]          // .*.............................
        vqrdmulh.s32 q0, q0, r7                // ........*......................
        vadd.u32 q3, q6, q5                    // ................*..............
        vmla.s32 q2, q0, modulus               // .........*.....................
        vldrw.u32 q0, [in]                     // ...*...........................
        vmul.s32 q1, q1, r2                    // ......................*........
        vsub.u32 q5, q0, q2                    // ..........*....................
        vmla.s32 q1, q4, modulus               // ........................*......
        vadd.u32 q0, q0, q2                    // ...........*...................
        vmul.s32 q7, q3, r9                    // .................*.............
        vsub.u32 q4, q5, q1                    // .........................*.....
        vqrdmulh.s32 q2, q3, r8                // ..................*............
        vadd.u32 q3, q5, q1                    // ..........................*....
        vmla.s32 q7, q2, modulus               // ...................*...........
        vldrw.u32 q5, [in, #112]               // ......e........................
        vadd.u32 q1, q0, q7                    // .....................*.........
        vldrw.u32 q6, [in, #80]                // ....e..........................
        vsub.u32 q2, q0, q7                    // ....................*..........
        ldrd r2, r1, [root_ptr, #16]           // ..e............................
        ldrd r9, r7, [root_ptr] , #24          // e..............................
        vst40.u32 {q1,q2,q3,q4}, [in]          // ...........................*...
        vqrdmulh.s32 q0, q5, r7                // .............e.................
        vst41.u32 {q1,q2,q3,q4}, [in]          // ............................*..
        vmul.s32 q5, q5, r9                    // ............e..................
        vst42.u32 {q1,q2,q3,q4}, [in]          // .............................*.
        vmla.s32 q5, q0, modulus               // ..............e................
        vst43.u32 {q1,q2,q3,q4}, [in]!         // ..............................*
        
        // original source code
        // ldrd root0, root0_twisted, [root_ptr] , #24       // .....e......................................
        // ldrd root1, root1_twisted, [root_ptr, #-16]       // .................*..........................
        // ldrd root2, root2_twisted, [root_ptr, #-8]        // ....e.......................................
        // vldrw.u32 data0, [in]                             // .....................*......................
        // vldrw.u32 data1, [in, #16]                        // ..e.........................................
        // vldrw.u32 data2, [in, #32]                        // ...............*............................
        // vldrw.u32 data3, [in, #48]                        // e...........................................
        // vmul.s32 tmp, data2, root0                        // ................*...........................
        // vqrdmulh.s32 data2, data2, root0_twisted          // ..................*.........................
        // vmla.s32 tmp, data2, modulus                      // ....................*.......................
        // vsub.u32 data2, data0, tmp                        // .......................*....................
        // vadd.u32 data0, data0, tmp                        // .........................*..................
        // vmul.s32 tmp, data3, root0                        // .........e..................................
        // vqrdmulh.s32 data3, data3, root0_twisted          // .......e....................................
        // vmla.s32 tmp, data3, modulus                      // ...........e................................
        // vsub.u32 data3, data1, tmp                        // .............*..............................
        // vadd.u32 data1, data1, tmp                        // ...................*........................
        // vmul.s32 tmp, data1, root1                        // ..........................*.................
        // vqrdmulh.s32 data1, data1, root1_twisted          // ............................*...............
        // vmla.s32 tmp, data1, modulus                      // ..............................*.............
        // vsub.u32 data1, data0, tmp                        // ..................................*.........
        // vadd.u32 data0, data0, tmp                        // ................................*...........
        // vmul.s32 tmp, data3, root2                        // ......................*.....................
        // vqrdmulh.s32 data3, data3, root2_twisted          // ..............*.............................
        // vmla.s32 tmp, data3, modulus                      // ........................*...................
        // vsub.u32 data3, data2, tmp                        // ...........................*................
        // vadd.u32 data2, data2, tmp                        // .............................*..............
        // vst40.u32 {data0,data1,data2,data3}, [in]         // .....................................*......
        // vst41.u32 {data0,data1,data2,data3}, [in]         // .......................................*....
        // vst42.u32 {data0,data1,data2,data3}, [in]         // .........................................*..
        // vst43.u32 {data0,data1,data2,data3}, [in]!        // ...........................................*
        
        le lr, layer56_loop
        vldrw.u32 q3, [in, #32]                // ..*............................
        vmul.s32 q7, q3, r9                    // ...*...........................
        vldrw.u32 q4, [in]                     // ........*......................
        vqrdmulh.s32 q0, q3, r7                // .....*.........................
        vadd.u32 q3, q6, q5                    // ......*........................
        vmla.s32 q7, q0, modulus               // .......*.......................
        vsub.u32 q2, q6, q5                    // *..............................
        vqrdmulh.s32 q0, q2, r1                // .*.............................
        vadd.u32 q1, q4, q7                    // ............*..................
        vmul.s32 q2, q2, r2                    // .........*.....................
        ldrd r9, r5, [root_ptr, #-16]          // ....*..........................
        vmla.s32 q2, q0, modulus               // ...........*...................
        vsub.u32 q5, q4, q7                    // ..........*....................
        vmul.s32 q0, q3, r9                    // .............*.................
        vsub.u32 q4, q5, q2                    // ..............*................
        vqrdmulh.s32 q7, q3, r5                // ...............*...............
        vadd.u32 q3, q5, q2                    // ................*..............
        vmla.s32 q0, q7, modulus               // .................*.............
        nop                                    // .............................*.
        vsub.u32 q2, q1, q0                    // ...................*...........
        nop                                    // ...........................*...
        vadd.u32 q1, q1, q0                    // ..................*............
        nop                                    // ..........................*....
        nop                                    // ............................*..
        vst40.u32 {q1,q2,q3,q4}, [in]          // ....................*..........
        nop                                    // ........................*......
        vst41.u32 {q1,q2,q3,q4}, [in]          // .....................*.........
        nop                                    // ..............................*
        vst42.u32 {q1,q2,q3,q4}, [in]          // ......................*........
        nop                                    // .........................*.....
        vst43.u32 {q1,q2,q3,q4}, [in]!         // .......................*.......
        
        // original source code
        // vsub.u32 q1, q6, q5                 // ......*........................
        // vqrdmulh.s32 q4, q1, r1             // .......*.......................
        // vldrw.u32 q0, [in, #32]             // *..............................
        // vmul.s32 q2, q0, r9                 // .*.............................
        // ldrd r9, r8, [root_ptr, #-16]       // ..........*....................
        // vqrdmulh.s32 q0, q0, r7             // ...*...........................
        // vadd.u32 q3, q6, q5                 // ....*..........................
        // vmla.s32 q2, q0, modulus            // .....*.........................
        // vldrw.u32 q0, [in]                  // ..*............................
        // vmul.s32 q1, q1, r2                 // .........*.....................
        // vsub.u32 q5, q0, q2                 // ............*..................
        // vmla.s32 q1, q4, modulus            // ...........*...................
        // vadd.u32 q0, q0, q2                 // ........*......................
        // vmul.s32 q7, q3, r9                 // .............*.................
        // vsub.u32 q4, q5, q1                 // ..............*................
        // vqrdmulh.s32 q2, q3, r8             // ...............*...............
        // vadd.u32 q3, q5, q1                 // ................*..............
        // vmla.s32 q7, q2, modulus            // .................*.............
        // vadd.u32 q1, q0, q7                 // .....................*.........
        // vsub.u32 q2, q0, q7                 // ...................*...........
        // vst40.u32 {q1,q2,q3,q4}, [in]       // ........................*......
        // vst41.u32 {q1,q2,q3,q4}, [in]       // ..........................*....
        // vst42.u32 {q1,q2,q3,q4}, [in]       // ............................*..
        // vst43.u32 {q1,q2,q3,q4}, [in]!      // ..............................*
        // nop                                 // .........................*.....
        // nop                                 // .............................*.
        // nop                                 // ......................*........
        // nop                                 // ....................*..........
        // nop                                 // .......................*.......
        // nop                                 // ..................*............
        // nop                                 // ...........................*...
        
layer56_loop_end:

        sub in, in, #(4*256)

        /* Layers 7,8 */

        .unreq root0
        .unreq root0_twisted
        .unreq root1
        .unreq root1_twisted
        .unreq root2
        .unreq root2_twisted

        root0         .req q5
        root0_twisted .req q6
        root1         .req q5
        root1_twisted .req q6
        root2         .req q5
        root2_twisted .req q6

        mov lr, #16
        vldrw.u32 q1, [root_ptr] , #96          // *........
        nop                                     // ........*
        vldrw.u32 q5, [in, #48]                 // ..*......
        vmul.s32 q2, q5, q1                     // ...*.....
        vldrw.u32 q7, [root_ptr, #-80]          // .*.......
        vqrdmulh.s32 q0, q5, q7                 // ......*..
        vldrw.u32 q5, [in, #16]                 // ....*....
        vmla.s32 q2, q0, modulus                // .......*.
        vldrw.u32 q0, [root_ptr, #-64]          // .....*...
        
        // original source code
        // vldrw.u32 q1, [root_ptr] , #96       // *........
        // vldrw.u32 q7, [root_ptr, #-80]       // ....*....
        // vldrw.u32 q3, [in, #48]              // ..*......
        // vmul.s32 q2, q3, q1                  // ...*.....
        // vldrw.u32 q5, [in, #16]              // ......*..
        // vldrw.u32 q0, [root_ptr, #-64]       // ........*
        // vqrdmulh.s32 q3, q3, q7              // .....*...
        // vmla.s32 q2, q3, modulus             // .......*.
        // nop                                  // .*.......
        
        sub lr, lr, #1
layer78_loop:
        vadd.u32 q4, q5, q2                     // ...............*..................
        vmul.s32 q0, q4, q0                     // ..................*...............
        vsub.u32 q5, q5, q2                     // ..............*...................
        vldrw.u32 q2, [root_ptr, #-16]          // ........................*.........
        vqrdmulh.s32 q6, q5, q2                 // ..........................*.......
        vldrw.u32 q3, [in, #32]                 // ..*...............................
        vmul.s32 q2, q3, q1                     // ......*...........................
        vldrw.u32 q1, [root_ptr, #-48]          // .................*................
        vqrdmulh.s32 q1, q4, q1                 // ...................*..............
        vldrw.u32 q4, [root_ptr, #-32]          // .......................*..........
        vmla.s32 q0, q1, modulus                // ....................*.............
        vldrw.u32 q1, [root_ptr] , #96          // ....e.............................
        vqrdmulh.s32 q7, q3, q7                 // .......*..........................
        vldrw.u32 q3, [in] , #64                // *.................................
        vmla.s32 q2, q7, modulus                // ........*.........................
        vldrw.u32 q7, [root_ptr, #-80]          // .....e............................
        vmul.s32 q5, q5, q4                     // .........................*........
        vadd.u32 q4, q3, q2                     // ..........*.......................
        vmla.s32 q5, q6, modulus                // ...........................*......
        vsub.u32 q6, q3, q2                     // .........*........................
        vldrw.u32 q3, [in, #48]                 // ...e..............................
        vadd.u32 q2, q6, q5                     // .............................*....
        vstrw.u32 q2, [in, #-32]                // ................................*.
        vsub.u32 q2, q4, q0                     // .....................*............
        vstrw.u32 q2, [in, #-48]                // ...............................*..
        vmul.s32 q2, q3, q1                     // ...........e......................
        vsub.u32 q6, q6, q5                     // ............................*.....
        vldrw.u32 q5, [in, #16]                 // .e................................
        vadd.u32 q4, q4, q0                     // ......................*...........
        vldrw.u32 q0, [root_ptr, #-64]          // ................e.................
        vqrdmulh.s32 q3, q3, q7                 // ............e.....................
        vstrw.u32 q6, [in, #-16]                // .................................*
        vmla.s32 q2, q3, modulus                // .............e....................
        vstrw.u32 q4, [in, #-64]                // ..............................*...
        
        // original source code
        // vldrw.u32 data0, [in] , #64                     // ....................................*....................
        // vldrw.u32 data1, [in, #-48]                     // ................e........................................
        // vldrw.u32 data2, [in, #-32]                     // ............................*............................
        // vldrw.u32 data3, [in, #-16]                     // .........e...............................................
        // vldrw.u32 root0, [root_ptr] , #96               // e........................................................
        // vldrw.u32 root0_twisted, [root_ptr, #-80]       // ....e....................................................
        // vmul.s32 tmp, data2, root0                      // .............................*...........................
        // vqrdmulh.s32 data2, data2, root0_twisted        // ...................................*.....................
        // vmla.s32 tmp, data2, modulus                    // .....................................*...................
        // vsub.u32 data2, data0, tmp                      // ..........................................*..............
        // vadd.u32 data0, data0, tmp                      // ........................................*................
        // vmul.s32 tmp, data3, root0                      // ..............e..........................................
        // vqrdmulh.s32 data3, data3, root0_twisted        // ...................e.....................................
        // vmla.s32 tmp, data3, modulus                    // .....................e...................................
        // vsub.u32 data3, data1, tmp                      // .........................*...............................
        // vadd.u32 data1, data1, tmp                      // .......................*.................................
        // vldrw.u32 root1, [root_ptr, #-64]               // ..................e......................................
        // vldrw.u32 root1_twisted, [root_ptr, #-48]       // ..............................*..........................
        // vmul.s32 tmp, data1, root1                      // ........................*................................
        // vqrdmulh.s32 data1, data1, root1_twisted        // ...............................*.........................
        // vmla.s32 tmp, data1, modulus                    // .................................*.......................
        // vsub.u32 data1, data0, tmp                      // ..............................................*..........
        // vadd.u32 data0, data0, tmp                      // ...................................................*.....
        // vldrw.u32 root2, [root_ptr, #-32]               // ................................*........................
        // vldrw.u32 root2_twisted, [root_ptr, #-16]       // ..........................*..............................
        // vmul.s32 tmp, data3, root2                      // .......................................*.................
        // vqrdmulh.s32 data3, data3, root2_twisted        // ...........................*.............................
        // vmla.s32 tmp, data3, modulus                    // .........................................*...............
        // vsub.u32 data3, data2, tmp                      // .................................................*.......
        // vadd.u32 data2, data2, tmp                      // ............................................*............
        // vstrw.u32 data0, [in, #-64]                     // ........................................................*
        // vstrw.u32 data1, [in, #-48]                     // ...............................................*.........
        // vstrw.u32 data2, [in, #-32]                     // .............................................*...........
        // vstrw.u32 data3, [in, #-16]                     // ......................................................*..
        
        le lr, layer78_loop
        vadd.u32 q3, q5, q2                     // *..........................
        vmul.s32 q0, q3, q0                     // .*.........................
        vsub.u32 q5, q5, q2                     // ..*........................
        vldrw.u32 q2, [root_ptr, #-16]          // ...*.......................
        vqrdmulh.s32 q4, q5, q2                 // ....*......................
        vldrw.u32 q6, [root_ptr, #-32]          // .........*.................
        vmul.s32 q5, q5, q6                     // ..............*............
        vldrw.u32 q6, [in, #32]                 // .....*.....................
        vqrdmulh.s32 q7, q6, q7                 // ...........*...............
        nop                                     // ..........................*
        vmul.s32 q2, q6, q1                     // ......*....................
        vldrw.u32 q6, [in] , #64                // ............*..............
        vmla.s32 q2, q7, modulus                // .............*.............
        vldrw.u32 q1, [root_ptr, #-48]          // .......*...................
        vmla.s32 q5, q4, modulus                // ................*..........
        vadd.u32 q4, q6, q2                     // ...............*...........
        vqrdmulh.s32 q1, q3, q1                 // ........*..................
        vsub.u32 q6, q6, q2                     // .................*.........
        vmla.s32 q0, q1, modulus                // ..........*................
        vadd.u32 q2, q6, q5                     // ..................*........
        vstrw.u32 q2, [in, #-32]                // ...................*.......
        vsub.u32 q6, q6, q5                     // ......................*....
        vstrw.u32 q6, [in, #-16]                // ........................*..
        vsub.u32 q2, q4, q0                     // ....................*......
        vstrw.u32 q2, [in, #-48]                // .....................*.....
        vadd.u32 q4, q4, q0                     // .......................*...
        vstrw.u32 q4, [in, #-64]                // .........................*.
        
        // original source code
        // vadd.u32 q4, q5, q2                  // *..........................
        // vmul.s32 q0, q4, q0                  // .*.........................
        // vsub.u32 q5, q5, q2                  // ..*........................
        // vldrw.u32 q2, [root_ptr, #-16]       // ...*.......................
        // vqrdmulh.s32 q6, q5, q2              // ....*......................
        // vldrw.u32 q3, [in, #32]              // .......*...................
        // vmul.s32 q2, q3, q1                  // ..........*................
        // vldrw.u32 q1, [root_ptr, #-48]       // .............*.............
        // vqrdmulh.s32 q1, q4, q1              // ................*..........
        // vldrw.u32 q4, [root_ptr, #-32]       // .....*.....................
        // vmla.s32 q0, q1, modulus             // ..................*........
        // vqrdmulh.s32 q7, q3, q7              // ........*..................
        // vldrw.u32 q3, [in] , #64             // ...........*...............
        // vmla.s32 q2, q7, modulus             // ............*..............
        // vmul.s32 q5, q5, q4                  // ......*....................
        // vadd.u32 q4, q3, q2                  // ...............*...........
        // vmla.s32 q5, q6, modulus             // ..............*............
        // vsub.u32 q6, q3, q2                  // .................*.........
        // vadd.u32 q2, q6, q5                  // ...................*.......
        // vstrw.u32 q2, [in, #-32]             // ....................*......
        // vsub.u32 q2, q4, q0                  // .......................*...
        // vstrw.u32 q2, [in, #-48]             // ........................*..
        // vsub.u32 q6, q6, q5                  // .....................*.....
        // vadd.u32 q4, q4, q0                  // .........................*.
        // vstrw.u32 q6, [in, #-16]             // ......................*....
        // vstrw.u32 q4, [in, #-64]             // ..........................*
        // nop                                  // .........*.................
        
layer78_loop_end:

        // Restore MVE vector registers
        vpop {d8-d15}
        // Restore GPRs
        pop {r4-r11,lr}
        bx lr