intt_n256_l8_s32_bar.s

///
/// Copyright (c) 2021 Arm Limited
/// Copyright (c) 2022 Hanno Becker
/// Copyright (c) 2023 Amin Abdulrahman, Matthias Kannwischer
/// SPDX-License-Identifier: MIT
///
/// Permission is hereby granted, free of charge, to any person obtaining a copy
/// of this software and associated documentation files (the "Software"), to deal
/// in the Software without restriction, including without limitation the rights
/// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
/// copies of the Software, and to permit persons to whom the Software is
/// furnished to do so, subject to the following conditions:
///
/// The above copyright notice and this permission notice shall be included in all
/// copies or substantial portions of the Software.
///
/// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
/// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
/// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
/// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
/// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
/// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
/// SOFTWARE.
///

#define INVNTT_REDUCE_AFTER_L56
#define INVNTT_REDUCE_AFTER_L34

.data
roots_inv:
#include "intt_n256_l8_s32_twiddles.s"
.text

// Barrett multiplication
.macro mulmod dst, src, const, const_twisted
        vmul.s32       \dst,  \src, \const
        vqrdmulh.s32   \src,  \src, \const_twisted
        vmla.s32       \dst,  \src, modulus
.endm

.macro gs_butterfly a, b, root, root_twisted
        vsub.u32       tmp, \a,  \b
        vadd.u32       \a,  \a,  \b
        mulmod         \b,  tmp, \root, \root_twisted
.endm

.align 4
roots_addr: .word roots_inv
.syntax unified
.type invntt_n256_u32_33556993_28678040_complete_manual, %function
.global invntt_n256_u32_33556993_28678040_complete_manual
invntt_n256_u32_33556993_28678040_complete_manual:

        push {r4-r11,lr}
        // Save MVE vector registers
        vpush {d8-d15}

        modulus     .req r12
        root_ptr    .req r11

        .equ modulus_const, -33556993
        movw modulus, #:lower16:modulus_const
        movt modulus, #:upper16:modulus_const
        ldr  root_ptr, roots_addr

        in .req r0

        data0 .req q0
        data1 .req q1
        data2 .req q2
        data3 .req q3

        root0         .req q5
        root0_twisted .req q6
        root1         .req q5
        root1_twisted .req q6
        root2         .req q5
        root2_twisted .req q6


        tmp .req q4

        /* Layers 7,8 */

        mov lr, #16
        vldrw.u32 q1, [in, #16]                // *....
        nop                                    // ...*.
        vldrw.u32 q5, [root_ptr, #48]          // ..*..
        nop                                    // ....*
        vldrw.u32 q3, [in]                     // .*...
        
        // original source code
        // vldrw.u32 q1, [in, #16]             // *....
        // vldrw.u32 q3, [in]                  // ....*
        // vldrw.u32 q5, [root_ptr, #48]       // ..*..
        // nop                                 // .*...
        // nop                                 // ...*.
        
        sub lr, lr, #1
layer78_loop:
        vsub.u32 q2, q3, q1                    // ......*...........................
        vqrdmulh.s32 q4, q2, q5                // .........*........................
        vldrw.u32 q7, [root_ptr, #32]          // ....*.............................
        vmul.s32 q2, q2, q7                    // ........*.........................
        vldrw.u32 q5, [in, #32]                // ..*...............................
        vmla.s32 q2, q4, modulus               // ..........*.......................
        vldrw.u32 q7, [in, #48]                // ...*..............................
        vsub.u32 q4, q5, q7                    // .............*....................
        vldrw.u32 q6, [root_ptr, #80]          // ............*.....................
        vqrdmulh.s32 q6, q4, q6                // ................*.................
        vldrw.u32 q0, [root_ptr, #64]          // ...........*......................
        vmul.s32 q0, q4, q0                    // ...............*..................
        vadd.u32 q4, q5, q7                    // ..............*...................
        vmla.s32 q0, q6, modulus               // .................*................
        vldrw.u32 q6, [root_ptr, #16]          // ...................*..............
        vadd.u32 q7, q2, q0                    // ..........................*.......
        vstrw.u32 q7, [in, #16]                // ...............................*..
        vsub.u32 q5, q2, q0                    // .........................*........
        vqrdmulh.s32 q7, q5, q6                // ............................*.....
        vadd.u32 q0, q3, q1                    // .......*..........................
        vldrw.u32 q1, [in, #80]                // .e................................
        vadd.u32 q3, q0, q4                    // .....................*............
        vldrw.u32 q2, [root_ptr] , #96         // ..................*...............
        vmul.s32 q5, q5, q2                    // ...........................*......
        vstrw.u32 q3, [in] , #64               // ..............................*...
        vmla.s32 q5, q7, modulus               // .............................*....
        vstrw.u32 q5, [in, #-16]               // .................................*
        vsub.u32 q4, q0, q4                    // ....................*.............
        vmul.s32 q0, q4, q2                    // ......................*...........
        vldrw.u32 q3, [in]                     // e.................................
        vqrdmulh.s32 q7, q4, q6                // .......................*..........
        vldrw.u32 q5, [root_ptr, #48]          // .....e............................
        vmla.s32 q0, q7, modulus               // ........................*.........
        vstrw.u32 q0, [in, #-32]               // ................................*.
        
        // original source code
        // vldrw.u32 data0, [in]                           // .........e......................................
        // vldrw.u32 data1, [in, #16]                      // e...............................................
        // vldrw.u32 data2, [in, #32]                      // ..................*.............................
        // vldrw.u32 data3, [in, #48]                      // ....................*...........................
        // vldrw.u32 root1, [root_ptr, #32]                // ................*...............................
        // vldrw.u32 root1_twisted, [root_ptr, #48]        // ...........e....................................
        // vsub.u32 tmp, data0, data1                      // ..............*.................................
        // vadd.u32 data0, data0, data1                    // .................................*..............
        // vmul.s32 data1, tmp, root1                      // .................*..............................
        // vqrdmulh.s32 tmp, tmp, root1_twisted            // ...............*................................
        // vmla.s32 data1, tmp, modulus                    // ...................*............................
        // vldrw.u32 root2, [root_ptr, #64]                // ........................*.......................
        // vldrw.u32 root2_twisted, [root_ptr, #80]        // ......................*.........................
        // vsub.u32 tmp, data2, data3                      // .....................*..........................
        // vadd.u32 data2, data2, data3                    // ..........................*.....................
        // vmul.s32 data3, tmp, root2                      // .........................*......................
        // vqrdmulh.s32 tmp, tmp, root2_twisted            // .......................*........................
        // vmla.s32 data3, tmp, modulus                    // ...........................*....................
        // vldrw.u32 root0, [root_ptr] , #96               // ....................................*...........
        // vldrw.u32 root0_twisted, [root_ptr, #-80]       // ............................*...................
        // vsub.u32 tmp, data0, data2                      // .........................................*......
        // vadd.u32 data0, data0, data2                    // ...................................*............
        // vmul.s32 data2, tmp, root0                      // ..........................................*.....
        // vqrdmulh.s32 tmp, tmp, root0_twisted            // ............................................*...
        // vmla.s32 data2, tmp, modulus                    // ..............................................*.
        // vsub.u32 tmp, data1, data3                      // ...............................*................
        // vadd.u32 data1, data1, data3                    // .............................*..................
        // vmul.s32 data3, tmp, root0                      // .....................................*..........
        // vqrdmulh.s32 tmp, tmp, root0_twisted            // ................................*...............
        // vmla.s32 data3, tmp, modulus                    // .......................................*........
        // vstrw.u32 data0, [in] , #64                     // ......................................*.........
        // vstrw.u32 data1, [in, #-48]                     // ..............................*.................
        // vstrw.u32 data2, [in, #-32]                     // ...............................................*
        // vstrw.u32 data3, [in, #-16]                     // ........................................*.......
        
        le lr, layer78_loop
        vsub.u32 q2, q3, q1                    // *..............................
        vqrdmulh.s32 q0, q2, q5                // .*.............................
        vldrw.u32 q6, [root_ptr, #32]          // ..*............................
        vmul.s32 q2, q2, q6                    // ...*...........................
        vldrw.u32 q5, [in, #32]                // ....*..........................
        vmla.s32 q2, q0, modulus               // .....*.........................
        vldrw.u32 q4, [in, #48]                // ......*........................
        vsub.u32 q6, q5, q4                    // .......*.......................
        vldrw.u32 q0, [root_ptr, #80]          // ........*......................
        vqrdmulh.s32 q7, q6, q0                // .........*.....................
        vldrw.u32 q0, [root_ptr, #64]          // ..........*....................
        vmul.s32 q0, q6, q0                    // ...........*...................
        vadd.u32 q4, q5, q4                    // ............*..................
        vmla.s32 q0, q7, modulus               // .............*.................
        vldrw.u32 q6, [root_ptr, #16]          // ..............*................
        vadd.u32 q7, q2, q0                    // ...............*...............
        vstrw.u32 q7, [in, #16]                // ................*..............
        vsub.u32 q5, q2, q0                    // .................*.............
        vqrdmulh.s32 q0, q5, q6                // ..................*............
        vadd.u32 q2, q3, q1                    // ...................*...........
        vldrw.u32 q7, [root_ptr] , #96         // .....................*.........
        vmul.s32 q5, q5, q7                    // ......................*........
        vadd.u32 q3, q2, q4                    // ....................*..........
        vmla.s32 q5, q0, modulus               // ........................*......
        vsub.u32 q0, q2, q4                    // ..........................*....
        vmul.s32 q7, q0, q7                    // ...........................*...
        vstrw.u32 q3, [in] , #64               // .......................*.......
        vqrdmulh.s32 q0, q0, q6                // ............................*..
        vstrw.u32 q5, [in, #-16]               // .........................*.....
        vmla.s32 q7, q0, modulus               // .............................*.
        vstrw.u32 q7, [in, #-32]               // ..............................*
        
        // original source code
        // vsub.u32 q2, q3, q1                 // *..............................
        // vqrdmulh.s32 q4, q2, q5             // .*.............................
        // vldrw.u32 q7, [root_ptr, #32]       // ..*............................
        // vmul.s32 q2, q2, q7                 // ...*...........................
        // vldrw.u32 q5, [in, #32]             // ....*..........................
        // vmla.s32 q2, q4, modulus            // .....*.........................
        // vldrw.u32 q7, [in, #48]             // ......*........................
        // vsub.u32 q4, q5, q7                 // .......*.......................
        // vldrw.u32 q6, [root_ptr, #80]       // ........*......................
        // vqrdmulh.s32 q6, q4, q6             // .........*.....................
        // vldrw.u32 q0, [root_ptr, #64]       // ..........*....................
        // vmul.s32 q0, q4, q0                 // ...........*...................
        // vadd.u32 q4, q5, q7                 // ............*..................
        // vmla.s32 q0, q6, modulus            // .............*.................
        // vldrw.u32 q6, [root_ptr, #16]       // ..............*................
        // vadd.u32 q7, q2, q0                 // ...............*...............
        // vstrw.u32 q7, [in, #16]             // ................*..............
        // vsub.u32 q5, q2, q0                 // .................*.............
        // vqrdmulh.s32 q7, q5, q6             // ..................*............
        // vadd.u32 q0, q3, q1                 // ...................*...........
        // vadd.u32 q3, q0, q4                 // ......................*........
        // vldrw.u32 q2, [root_ptr] , #96      // ....................*..........
        // vmul.s32 q5, q5, q2                 // .....................*.........
        // vstrw.u32 q3, [in] , #64            // ..........................*....
        // vmla.s32 q5, q7, modulus            // .......................*.......
        // vstrw.u32 q5, [in, #-16]            // ............................*..
        // vsub.u32 q4, q0, q4                 // ........................*......
        // vmul.s32 q0, q4, q2                 // .........................*.....
        // vqrdmulh.s32 q7, q4, q6             // ...........................*...
        // vmla.s32 q0, q7, modulus            // .............................*.
        // vstrw.u32 q0, [in, #-32]            // ..............................*
        
layer78_loop_end:

        sub in, in, #(4*256)

        .unreq root0
        .unreq root0_twisted
        .unreq root1
        .unreq root1_twisted
        .unreq root2
        .unreq root2_twisted

        root0         .req r2
        root0_twisted .req r3
        root1         .req r4
        root1_twisted .req r5
        root2         .req r6
        root2_twisted .req r7

        /* Layers 5,6 */

        mov lr, #16
        ldrd r1, r2, [root_ptr, #16]          // *.......
        vld40.u32 {q2,q3,q4,q5}, [in]         // .*......
        nop                                   // .....*..
        vld41.u32 {q2,q3,q4,q5}, [in]         // ..*.....
        nop                                   // .......*
        vld42.u32 {q2,q3,q4,q5}, [in]         // ...*....
        nop                                   // ......*.
        vld43.u32 {q2,q3,q4,q5}, [in]         // ....*...
        
        // original source code
        // ldrd r1, r2, [root_ptr, #16]       // *.......
        // vld40.u32 {q2,q3,q4,q5}, [in]      // .*......
        // vld41.u32 {q2,q3,q4,q5}, [in]      // ...*....
        // vld42.u32 {q2,q3,q4,q5}, [in]      // .....*..
        // vld43.u32 {q2,q3,q4,q5}, [in]      // .......*
        // nop                                // ..*.....
        // nop                                // ......*.
        // nop                                // ....*...
        
        sub lr, lr, #1
layer56_loop:
        vsub.u32 q7, q4, q5                   // ............*..................
        vmul.s32 q1, q7, r1                   // ..............*................
        ldrd r7, r9, [root_ptr, #8]           // .*.............................
        vadd.u32 q6, q2, q3                   // ........*......................
        vqrdmulh.s32 q0, q7, r2               // ...............*...............
        vsub.u32 q2, q2, q3                   // .......*.......................
        vmul.s32 q7, q2, r7                   // .........*.....................
        vadd.u32 q3, q4, q5                   // .............*.................
        vmla.s32 q1, q0, modulus              // ................*..............
        ldrd r1, r2, [root_ptr, #40]          // ..e............................
        vadd.u32 q0, q6, q3                   // ..................*............
        vqrdmulh.s32 q4, q2, r9               // ..........*....................
        ldrd r9, r7, [root_ptr] , #24         // *..............................
        vmla.s32 q7, q4, modulus              // ...........*...................
        vstrw.u32 q0, [in] , #64              // ...........................*...
        vadd.u32 q0, q7, q1                   // .......................*.......
        vstrw.u32 q0, [in, #-48]              // ............................*..
        vsub.u32 q0, q6, q3                   // .................*.............
        vmul.s32 q6, q0, r9                   // ...................*...........
        vld40.u32 {q2,q3,q4,q5}, [in]         // ...e...........................
        vqrdmulh.s32 q0, q0, r7               // ....................*..........
        vld41.u32 {q2,q3,q4,q5}, [in]         // ....e..........................
        vsub.u32 q1, q7, q1                   // ......................*........
        vqrdmulh.s32 q7, q1, r7               // .........................*.....
        vld42.u32 {q2,q3,q4,q5}, [in]         // .....e.........................
        vmul.s32 q1, q1, r9                   // ........................*......
        vld43.u32 {q2,q3,q4,q5}, [in]         // ......e........................
        vmla.s32 q6, q0, modulus              // .....................*.........
        vstrw.u32 q6, [in, #-32]              // .............................*.
        vmla.s32 q1, q7, modulus              // ..........................*....
        vstrw.u32 q1, [in, #-16]              // ..............................*
        
        // original source code
        // ldrd root0, root0_twisted, [root_ptr] , #24       // ..................................*..................
        // ldrd root1, root1_twisted, [root_ptr, #-16]       // ........................*............................
        // ldrd root2, root2_twisted, [root_ptr, #-8]        // e....................................................
        // vld40.u32 {data0,data1,data2,data3}, [in]         // ..........e..........................................
        // vld41.u32 {data0,data1,data2,data3}, [in]         // ............e........................................
        // vld42.u32 {data0,data1,data2,data3}, [in]         // ...............e.....................................
        // vld43.u32 {data0,data1,data2,data3}, [in]         // .................e...................................
        // vsub.u32 tmp, data0, data1                        // ...........................*.........................
        // vadd.u32 data0, data0, data1                      // .........................*...........................
        // vmul.s32 data1, tmp, root1                        // ............................*........................
        // vqrdmulh.s32 tmp, tmp, root1_twisted              // .................................*...................
        // vmla.s32 data1, tmp, modulus                      // ...................................*.................
        // vsub.u32 tmp, data2, data3                        // ......................*..............................
        // vadd.u32 data2, data2, data3                      // .............................*.......................
        // vmul.s32 data3, tmp, root2                        // .......................*.............................
        // vqrdmulh.s32 tmp, tmp, root2_twisted              // ..........................*..........................
        // vmla.s32 data3, tmp, modulus                      // ..............................*......................
        // vsub.u32 tmp, data0, data2                        // .......................................*.............
        // vadd.u32 data0, data0, data2                      // ................................*....................
        // vmul.s32 data2, tmp, root0                        // ........................................*............
        // vqrdmulh.s32 tmp, tmp, root0_twisted              // ..........................................*..........
        // vmla.s32 data2, tmp, modulus                      // .................................................*...
        // vsub.u32 tmp, data1, data3                        // ............................................*........
        // vadd.u32 data1, data1, data3                      // .....................................*...............
        // vmul.s32 data3, tmp, root0                        // ...............................................*.....
        // vqrdmulh.s32 tmp, tmp, root0_twisted              // .............................................*.......
        // vmla.s32 data3, tmp, modulus                      // ...................................................*.
        // vstrw.u32 data0, [in] , #64                       // ....................................*................
        // vstrw.u32 data1, [in, #-48]                       // ......................................*..............
        // vstrw.u32 data2, [in, #-32]                       // ..................................................*..
        // vstrw.u32 data3, [in, #-16]                       // ....................................................*
        
        le lr, layer56_loop
        vsub.u32 q1, q4, q5                    // *...........................
        vqrdmulh.s32 q7, q1, r2                // ....*.......................
        ldrd r9, r7, [root_ptr, #8]            // ..*.........................
        vmul.s32 q1, q1, r1                    // .*..........................
        vadd.u32 q0, q2, q3                    // ...*........................
        vmla.s32 q1, q7, modulus               // ........*...................
        vsub.u32 q2, q2, q3                    // .....*......................
        vmul.s32 q7, q2, r9                    // ......*.....................
        vadd.u32 q3, q4, q5                    // .......*....................
        vqrdmulh.s32 q4, q2, r7                // ..........*.................
        nop                                    // ...........................*
        vmla.s32 q7, q4, modulus               // ............*...............
        ldrd r8, r10, [root_ptr] , #24         // ...........*................
        vadd.u32 q6, q7, q1                    // ..............*.............
        vstrw.u32 q6, [in, #16]                // ...............*............
        vsub.u32 q6, q7, q1                    // ...................*........
        vqrdmulh.s32 q7, q6, r10               // ....................*.......
        vadd.u32 q1, q0, q3                    // .........*..................
        vmul.s32 q6, q6, r8                    // .....................*......
        vstrw.u32 q1, [in] , #64               // .............*..............
        vmla.s32 q6, q7, modulus               // ........................*...
        vsub.u32 q0, q0, q3                    // ................*...........
        vmul.s32 q1, q0, r8                    // .................*..........
        vstrw.u32 q6, [in, #-16]               // .........................*..
        vqrdmulh.s32 q0, q0, r10               // ..................*.........
        nop                                    // ..........................*.
        vmla.s32 q1, q0, modulus               // ......................*.....
        vstrw.u32 q1, [in, #-32]               // .......................*....
        
        // original source code
        // vsub.u32 q7, q4, q5                // *...........................
        // vmul.s32 q1, q7, r1                // ...*........................
        // ldrd r7, r9, [root_ptr, #8]        // ..*.........................
        // vadd.u32 q6, q2, q3                // ....*.......................
        // vqrdmulh.s32 q0, q7, r2            // .*..........................
        // vsub.u32 q2, q2, q3                // ......*.....................
        // vmul.s32 q7, q2, r7                // .......*....................
        // vadd.u32 q3, q4, q5                // ........*...................
        // vmla.s32 q1, q0, modulus           // .....*......................
        // vadd.u32 q0, q6, q3                // .................*..........
        // vqrdmulh.s32 q4, q2, r9            // .........*..................
        // ldrd r9, r7, [root_ptr] , #24      // ............*...............
        // vmla.s32 q7, q4, modulus           // ...........*................
        // vstrw.u32 q0, [in] , #64           // ...................*........
        // vadd.u32 q0, q7, q1                // .............*..............
        // vstrw.u32 q0, [in, #-48]           // ..............*.............
        // vsub.u32 q0, q6, q3                // .....................*......
        // vmul.s32 q6, q0, r9                // ......................*.....
        // vqrdmulh.s32 q0, q0, r7            // ........................*...
        // vsub.u32 q1, q7, q1                // ...............*............
        // vqrdmulh.s32 q7, q1, r7            // ................*...........
        // vmul.s32 q1, q1, r9                // ..................*.........
        // vmla.s32 q6, q0, modulus           // ..........................*.
        // vstrw.u32 q6, [in, #-32]           // ...........................*
        // vmla.s32 q1, q7, modulus           // ....................*.......
        // vstrw.u32 q1, [in, #-16]           // .......................*....
        // nop                                // .........................*..
        // nop                                // ..........*.................
        
layer56_loop_end:

        sub in, in, #(4*256)

        // TEMPORARY: Barrett reduction
        barrett_const .req r1
        .equ const_barrett, 63
        movw barrett_const, #:lower16:const_barrett
        movt barrett_const, #:upper16:const_barrett
        mov lr, #64
1:
        vldrw.u32 data0, [in]
        vqrdmulh.s32 tmp, data0, barrett_const
        vmla.s32 data0, tmp, modulus
        vstrw.u32 data0, [in], #16
        le lr, 1b
2:
        sub in, in, #(4*256)
        .unreq barrett_const

        /* Layers 3,4 */

        // 4 butterfly blocks per root config, 4 root configs
        // loop over root configs

        count .req r1
        mov count, #4

out_start:
        ldrd root0, root0_twisted, [root_ptr], #+8
        ldrd root1, root1_twisted, [root_ptr], #+8
        ldrd root2, root2_twisted, [root_ptr], #+8

        mov lr, #4
        vldrw.u32 q7, [in, #128]          // *..
        nop                               // ..*
        vldrw.u32 q0, [in, #192]          // .*.
        
        // original source code
        // vldrw.u32 q7, [in, #128]       // *..
        // vldrw.u32 q0, [in, #192]       // ..*
        // nop                            // .*.
        
        sub lr, lr, #1
layer34_loop:
        vsub.u32 q4, q7, q0                        // .........*..................
        vqrdmulh.s32 q1, q4, root2_twisted         // ............*...............
        vadd.u32 q7, q7, q0                        // ..........*.................
        vldrw.u32 q5, [in, #64]                    // .*..........................
        vmul.s32 q0, q4, root2                     // ...........*................
        vldrw.u32 q3, [in]                         // *...........................
        vmla.s32 q0, q1, modulus                   // .............*..............
        vsub.u32 q1, q3, q5                        // ....*.......................
        vmul.s32 q2, q1, root1                     // ......*.....................
        vadd.u32 q4, q3, q5                        // .....*......................
        vqrdmulh.s32 q3, q1, root1_twisted         // .......*....................
        vadd.u32 q6, q4, q7                        // ...............*............
        vmla.s32 q2, q3, modulus                   // ........*...................
        vstrw.u32 q6, [in] , #16                   // ........................*...
        vsub.u32 q5, q2, q0                        // ...................*........
        vqrdmulh.s32 q3, q5, root0_twisted         // ......................*.....
        vadd.u32 q1, q2, q0                        // ....................*.......
        vmul.s32 q2, q5, root0                     // .....................*......
        vstrw.u32 q1, [in, #48]                    // .........................*..
        vmla.s32 q2, q3, modulus                   // .......................*....
        vstrw.u32 q2, [in, #176]                   // ...........................*
        vsub.u32 q2, q4, q7                        // ..............*.............
        vqrdmulh.s32 q5, q2, root0_twisted         // .................*..........
        vldrw.u32 q7, [in, #128]                   // ..e.........................
        vmul.s32 q2, q2, root0                     // ................*...........
        vldrw.u32 q0, [in, #192]                   // ...e........................
        vmla.s32 q2, q5, modulus                   // ..................*.........
        vstrw.u32 q2, [in, #112]                   // ..........................*.
        
        // original source code
        // vldrw.u32 data0, [in]                     // ..........*......................
        // vldrw.u32 data1, [in, #64]                // ........*........................
        // vldrw.u32 data2, [in, #128]               // e................................
        // vldrw.u32 data3, [in, #192]               // ..e..............................
        // vsub.u32 tmp, data0, data1                // ............*....................
        // vadd.u32 data0, data0, data1              // ..............*..................
        // vmul.s32 data1, tmp, root1                // .............*...................
        // vqrdmulh.s32 tmp, tmp, root1_twisted      // ...............*.................
        // vmla.s32 data1, tmp, modulus              // .................*...............
        // vsub.u32 tmp, data2, data3                // .....*...........................
        // vadd.u32 data2, data2, data3              // .......*.........................
        // vmul.s32 data3, tmp, root2                // .........*.......................
        // vqrdmulh.s32 tmp, tmp, root2_twisted      // ......*..........................
        // vmla.s32 data3, tmp, modulus              // ...........*.....................
        // vsub.u32 tmp, data0, data2                // ..........................*......
        // vadd.u32 data0, data0, data2              // ................*................
        // vmul.s32 data2, tmp, root0                // .............................*...
        // vqrdmulh.s32 tmp, tmp, root0_twisted      // ...........................*.....
        // vmla.s32 data2, tmp, modulus              // ...............................*.
        // vsub.u32 tmp, data1, data3                // ...................*.............
        // vadd.u32 data1, data1, data3              // .....................*...........
        // vmul.s32 data3, tmp, root0                // ......................*..........
        // vqrdmulh.s32 tmp, tmp, root0_twisted      // ....................*............
        // vmla.s32 data3, tmp, modulus              // ........................*........
        // vstrw.u32 data0, [in] , #16               // ..................*..............
        // vstrw.u32 data1, [in, #48]                // .......................*.........
        // vstrw.u32 data2, [in, #112]               // ................................*
        // vstrw.u32 data3, [in, #176]               // .........................*.......
        
        le lr, layer34_loop
        vsub.u32 q3, q7, q0                        // *.........................
        vqrdmulh.s32 q5, q3, root2_twisted         // .*........................
        vadd.u32 q7, q7, q0                        // ..*.......................
        vmul.s32 q0, q3, root2                     // ....*.....................
        vldrw.u32 q1, [in, #64]                    // ...*......................
        vmla.s32 q0, q5, modulus                   // ......*...................
        vldrw.u32 q3, [in]                         // .....*....................
        vsub.u32 q5, q3, q1                        // .......*..................
        vmul.s32 q2, q5, root1                     // ........*.................
        vadd.u32 q3, q3, q1                        // .........*................
        vqrdmulh.s32 q5, q5, root1_twisted         // ..........*...............
        vadd.u32 q1, q3, q7                        // ...........*..............
        vmla.s32 q2, q5, modulus                   // ............*.............
        vsub.u32 q5, q3, q7                        // .....................*....
        vqrdmulh.s32 q3, q5, root0_twisted         // ......................*...
        vstrw.u32 q1, [in] , #16                   // .............*............
        vmul.s32 q5, q5, root0                     // .......................*..
        vsub.u32 q1, q2, q0                        // ..............*...........
        vmla.s32 q5, q3, modulus                   // ........................*.
        vstrw.u32 q5, [in, #112]                   // .........................*
        vmul.s32 q5, q1, root0                     // .................*........
        vadd.u32 q3, q2, q0                        // ................*.........
        vqrdmulh.s32 q1, q1, root0_twisted         // ...............*..........
        vstrw.u32 q3, [in, #48]                    // ..................*.......
        vmla.s32 q5, q1, modulus                   // ...................*......
        vstrw.u32 q5, [in, #176]                   // ....................*.....
        
        // original source code
        // vsub.u32 q4, q7, q0                     // *.........................
        // vqrdmulh.s32 q1, q4, root2_twisted      // .*........................
        // vadd.u32 q7, q7, q0                     // ..*.......................
        // vldrw.u32 q5, [in, #64]                 // ....*.....................
        // vmul.s32 q0, q4, root2                  // ...*......................
        // vldrw.u32 q3, [in]                      // ......*...................
        // vmla.s32 q0, q1, modulus                // .....*....................
        // vsub.u32 q1, q3, q5                     // .......*..................
        // vmul.s32 q2, q1, root1                  // ........*.................
        // vadd.u32 q4, q3, q5                     // .........*................
        // vqrdmulh.s32 q3, q1, root1_twisted      // ..........*...............
        // vadd.u32 q6, q4, q7                     // ...........*..............
        // vmla.s32 q2, q3, modulus                // ............*.............
        // vstrw.u32 q6, [in] , #16                // ...............*..........
        // vsub.u32 q5, q2, q0                     // .................*........
        // vqrdmulh.s32 q3, q5, root0_twisted      // ......................*...
        // vadd.u32 q1, q2, q0                     // .....................*....
        // vmul.s32 q2, q5, root0                  // ....................*.....
        // vstrw.u32 q1, [in, #48]                 // .......................*..
        // vmla.s32 q2, q3, modulus                // ........................*.
        // vstrw.u32 q2, [in, #176]                // .........................*
        // vsub.u32 q2, q4, q7                     // .............*............
        // vqrdmulh.s32 q5, q2, root0_twisted      // ..............*...........
        // vmul.s32 q2, q2, root0                  // ................*.........
        // vmla.s32 q2, q5, modulus                // ..................*.......
        // vstrw.u32 q2, [in, #112]                // ...................*......
        
layer34_loop_end:
        add in, in, #(4*64 - 4*16)

        subs count, count, #1
        bne out_start

        sub in, in, #(4*256)

        // TEMPORARY: Barrett reduction
        barrett_const .req r1
        .equ const_barrett, 63
        movw barrett_const, #:lower16:const_barrett
        movt barrett_const, #:upper16:const_barrett
        mov lr, #64
1:
        vldrw.u32 data0, [in]
        vqrdmulh.s32 tmp, data0, barrett_const
        vmla.s32 data0, tmp, modulus
        vstrw.u32 data0, [in], #16
        le lr, 1b
2:
        sub in, in, #(4*256)
        .unreq barrett_const

        in_low       .req r0
        in_high      .req r1
        add in_high, in_low, #(4*128)

        /* Layers 1,2 */

        ldrd root0, root0_twisted, [root_ptr], #+8
        ldrd root1, root1_twisted, [root_ptr], #+8
        ldrd root2, root2_twisted, [root_ptr], #+8

        mov lr, #16
        vldrw.u32 q7, [in_high]                // *..
        nop                                    // ..*
        vldrw.u32 q0, [in_high, #256]          // .*.
        
        // original source code
        // vldrw.u32 q7, [in_high]             // *..
        // vldrw.u32 q0, [in_high, #256]       // ..*
        // nop                                 // .*.
        
        sub lr, lr, #1
layer12_loop:
        vsub.u32 q4, q7, q0                        // .........*..................
        vqrdmulh.s32 q1, q4, root2_twisted         // ............*...............
        vadd.u32 q7, q7, q0                        // ..........*.................
        vldrw.u32 q5, [in_low, #256]               // .*..........................
        vmul.s32 q0, q4, root2                     // ...........*................
        vldrw.u32 q3, [in_low]                     // *...........................
        vmla.s32 q0, q1, modulus                   // .............*..............
        vsub.u32 q1, q3, q5                        // ....*.......................
        vmul.s32 q2, q1, root1                     // ......*.....................
        vadd.u32 q4, q3, q5                        // .....*......................
        vqrdmulh.s32 q3, q1, root1_twisted         // .......*....................
        vadd.u32 q6, q4, q7                        // ...............*............
        vmla.s32 q2, q3, modulus                   // ........*...................
        vstrw.u32 q6, [in_low] , #16               // ........................*...
        vsub.u32 q5, q2, q0                        // ...................*........
        vqrdmulh.s32 q3, q5, root0_twisted         // ......................*.....
        vadd.u32 q1, q2, q0                        // ....................*.......
        vmul.s32 q2, q5, root0                     // .....................*......
        vstrw.u32 q1, [in_low, #240]               // .........................*..
        vmla.s32 q2, q3, modulus                   // .......................*....
        vstrw.u32 q2, [in_high, #256]              // ...........................*
        vsub.u32 q2, q4, q7                        // ..............*.............
        vqrdmulh.s32 q5, q2, root0_twisted         // .................*..........
        vldrw.u32 q7, [in_high, #16]               // ..e.........................
        vmul.s32 q2, q2, root0                     // ................*...........
        vldrw.u32 q0, [in_high, #272]              // ...e........................
        vmla.s32 q2, q5, modulus                   // ..................*.........
        vstrw.u32 q2, [in_high] , #16              // ..........................*.
        
        // original source code
        // vldrw.u32 data0, [in_low]                 // ..........*......................
        // vldrw.u32 data1, [in_low, #256]           // ........*........................
        // vldrw.u32 data2, [in_high]                // e................................
        // vldrw.u32 data3, [in_high, #256]          // ..e..............................
        // vsub.u32 tmp, data0, data1                // ............*....................
        // vadd.u32 data0, data0, data1              // ..............*..................
        // vmul.s32 data1, tmp, root1                // .............*...................
        // vqrdmulh.s32 tmp, tmp, root1_twisted      // ...............*.................
        // vmla.s32 data1, tmp, modulus              // .................*...............
        // vsub.u32 tmp, data2, data3                // .....*...........................
        // vadd.u32 data2, data2, data3              // .......*.........................
        // vmul.s32 data3, tmp, root2                // .........*.......................
        // vqrdmulh.s32 tmp, tmp, root2_twisted      // ......*..........................
        // vmla.s32 data3, tmp, modulus              // ...........*.....................
        // vsub.u32 tmp, data0, data2                // ..........................*......
        // vadd.u32 data0, data0, data2              // ................*................
        // vmul.s32 data2, tmp, root0                // .............................*...
        // vqrdmulh.s32 tmp, tmp, root0_twisted      // ...........................*.....
        // vmla.s32 data2, tmp, modulus              // ...............................*.
        // vsub.u32 tmp, data1, data3                // ...................*.............
        // vadd.u32 data1, data1, data3              // .....................*...........
        // vmul.s32 data3, tmp, root0                // ......................*..........
        // vqrdmulh.s32 tmp, tmp, root0_twisted      // ....................*............
        // vmla.s32 data3, tmp, modulus              // ........................*........
        // vstrw.u32 data0, [in_low] , #16           // ..................*..............
        // vstrw.u32 data1, [in_low, #240]           // .......................*.........
        // vstrw.u32 data2, [in_high] , #16          // ................................*
        // vstrw.u32 data3, [in_high, #240]          // .........................*.......
        
        le lr, layer12_loop
        vsub.u32 q3, q7, q0                        // *.........................
        vqrdmulh.s32 q5, q3, root2_twisted         // .*........................
        vadd.u32 q7, q7, q0                        // ..*.......................
        vmul.s32 q0, q3, root2                     // ....*.....................
        vldrw.u32 q1, [in_low, #256]               // ...*......................
        vmla.s32 q0, q5, modulus                   // ......*...................
        vldrw.u32 q3, [in_low]                     // .....*....................
        vsub.u32 q5, q3, q1                        // .......*..................
        vmul.s32 q2, q5, root1                     // ........*.................
        vadd.u32 q3, q3, q1                        // .........*................
        vqrdmulh.s32 q5, q5, root1_twisted         // ..........*...............
        vadd.u32 q1, q3, q7                        // ...........*..............
        vmla.s32 q2, q5, modulus                   // ............*.............
        vsub.u32 q5, q3, q7                        // .....................*....
        vqrdmulh.s32 q3, q5, root0_twisted         // ......................*...
        vstrw.u32 q1, [in_low] , #16               // .............*............
        vmul.s32 q5, q5, root0                     // .......................*..
        vsub.u32 q1, q2, q0                        // ..............*...........
        vmla.s32 q5, q3, modulus                   // ........................*.
        vstrw.u32 q5, [in_high] , #16              // .........................*
        vmul.s32 q5, q1, root0                     // .................*........
        vadd.u32 q3, q2, q0                        // ................*.........
        vqrdmulh.s32 q1, q1, root0_twisted         // ...............*..........
        vstrw.u32 q3, [in_low, #240]               // ..................*.......
        vmla.s32 q5, q1, modulus                   // ...................*......
        vstrw.u32 q5, [in_high, #240]              // ....................*.....
        
        // original source code
        // vsub.u32 q4, q7, q0                     // *.........................
        // vqrdmulh.s32 q1, q4, root2_twisted      // .*........................
        // vadd.u32 q7, q7, q0                     // ..*.......................
        // vldrw.u32 q5, [in_low, #256]            // ....*.....................
        // vmul.s32 q0, q4, root2                  // ...*......................
        // vldrw.u32 q3, [in_low]                  // ......*...................
        // vmla.s32 q0, q1, modulus                // .....*....................
        // vsub.u32 q1, q3, q5                     // .......*..................
        // vmul.s32 q2, q1, root1                  // ........*.................
        // vadd.u32 q4, q3, q5                     // .........*................
        // vqrdmulh.s32 q3, q1, root1_twisted      // ..........*...............
        // vadd.u32 q6, q4, q7                     // ...........*..............
        // vmla.s32 q2, q3, modulus                // ............*.............
        // vstrw.u32 q6, [in_low] , #16            // ...............*..........
        // vsub.u32 q5, q2, q0                     // .................*........
        // vqrdmulh.s32 q3, q5, root0_twisted      // ......................*...
        // vadd.u32 q1, q2, q0                     // .....................*....
        // vmul.s32 q2, q5, root0                  // ....................*.....
        // vstrw.u32 q1, [in_low, #240]            // .......................*..
        // vmla.s32 q2, q3, modulus                // ........................*.
        // vstrw.u32 q2, [in_high, #256]           // .........................*
        // vsub.u32 q2, q4, q7                     // .............*............
        // vqrdmulh.s32 q5, q2, root0_twisted      // ..............*...........
        // vmul.s32 q2, q2, root0                  // ................*.........
        // vmla.s32 q2, q5, modulus                // ..................*.......
        // vstrw.u32 q2, [in_high] , #16           // ...................*......
        
layer12_loop_end:

        // Restore MVE vector registers
        vpop {d8-d15}
        // Restore GPRs
        pop {r4-r11,lr}
        bx lr