/*++

Copyright (c) Microsoft Corporation. All rights reserved.

Licensed under the MIT License.

Module Name:

    DepthwiseQConvSymS8KernelNeon.asm

Abstract:

    This module implements the kernels for the depthwise convolution
    operation with symmetrically quantized integer values

--*/

#include "kxarm64.h"

//
// Stack frame layout for the depthwise conv kernel.
// d8-d15, x19-x30 need to be preserved if used
//

#define  ConvSymDepthwiseKernelFrame_SavedRegisters        (4 * 8)
#define  ConvSymDepthwiseKernelFrame_PostProcessParams     0 + ConvSymDepthwiseKernelFrame_SavedRegisters
#define  ConvSymDepthwiseKernelFrame_KernelFlags           8 + ConvSymDepthwiseKernelFrame_SavedRegisters

#define  ConvSymDepthwisePostProcessParams_Bias            0
#define  ConvSymDepthwisePostProcessParams_Scale           8
#define  ConvSymDepthwisePostProcessParams_Min             16
#define  ConvSymDepthwisePostProcessParams_Max             20
#define  ConvSymDepthwisePostProcessParams_ZeroPoint       24

#define  MLAS_CONV_SYM_FLAG_INPUT_DIRECT                   1
#define  MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE              2

        TEXTAREA

/*++

Routine Description:

    This routine is the inner kernel to compute a depthwise convolution for the
    elements of an output row for a set of filter rows.

Arguments:

    Input (x0) - Supplies the address of the indirection buffer.
 
    Filter (x1) - Supplies the address of the filter buffer.

    Output (x2) - Supplies the address of the output buffer.

    KernelSize (x3) - Supplies the size of the kernel.
 
    Channels (x4) - Supplies the number of input and output channels.
 
    ChannelOffset (x5) - Supplies the byte offset from the indirection buffer base
        address for this iteration.
 
    ChannelCount (x6) - Supplies the number of channels this iteration produces.
 
        This implementation requires the count to be 16 or 8
 
    OutputCount (x7)- Supplies the number of output elements this iteration produces.
 
        This implementation requires the count to be in the range 1 to 2.
 
    PostProcessParams - Supplies the address of the post process parameter block.
 
    KernelFlags - Supplies additional flags controlling the operation.

Return Value:

    None.

--*/

        NESTED_ENTRY MlasConvSymDepthwiseS8KernelNeon

        PROLOG_SAVE_REG_PAIR      d12,d13,#-ConvSymDepthwiseKernelFrame_SavedRegisters!
        PROLOG_NOP       ldr      x8,[sp,#ConvSymDepthwiseKernelFrame_PostProcessParams]
        PROLOG_SAVE_REG_PAIR      d14,d15,#16
        cmp     x7,2
        add     x9,x0,x3,lsl#3              // x9 -> &A1
        add     x14,x0,x3,lsl#4             // x14 -> &A2
        add     x15,x9,x3,lsl#4             // x15 -> &A3
        ldr     x16,[x8,#ConvSymDepthwisePostProcessParams_Bias]
        csel    x9,x0,x9,lo                 // x9 -> &A0 if OutputCount < 2
        csel    x14,x0,x14,ls               // x14 -> &A0 if OutputCount <= 2
        ldr     x11,[x9],#8                 // x11 -> A1 iter 0
        cmp     x7,4
        ldp     q24,q25,[x16],#32           // init accumulators with bias
        csel    x15,x0,x15,lo               // x15 -> &A0 if OutputCount < 4
        cmp     x6,16
        ldr     x10,[x0],#8                 // x10 -> A0 iter 0
        b.lo    Process8Channels

//
// Process an input block of length Channels for each element of the kernel.
//
// Filter:  v0,
//          v1       // unroll
// Input:
// x0  -> x10 -> v4
//     -> x12 -> v2  // unroll
// x9  -> x11 -> v6
//     -> x13 -> v3  // unroll
// x14 -> x10 -> v4
//     -> x12 -> v2  // unroll
// x15 -> x11 -> v6
//     -> x13 -> v3  // unroll
//

Process16Channels
        cmp     x3,1
        ldp     q26,q27,[x16]
        b.eq    ProcC16P1

        ldr     x12,[x0],#8                 // x12 -> A0 iter 1
        ldr     x13,[x9],#8                 // x13 -> A1 iter 1
        mov     v28.16b,v24.16b
        mov     v29.16b,v25.16b
        ld1     {v0.16b},[x1],x4            // filter iter 0
        ld1     {v1.16b},[x1],x4            // filter iter 1
        mov     v16.16b,v24.16b
        mov     v17.16b,v25.16b
        ldr     q4,[x10,x5]                 // A0 iter 0
        mov     v20.16b,v24.16b
        ldr     x10,[x14],#8                // x10 -> A2 iter 0
        mov     v21.16b,v25.16b
        ldr     q6,[x11,x5]                 // A1 iter 0
        mov     v30.16b,v26.16b
        ldr     x11,[x15],#8                // x11 -> A3 iter 0
        mov     v31.16b,v27.16b
        ldr     q2,[x12,x5]                 // A0 iter 1
        subs    x3,x3,2                     // decrement input blocks remaining
        mov     v18.16b,v26.16b
        ldr     x12,[x14],#8                // x12 -> A2 iter 1
        mov     v19.16b,v27.16b
        ldr     q3,[x13,x5]                 // A1 iter 1
        mov     v22.16b,v26.16b
        ldr     x13,[x15],#8                // x13 -> A3 iter 1
        mov     v23.16b,v27.16b

BlockLoopC16

        //
        // Process 2 pixels, and load next two pixels
        //
        smull   v12.8h,v0.8b,v4.8b
        smull2  v13.8h,v0.16b,v4.16b
        ldr     q4,[x10,x5]                 // A2 iter 0
        b.eq    EpilogueC16P2
        smull   v14.8h,v0.8b,v6.8b
        ldr     x10,[x0],#8                 // x10 -> A0 iter 2
        smull2  v15.8h,v0.16b,v6.16b
        cmp     x3,1
        ldr     q6,[x11,x5]                 // A3 iter 0
        smlal   v12.8h,v1.8b,v2.8b
        ldr     x11,[x9],#8                 // x11 -> A1 iter 2
        smlal2  v13.8h,v1.16b,v2.16b
        b.eq    EpilogueC16P3               // 3 pixel remains      
        ldr     q2,[x12,x5]                 // A2 iter 1
        smlal   v14.8h,v1.8b,v3.8b
        ldr     x12,[x0],#8                 // x12 -> A0 iter 3
        smlal2  v15.8h,v1.16b,v3.16b
        ldr     q3,[x13,x5]                 // A3 iter 1
        saddw   v24.4s,v24.4s,v12.4h
        saddw2  v25.4s,v25.4s,v12.8h
        ldr     x13,[x9],#8                 // x13 -> A1 iter 3
        saddw   v26.4s,v26.4s,v13.4h
        saddw2  v27.4s,v27.4s,v13.8h
        saddw   v28.4s,v28.4s,v14.4h
        saddw2  v29.4s,v29.4s,v14.8h
        saddw   v30.4s,v30.4s,v15.4h
        saddw2  v31.4s,v31.4s,v15.8h
        subs    x3,x3,2                     // decrement input blocks remaining
        smull   v12.8h,v0.8b,v4.8b
        smull2  v13.8h,v0.16b,v4.16b
        ldr     q4,[x10,x5]                 // A0 iter 2
        smull   v14.8h,v0.8b,v6.8b
        ldr     x10,[x14],#8                // x10 -> A2 iter 2
        smull2  v15.8h,v0.16b,v6.16b
        ldr     q6,[x11,x5]                 // A1 iter 2
        ld1     {v0.16b},[x1],x4            // filter iter 2
        smlal   v12.8h,v1.8b,v2.8b
        ldr     x11,[x15],#8                // x11 -> A3 iter 2
        smlal2  v13.8h,v1.16b,v2.16b
        ldr     q2,[x12,x5]                 // A0 iter 3
        smlal   v14.8h,v1.8b,v3.8b
        ldr     x12,[x14],#8                // x12 -> A2 iter 3
        smlal2  v15.8h,v1.16b,v3.16b
        ldr     q3,[x13,x5]                 // A1 iter 3
        saddw   v16.4s,v16.4s,v12.4h
        saddw2  v17.4s,v17.4s,v12.8h
        ld1     {v1.16b},[x1],x4            // filter iter 3
        saddw   v18.4s,v18.4s,v13.4h
        saddw2  v19.4s,v19.4s,v13.8h
        ldr     x13,[x15],#8                // x13 -> A3 iter 3
        saddw   v20.4s,v20.4s,v14.4h
        saddw2  v21.4s,v21.4s,v14.8h
        saddw   v22.4s,v22.4s,v15.4h
        saddw2  v23.4s,v23.4s,v15.8h
        b       BlockLoopC16

EpilogueC16P2
        //
        // Loop epilogue (process last 2 pixels) mixed
        // with loading of dequantization params
        //
        smull   v14.8h,v0.8b,v6.8b
        smull2  v15.8h,v0.16b,v6.16b
        ldr     q6,[x11,x5]                 // A3 iter 0
        smlal   v12.8h,v1.8b,v2.8b
        smlal2  v13.8h,v1.16b,v2.16b
        ldr     q2,[x12,x5]                 // A2 iter 1
        smlal   v14.8h,v1.8b,v3.8b
        smlal2  v15.8h,v1.16b,v3.16b
        ldr     q3,[x13,x5]                 // A3 iter 1
        saddw   v24.4s,v24.4s,v12.4h
        saddw2  v25.4s,v25.4s,v12.8h
        saddw   v26.4s,v26.4s,v13.4h
        saddw2  v27.4s,v27.4s,v13.8h
        saddw   v28.4s,v28.4s,v14.4h
        saddw2  v29.4s,v29.4s,v14.8h
        saddw   v30.4s,v30.4s,v15.4h
        saddw2  v31.4s,v31.4s,v15.8h
        ldr     w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags]
        ldr     x12,[x8,#ConvSymDepthwisePostProcessParams_Scale]
        smull   v12.8h,v0.8b,v4.8b
        smull2  v13.8h,v0.16b,v4.16b
        ldr     w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
        smull   v14.8h,v0.8b,v6.8b
        smull2  v15.8h,v0.16b,v6.16b
        smlal   v12.8h,v1.8b,v2.8b
        smlal2  v13.8h,v1.16b,v2.16b
        smlal   v14.8h,v1.8b,v3.8b
        smlal2  v15.8h,v1.16b,v3.16b
        tst     w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
        ld1r    {v4.4s},[x12]               // load scale val
        b.eq    SkipScaleVecLoad2
        ldp     q4,q5,[x12],#32             // load scale vector if per channel
        ldp     q6,q3,[x12]
SkipScaleVecLoad2
        saddw   v16.4s,v16.4s,v12.4h
        saddw2  v17.4s,v17.4s,v12.8h
        saddw   v18.4s,v18.4s,v13.4h
        saddw2  v19.4s,v19.4s,v13.8h
        saddw   v20.4s,v20.4s,v14.4h
        saddw2  v21.4s,v21.4s,v14.8h
        saddw   v22.4s,v22.4s,v15.4h
        saddw2  v23.4s,v23.4s,v15.8h
        b       Dequantization

ProcC16P1
        //
        // Channel 16 kernel size 1
        // TODO!! is this reachable at all?
        //
        ldr     x12,[x14],#8                // x12 -> A2
        ldr     x13,[x15],#8                // x13 -> A3
        mov     v28.16b,v24.16b
        mov     v29.16b,v25.16b
        ld1     {v0.16b},[x1]
        mov     v16.16b,v24.16b
        mov     v17.16b,v25.16b
        ldr     q4,[x10,x5]
        mov     v20.16b,v24.16b
        mov     v21.16b,v25.16b
        ldr     q6,[x11,x5]
        mov     v30.16b,v26.16b
        mov     v31.16b,v27.16b
        ldr     q2,[x12,x5]
        subs    x3,x3,2                     // decrement input blocks remaining
        mov     v18.16b,v26.16b
        mov     v19.16b,v27.16b
        ldr     q3,[x13,x5]
        mov     v22.16b,v26.16b
        mov     v23.16b,v27.16b
        b       EpilogueC16P1

EpilogueC16P3
        //
        // Loop epilogue (process last 2 pixels) mixed
        // with loading of dequantization params
        //
        ldr     q2,[x12,x5]                 // A2 iter 1
        smlal   v14.8h,v1.8b,v3.8b
        ldr     x12,[x14],#8                // x12 -> A2 iter 2
        smlal2  v15.8h,v1.16b,v3.16b
        ldr     q3,[x13,x5]                 // A3 iter 1
        saddw   v24.4s,v24.4s,v12.4h
        saddw2  v25.4s,v25.4s,v12.8h
        ldr     x13,[x15],#8                // x13 -> A3 iter 2
        saddw   v26.4s,v26.4s,v13.4h
        saddw2  v27.4s,v27.4s,v13.8h
        saddw   v28.4s,v28.4s,v14.4h
        saddw2  v29.4s,v29.4s,v14.8h
        saddw   v30.4s,v30.4s,v15.4h
        saddw2  v31.4s,v31.4s,v15.8h
        smull   v12.8h,v0.8b,v4.8b
        smull2  v13.8h,v0.16b,v4.16b
        ldr     q4,[x10,x5]                 // A0 iter 2
        smull   v14.8h,v0.8b,v6.8b
        smull2  v15.8h,v0.16b,v6.16b
        ld1     {v0.16b},[x1]               // filter iter 2
        ldr     q6,[x11,x5]                 // A1 iter 2
        smlal   v12.8h,v1.8b,v2.8b
        smlal2  v13.8h,v1.16b,v2.16b
        ldr     q2,[x12,x5]                 // A2 iter 2
        smlal   v14.8h,v1.8b,v3.8b
        smlal2  v15.8h,v1.16b,v3.16b
        ldr     q3,[x13,x5]                 // A3 iter 2
        saddw   v16.4s,v16.4s,v12.4h
        saddw2  v17.4s,v17.4s,v12.8h
        saddw   v18.4s,v18.4s,v13.4h
        saddw2  v19.4s,v19.4s,v13.8h
        saddw   v20.4s,v20.4s,v14.4h
        saddw2  v21.4s,v21.4s,v14.8h
        saddw   v22.4s,v22.4s,v15.4h
        saddw2  v23.4s,v23.4s,v15.8h

EpilogueC16P1
        //
        // Loop epilogue (process last single pixel) mixed with loading of dequantization params
        //
        ldr     w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags]
        ldr     x12,[x8,#ConvSymDepthwisePostProcessParams_Scale]
        smull   v12.8h,v0.8b,v4.8b
        smull2  v13.8h,v0.16b,v4.16b
        ldr     w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
        smull   v14.8h,v0.8b,v6.8b
        smull2  v15.8h,v0.16b,v6.16b
        saddw   v24.4s,v24.4s,v12.4h
        saddw2  v25.4s,v25.4s,v12.8h
        saddw   v26.4s,v26.4s,v13.4h
        saddw2  v27.4s,v27.4s,v13.8h
        saddw   v28.4s,v28.4s,v14.4h
        saddw2  v29.4s,v29.4s,v14.8h
        saddw   v30.4s,v30.4s,v15.4h
        saddw2  v31.4s,v31.4s,v15.8h
        smull   v12.8h,v0.8b,v2.8b
        smull2  v13.8h,v0.16b,v2.16b
        smull   v14.8h,v0.8b,v3.8b
        smull2  v15.8h,v0.16b,v3.16b
        tst     w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
        ld1r    {v4.4s},[x12]               // load scale val
        b.eq    SkipScaleVecLoad
        ldp     q4,q5,[x12],#32             // load scale vector if per channel
        ldp     q6,q3,[x12]
SkipScaleVecLoad
        saddw   v16.4s,v16.4s,v12.4h
        saddw2  v17.4s,v17.4s,v12.8h
        saddw   v18.4s,v18.4s,v13.4h
        saddw2  v19.4s,v19.4s,v13.8h
        saddw   v20.4s,v20.4s,v14.4h
        saddw2  v21.4s,v21.4s,v14.8h
        saddw   v22.4s,v22.4s,v15.4h
        saddw2  v23.4s,v23.4s,v15.8h

Dequantization
        scvtf   v24.4s,v24.4s               // convert to float
        scvtf   v25.4s,v25.4s
        scvtf   v26.4s,v26.4s
        scvtf   v27.4s,v27.4s
        scvtf   v28.4s,v28.4s
        scvtf   v29.4s,v29.4s
        scvtf   v30.4s,v30.4s
        scvtf   v31.4s,v31.4s
        scvtf   v16.4s,v16.4s
        scvtf   v17.4s,v17.4s
        scvtf   v18.4s,v18.4s
        scvtf   v19.4s,v19.4s
        scvtf   v20.4s,v20.4s
        scvtf   v21.4s,v21.4s
        scvtf   v22.4s,v22.4s
        scvtf   v23.4s,v23.4s
        b.ne    SkipScaleBroadcast
        mov     v5.16b,v4.16b               // broadcast scale val if not per channel
        mov     v6.16b,v4.16b
        mov     v3.16b,v4.16b
SkipScaleBroadcast
        fmul    v24.4s,v24.4s,v4.4s         // multiply by scale
        fmul    v25.4s,v25.4s,v5.4s
        fmul    v26.4s,v26.4s,v6.4s
        fmul    v27.4s,v27.4s,v3.4s
        fmul    v28.4s,v28.4s,v4.4s
        fmul    v29.4s,v29.4s,v5.4s
        fmul    v30.4s,v30.4s,v6.4s
        fmul    v31.4s,v31.4s,v3.4s
        fmul    v16.4s,v16.4s,v4.4s
        fmul    v17.4s,v17.4s,v5.4s
        fmul    v18.4s,v18.4s,v6.4s
        fmul    v19.4s,v19.4s,v3.4s
        fmul    v20.4s,v20.4s,v4.4s
        fmul    v21.4s,v21.4s,v5.4s
        fmul    v22.4s,v22.4s,v6.4s
        fmul    v23.4s,v23.4s,v3.4s
        fcvtns  v24.4s,v24.4s               // convert to int
        fcvtns  v25.4s,v25.4s
        fcvtns  v26.4s,v26.4s
        fcvtns  v27.4s,v27.4s
        fcvtns  v28.4s,v28.4s
        fcvtns  v29.4s,v29.4s
        fcvtns  v30.4s,v30.4s
        fcvtns  v31.4s,v31.4s
        fcvtns  v16.4s,v16.4s
        fcvtns  v17.4s,v17.4s
        fcvtns  v18.4s,v18.4s
        fcvtns  v19.4s,v19.4s
        fcvtns  v20.4s,v20.4s
        fcvtns  v21.4s,v21.4s
        fcvtns  v22.4s,v22.4s
        fcvtns  v23.4s,v23.4s
        sqxtn   v24.4h,v24.4s               // shorten to int16
        sqxtn   v26.4h,v26.4s
        sqxtn2  v24.8h,v25.4s
        sqxtn2  v26.8h,v27.4s
        sqxtn   v28.4h,v28.4s
        sqxtn   v30.4h,v30.4s
        sqxtn2  v28.8h,v29.4s
        sqxtn2  v30.8h,v31.4s
        dup     v0.8h,w15
        sqxtn   v16.4h,v16.4s
        sqxtn   v18.4h,v18.4s
        sqxtn2  v16.8h,v17.4s
        sqxtn2  v18.8h,v19.4s
        sqxtn   v20.4h,v20.4s
        sqxtn   v22.4h,v22.4s
        sqxtn2  v20.8h,v21.4s
        sqxtn2  v22.8h,v23.4s
        sqadd   v24.8h,v24.8h,v0.8h         // add zero point
        sqadd   v26.8h,v26.8h,v0.8h
        sqadd   v28.8h,v28.8h,v0.8h
        sqadd   v30.8h,v30.8h,v0.8h
        sqadd   v16.8h,v16.8h,v0.8h
        sqadd   v18.8h,v18.8h,v0.8h
        sqadd   v20.8h,v20.8h,v0.8h
        sqadd   v22.8h,v22.8h,v0.8h
        sqxtn   v24.8b,v24.8h               // shorten to int8
        sqxtn2  v24.16b,v26.8h
        sqxtn   v28.8b,v28.8h
        sqxtn2  v28.16b,v30.8h
        sqxtn   v16.8b,v16.8h
        sqxtn2  v16.16b,v18.8h
        sqxtn   v20.8b,v20.8h
        sqxtn2  v20.16b,v22.8h
        cmp     x7,2                        // OutputCount < 2 ?
        st1     {v24.16b},[x2],x4
        b.lo    ExitKernel                  // exit if OutputCount < 2
        st1     {v28.16b},[x2],x4
        b.ls    ExitKernel                  // exit if OutputCount <=2
        cmp     x7,4                        // OutputCount < 4 ?
        st1     {v16.16b},[x2],x4
        b.lo    ExitKernel                  // exit if OutputCount < 4
        str     q20,[x2]

ExitKernel
        EPILOG_RESTORE_REG_PAIR d14,d15,#16
        EPILOG_RESTORE_REG_PAIR d12,d13,#ConvSymDepthwiseKernelFrame_SavedRegisters!
        EPILOG_RETURN

Process8Channels
        cmp     x3,1
        b.eq    ProcC8P1

        ldr     x12,[x0],#8                 // x12 -> A0 iter 1
        ldr     x13,[x9],#8                 // x13 -> A1 iter 1
        ld1     {v0.8b},[x1],x4             // filter iter 0
        ld1     {v1.8b},[x1],x4             // filter iter 1
        ldr     d4,[x10,x5]                 // A0 iter 0
        ldr     x10,[x14],#8                // x10 -> A2 iter 0
        mov     v28.16b,v24.16b
        ldr     d6,[x11,x5]                 // A1 iter 0
        mov     v29.16b,v25.16b
        ldr     x11,[x15],#8                // x11 -> A3 iter 0
        mov     v16.16b,v24.16b
        ldr     d2,[x12,x5]                 // A0 iter 1
        mov     v17.16b,v25.16b
        ldr     x12,[x14],#8                // x12 -> A2 iter 1
        subs    x3,x3,2                     // decrement input blocks remaining
        ldr     d3,[x13,x5]                 // A1 iter 1
        mov     v20.16b,v24.16b
        ldr     x13,[x15],#8                // x13 -> A3 iter 1
        mov     v21.16b,v25.16b

BlockLoopC8
        //
        // Process 2 pixels, and load next two pixels
        //
        smull   v12.8h,v0.8b,v4.8b
        ldr     d4,[x10,x5]                 // A2 iter 0
        smull   v14.8h,v0.8b,v6.8b
        b.eq    EpilogueC8P2
        ldr     x10,[x0],#8                 // x10 -> A0 iter 2
        ldr     d6,[x11,x5]                 // A3 iter 0
        cmp     x3,1
        smlal   v12.8h,v1.8b,v2.8b
        ldr     x11,[x9],#8                 // x11 -> A1 iter 2
        smlal   v14.8h,v1.8b,v3.8b
        ldr     d2,[x12,x5]                 // A2 iter 1
        b.eq    EpilogueC8P3                // 3 pixel remains      
        ldr     d3,[x13,x5]                 // A3 iter 1
        saddw   v24.4s,v24.4s,v12.4h
        ldr     x12,[x0],#8                 // x12 -> A0 iter 3
        saddw2  v25.4s,v25.4s,v12.8h
        ldr     x13,[x9],#8                 // x13 -> A1 iter 3
        saddw   v28.4s,v28.4s,v14.4h
        saddw2  v29.4s,v29.4s,v14.8h
        subs    x3,x3,2                     // decrement input blocks remaining
        smull   v12.8h,v0.8b,v4.8b
        ldr     d4,[x10,x5]                 // A0 iter 2
        smull   v14.8h,v0.8b,v6.8b
        ldr     x10,[x14],#8                // x10 -> A2 iter 2
        ldr     d6,[x11,x5]                 // A1 iter 2
        ld1     {v0.8b},[x1],x4             // filter iter 2
        smlal   v12.8h,v1.8b,v2.8b
        ldr     x11,[x15],#8                // x11 -> A3 iter 2
        ldr     d2,[x12,x5]                 // A0 iter 3
        smlal   v14.8h,v1.8b,v3.8b
        ldr     x12,[x14],#8                // x12 -> A2 iter 3
        saddw   v16.4s,v16.4s,v12.4h
        ldr     d3,[x13,x5]                 // A1 iter 3
        saddw2  v17.4s,v17.4s,v12.8h
        ld1     {v1.8b},[x1],x4             // filter iter 3
        saddw   v20.4s,v20.4s,v14.4h
        ldr     x13,[x15],#8                // x13 -> A3 iter 3
        saddw2  v21.4s,v21.4s,v14.8h
        b       BlockLoopC8

EpilogueC8P2
        //
        // Loop epilogue (process last 2 pixels) mixed
        // with loading of dequantization params
        //
        ldr     d6,[x11,x5]                 // A3 iter 0
        smlal   v12.8h,v1.8b,v2.8b
        ldr     d2,[x12,x5]                 // A2 iter 1
        smlal   v14.8h,v1.8b,v3.8b
        ldr     d3,[x13,x5]                 // A3 iter 1
        saddw   v24.4s,v24.4s,v12.4h
        saddw2  v25.4s,v25.4s,v12.8h
        saddw   v28.4s,v28.4s,v14.4h
        saddw2  v29.4s,v29.4s,v14.8h
        ldr     w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags]
        smull   v12.8h,v0.8b,v4.8b
        ldr     x12,[x8,#ConvSymDepthwisePostProcessParams_Scale]
        smull   v14.8h,v0.8b,v6.8b
        ldr     w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
        smlal   v12.8h,v1.8b,v2.8b
        smlal   v14.8h,v1.8b,v3.8b
        tst     w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
        ld1r    {v4.4s},[x12]               // load scale val
        b.eq    SkipScaleVecLoad2C8
        ldp     q4,q5,[x12],#32             // load scale vector if per channel
SkipScaleVecLoad2C8
        saddw   v16.4s,v16.4s,v12.4h
        saddw2  v17.4s,v17.4s,v12.8h
        saddw   v20.4s,v20.4s,v14.4h
        saddw2  v21.4s,v21.4s,v14.8h
        b       DequantC8

ProcC8P1
        //
        // Channel 8 kernel size 1
        // TODO!! is this reachable at all?
        //
        ldr     x12,[x14],#8                // x12 -> A2
        mov     v28.16b,v24.16b
        ldr     x13,[x15],#8                // x13 -> A3
        mov     v29.16b,v25.16b
        ld1     {v0.8b},[x1]
        mov     v16.16b,v24.16b
        ldr     d4,[x10,x5]
        mov     v17.16b,v25.16b
        ldr     d6,[x11,x5]
        mov     v20.16b,v24.16b
        ldr     d2,[x12,x5]
        subs    x3,x3,2                     // decrement input blocks remaining
        ldr     d3,[x13,x5]
        mov     v21.16b,v25.16b
        b       EpilogueC8P1

EpilogueC8P3
        //
        // Loop epilogue (process 2 of last 3 pixels)
        //
        ldr     x12,[x14],#8                // x12 -> A2 iter 2
        ldr     d3,[x13,x5]                 // A3 iter 1
        saddw   v24.4s,v24.4s,v12.4h
        saddw2  v25.4s,v25.4s,v12.8h
        ldr     x13,[x15],#8                // x13 -> A3 iter 2
        saddw   v28.4s,v28.4s,v14.4h
        saddw2  v29.4s,v29.4s,v14.8h
        smull   v12.8h,v0.8b,v4.8b
        ldr     d4,[x10,x5]                 // A0 iter 2
        smull   v14.8h,v0.8b,v6.8b
        ld1     {v0.8b},[x1]                // filter iter 2
        ldr     d6,[x11,x5]                 // A1 iter 2
        smlal   v12.8h,v1.8b,v2.8b
        ldr     d2,[x12,x5]                 // A2 iter 2
        smlal   v14.8h,v1.8b,v3.8b
        ldr     d3,[x13,x5]                 // A3 iter 2
        saddw   v16.4s,v16.4s,v12.4h
        saddw2  v17.4s,v17.4s,v12.8h
        saddw   v20.4s,v20.4s,v14.4h
        saddw2  v21.4s,v21.4s,v14.8h

EpilogueC8P1
        //
        // Loop epilogue (process last single pixel) mixed with loading of dequantization params
        //
        ldr     w9,[sp,#ConvSymDepthwiseKernelFrame_KernelFlags]
        ldr     x12,[x8,#ConvSymDepthwisePostProcessParams_Scale]
        smull   v12.8h,v0.8b,v4.8b
        ldr     w15,[x8,#ConvSymDepthwisePostProcessParams_ZeroPoint]
        smull   v14.8h,v0.8b,v6.8b
        saddw   v24.4s,v24.4s,v12.4h
        saddw2  v25.4s,v25.4s,v12.8h
        saddw   v28.4s,v28.4s,v14.4h
        saddw2  v29.4s,v29.4s,v14.8h
        smull   v12.8h,v0.8b,v2.8b
        smull   v14.8h,v0.8b,v3.8b
        tst     w9,#MLAS_CONV_SYM_FLAG_PER_CHANNEL_SCALE
        ld1r    {v4.4s},[x12]               // load scale val
        b.eq    SkipScaleVecLoadC8
        ldp     q4,q5,[x12]                 // load scale vector if per channel
SkipScaleVecLoadC8
        saddw   v16.4s,v16.4s,v12.4h
        saddw2  v17.4s,v17.4s,v12.8h
        saddw   v20.4s,v20.4s,v14.4h
        saddw2  v21.4s,v21.4s,v14.8h

DequantC8
        scvtf   v24.4s,v24.4s               // convert to float
        scvtf   v25.4s,v25.4s
        scvtf   v28.4s,v28.4s
        scvtf   v29.4s,v29.4s
        scvtf   v16.4s,v16.4s
        scvtf   v17.4s,v17.4s
        scvtf   v20.4s,v20.4s
        scvtf   v21.4s,v21.4s
        b.ne    SkipScaleBroadcastC8
        mov     v5.16b,v4.16b               // broadcast scale val if not per channel
SkipScaleBroadcastC8
        fmul    v24.4s,v24.4s,v4.4s         // multiply by scale
        fmul    v25.4s,v25.4s,v5.4s
        fmul    v28.4s,v28.4s,v4.4s
        fmul    v29.4s,v29.4s,v5.4s
        fmul    v16.4s,v16.4s,v4.4s
        fmul    v17.4s,v17.4s,v5.4s
        fmul    v20.4s,v20.4s,v4.4s
        fmul    v21.4s,v21.4s,v5.4s
        fcvtns  v24.4s,v24.4s               // convert to int
        fcvtns  v25.4s,v25.4s
        fcvtns  v28.4s,v28.4s
        fcvtns  v29.4s,v29.4s
        fcvtns  v16.4s,v16.4s
        fcvtns  v17.4s,v17.4s
        fcvtns  v20.4s,v20.4s
        fcvtns  v21.4s,v21.4s
        dup     v0.8h,w15
        sqxtn   v24.4h,v24.4s               // shorten to int16
        sqxtn2  v24.8h,v25.4s
        sqxtn   v28.4h,v28.4s
        sqxtn2  v28.8h,v29.4s
        sqxtn   v16.4h,v16.4s
        sqxtn2  v16.8h,v17.4s
        sqxtn   v20.4h,v20.4s
        sqxtn2  v20.8h,v21.4s
        sqadd   v24.8h,v24.8h,v0.8h         // add zero point
        sqadd   v28.8h,v28.8h,v0.8h
        sqadd   v16.8h,v16.8h,v0.8h
        sqadd   v20.8h,v20.8h,v0.8h
        sqxtn   v24.8b,v24.8h               // shorten to int8
        sqxtn   v28.8b,v28.8h
        sqxtn   v16.8b,v16.8h
        sqxtn   v20.8b,v20.8h
        cmp     x7,2                        // OutputCount < 2 ?
        st1     {v24.8b},[x2],x4
        b.lo    ExitKernel                  // exit if OutputCount < 2
        st1     {v28.8b},[x2],x4
        b.ls    ExitKernel                  // exit if OutputCount <=2
        cmp     x7,4                        // OutputCount < 4 ?
        st1     {v16.8b},[x2],x4
        b.lo    ExitKernel                  // exit if OutputCount < 4
        str     d20,[x2]
        b       ExitKernel
        NESTED_END MlasConvSymDepthwiseS8KernelNeon

        END
