/vendor/github.com/minio/sha256-simd/sha256blockAvx512_amd64.asm at ba9e4e25705b04fbcd6ebe61c8dd0aa075dc3b0e


// 16x Parallel implementation of SHA256 for AVX512

//
// Minio Cloud Storage, (C) 2017 Minio, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//
// This code is based on the Intel Multi-Buffer Crypto for IPSec library
// and more specifically the following implementation:
// https://github.com/intel/intel-ipsec-mb/blob/master/avx512/sha256_x16_avx512.asm
//
// For Golang it has been converted into Plan 9 assembly with the help of
// github.com/minio/asm2plan9s to assemble the AVX512 instructions
//

// Copyright (c) 2017, Intel Corporation
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
//     * Redistributions of source code must retain the above copyright notice,
//       this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above copyright
//       notice, this list of conditions and the following disclaimer in the
//       documentation and/or other materials provided with the distribution.
//     * Neither the name of Intel Corporation nor the names of its contributors
//       may be used to endorse or promote products derived from this software
//       without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

#define SHA256_DIGEST_ROW_SIZE 64

// arg1
#define STATE rdi
#define STATE_P9 DI
// arg2
#define INP_SIZE rsi
#define INP_SIZE_P9 SI

#define IDX rcx
#define TBL rdx
#define TBL_P9 DX

#define INPUT rax
#define INPUT_P9 AX

#define inp0	r9
#define SCRATCH_P9 R12
#define SCRATCH  r12
#define maskp    r13
#define MASKP_P9 R13
#define mask     r14
#define MASK_P9  R14

#define A       zmm0
#define B       zmm1
#define C       zmm2
#define D       zmm3
#define E       zmm4
#define F       zmm5
#define G       zmm6
#define H       zmm7
#define T1      zmm8
#define TMP0    zmm9
#define TMP1    zmm10
#define TMP2    zmm11
#define TMP3    zmm12
#define TMP4    zmm13
#define TMP5    zmm14
#define TMP6    zmm15

#define W0      zmm16
#define W1      zmm17
#define W2      zmm18
#define W3      zmm19
#define W4      zmm20
#define W5      zmm21
#define W6      zmm22
#define W7      zmm23
#define W8      zmm24
#define W9      zmm25
#define W10     zmm26
#define W11     zmm27
#define W12     zmm28
#define W13     zmm29
#define W14     zmm30
#define W15     zmm31


#define TRANSPOSE16(_r0, _r1, _r2, _r3, _r4, _r5, _r6, _r7, _r8, _r9, _r10, _r11, _r12, _r13, _r14, _r15, _t0, _t1) \
    \
    \ // input   r0  = {a15 a14 a13 a12   a11 a10  a9  a8    a7  a6  a5  a4    a3  a2  a1  a0}
    \ //         r1  = {b15 b14 b13 b12   b11 b10  b9  b8    b7  b6  b5  b4    b3  b2  b1  b0}
    \ //         r2  = {c15 c14 c13 c12   c11 c10  c9  c8    c7  c6  c5  c4    c3  c2  c1  c0}
    \ //         r3  = {d15 d14 d13 d12   d11 d10  d9  d8    d7  d6  d5  d4    d3  d2  d1  d0}
    \ //         r4  = {e15 e14 e13 e12   e11 e10  e9  e8    e7  e6  e5  e4    e3  e2  e1  e0}
    \ //         r5  = {f15 f14 f13 f12   f11 f10  f9  f8    f7  f6  f5  f4    f3  f2  f1  f0}
    \ //         r6  = {g15 g14 g13 g12   g11 g10  g9  g8    g7  g6  g5  g4    g3  g2  g1  g0}
    \ //         r7  = {h15 h14 h13 h12   h11 h10  h9  h8    h7  h6  h5  h4    h3  h2  h1  h0}
    \ //         r8  = {i15 i14 i13 i12   i11 i10  i9  i8    i7  i6  i5  i4    i3  i2  i1  i0}
    \ //         r9  = {j15 j14 j13 j12   j11 j10  j9  j8    j7  j6  j5  j4    j3  j2  j1  j0}
    \ //         r10 = {k15 k14 k13 k12   k11 k10  k9  k8    k7  k6  k5  k4    k3  k2  k1  k0}
    \ //         r11 = {l15 l14 l13 l12   l11 l10  l9  l8    l7  l6  l5  l4    l3  l2  l1  l0}
    \ //         r12 = {m15 m14 m13 m12   m11 m10  m9  m8    m7  m6  m5  m4    m3  m2  m1  m0}
    \ //         r13 = {n15 n14 n13 n12   n11 n10  n9  n8    n7  n6  n5  n4    n3  n2  n1  n0}
    \ //         r14 = {o15 o14 o13 o12   o11 o10  o9  o8    o7  o6  o5  o4    o3  o2  o1  o0}
    \ //         r15 = {p15 p14 p13 p12   p11 p10  p9  p8    p7  p6  p5  p4    p3  p2  p1  p0}
    \
    \ // output  r0  = { p0  o0  n0  m0    l0  k0  j0  i0    h0  g0  f0  e0    d0  c0  b0  a0}
    \ //         r1  = { p1  o1  n1  m1    l1  k1  j1  i1    h1  g1  f1  e1    d1  c1  b1  a1}
    \ //         r2  = { p2  o2  n2  m2    l2  k2  j2  i2    h2  g2  f2  e2    d2  c2  b2  a2}
    \ //         r3  = { p3  o3  n3  m3    l3  k3  j3  i3    h3  g3  f3  e3    d3  c3  b3  a3}
    \ //         r4  = { p4  o4  n4  m4    l4  k4  j4  i4    h4  g4  f4  e4    d4  c4  b4  a4}
    \ //         r5  = { p5  o5  n5  m5    l5  k5  j5  i5    h5  g5  f5  e5    d5  c5  b5  a5}
    \ //         r6  = { p6  o6  n6  m6    l6  k6  j6  i6    h6  g6  f6  e6    d6  c6  b6  a6}
    \ //         r7  = { p7  o7  n7  m7    l7  k7  j7  i7    h7  g7  f7  e7    d7  c7  b7  a7}
    \ //         r8  = { p8  o8  n8  m8    l8  k8  j8  i8    h8  g8  f8  e8    d8  c8  b8  a8}
    \ //         r9  = { p9  o9  n9  m9    l9  k9  j9  i9    h9  g9  f9  e9    d9  c9  b9  a9}
    \ //         r10 = {p10 o10 n10 m10   l10 k10 j10 i10   h10 g10 f10 e10   d10 c10 b10 a10}
    \ //         r11 = {p11 o11 n11 m11   l11 k11 j11 i11   h11 g11 f11 e11   d11 c11 b11 a11}
    \ //         r12 = {p12 o12 n12 m12   l12 k12 j12 i12   h12 g12 f12 e12   d12 c12 b12 a12}
    \ //         r13 = {p13 o13 n13 m13   l13 k13 j13 i13   h13 g13 f13 e13   d13 c13 b13 a13}
    \ //         r14 = {p14 o14 n14 m14   l14 k14 j14 i14   h14 g14 f14 e14   d14 c14 b14 a14}
    \ //         r15 = {p15 o15 n15 m15   l15 k15 j15 i15   h15 g15 f15 e15   d15 c15 b15 a15}
    \
    \ // process top half
    vshufps _t0, _r0, _r1, 0x44      \ // t0 = {b13 b12 a13 a12   b9  b8  a9  a8   b5 b4 a5 a4   b1 b0 a1 a0}
    vshufps _r0, _r0, _r1, 0xEE      \ // r0 = {b15 b14 a15 a14   b11 b10 a11 a10  b7 b6 a7 a6   b3 b2 a3 a2}
    vshufps _t1, _r2, _r3, 0x44      \ // t1 = {d13 d12 c13 c12   d9  d8  c9  c8   d5 d4 c5 c4   d1 d0 c1 c0}
    vshufps _r2, _r2, _r3, 0xEE      \ // r2 = {d15 d14 c15 c14   d11 d10 c11 c10  d7 d6 c7 c6   d3 d2 c3 c2}
                                     \
    vshufps	_r3, _t0, _t1, 0xDD      \ // r3 = {d13 c13 b13 a13   d9  c9  b9  a9   d5 c5 b5 a5   d1 c1 b1 a1}
    vshufps	_r1, _r0, _r2, 0x88      \ // r1 = {d14 c14 b14 a14   d10 c10 b10 a10  d6 c6 b6 a6   d2 c2 b2 a2}
    vshufps	_r0, _r0, _r2, 0xDD      \ // r0 = {d15 c15 b15 a15   d11 c11 b11 a11  d7 c7 b7 a7   d3 c3 b3 a3}
    vshufps	_t0, _t0, _t1, 0x88      \ // t0 = {d12 c12 b12 a12   d8  c8  b8  a8   d4 c4 b4 a4   d0 c0 b0 a0}
                                     \
    \ // use r2 in place of t0
    vshufps _r2, _r4, _r5, 0x44      \ // r2 = {f13 f12 e13 e12   f9  f8  e9  e8   f5 f4 e5 e4   f1 f0 e1 e0}
    vshufps _r4, _r4, _r5, 0xEE      \ // r4 = {f15 f14 e15 e14   f11 f10 e11 e10  f7 f6 e7 e6   f3 f2 e3 e2}
    vshufps _t1, _r6, _r7, 0x44      \ // t1 = {h13 h12 g13 g12   h9  h8  g9  g8   h5 h4 g5 g4   h1 h0 g1 g0}
    vshufps _r6, _r6, _r7, 0xEE      \ // r6 = {h15 h14 g15 g14   h11 h10 g11 g10  h7 h6 g7 g6   h3 h2 g3 g2}
                                     \
    vshufps _r7, _r2, _t1, 0xDD      \ // r7 = {h13 g13 f13 e13   h9  g9  f9  e9   h5 g5 f5 e5   h1 g1 f1 e1}
    vshufps _r5, _r4, _r6, 0x88      \ // r5 = {h14 g14 f14 e14   h10 g10 f10 e10  h6 g6 f6 e6   h2 g2 f2 e2}
    vshufps _r4, _r4, _r6, 0xDD      \ // r4 = {h15 g15 f15 e15   h11 g11 f11 e11  h7 g7 f7 e7   h3 g3 f3 e3}
    vshufps _r2, _r2, _t1, 0x88      \ // r2 = {h12 g12 f12 e12   h8  g8  f8  e8   h4 g4 f4 e4   h0 g0 f0 e0}
                                     \
    \ // use r6 in place of t0
    vshufps _r6, _r8, _r9,    0x44   \ // r6  = {j13 j12 i13 i12   j9  j8  i9  i8   j5 j4 i5 i4   j1 j0 i1 i0}
    vshufps _r8, _r8, _r9,    0xEE   \ // r8  = {j15 j14 i15 i14   j11 j10 i11 i10  j7 j6 i7 i6   j3 j2 i3 i2}
    vshufps _t1, _r10, _r11,  0x44   \ // t1  = {l13 l12 k13 k12   l9  l8  k9  k8   l5 l4 k5 k4   l1 l0 k1 k0}
    vshufps _r10, _r10, _r11, 0xEE   \ // r10 = {l15 l14 k15 k14   l11 l10 k11 k10  l7 l6 k7 k6   l3 l2 k3 k2}
                                     \
    vshufps _r11, _r6, _t1, 0xDD     \ // r11 = {l13 k13 j13 113   l9  k9  j9  i9   l5 k5 j5 i5   l1 k1 j1 i1}
    vshufps _r9, _r8, _r10, 0x88     \ // r9  = {l14 k14 j14 114   l10 k10 j10 i10  l6 k6 j6 i6   l2 k2 j2 i2}
    vshufps _r8, _r8, _r10, 0xDD     \ // r8  = {l15 k15 j15 115   l11 k11 j11 i11  l7 k7 j7 i7   l3 k3 j3 i3}
    vshufps _r6, _r6, _t1,  0x88     \ // r6  = {l12 k12 j12 112   l8  k8  j8  i8   l4 k4 j4 i4   l0 k0 j0 i0}
                                     \
    \ // use r10 in place of t0
    vshufps _r10, _r12, _r13, 0x44   \ // r10 = {n13 n12 m13 m12   n9  n8  m9  m8   n5 n4 m5 m4   n1 n0 a1 m0}
    vshufps _r12, _r12, _r13, 0xEE   \ // r12 = {n15 n14 m15 m14   n11 n10 m11 m10  n7 n6 m7 m6   n3 n2 a3 m2}
    vshufps _t1, _r14, _r15,  0x44   \ // t1  = {p13 p12 013 012   p9  p8  09  08   p5 p4 05 04   p1 p0 01 00}
    vshufps _r14, _r14, _r15, 0xEE   \ // r14 = {p15 p14 015 014   p11 p10 011 010  p7 p6 07 06   p3 p2 03 02}
                                     \
    vshufps _r15, _r10, _t1,  0xDD   \ // r15 = {p13 013 n13 m13   p9  09  n9  m9   p5 05 n5 m5   p1 01 n1 m1}
    vshufps _r13, _r12, _r14, 0x88   \ // r13 = {p14 014 n14 m14   p10 010 n10 m10  p6 06 n6 m6   p2 02 n2 m2}
    vshufps _r12, _r12, _r14, 0xDD   \ // r12 = {p15 015 n15 m15   p11 011 n11 m11  p7 07 n7 m7   p3 03 n3 m3}
    vshufps _r10, _r10, _t1,  0x88   \ // r10 = {p12 012 n12 m12   p8  08  n8  m8   p4 04 n4 m4   p0 00 n0 m0}
                                     \
    \ // At this point, the registers that contain interesting data are:
    \ // t0, r3, r1, r0, r2, r7, r5, r4, r6, r11, r9, r8, r10, r15, r13, r12
    \ // Can use t1 and r14 as scratch registers
    LEAQ PSHUFFLE_TRANSPOSE16_MASK1<>(SB), BX \
    LEAQ PSHUFFLE_TRANSPOSE16_MASK2<>(SB), R8 \
                                     \
    vmovdqu32 _r14, [rbx]            \
    vpermi2q  _r14, _t0, _r2         \ // r14 = {h8  g8  f8  e8   d8  c8  b8  a8   h0 g0 f0 e0	 d0 c0 b0 a0}
    vmovdqu32 _t1,  [r8]             \
    vpermi2q  _t1,  _t0, _r2         \ // t1  = {h12 g12 f12 e12  d12 c12 b12 a12  h4 g4 f4 e4	 d4 c4 b4 a4}
                                     \
    vmovdqu32 _r2, [rbx]             \
    vpermi2q  _r2, _r3, _r7          \ // r2  = {h9  g9  f9  e9   d9  c9  b9  a9   h1 g1 f1 e1	 d1 c1 b1 a1}
    vmovdqu32 _t0, [r8]              \
    vpermi2q  _t0, _r3, _r7          \ // t0  = {h13 g13 f13 e13  d13 c13 b13 a13  h5 g5 f5 e5	 d5 c5 b5 a5}
                                     \
    vmovdqu32 _r3, [rbx]             \
    vpermi2q  _r3, _r1, _r5          \ // r3  = {h10 g10 f10 e10  d10 c10 b10 a10  h2 g2 f2 e2	 d2 c2 b2 a2}
    vmovdqu32 _r7, [r8]              \
    vpermi2q  _r7, _r1, _r5          \ // r7  = {h14 g14 f14 e14  d14 c14 b14 a14  h6 g6 f6 e6	 d6 c6 b6 a6}
                                     \
    vmovdqu32 _r1, [rbx]             \
    vpermi2q  _r1, _r0, _r4          \ // r1  = {h11 g11 f11 e11  d11 c11 b11 a11  h3 g3 f3 e3	 d3 c3 b3 a3}
    vmovdqu32 _r5, [r8]              \
    vpermi2q  _r5, _r0, _r4          \ // r5  = {h15 g15 f15 e15  d15 c15 b15 a15  h7 g7 f7 e7	 d7 c7 b7 a7}
                                     \
    vmovdqu32 _r0, [rbx]             \
    vpermi2q  _r0, _r6, _r10         \ // r0  = {p8  o8  n8  m8   l8  k8  j8  i8   p0 o0 n0 m0	 l0 k0 j0 i0}
    vmovdqu32 _r4, [r8]              \
    vpermi2q  _r4, _r6, _r10         \ // r4  = {p12 o12 n12 m12  l12 k12 j12 i12  p4 o4 n4 m4	 l4 k4 j4 i4}
                                     \
    vmovdqu32 _r6, [rbx]             \
    vpermi2q  _r6, _r11, _r15        \ // r6  = {p9  o9  n9  m9   l9  k9  j9  i9   p1 o1 n1 m1	 l1 k1 j1 i1}
    vmovdqu32 _r10, [r8]             \
    vpermi2q  _r10, _r11, _r15       \ // r10 = {p13 o13 n13 m13  l13 k13 j13 i13  p5 o5 n5 m5	 l5 k5 j5 i5}
                                     \
    vmovdqu32 _r11, [rbx]            \
    vpermi2q  _r11, _r9, _r13        \ // r11 = {p10 o10 n10 m10  l10 k10 j10 i10  p2 o2 n2 m2	 l2 k2 j2 i2}
    vmovdqu32 _r15, [r8]             \
    vpermi2q  _r15, _r9, _r13        \ // r15 = {p14 o14 n14 m14  l14 k14 j14 i14  p6 o6 n6 m6	 l6 k6 j6 i6}
                                     \
    vmovdqu32 _r9, [rbx]             \
    vpermi2q  _r9, _r8, _r12         \ // r9  = {p11 o11 n11 m11  l11 k11 j11 i11  p3 o3 n3 m3	 l3 k3 j3 i3}
    vmovdqu32 _r13, [r8]             \
    vpermi2q  _r13, _r8, _r12        \ // r13 = {p15 o15 n15 m15  l15 k15 j15 i15  p7 o7 n7 m7	 l7 k7 j7 i7}
                                     \
    \ // At this point r8 and r12 can be used as scratch registers
    vshuff64x2 _r8, _r14, _r0, 0xEE  \ // r8  = {p8  o8  n8  m8   l8  k8  j8  i8   h8 g8 f8 e8   d8 c8 b8 a8}
    vshuff64x2 _r0, _r14, _r0, 0x44  \ // r0  = {p0  o0  n0  m0   l0  k0  j0  i0   h0 g0 f0 e0   d0 c0 b0 a0}
                                     \
    vshuff64x2 _r12, _t1, _r4, 0xEE  \ // r12 = {p12 o12 n12 m12  l12 k12 j12 i12  h12 g12 f12 e12  d12 c12 b12 a12}
    vshuff64x2 _r4, _t1, _r4, 0x44   \ // r4  = {p4  o4  n4  m4   l4  k4  j4  i4   h4 g4 f4 e4   d4 c4 b4 a4}
                                     \
    vshuff64x2 _r14, _r7, _r15, 0xEE \ // r14 = {p14 o14 n14 m14  l14 k14 j14 i14  h14 g14 f14 e14  d14 c14 b14 a14}
    vshuff64x2 _t1, _r7, _r15, 0x44  \ // t1  = {p6  o6  n6  m6   l6  k6  j6  i6   h6 g6 f6 e6   d6 c6 b6 a6}
                                     \
    vshuff64x2 _r15, _r5, _r13, 0xEE \ // r15 = {p15 o15 n15 m15  l15 k15 j15 i15  h15 g15 f15 e15  d15 c15 b15 a15}
    vshuff64x2 _r7, _r5, _r13, 0x44  \ // r7  = {p7  o7  n7  m7   l7  k7  j7  i7   h7 g7 f7 e7   d7 c7 b7 a7}
                                     \
    vshuff64x2 _r13, _t0, _r10, 0xEE \ // r13 = {p13 o13 n13 m13  l13 k13 j13 i13  h13 g13 f13 e13  d13 c13 b13 a13}
    vshuff64x2 _r5, _t0, _r10, 0x44  \ // r5  = {p5  o5  n5  m5   l5  k5  j5  i5   h5 g5 f5 e5   d5 c5 b5 a5}
                                     \
    vshuff64x2 _r10, _r3, _r11, 0xEE \ // r10 = {p10 o10 n10 m10  l10 k10 j10 i10  h10 g10 f10 e10  d10 c10 b10 a10}
    vshuff64x2 _t0, _r3, _r11, 0x44  \ // t0  = {p2  o2  n2  m2   l2  k2  j2  i2   h2 g2 f2 e2   d2 c2 b2 a2}
                                     \
    vshuff64x2 _r11, _r1, _r9, 0xEE  \ // r11 = {p11 o11 n11 m11  l11 k11 j11 i11  h11 g11 f11 e11  d11 c11 b11 a11}
    vshuff64x2 _r3, _r1, _r9, 0x44   \ // r3  = {p3  o3  n3  m3   l3  k3  j3  i3   h3 g3 f3 e3   d3 c3 b3 a3}
                                     \
    vshuff64x2 _r9, _r2, _r6, 0xEE   \ // r9  = {p9  o9  n9  m9   l9  k9  j9  i9   h9 g9 f9 e9   d9 c9 b9 a9}
    vshuff64x2 _r1, _r2, _r6, 0x44   \ // r1  = {p1  o1  n1  m1   l1  k1  j1  i1   h1 g1 f1 e1   d1 c1 b1 a1}
                                     \
    vmovdqu32 _r2, _t0               \ // r2  = {p2  o2  n2  m2   l2  k2  j2  i2   h2 g2 f2 e2   d2 c2 b2 a2}
    vmovdqu32 _r6, _t1               \ // r6  = {p6  o6  n6  m6   l6  k6  j6  i6   h6 g6 f6 e6   d6 c6 b6 a6}


//  CH(A, B, C) = (A&B) ^ (~A&C)
// MAJ(E, F, G) = (E&F) ^ (E&G) ^ (F&G)
// SIGMA0 = ROR_2  ^ ROR_13 ^ ROR_22
// SIGMA1 = ROR_6  ^ ROR_11 ^ ROR_25
// sigma0 = ROR_7  ^ ROR_18 ^ SHR_3
// sigma1 = ROR_17 ^ ROR_19 ^ SHR_10

// Main processing loop per round
#define PROCESS_LOOP(_WT, _ROUND, _A, _B, _C, _D, _E, _F, _G, _H)  \
    \ // T1 = H + SIGMA1(E) + CH(E, F, G) + Kt + Wt
    \ // T2 = SIGMA0(A) + MAJ(A, B, C)
    \ // H=G, G=F, F=E, E=D+T1, D=C, C=B, B=A, A=T1+T2
    \
    \ // H becomes T2, then add T1 for A
    \ // D becomes D + T1 for E
    \
    vpaddd      T1, _H, TMP3           \ // T1 = H + Kt
    vmovdqu32   TMP0, _E               \
    vprord      TMP1, _E, 6            \ // ROR_6(E)
    vprord      TMP2, _E, 11           \ // ROR_11(E)
    vprord      TMP3, _E, 25           \ // ROR_25(E)
    vpternlogd  TMP0, _F, _G, 0xCA     \ // TMP0 = CH(E,F,G)
    vpaddd      T1, T1, _WT            \ // T1 = T1 + Wt
    vpternlogd  TMP1, TMP2, TMP3, 0x96 \ // TMP1 = SIGMA1(E)
    vpaddd      T1, T1, TMP0           \ // T1 = T1 + CH(E,F,G)
    vpaddd      T1, T1, TMP1           \ // T1 = T1 + SIGMA1(E)
    vpaddd      _D, _D, T1             \ // D = D + T1
                                       \
    vprord      _H, _A, 2              \ // ROR_2(A)
    vprord      TMP2, _A, 13           \ // ROR_13(A)
    vprord      TMP3, _A, 22           \ // ROR_22(A)
    vmovdqu32   TMP0, _A               \
    vpternlogd  TMP0, _B, _C, 0xE8     \ // TMP0 = MAJ(A,B,C)
    vpternlogd  _H, TMP2, TMP3, 0x96   \ // H(T2) = SIGMA0(A)
    vpaddd      _H, _H, TMP0           \ // H(T2) = SIGMA0(A) + MAJ(A,B,C)
    vpaddd      _H, _H, T1             \ // H(A) = H(T2) + T1
                                       \
    vmovdqu32   TMP3, [TBL + ((_ROUND+1)*64)] \ // Next Kt


#define MSG_SCHED_ROUND_16_63(_WT, _WTp1, _WTp9, _WTp14) \
    vprord      TMP4, _WTp14, 17                         \ // ROR_17(Wt-2)
    vprord      TMP5, _WTp14, 19                         \ // ROR_19(Wt-2)
    vpsrld      TMP6, _WTp14, 10                         \ // SHR_10(Wt-2)
    vpternlogd  TMP4, TMP5, TMP6, 0x96                   \ // TMP4 = sigma1(Wt-2)
                                                         \
    vpaddd      _WT, _WT, TMP4	                         \ // Wt = Wt-16 + sigma1(Wt-2)
    vpaddd      _WT, _WT, _WTp9	                         \ // Wt = Wt-16 + sigma1(Wt-2) + Wt-7
                                                         \
    vprord      TMP4, _WTp1, 7                           \ // ROR_7(Wt-15)
    vprord      TMP5, _WTp1, 18                          \ // ROR_18(Wt-15)
    vpsrld      TMP6, _WTp1, 3                           \ // SHR_3(Wt-15)
    vpternlogd  TMP4, TMP5, TMP6, 0x96                   \ // TMP4 = sigma0(Wt-15)
                                                         \
    vpaddd      _WT, _WT, TMP4	                         \ // Wt = Wt-16 + sigma1(Wt-2) +
                                                         \ //      Wt-7 + sigma0(Wt-15) +


// Note this is reading in a block of data for one lane
// When all 16 are read, the data must be transposed to build msg schedule
#define MSG_SCHED_ROUND_00_15(_WT, OFFSET, LABEL)             \
    TESTQ $(1<<OFFSET), MASK_P9                               \
    JE    LABEL                                               \
    MOVQ  OFFSET*24(INPUT_P9), R9                             \
    vmovups _WT, [inp0+IDX]                                   \
LABEL:                                                        \

#define MASKED_LOAD(_WT, OFFSET, LABEL) \
    TESTQ $(1<<OFFSET), MASK_P9         \
    JE    LABEL                         \
    MOVQ  OFFSET*24(INPUT_P9), R9       \
    vmovups _WT,[inp0+IDX]              \
LABEL:                                  \

TEXT ·sha256_x16_avx512(SB), 7, $0
    MOVQ  digests+0(FP), STATE_P9       //
    MOVQ  scratch+8(FP), SCRATCH_P9
    MOVQ  mask_len+32(FP), INP_SIZE_P9  // number of blocks to process
    MOVQ  mask+24(FP), MASKP_P9
    MOVQ (MASKP_P9), MASK_P9
    kmovq k1, mask
    LEAQ  inputs+48(FP), INPUT_P9

    // Initialize digests
    vmovdqu32 A, [STATE + 0*SHA256_DIGEST_ROW_SIZE]
    vmovdqu32 B, [STATE + 1*SHA256_DIGEST_ROW_SIZE]
    vmovdqu32 C, [STATE + 2*SHA256_DIGEST_ROW_SIZE]
    vmovdqu32 D, [STATE + 3*SHA256_DIGEST_ROW_SIZE]
    vmovdqu32 E, [STATE + 4*SHA256_DIGEST_ROW_SIZE]
    vmovdqu32 F, [STATE + 5*SHA256_DIGEST_ROW_SIZE]
    vmovdqu32 G, [STATE + 6*SHA256_DIGEST_ROW_SIZE]
    vmovdqu32 H, [STATE + 7*SHA256_DIGEST_ROW_SIZE]

    MOVQ  table+16(FP), TBL_P9

    xor IDX, IDX

    // Read in first block of input data
    MASKED_LOAD( W0,  0, skipInput0)
    MASKED_LOAD( W1,  1, skipInput1)
    MASKED_LOAD( W2,  2, skipInput2)
    MASKED_LOAD( W3,  3, skipInput3)
    MASKED_LOAD( W4,  4, skipInput4)
    MASKED_LOAD( W5,  5, skipInput5)
    MASKED_LOAD( W6,  6, skipInput6)
    MASKED_LOAD( W7,  7, skipInput7)
    MASKED_LOAD( W8,  8, skipInput8)
    MASKED_LOAD( W9,  9, skipInput9)
    MASKED_LOAD(W10, 10, skipInput10)
    MASKED_LOAD(W11, 11, skipInput11)
    MASKED_LOAD(W12, 12, skipInput12)
    MASKED_LOAD(W13, 13, skipInput13)
    MASKED_LOAD(W14, 14, skipInput14)
    MASKED_LOAD(W15, 15, skipInput15)

lloop:
    LEAQ PSHUFFLE_BYTE_FLIP_MASK<>(SB), TBL_P9
    vmovdqu32 TMP2, [TBL]

    // Get first K from table
    MOVQ  table+16(FP), TBL_P9
    vmovdqu32	TMP3, [TBL]

    // Save digests for later addition
    vmovdqu32 [SCRATCH + 64*0], A
    vmovdqu32 [SCRATCH + 64*1], B
    vmovdqu32 [SCRATCH + 64*2], C
    vmovdqu32 [SCRATCH + 64*3], D
    vmovdqu32 [SCRATCH + 64*4], E
    vmovdqu32 [SCRATCH + 64*5], F
    vmovdqu32 [SCRATCH + 64*6], G
    vmovdqu32 [SCRATCH + 64*7], H

    add IDX, 64

    // Transpose input data
    TRANSPOSE16(W0, W1, W2, W3, W4, W5, W6, W7, W8, W9, W10, W11, W12, W13, W14, W15, TMP0, TMP1)

    vpshufb W0, W0, TMP2
    vpshufb W1, W1, TMP2
    vpshufb W2, W2, TMP2
    vpshufb W3, W3, TMP2
    vpshufb W4, W4, TMP2
    vpshufb W5, W5, TMP2
    vpshufb W6, W6, TMP2
    vpshufb W7, W7, TMP2
    vpshufb W8, W8, TMP2
    vpshufb W9, W9, TMP2
    vpshufb W10, W10, TMP2
    vpshufb W11, W11, TMP2
    vpshufb W12, W12, TMP2
    vpshufb W13, W13, TMP2
    vpshufb W14, W14, TMP2
    vpshufb W15, W15, TMP2

    // MSG Schedule for W0-W15 is now complete in registers
    // Process first 48 rounds
    // Calculate next Wt+16 after processing is complete and Wt is unneeded

    PROCESS_LOOP( W0,  0, A, B, C, D, E, F, G, H)
    MSG_SCHED_ROUND_16_63( W0,  W1,  W9, W14)
    PROCESS_LOOP( W1,  1, H, A, B, C, D, E, F, G)
    MSG_SCHED_ROUND_16_63( W1,  W2, W10, W15)
    PROCESS_LOOP( W2,  2, G, H, A, B, C, D, E, F)
    MSG_SCHED_ROUND_16_63( W2,  W3, W11,  W0)
    PROCESS_LOOP( W3,  3, F, G, H, A, B, C, D, E)
    MSG_SCHED_ROUND_16_63( W3,  W4, W12,  W1)
    PROCESS_LOOP( W4,  4, E, F, G, H, A, B, C, D)
    MSG_SCHED_ROUND_16_63( W4,  W5, W13,  W2)
    PROCESS_LOOP( W5,  5, D, E, F, G, H, A, B, C)
    MSG_SCHED_ROUND_16_63( W5,  W6, W14,  W3)
    PROCESS_LOOP( W6,  6, C, D, E, F, G, H, A, B)
    MSG_SCHED_ROUND_16_63( W6,  W7, W15,  W4)
    PROCESS_LOOP( W7,  7, B, C, D, E, F, G, H, A)
    MSG_SCHED_ROUND_16_63( W7,  W8,  W0,  W5)
    PROCESS_LOOP( W8,  8, A, B, C, D, E, F, G, H)
    MSG_SCHED_ROUND_16_63( W8,  W9,  W1,  W6)
    PROCESS_LOOP( W9,  9, H, A, B, C, D, E, F, G)
    MSG_SCHED_ROUND_16_63( W9, W10,  W2,  W7)
    PROCESS_LOOP(W10, 10, G, H, A, B, C, D, E, F)
    MSG_SCHED_ROUND_16_63(W10, W11,  W3,  W8)
    PROCESS_LOOP(W11, 11, F, G, H, A, B, C, D, E)
    MSG_SCHED_ROUND_16_63(W11, W12,  W4,  W9)
    PROCESS_LOOP(W12, 12, E, F, G, H, A, B, C, D)
    MSG_SCHED_ROUND_16_63(W12, W13,  W5, W10)
    PROCESS_LOOP(W13, 13, D, E, F, G, H, A, B, C)
    MSG_SCHED_ROUND_16_63(W13, W14,  W6, W11)
    PROCESS_LOOP(W14, 14, C, D, E, F, G, H, A, B)
    MSG_SCHED_ROUND_16_63(W14, W15,  W7, W12)
    PROCESS_LOOP(W15, 15, B, C, D, E, F, G, H, A)
    MSG_SCHED_ROUND_16_63(W15,  W0,  W8, W13)
    PROCESS_LOOP( W0, 16, A, B, C, D, E, F, G, H)
    MSG_SCHED_ROUND_16_63( W0,  W1,  W9, W14)
    PROCESS_LOOP( W1, 17, H, A, B, C, D, E, F, G)
    MSG_SCHED_ROUND_16_63( W1,  W2, W10, W15)
    PROCESS_LOOP( W2, 18, G, H, A, B, C, D, E, F)
    MSG_SCHED_ROUND_16_63( W2,  W3, W11,  W0)
    PROCESS_LOOP( W3, 19, F, G, H, A, B, C, D, E)
    MSG_SCHED_ROUND_16_63( W3,  W4, W12,  W1)
    PROCESS_LOOP( W4, 20, E, F, G, H, A, B, C, D)
    MSG_SCHED_ROUND_16_63( W4,  W5, W13,  W2)
    PROCESS_LOOP( W5, 21, D, E, F, G, H, A, B, C)
    MSG_SCHED_ROUND_16_63( W5,  W6, W14,  W3)
    PROCESS_LOOP( W6, 22, C, D, E, F, G, H, A, B)
    MSG_SCHED_ROUND_16_63( W6,  W7, W15,  W4)
    PROCESS_LOOP( W7, 23, B, C, D, E, F, G, H, A)
    MSG_SCHED_ROUND_16_63( W7,  W8,  W0,  W5)
    PROCESS_LOOP( W8, 24, A, B, C, D, E, F, G, H)
    MSG_SCHED_ROUND_16_63( W8,  W9,  W1,  W6)
    PROCESS_LOOP( W9, 25, H, A, B, C, D, E, F, G)
    MSG_SCHED_ROUND_16_63( W9, W10,  W2,  W7)
    PROCESS_LOOP(W10, 26, G, H, A, B, C, D, E, F)
    MSG_SCHED_ROUND_16_63(W10, W11,  W3,  W8)
    PROCESS_LOOP(W11, 27, F, G, H, A, B, C, D, E)
    MSG_SCHED_ROUND_16_63(W11, W12,  W4,  W9)
    PROCESS_LOOP(W12, 28, E, F, G, H, A, B, C, D)
    MSG_SCHED_ROUND_16_63(W12, W13,  W5, W10)
    PROCESS_LOOP(W13, 29, D, E, F, G, H, A, B, C)
    MSG_SCHED_ROUND_16_63(W13, W14,  W6, W11)
    PROCESS_LOOP(W14, 30, C, D, E, F, G, H, A, B)
    MSG_SCHED_ROUND_16_63(W14, W15,  W7, W12)
    PROCESS_LOOP(W15, 31, B, C, D, E, F, G, H, A)
    MSG_SCHED_ROUND_16_63(W15,  W0,  W8, W13)
    PROCESS_LOOP( W0, 32, A, B, C, D, E, F, G, H)
    MSG_SCHED_ROUND_16_63( W0,  W1,  W9, W14)
    PROCESS_LOOP( W1, 33, H, A, B, C, D, E, F, G)
    MSG_SCHED_ROUND_16_63( W1,  W2, W10, W15)
    PROCESS_LOOP( W2, 34, G, H, A, B, C, D, E, F)
    MSG_SCHED_ROUND_16_63( W2,  W3, W11,  W0)
    PROCESS_LOOP( W3, 35, F, G, H, A, B, C, D, E)
    MSG_SCHED_ROUND_16_63( W3,  W4, W12,  W1)
    PROCESS_LOOP( W4, 36, E, F, G, H, A, B, C, D)
    MSG_SCHED_ROUND_16_63( W4,  W5, W13,  W2)
    PROCESS_LOOP( W5, 37, D, E, F, G, H, A, B, C)
    MSG_SCHED_ROUND_16_63( W5,  W6, W14,  W3)
    PROCESS_LOOP( W6, 38, C, D, E, F, G, H, A, B)
    MSG_SCHED_ROUND_16_63( W6,  W7, W15,  W4)
    PROCESS_LOOP( W7, 39, B, C, D, E, F, G, H, A)
    MSG_SCHED_ROUND_16_63( W7,  W8,  W0,  W5)
    PROCESS_LOOP( W8, 40, A, B, C, D, E, F, G, H)
    MSG_SCHED_ROUND_16_63( W8,  W9,  W1,  W6)
    PROCESS_LOOP( W9, 41, H, A, B, C, D, E, F, G)
    MSG_SCHED_ROUND_16_63( W9, W10,  W2,  W7)
    PROCESS_LOOP(W10, 42, G, H, A, B, C, D, E, F)
    MSG_SCHED_ROUND_16_63(W10, W11,  W3,  W8)
    PROCESS_LOOP(W11, 43, F, G, H, A, B, C, D, E)
    MSG_SCHED_ROUND_16_63(W11, W12,  W4,  W9)
    PROCESS_LOOP(W12, 44, E, F, G, H, A, B, C, D)
    MSG_SCHED_ROUND_16_63(W12, W13,  W5, W10)
    PROCESS_LOOP(W13, 45, D, E, F, G, H, A, B, C)
    MSG_SCHED_ROUND_16_63(W13, W14,  W6, W11)
    PROCESS_LOOP(W14, 46, C, D, E, F, G, H, A, B)
    MSG_SCHED_ROUND_16_63(W14, W15,  W7, W12)
    PROCESS_LOOP(W15, 47, B, C, D, E, F, G, H, A)
    MSG_SCHED_ROUND_16_63(W15,  W0,  W8, W13)

    // Check if this is the last block
    sub INP_SIZE, 1
    JE  lastLoop

    // Load next mask for inputs
    ADDQ $8, MASKP_P9
    MOVQ (MASKP_P9), MASK_P9

    // Process last 16 rounds
    // Read in next block msg data for use in first 16 words of msg sched

    PROCESS_LOOP( W0, 48, A, B, C, D, E, F, G, H)
    MSG_SCHED_ROUND_00_15( W0,  0, skipNext0)
    PROCESS_LOOP( W1, 49, H, A, B, C, D, E, F, G)
    MSG_SCHED_ROUND_00_15( W1,  1, skipNext1)
    PROCESS_LOOP( W2, 50, G, H, A, B, C, D, E, F)
    MSG_SCHED_ROUND_00_15( W2,  2, skipNext2)
    PROCESS_LOOP( W3, 51, F, G, H, A, B, C, D, E)
    MSG_SCHED_ROUND_00_15( W3,  3, skipNext3)
    PROCESS_LOOP( W4, 52, E, F, G, H, A, B, C, D)
    MSG_SCHED_ROUND_00_15( W4,  4, skipNext4)
    PROCESS_LOOP( W5, 53, D, E, F, G, H, A, B, C)
    MSG_SCHED_ROUND_00_15( W5,  5, skipNext5)
    PROCESS_LOOP( W6, 54, C, D, E, F, G, H, A, B)
    MSG_SCHED_ROUND_00_15( W6,  6, skipNext6)
    PROCESS_LOOP( W7, 55, B, C, D, E, F, G, H, A)
    MSG_SCHED_ROUND_00_15( W7,  7, skipNext7)
    PROCESS_LOOP( W8, 56, A, B, C, D, E, F, G, H)
    MSG_SCHED_ROUND_00_15( W8,  8, skipNext8)
    PROCESS_LOOP( W9, 57, H, A, B, C, D, E, F, G)
    MSG_SCHED_ROUND_00_15( W9,  9, skipNext9)
    PROCESS_LOOP(W10, 58, G, H, A, B, C, D, E, F)
    MSG_SCHED_ROUND_00_15(W10, 10, skipNext10)
    PROCESS_LOOP(W11, 59, F, G, H, A, B, C, D, E)
    MSG_SCHED_ROUND_00_15(W11, 11, skipNext11)
    PROCESS_LOOP(W12, 60, E, F, G, H, A, B, C, D)
    MSG_SCHED_ROUND_00_15(W12, 12, skipNext12)
    PROCESS_LOOP(W13, 61, D, E, F, G, H, A, B, C)
    MSG_SCHED_ROUND_00_15(W13, 13, skipNext13)
    PROCESS_LOOP(W14, 62, C, D, E, F, G, H, A, B)
    MSG_SCHED_ROUND_00_15(W14, 14, skipNext14)
    PROCESS_LOOP(W15, 63, B, C, D, E, F, G, H, A)
    MSG_SCHED_ROUND_00_15(W15, 15, skipNext15)

    // Add old digest
    vmovdqu32  TMP2, A
    vmovdqu32 A, [SCRATCH + 64*0]
    vpaddd A{k1}, A, TMP2
    vmovdqu32  TMP2, B
    vmovdqu32 B, [SCRATCH + 64*1]
    vpaddd B{k1}, B, TMP2
    vmovdqu32  TMP2, C
    vmovdqu32 C, [SCRATCH + 64*2]
    vpaddd C{k1}, C, TMP2
    vmovdqu32  TMP2, D
    vmovdqu32 D, [SCRATCH + 64*3]
    vpaddd D{k1}, D, TMP2
    vmovdqu32  TMP2, E
    vmovdqu32 E, [SCRATCH + 64*4]
    vpaddd E{k1}, E, TMP2
    vmovdqu32  TMP2, F
    vmovdqu32 F, [SCRATCH + 64*5]
    vpaddd F{k1}, F, TMP2
    vmovdqu32  TMP2, G
    vmovdqu32 G, [SCRATCH + 64*6]
    vpaddd G{k1}, G, TMP2
    vmovdqu32  TMP2, H
    vmovdqu32 H, [SCRATCH + 64*7]
    vpaddd H{k1}, H, TMP2

    kmovq k1, mask
    JMP lloop

lastLoop:
    // Process last 16 rounds
    PROCESS_LOOP( W0, 48, A, B, C, D, E, F, G, H)
    PROCESS_LOOP( W1, 49, H, A, B, C, D, E, F, G)
    PROCESS_LOOP( W2, 50, G, H, A, B, C, D, E, F)
    PROCESS_LOOP( W3, 51, F, G, H, A, B, C, D, E)
    PROCESS_LOOP( W4, 52, E, F, G, H, A, B, C, D)
    PROCESS_LOOP( W5, 53, D, E, F, G, H, A, B, C)
    PROCESS_LOOP( W6, 54, C, D, E, F, G, H, A, B)
    PROCESS_LOOP( W7, 55, B, C, D, E, F, G, H, A)
    PROCESS_LOOP( W8, 56, A, B, C, D, E, F, G, H)
    PROCESS_LOOP( W9, 57, H, A, B, C, D, E, F, G)
    PROCESS_LOOP(W10, 58, G, H, A, B, C, D, E, F)
    PROCESS_LOOP(W11, 59, F, G, H, A, B, C, D, E)
    PROCESS_LOOP(W12, 60, E, F, G, H, A, B, C, D)
    PROCESS_LOOP(W13, 61, D, E, F, G, H, A, B, C)
    PROCESS_LOOP(W14, 62, C, D, E, F, G, H, A, B)
    PROCESS_LOOP(W15, 63, B, C, D, E, F, G, H, A)

    // Add old digest
    vmovdqu32  TMP2, A
    vmovdqu32 A, [SCRATCH + 64*0]
    vpaddd A{k1}, A, TMP2
    vmovdqu32  TMP2, B
    vmovdqu32 B, [SCRATCH + 64*1]
    vpaddd B{k1}, B, TMP2
    vmovdqu32  TMP2, C
    vmovdqu32 C, [SCRATCH + 64*2]
    vpaddd C{k1}, C, TMP2
    vmovdqu32  TMP2, D
    vmovdqu32 D, [SCRATCH + 64*3]
    vpaddd D{k1}, D, TMP2
    vmovdqu32  TMP2, E
    vmovdqu32 E, [SCRATCH + 64*4]
    vpaddd E{k1}, E, TMP2
    vmovdqu32  TMP2, F
    vmovdqu32 F, [SCRATCH + 64*5]
    vpaddd F{k1}, F, TMP2
    vmovdqu32  TMP2, G
    vmovdqu32 G, [SCRATCH + 64*6]
    vpaddd G{k1}, G, TMP2
    vmovdqu32  TMP2, H
    vmovdqu32 H, [SCRATCH + 64*7]
    vpaddd H{k1}, H, TMP2

    // Write out digest
    vmovdqu32 [STATE + 0*SHA256_DIGEST_ROW_SIZE], A
    vmovdqu32 [STATE + 1*SHA256_DIGEST_ROW_SIZE], B
    vmovdqu32 [STATE + 2*SHA256_DIGEST_ROW_SIZE], C
    vmovdqu32 [STATE + 3*SHA256_DIGEST_ROW_SIZE], D
    vmovdqu32 [STATE + 4*SHA256_DIGEST_ROW_SIZE], E
    vmovdqu32 [STATE + 5*SHA256_DIGEST_ROW_SIZE], F
    vmovdqu32 [STATE + 6*SHA256_DIGEST_ROW_SIZE], G
    vmovdqu32 [STATE + 7*SHA256_DIGEST_ROW_SIZE], H

    VZEROUPPER
    RET

//
// Tables
//

DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x000(SB)/8, $0x0405060700010203
DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x008(SB)/8, $0x0c0d0e0f08090a0b
DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x010(SB)/8, $0x0405060700010203
DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x018(SB)/8, $0x0c0d0e0f08090a0b
DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x020(SB)/8, $0x0405060700010203
DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x028(SB)/8, $0x0c0d0e0f08090a0b
DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x030(SB)/8, $0x0405060700010203
DATA PSHUFFLE_BYTE_FLIP_MASK<>+0x038(SB)/8, $0x0c0d0e0f08090a0b
GLOBL PSHUFFLE_BYTE_FLIP_MASK<>(SB), 8, $64

DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x000(SB)/8, $0x0000000000000000
DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x008(SB)/8, $0x0000000000000001
DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x010(SB)/8, $0x0000000000000008
DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x018(SB)/8, $0x0000000000000009
DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x020(SB)/8, $0x0000000000000004
DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x028(SB)/8, $0x0000000000000005
DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x030(SB)/8, $0x000000000000000C
DATA PSHUFFLE_TRANSPOSE16_MASK1<>+0x038(SB)/8, $0x000000000000000D
GLOBL PSHUFFLE_TRANSPOSE16_MASK1<>(SB), 8, $64

DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x000(SB)/8, $0x0000000000000002
DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x008(SB)/8, $0x0000000000000003
DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x010(SB)/8, $0x000000000000000A
DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x018(SB)/8, $0x000000000000000B
DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x020(SB)/8, $0x0000000000000006
DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x028(SB)/8, $0x0000000000000007
DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x030(SB)/8, $0x000000000000000E
DATA PSHUFFLE_TRANSPOSE16_MASK2<>+0x038(SB)/8, $0x000000000000000F
GLOBL PSHUFFLE_TRANSPOSE16_MASK2<>(SB), 8, $64