about summary refs log tree commit diff
path: root/vendor/golang.org/x/crypto/blake2b/blake2b_amd64.s
blob: adfac00c15c9c5f89160f1a6526bb63d26a74091 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build amd64 && gc && !purego

#include "textflag.h"

DATA ·iv0<>+0x00(SB)/8, $0x6a09e667f3bcc908
DATA ·iv0<>+0x08(SB)/8, $0xbb67ae8584caa73b
GLOBL ·iv0<>(SB), (NOPTR+RODATA), $16

DATA ·iv1<>+0x00(SB)/8, $0x3c6ef372fe94f82b
DATA ·iv1<>+0x08(SB)/8, $0xa54ff53a5f1d36f1
GLOBL ·iv1<>(SB), (NOPTR+RODATA), $16

DATA ·iv2<>+0x00(SB)/8, $0x510e527fade682d1
DATA ·iv2<>+0x08(SB)/8, $0x9b05688c2b3e6c1f
GLOBL ·iv2<>(SB), (NOPTR+RODATA), $16

DATA ·iv3<>+0x00(SB)/8, $0x1f83d9abfb41bd6b
DATA ·iv3<>+0x08(SB)/8, $0x5be0cd19137e2179
GLOBL ·iv3<>(SB), (NOPTR+RODATA), $16

DATA ·c40<>+0x00(SB)/8, $0x0201000706050403
DATA ·c40<>+0x08(SB)/8, $0x0a09080f0e0d0c0b
GLOBL ·c40<>(SB), (NOPTR+RODATA), $16

DATA ·c48<>+0x00(SB)/8, $0x0100070605040302
DATA ·c48<>+0x08(SB)/8, $0x09080f0e0d0c0b0a
GLOBL ·c48<>(SB), (NOPTR+RODATA), $16

#define SHUFFLE(v2, v3, v4, v5, v6, v7, t1, t2) \
	MOVO       v4, t1; \
	MOVO       v5, v4; \
	MOVO       t1, v5; \
	MOVO       v6, t1; \
	PUNPCKLQDQ v6, t2; \
	PUNPCKHQDQ v7, v6; \
	PUNPCKHQDQ t2, v6; \
	PUNPCKLQDQ v7, t2; \
	MOVO       t1, v7; \
	MOVO       v2, t1; \
	PUNPCKHQDQ t2, v7; \
	PUNPCKLQDQ v3, t2; \
	PUNPCKHQDQ t2, v2; \
	PUNPCKLQDQ t1, t2; \
	PUNPCKHQDQ t2, v3

#define SHUFFLE_INV(v2, v3, v4, v5, v6, v7, t1, t2) \
	MOVO       v4, t1; \
	MOVO       v5, v4; \
	MOVO       t1, v5; \
	MOVO       v2, t1; \
	PUNPCKLQDQ v2, t2; \
	PUNPCKHQDQ v3, v2; \
	PUNPCKHQDQ t2, v2; \
	PUNPCKLQDQ v3, t2; \
	MOVO       t1, v3; \
	MOVO       v6, t1; \
	PUNPCKHQDQ t2, v3; \
	PUNPCKLQDQ v7, t2; \
	PUNPCKHQDQ t2, v6; \
	PUNPCKLQDQ t1, t2; \
	PUNPCKHQDQ t2, v7

#define HALF_ROUND(v0, v1, v2, v3, v4, v5, v6, v7, m0, m1, m2, m3, t0, c40, c48) \
	PADDQ  m0, v0;        \
	PADDQ  m1, v1;        \
	PADDQ  v2, v0;        \
	PADDQ  v3, v1;        \
	PXOR   v0, v6;        \
	PXOR   v1, v7;        \
	PSHUFD $0xB1, v6, v6; \
	PSHUFD $0xB1, v7, v7; \
	PADDQ  v6, v4;        \
	PADDQ  v7, v5;        \
	PXOR   v4, v2;        \
	PXOR   v5, v3;        \
	PSHUFB c40, v2;       \
	PSHUFB c40, v3;       \
	PADDQ  m2, v0;        \
	PADDQ  m3, v1;        \
	PADDQ  v2, v0;        \
	PADDQ  v3, v1;        \
	PXOR   v0, v6;        \
	PXOR   v1, v7;        \
	PSHUFB c48, v6;       \
	PSHUFB c48, v7;       \
	PADDQ  v6, v4;        \
	PADDQ  v7, v5;        \
	PXOR   v4, v2;        \
	PXOR   v5, v3;        \
	MOVOU  v2, t0;        \
	PADDQ  v2, t0;        \
	PSRLQ  $63, v2;       \
	PXOR   t0, v2;        \
	MOVOU  v3, t0;        \
	PADDQ  v3, t0;        \
	PSRLQ  $63, v3;       \
	PXOR   t0, v3

#define LOAD_MSG(m0, m1, m2, m3, src, i0, i1, i2, i3, i4, i5, i6, i7) \
	MOVQ   i0*8(src), m0;     \
	PINSRQ $1, i1*8(src), m0; \
	MOVQ   i2*8(src), m1;     \
	PINSRQ $1, i3*8(src), m1; \
	MOVQ   i4*8(src), m2;     \
	PINSRQ $1, i5*8(src), m2; \
	MOVQ   i6*8(src), m3;     \
	PINSRQ $1, i7*8(src), m3

// func hashBlocksSSE4(h *[8]uint64, c *[2]uint64, flag uint64, blocks []byte)
TEXT ·hashBlocksSSE4(SB), 4, $288-48 // frame size = 272 + 16 byte alignment
	MOVQ h+0(FP), AX
	MOVQ c+8(FP), BX
	MOVQ flag+16(FP), CX
	MOVQ blocks_base+24(FP), SI
	MOVQ blocks_len+32(FP), DI

	MOVQ SP, R10
	ADDQ $15, R10
	ANDQ $~15, R10

	MOVOU ·iv3<>(SB), X0
	MOVO  X0, 0(R10)
	XORQ  CX, 0(R10)     // 0(R10) = ·iv3 ^ (CX || 0)

	MOVOU ·c40<>(SB), X13
	MOVOU ·c48<>(SB), X14

	MOVOU 0(AX), X12
	MOVOU 16(AX), X15

	MOVQ 0(BX), R8
	MOVQ 8(BX), R9

loop:
	ADDQ $128, R8
	CMPQ R8, $128
	JGE  noinc
	INCQ R9

noinc:
	MOVQ R8, X8
	PINSRQ $1, R9, X8

	MOVO X12, X0
	MOVO X15, X1
	MOVOU 32(AX), X2
	MOVOU 48(AX), X3
	MOVOU ·iv0<>(SB), X4
	MOVOU ·iv1<>(SB), X5
	MOVOU ·iv2<>(SB), X6

	PXOR X8, X6
	MOVO 0(R10), X7

	LOAD_MSG(X8, X9, X10, X11, SI, 0, 2, 4, 6, 1, 3, 5, 7)
	MOVO X8, 16(R10)
	MOVO X9, 32(R10)
	MOVO X10, 48(R10)
	MOVO X11, 64(R10)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
	LOAD_MSG(X8, X9, X10, X11, SI, 8, 10, 12, 14, 9, 11, 13, 15)
	MOVO X8, 80(R10)
	MOVO X9, 96(R10)
	MOVO X10, 112(R10)
	MOVO X11, 128(R10)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)

	LOAD_MSG(X8, X9, X10, X11, SI, 14, 4, 9, 13, 10, 8, 15, 6)
	MOVO X8, 144(R10)
	MOVO X9, 160(R10)
	MOVO X10, 176(R10)
	MOVO X11, 192(R10)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
	LOAD_MSG(X8, X9, X10, X11, SI, 1, 0, 11, 5, 12, 2, 7, 3)
	MOVO X8, 208(R10)
	MOVO X9, 224(R10)
	MOVO X10, 240(R10)
	MOVO X11, 256(R10)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)

	LOAD_MSG(X8, X9, X10, X11, SI, 11, 12, 5, 15, 8, 0, 2, 13)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
	LOAD_MSG(X8, X9, X10, X11, SI, 10, 3, 7, 9, 14, 6, 1, 4)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)

	LOAD_MSG(X8, X9, X10, X11, SI, 7, 3, 13, 11, 9, 1, 12, 14)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
	LOAD_MSG(X8, X9, X10, X11, SI, 2, 5, 4, 15, 6, 10, 0, 8)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)

	LOAD_MSG(X8, X9, X10, X11, SI, 9, 5, 2, 10, 0, 7, 4, 15)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
	LOAD_MSG(X8, X9, X10, X11, SI, 14, 11, 6, 3, 1, 12, 8, 13)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)

	LOAD_MSG(X8, X9, X10, X11, SI, 2, 6, 0, 8, 12, 10, 11, 3)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
	LOAD_MSG(X8, X9, X10, X11, SI, 4, 7, 15, 1, 13, 5, 14, 9)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)

	LOAD_MSG(X8, X9, X10, X11, SI, 12, 1, 14, 4, 5, 15, 13, 10)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
	LOAD_MSG(X8, X9, X10, X11, SI, 0, 6, 9, 8, 7, 3, 2, 11)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)

	LOAD_MSG(X8, X9, X10, X11, SI, 13, 7, 12, 3, 11, 14, 1, 9)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
	LOAD_MSG(X8, X9, X10, X11, SI, 5, 15, 8, 2, 0, 4, 6, 10)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)

	LOAD_MSG(X8, X9, X10, X11, SI, 6, 14, 11, 0, 15, 9, 3, 8)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
	LOAD_MSG(X8, X9, X10, X11, SI, 12, 13, 1, 10, 2, 7, 4, 5)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)

	LOAD_MSG(X8, X9, X10, X11, SI, 10, 8, 7, 1, 2, 4, 6, 5)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
	LOAD_MSG(X8, X9, X10, X11, SI, 15, 9, 3, 13, 11, 14, 12, 0)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, X8, X9, X10, X11, X11, X13, X14)
	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)

	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, 16(R10), 32(R10), 48(R10), 64(R10), X11, X13, X14)
	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, 80(R10), 96(R10), 112(R10), 128(R10), X11, X13, X14)
	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)

	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, 144(R10), 160(R10), 176(R10), 192(R10), X11, X13, X14)
	SHUFFLE(X2, X3, X4, X5, X6, X7, X8, X9)
	HALF_ROUND(X0, X1, X2, X3, X4, X5, X6, X7, 208(R10), 224(R10), 240(R10), 256(R10), X11, X13, X14)
	SHUFFLE_INV(X2, X3, X4, X5, X6, X7, X8, X9)

	MOVOU 32(AX), X10
	MOVOU 48(AX), X11
	PXOR  X0, X12
	PXOR  X1, X15
	PXOR  X2, X10
	PXOR  X3, X11
	PXOR  X4, X12
	PXOR  X5, X15
	PXOR  X6, X10
	PXOR  X7, X11
	MOVOU X10, 32(AX)
	MOVOU X11, 48(AX)

	LEAQ 128(SI), SI
	SUBQ $128, DI
	JNE  loop

	MOVOU X12, 0(AX)
	MOVOU X15, 16(AX)

	MOVQ R8, 0(BX)
	MOVQ R9, 8(BX)

	RET