/* x86 */
#if defined(X86ASM_SSSE3) && (!defined(SCRYPT_CHOOSE_COMPILETIME) || !defined(SCRYPT_CHACHA_INCLUDED)) && !defined(CPU_X86_FORCE_INTRINSICS)

#define SCRYPT_CHACHA_SSSE3

asm_naked_fn_proto(void, scrypt_ChunkMix_ssse3)(uint32_t *Bout/*[chunkBytes]*/, uint32_t *Bin/*[chunkBytes]*/, uint32_t *Bxor/*[chunkBytes]*/, uint32_t r)
asm_naked_fn(scrypt_ChunkMix_ssse3)
	a1(push ebx)
	a1(push edi)
	a1(push esi)
	a1(push ebp)
	a2(mov ebp,esp)
	a2(mov edi,[ebp+20])
	a2(mov esi,[ebp+24])
	a2(mov eax,[ebp+28])
	a2(mov ebx,[ebp+32])
	a2(sub esp,64)
	a2(and esp,~63)
	a2(lea edx,[ebx*2])
	a2(shl edx,6)
	a2(lea ecx,[edx-64])
	a2(and eax, eax)
	a2(mov ebx, 0x01000302)
	a2(movd xmm4, ebx)
	a2(mov ebx, 0x05040706)
	a2(movd xmm0, ebx)
	a2(mov ebx, 0x09080b0a)
	a2(movd xmm1, ebx)
	a2(mov ebx, 0x0d0c0f0e)
	a2(movd xmm2, ebx)
	a2(mov ebx, 0x02010003)
	a2(movd xmm5, ebx)
	a2(mov ebx, 0x06050407)
	a2(movd xmm3, ebx)
	a2(mov ebx, 0x0a09080b)
	a2(movd xmm6, ebx)
	a2(mov ebx, 0x0e0d0c0f)
	a2(movd xmm7, ebx)
	a2(punpckldq xmm4, xmm0)
	a2(punpckldq xmm5, xmm3)
	a2(punpckldq xmm1, xmm2)
	a2(punpckldq xmm6, xmm7)
	a2(punpcklqdq xmm4, xmm1)
	a2(punpcklqdq xmm5, xmm6)
	a2(movdqa xmm0,[ecx+esi+0])
	a2(movdqa xmm1,[ecx+esi+16])
	a2(movdqa xmm2,[ecx+esi+32])
	a2(movdqa xmm3,[ecx+esi+48])
	aj(jz scrypt_ChunkMix_ssse3_no_xor1)
	a2(pxor xmm0,[ecx+eax+0])
	a2(pxor xmm1,[ecx+eax+16])
	a2(pxor xmm2,[ecx+eax+32])
	a2(pxor xmm3,[ecx+eax+48])
	a1(scrypt_ChunkMix_ssse3_no_xor1:)
	a2(xor ecx,ecx)
	a2(xor ebx,ebx)
	a1(scrypt_ChunkMix_ssse3_loop:)
		a2(and eax, eax)
		a2(pxor xmm0,[esi+ecx+0])
		a2(pxor xmm1,[esi+ecx+16])
		a2(pxor xmm2,[esi+ecx+32])
		a2(pxor xmm3,[esi+ecx+48])
		aj(jz scrypt_ChunkMix_ssse3_no_xor2)
		a2(pxor xmm0,[eax+ecx+0])
		a2(pxor xmm1,[eax+ecx+16])
		a2(pxor xmm2,[eax+ecx+32])
		a2(pxor xmm3,[eax+ecx+48])
		a1(scrypt_ChunkMix_ssse3_no_xor2:)
		a2(movdqa [esp+0],xmm0)
		a2(movdqa [esp+16],xmm1)
		a2(movdqa [esp+32],xmm2)
		a2(movdqa xmm7,xmm3)
		a2(mov eax,8)
		a1(scrypt_chacha_ssse3_loop: )
			a2(paddd xmm0,xmm1)
			a2(pxor  xmm3,xmm0)
			a2(pshufb xmm3,xmm4)
			a2(paddd xmm2,xmm3)
			a2(pxor  xmm1,xmm2)
			a2(movdqa xmm6,xmm1)
			a2(pslld xmm1,12)
			a2(psrld xmm6,20)
			a2(pxor  xmm1,xmm6)
			a2(paddd xmm0,xmm1)
			a2(pxor  xmm3,xmm0)
			a2(pshufb xmm3,xmm5)
			a3(pshufd xmm0,xmm0,0x93)
			a2(paddd xmm2,xmm3)
			a3(pshufd xmm3,xmm3,0x4e)
			a2(pxor  xmm1,xmm2)
			a3(pshufd xmm2,xmm2,0x39)
			a2(movdqa xmm6,xmm1)
			a2(pslld xmm1,7)
			a2(psrld xmm6,25)
			a2(pxor  xmm1,xmm6)
			a2(sub eax,2)
			a2(paddd xmm0,xmm1)
			a2(pxor  xmm3,xmm0)
			a2(pshufb xmm3,xmm4)
			a2(paddd xmm2,xmm3)
			a2(pxor  xmm1,xmm2)
			a2(movdqa xmm6,xmm1)
			a2(pslld xmm1,12)
			a2(psrld xmm6,20)
			a2(pxor  xmm1,xmm6)
			a2(paddd xmm0,xmm1)
			a2(pxor  xmm3,xmm0)
			a2(pshufb xmm3,xmm5)
			a3(pshufd xmm0,xmm0,0x39)
			a2(paddd xmm2,xmm3)
			a3(pshufd xmm3,xmm3,0x4e)
			a2(pxor  xmm1,xmm2)
			a3(pshufd xmm2,xmm2,0x93)
			a2(movdqa xmm6,xmm1)
			a2(pslld xmm1,7)
			a2(psrld xmm6,25)
			a2(pxor  xmm1,xmm6)
			aj(ja scrypt_chacha_ssse3_loop)
		a2(paddd xmm0,[esp+0])
		a2(paddd xmm1,[esp+16])
		a2(paddd xmm2,[esp+32])
		a2(paddd xmm3,xmm7)
		a2(lea eax,[ebx+ecx])
		a2(xor ebx,edx)
		a2(and eax,~0x7f)
		a2(add ecx,64)
		a2(shr eax,1)
		a2(add eax, edi)
		a2(cmp ecx,edx)
		a2(movdqa [eax+0],xmm0)
		a2(movdqa [eax+16],xmm1)
		a2(movdqa [eax+32],xmm2)
		a2(movdqa [eax+48],xmm3)
		a2(mov eax,[ebp+28])
		aj(jne scrypt_ChunkMix_ssse3_loop)
	a2(mov esp,ebp)
	a1(pop ebp)
	a1(pop esi)
	a1(pop edi)
	a1(pop ebx)
	aret(16)
asm_naked_fn_end(scrypt_ChunkMix_ssse3)

#endif



/* x64 */
#if defined(X86_64ASM_SSSE3) && (!defined(SCRYPT_CHOOSE_COMPILETIME) || !defined(SCRYPT_CHACHA_INCLUDED)) && !defined(CPU_X86_FORCE_INTRINSICS)

#define SCRYPT_CHACHA_SSSE3

asm_naked_fn_proto(void, scrypt_ChunkMix_ssse3)(uint32_t *Bout/*[chunkBytes]*/, uint32_t *Bin/*[chunkBytes]*/, uint32_t *Bxor/*[chunkBytes]*/, uint32_t r)
asm_naked_fn(scrypt_ChunkMix_ssse3)
	a2(lea rcx,[ecx*2]) /* zero extend uint32_t by using ecx, win64 can leave garbage in the top half */
	a2(shl rcx,6)
	a2(lea r9,[rcx-64])
	a2(lea rax,[rsi+r9])
	a2(lea r9,[rdx+r9])
	a2(and rdx, rdx)
	a2(movdqa xmm0,[rax+0])
	a2(movdqa xmm1,[rax+16])
	a2(movdqa xmm2,[rax+32])
	a2(movdqa xmm3,[rax+48])
	a2(mov r8, 0x0504070601000302)
	a2(mov rax, 0x0d0c0f0e09080b0a)
	a2(movd xmm4, r8)
	a2(movd xmm6, rax)
	a2(mov r8, 0x0605040702010003)
	a2(mov rax, 0x0e0d0c0f0a09080b)
	a2(movd xmm5, r8)
	a2(movd xmm7, rax)
	a2(punpcklqdq xmm4, xmm6)
	a2(punpcklqdq xmm5, xmm7)
	aj(jz scrypt_ChunkMix_ssse3_no_xor1)
	a2(pxor xmm0,[r9+0])
	a2(pxor xmm1,[r9+16])
	a2(pxor xmm2,[r9+32])
	a2(pxor xmm3,[r9+48])
	a1(scrypt_ChunkMix_ssse3_no_xor1:)
	a2(xor r8,r8)
	a2(xor r9,r9)
	a1(scrypt_ChunkMix_ssse3_loop:)
		a2(and rdx, rdx)
		a2(pxor xmm0,[rsi+r9+0])
		a2(pxor xmm1,[rsi+r9+16])
		a2(pxor xmm2,[rsi+r9+32])
		a2(pxor xmm3,[rsi+r9+48])
		aj(jz scrypt_ChunkMix_ssse3_no_xor2)
		a2(pxor xmm0,[rdx+r9+0])
		a2(pxor xmm1,[rdx+r9+16])
		a2(pxor xmm2,[rdx+r9+32])
		a2(pxor xmm3,[rdx+r9+48])
		a1(scrypt_ChunkMix_ssse3_no_xor2:)
		a2(movdqa xmm8,xmm0)
		a2(movdqa xmm9,xmm1)
		a2(movdqa xmm10,xmm2)
		a2(movdqa xmm11,xmm3)
		a2(mov rax,8)
		a1(scrypt_chacha_ssse3_loop: )
			a2(paddd xmm0,xmm1)
			a2(pxor  xmm3,xmm0)
			a2(pshufb xmm3,xmm4)
			a2(paddd xmm2,xmm3)
			a2(pxor  xmm1,xmm2)
			a2(movdqa xmm12,xmm1)
			a2(pslld xmm1,12)
			a2(psrld xmm12,20)
			a2(pxor  xmm1,xmm12)
			a2(paddd xmm0,xmm1)
			a2(pxor  xmm3,xmm0)
			a2(pshufb xmm3,xmm5)
			a3(pshufd xmm0,xmm0,0x93)
			a2(paddd xmm2,xmm3)
			a3(pshufd xmm3,xmm3,0x4e)
			a2(pxor  xmm1,xmm2)
			a3(pshufd xmm2,xmm2,0x39)
			a2(movdqa xmm12,xmm1)
			a2(pslld xmm1,7)
			a2(psrld xmm12,25)
			a2(pxor  xmm1,xmm12)
			a2(sub rax,2)
			a2(paddd xmm0,xmm1)
			a2(pxor  xmm3,xmm0)
			a2(pshufb xmm3,xmm4)
			a2(paddd xmm2,xmm3)
			a2(pxor  xmm1,xmm2)
			a2(movdqa xmm12,xmm1)
			a2(pslld xmm1,12)
			a2(psrld xmm12,20)
			a2(pxor  xmm1,xmm12)
			a2(paddd xmm0,xmm1)
			a2(pxor  xmm3,xmm0)
			a2(pshufb xmm3,xmm5)
			a3(pshufd xmm0,xmm0,0x39)
			a2(paddd xmm2,xmm3)
			a3(pshufd xmm3,xmm3,0x4e)
			a2(pxor  xmm1,xmm2)
			a3(pshufd xmm2,xmm2,0x93)
			a2(movdqa xmm12,xmm1)
			a2(pslld xmm1,7)
			a2(psrld xmm12,25)
			a2(pxor  xmm1,xmm12)
			aj(ja scrypt_chacha_ssse3_loop)
		a2(paddd xmm0,xmm8)
		a2(paddd xmm1,xmm9)
		a2(paddd xmm2,xmm10)
		a2(paddd xmm3,xmm11)
		a2(lea rax,[r8+r9])
		a2(xor r8,rcx)
		a2(and rax,~0x7f)
		a2(add r9,64)
		a2(shr rax,1)
		a2(add rax, rdi)
		a2(cmp r9,rcx)
		a2(movdqa [rax+0],xmm0)
		a2(movdqa [rax+16],xmm1)
		a2(movdqa [rax+32],xmm2)
		a2(movdqa [rax+48],xmm3)
		aj(jne scrypt_ChunkMix_ssse3_loop)
	a1(ret)
asm_naked_fn_end(scrypt_ChunkMix_ssse3)

#endif


/* intrinsic */
#if defined(X86_INTRINSIC_SSSE3) && (!defined(SCRYPT_CHOOSE_COMPILETIME) || !defined(SCRYPT_CHACHA_INCLUDED))

#define SCRYPT_CHACHA_SSSE3

static void NOINLINE asm_calling_convention
scrypt_ChunkMix_ssse3(uint32_t *Bout/*[chunkBytes]*/, uint32_t *Bin/*[chunkBytes]*/, uint32_t *Bxor/*[chunkBytes]*/, uint32_t r) {
	uint32_t i, blocksPerChunk = r * 2, half = 0;
	xmmi *xmmp,x0,x1,x2,x3,x6,t0,t1,t2,t3;
	const xmmi x4 = *(xmmi *)&ssse3_rotl16_32bit, x5 = *(xmmi *)&ssse3_rotl8_32bit;
	size_t rounds;

	/* 1: X = B_{2r - 1} */
	xmmp = (xmmi *)scrypt_block(Bin, blocksPerChunk - 1);
	x0 = xmmp[0];
	x1 = xmmp[1];
	x2 = xmmp[2];
	x3 = xmmp[3];

	if (Bxor) {
		xmmp = (xmmi *)scrypt_block(Bxor, blocksPerChunk - 1);
		x0 = _mm_xor_si128(x0, xmmp[0]);
		x1 = _mm_xor_si128(x1, xmmp[1]);
		x2 = _mm_xor_si128(x2, xmmp[2]);
		x3 = _mm_xor_si128(x3, xmmp[3]);
	}

	/* 2: for i = 0 to 2r - 1 do */
	for (i = 0; i < blocksPerChunk; i++, half ^= r) {
		/* 3: X = H(X ^ B_i) */
		xmmp = (xmmi *)scrypt_block(Bin, i);
		x0 = _mm_xor_si128(x0, xmmp[0]);
		x1 = _mm_xor_si128(x1, xmmp[1]);
		x2 = _mm_xor_si128(x2, xmmp[2]);
		x3 = _mm_xor_si128(x3, xmmp[3]);

		if (Bxor) {
			xmmp = (xmmi *)scrypt_block(Bxor, i);
			x0 = _mm_xor_si128(x0, xmmp[0]);
			x1 = _mm_xor_si128(x1, xmmp[1]);
			x2 = _mm_xor_si128(x2, xmmp[2]);
			x3 = _mm_xor_si128(x3, xmmp[3]);
		}

		t0 = x0;
		t1 = x1;
		t2 = x2;
		t3 = x3;

		for (rounds = 8; rounds; rounds -= 2) {
			x0 = _mm_add_epi32(x0, x1);
			x3 = _mm_xor_si128(x3, x0);
			x3 = _mm_shuffle_epi8(x3, x4);
			x2 = _mm_add_epi32(x2, x3);
			x1 = _mm_xor_si128(x1, x2);
			x6 = x1;
			x1 = _mm_or_si128(_mm_slli_epi32(x1, 12), _mm_srli_epi32(x6, 20));
			x0 = _mm_add_epi32(x0, x1);
			x3 = _mm_xor_si128(x3, x0);
			x3 = _mm_shuffle_epi8(x3, x5);
			x0 = _mm_shuffle_epi32(x0, 0x93);
			x2 = _mm_add_epi32(x2, x3);
			x3 = _mm_shuffle_epi32(x3, 0x4e);
			x1 = _mm_xor_si128(x1, x2);
			x2 = _mm_shuffle_epi32(x2, 0x39);
			x6 = x1;
			x1 = _mm_or_si128(_mm_slli_epi32(x1, 7), _mm_srli_epi32(x6, 25));
			x0 = _mm_add_epi32(x0, x1);
			x3 = _mm_xor_si128(x3, x0);
			x3 = _mm_shuffle_epi8(x3, x4);
			x2 = _mm_add_epi32(x2, x3);
			x1 = _mm_xor_si128(x1, x2);
			x6 = x1;
			x1 = _mm_or_si128(_mm_slli_epi32(x1, 12), _mm_srli_epi32(x6, 20));
			x0 = _mm_add_epi32(x0, x1);
			x3 = _mm_xor_si128(x3, x0);
			x3 = _mm_shuffle_epi8(x3, x5);
			x0 = _mm_shuffle_epi32(x0, 0x39);
			x2 = _mm_add_epi32(x2, x3);
			x3 = _mm_shuffle_epi32(x3, 0x4e);
			x1 = _mm_xor_si128(x1, x2);
			x2 = _mm_shuffle_epi32(x2, 0x93);
			x6 = x1;
			x1 = _mm_or_si128(_mm_slli_epi32(x1, 7), _mm_srli_epi32(x6, 25));
		}

		x0 = _mm_add_epi32(x0, t0);
		x1 = _mm_add_epi32(x1, t1);
		x2 = _mm_add_epi32(x2, t2);
		x3 = _mm_add_epi32(x3, t3);

		/* 4: Y_i = X */
		/* 6: B'[0..r-1] = Y_even */
		/* 6: B'[r..2r-1] = Y_odd */
		xmmp = (xmmi *)scrypt_block(Bout, (i / 2) + half);
		xmmp[0] = x0;
		xmmp[1] = x1;
		xmmp[2] = x2;
		xmmp[3] = x3;
	}
}

#endif

#if defined(SCRYPT_CHACHA_SSSE3)
	#undef SCRYPT_MIX
	#define SCRYPT_MIX "ChaCha/8-SSSE3"
	#undef SCRYPT_CHACHA_INCLUDED
	#define SCRYPT_CHACHA_INCLUDED
#endif
