xref: /freebsd/sys/contrib/openzfs/module/zfs/vdev_raidz_math_aarch64_neon_common.h (revision 271171e0)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or https://opensource.org/licenses/CDDL-1.0.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (C) 2016 Romain Dolbeau. All rights reserved.
 */

#include <sys/types.h>
#include <sys/simd.h>

#ifdef __linux__
#define	__asm __asm__ __volatile__
#endif

#define	_REG_CNT(_0, _1, _2, _3, _4, _5, _6, _7, N, ...) N
#define	REG_CNT(r...) _REG_CNT(r, 8, 7, 6, 5, 4, 3, 2, 1)

#define	VR0_(REG, ...) "%[w"#REG"]"
#define	VR1_(_1, REG, ...) "%[w"#REG"]"
#define	VR2_(_1, _2, REG, ...) "%[w"#REG"]"
#define	VR3_(_1, _2, _3, REG, ...) "%[w"#REG"]"
#define	VR4_(_1, _2, _3, _4, REG, ...) "%[w"#REG"]"
#define	VR5_(_1, _2, _3, _4, _5, REG, ...) "%[w"#REG"]"
#define	VR6_(_1, _2, _3, _4, _5, _6, REG, ...) "%[w"#REG"]"
#define	VR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) "%[w"#REG"]"

/*
 * Here we need registers not used otherwise.
 * They will be used in unused ASM for the case
 * with more registers than required... but GCC
 * will still need to make sure the constraints
 * are correct, and duplicate constraints are illegal
 * ... and we use the "register" number as a name
 */

#define	VR0(r...) VR0_(r)
#define	VR1(r...) VR1_(r)
#define	VR2(r...) VR2_(r, 36)
#define	VR3(r...) VR3_(r, 36, 35)
#define	VR4(r...) VR4_(r, 36, 35, 34, 33)
#define	VR5(r...) VR5_(r, 36, 35, 34, 33, 32)
#define	VR6(r...) VR6_(r, 36, 35, 34, 33, 32, 31)
#define	VR7(r...) VR7_(r, 36, 35, 34, 33, 32, 31, 30)

#define	VR(X) "%[w"#X"]"

#define	RVR0_(REG, ...) [w##REG] "w" (w##REG)
#define	RVR1_(_1, REG, ...) [w##REG] "w" (w##REG)
#define	RVR2_(_1, _2, REG, ...) [w##REG] "w" (w##REG)
#define	RVR3_(_1, _2, _3, REG, ...) [w##REG] "w" (w##REG)
#define	RVR4_(_1, _2, _3, _4, REG, ...) [w##REG] "w" (w##REG)
#define	RVR5_(_1, _2, _3, _4, _5, REG, ...) [w##REG] "w" (w##REG)
#define	RVR6_(_1, _2, _3, _4, _5, _6, REG, ...) [w##REG] "w" (w##REG)
#define	RVR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) [w##REG] "w" (w##REG)

#define	RVR0(r...) RVR0_(r)
#define	RVR1(r...) RVR1_(r)
#define	RVR2(r...) RVR2_(r, 36)
#define	RVR3(r...) RVR3_(r, 36, 35)
#define	RVR4(r...) RVR4_(r, 36, 35, 34, 33)
#define	RVR5(r...) RVR5_(r, 36, 35, 34, 33, 32)
#define	RVR6(r...) RVR6_(r, 36, 35, 34, 33, 32, 31)
#define	RVR7(r...) RVR7_(r, 36, 35, 34, 33, 32, 31, 30)

#define	RVR(X) [w##X] "w" (w##X)

#define	WVR0_(REG, ...) [w##REG] "=w" (w##REG)
#define	WVR1_(_1, REG, ...) [w##REG] "=w" (w##REG)
#define	WVR2_(_1, _2, REG, ...) [w##REG] "=w" (w##REG)
#define	WVR3_(_1, _2, _3, REG, ...) [w##REG] "=w" (w##REG)
#define	WVR4_(_1, _2, _3, _4, REG, ...) [w##REG] "=w" (w##REG)
#define	WVR5_(_1, _2, _3, _4, _5, REG, ...) [w##REG] "=w" (w##REG)
#define	WVR6_(_1, _2, _3, _4, _5, _6, REG, ...) [w##REG] "=w" (w##REG)
#define	WVR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) [w##REG] "=w" (w##REG)

#define	WVR0(r...) WVR0_(r)
#define	WVR1(r...) WVR1_(r)
#define	WVR2(r...) WVR2_(r, 36)
#define	WVR3(r...) WVR3_(r, 36, 35)
#define	WVR4(r...) WVR4_(r, 36, 35, 34, 33)
#define	WVR5(r...) WVR5_(r, 36, 35, 34, 33, 32)
#define	WVR6(r...) WVR6_(r, 36, 35, 34, 33, 32, 31)
#define	WVR7(r...) WVR7_(r, 36, 35, 34, 33, 32, 31, 30)

#define	WVR(X) [w##X] "=w" (w##X)

#define	UVR0_(REG, ...) [w##REG] "+&w" (w##REG)
#define	UVR1_(_1, REG, ...) [w##REG] "+&w" (w##REG)
#define	UVR2_(_1, _2, REG, ...) [w##REG] "+&w" (w##REG)
#define	UVR3_(_1, _2, _3, REG, ...) [w##REG] "+&w" (w##REG)
#define	UVR4_(_1, _2, _3, _4, REG, ...) [w##REG] "+&w" (w##REG)
#define	UVR5_(_1, _2, _3, _4, _5, REG, ...) [w##REG] "+&w" (w##REG)
#define	UVR6_(_1, _2, _3, _4, _5, _6, REG, ...) [w##REG] "+&w" (w##REG)
#define	UVR7_(_1, _2, _3, _4, _5, _6, _7, REG, ...) [w##REG] "+&w" (w##REG)

#define	UVR0(r...) UVR0_(r)
#define	UVR1(r...) UVR1_(r)
#define	UVR2(r...) UVR2_(r, 36)
#define	UVR3(r...) UVR3_(r, 36, 35)
#define	UVR4(r...) UVR4_(r, 36, 35, 34, 33)
#define	UVR5(r...) UVR5_(r, 36, 35, 34, 33, 32)
#define	UVR6(r...) UVR6_(r, 36, 35, 34, 33, 32, 31)
#define	UVR7(r...) UVR7_(r, 36, 35, 34, 33, 32, 31, 30)

#define	UVR(X) [w##X] "+&w" (w##X)

#define	R_01(REG1, REG2, ...) REG1, REG2
#define	_R_23(_0, _1, REG2, REG3, ...) REG2, REG3
#define	R_23(REG...) _R_23(REG, 1, 2, 3)

#define	ZFS_ASM_BUG()	ASSERT(0)

#define	OFFSET(ptr, val)	(((unsigned char *)(ptr))+val)

extern const uint8_t gf_clmul_mod_lt[4*256][16];

#define	ELEM_SIZE 16

typedef struct v {
	uint8_t b[ELEM_SIZE] __attribute__((aligned(ELEM_SIZE)));
} v_t;

#define	XOR_ACC(src, r...)						\
{									\
	switch (REG_CNT(r)) {						\
	case 8:								\
		__asm(							\
		"ld1 { v21.4s },%[SRC0]\n"				\
		"ld1 { v20.4s },%[SRC1]\n"				\
		"ld1 { v19.4s },%[SRC2]\n"				\
		"ld1 { v18.4s },%[SRC3]\n"				\
		"eor " VR0(r) ".16b," VR0(r) ".16b,v21.16b\n"		\
		"eor " VR1(r) ".16b," VR1(r) ".16b,v20.16b\n"		\
		"eor " VR2(r) ".16b," VR2(r) ".16b,v19.16b\n"		\
		"eor " VR3(r) ".16b," VR3(r) ".16b,v18.16b\n"		\
		"ld1 { v21.4s },%[SRC4]\n"				\
		"ld1 { v20.4s },%[SRC5]\n"				\
		"ld1 { v19.4s },%[SRC6]\n"				\
		"ld1 { v18.4s },%[SRC7]\n"				\
		"eor " VR4(r) ".16b," VR4(r) ".16b,v21.16b\n"		\
		"eor " VR5(r) ".16b," VR5(r) ".16b,v20.16b\n"		\
		"eor " VR6(r) ".16b," VR6(r) ".16b,v19.16b\n"		\
		"eor " VR7(r) ".16b," VR7(r) ".16b,v18.16b\n"		\
		:	UVR0(r), UVR1(r), UVR2(r), UVR3(r),		\
			UVR4(r), UVR5(r), UVR6(r), UVR7(r)		\
		:	[SRC0] "Q" (*(OFFSET(src, 0))),			\
		[SRC1] "Q" (*(OFFSET(src, 16))),			\
		[SRC2] "Q" (*(OFFSET(src, 32))),			\
		[SRC3] "Q" (*(OFFSET(src, 48))),			\
		[SRC4] "Q" (*(OFFSET(src, 64))),			\
		[SRC5] "Q" (*(OFFSET(src, 80))),			\
		[SRC6] "Q" (*(OFFSET(src, 96))),			\
		[SRC7] "Q" (*(OFFSET(src, 112)))			\
		:	"v18", "v19", "v20", "v21");			\
		break;							\
	case 4:								\
		__asm(							\
		"ld1 { v21.4s },%[SRC0]\n"				\
		"ld1 { v20.4s },%[SRC1]\n"				\
		"ld1 { v19.4s },%[SRC2]\n"				\
		"ld1 { v18.4s },%[SRC3]\n"				\
		"eor " VR0(r) ".16b," VR0(r) ".16b,v21.16b\n"		\
		"eor " VR1(r) ".16b," VR1(r) ".16b,v20.16b\n"		\
		"eor " VR2(r) ".16b," VR2(r) ".16b,v19.16b\n"		\
		"eor " VR3(r) ".16b," VR3(r) ".16b,v18.16b\n"		\
		:	UVR0(r), UVR1(r), UVR2(r), UVR3(r)		\
		:	[SRC0] "Q" (*(OFFSET(src, 0))),			\
		[SRC1] "Q" (*(OFFSET(src, 16))),			\
		[SRC2] "Q" (*(OFFSET(src, 32))),			\
		[SRC3] "Q" (*(OFFSET(src, 48)))				\
		:	"v18", "v19", "v20", "v21");			\
		break;							\
	case 2:								\
		__asm(							\
		"ld1 { v21.4s },%[SRC0]\n"				\
		"ld1 { v20.4s },%[SRC1]\n"				\
		"eor " VR0(r) ".16b," VR0(r) ".16b,v21.16b\n"		\
		"eor " VR1(r) ".16b," VR1(r) ".16b,v20.16b\n"		\
		:	UVR0(r), UVR1(r)				\
		:	[SRC0] "Q" (*(OFFSET(src, 0))),			\
		[SRC1] "Q" (*(OFFSET(src, 16)))				\
		:	"v20", "v21");					\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	XOR(r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 8:								\
		__asm(							\
		"eor " VR4(r) ".16b," VR4(r) ".16b," VR0(r) ".16b\n"	\
		"eor " VR5(r) ".16b," VR5(r) ".16b," VR1(r) ".16b\n"	\
		"eor " VR6(r) ".16b," VR6(r) ".16b," VR2(r) ".16b\n"	\
		"eor " VR7(r) ".16b," VR7(r) ".16b," VR3(r) ".16b\n"	\
		:	UVR4(r), UVR5(r), UVR6(r), UVR7(r)		\
		:	RVR0(r), RVR1(r), RVR2(r), RVR3(r));		\
		break;							\
	case 4:								\
		__asm(							\
		"eor " VR2(r) ".16b," VR2(r) ".16b," VR0(r) ".16b\n"	\
		"eor " VR3(r) ".16b," VR3(r) ".16b," VR1(r) ".16b\n"	\
		:	UVR2(r), UVR3(r)				\
		:	RVR0(r), RVR1(r));				\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	ZERO(r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 8:								\
		__asm(							\
		"eor " VR0(r) ".16b," VR0(r) ".16b," VR0(r) ".16b\n"	\
		"eor " VR1(r) ".16b," VR1(r) ".16b," VR1(r) ".16b\n"	\
		"eor " VR2(r) ".16b," VR2(r) ".16b," VR2(r) ".16b\n"	\
		"eor " VR3(r) ".16b," VR3(r) ".16b," VR3(r) ".16b\n"	\
		"eor " VR4(r) ".16b," VR4(r) ".16b," VR4(r) ".16b\n"	\
		"eor " VR5(r) ".16b," VR5(r) ".16b," VR5(r) ".16b\n"	\
		"eor " VR6(r) ".16b," VR6(r) ".16b," VR6(r) ".16b\n"	\
		"eor " VR7(r) ".16b," VR7(r) ".16b," VR7(r) ".16b\n"	\
		:	WVR0(r), WVR1(r), WVR2(r), WVR3(r),		\
			WVR4(r), WVR5(r), WVR6(r), WVR7(r));		\
		break;							\
	case 4:								\
		__asm(							\
		"eor " VR0(r) ".16b," VR0(r) ".16b," VR0(r) ".16b\n"	\
		"eor " VR1(r) ".16b," VR1(r) ".16b," VR1(r) ".16b\n"	\
		"eor " VR2(r) ".16b," VR2(r) ".16b," VR2(r) ".16b\n"	\
		"eor " VR3(r) ".16b," VR3(r) ".16b," VR3(r) ".16b\n"	\
		:	WVR0(r), WVR1(r), WVR2(r), WVR3(r));		\
		break;							\
	case 2:								\
		__asm(							\
		"eor " VR0(r) ".16b," VR0(r) ".16b," VR0(r) ".16b\n"	\
		"eor " VR1(r) ".16b," VR1(r) ".16b," VR1(r) ".16b\n"	\
		:	WVR0(r), WVR1(r));				\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	COPY(r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 8:								\
		__asm(							\
		"mov " VR4(r) ".16b," VR0(r) ".16b\n"			\
		"mov " VR5(r) ".16b," VR1(r) ".16b\n"			\
		"mov " VR6(r) ".16b," VR2(r) ".16b\n"			\
		"mov " VR7(r) ".16b," VR3(r) ".16b\n"			\
		:	WVR4(r), WVR5(r), WVR6(r), WVR7(r)		\
		:	RVR0(r), RVR1(r), RVR2(r), RVR3(r));		\
		break;							\
	case 4:								\
		__asm(							\
		"mov " VR2(r) ".16b," VR0(r) ".16b\n"			\
		"mov " VR3(r) ".16b," VR1(r) ".16b\n"			\
		:	WVR2(r), WVR3(r)				\
		:	RVR0(r), RVR1(r));				\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	LOAD(src, r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 8:								\
		__asm(							\
		"ld1 { " VR0(r) ".4s },%[SRC0]\n"			\
		"ld1 { " VR1(r) ".4s },%[SRC1]\n"			\
		"ld1 { " VR2(r) ".4s },%[SRC2]\n"			\
		"ld1 { " VR3(r) ".4s },%[SRC3]\n"			\
		"ld1 { " VR4(r) ".4s },%[SRC4]\n"			\
		"ld1 { " VR5(r) ".4s },%[SRC5]\n"			\
		"ld1 { " VR6(r) ".4s },%[SRC6]\n"			\
		"ld1 { " VR7(r) ".4s },%[SRC7]\n"			\
		:	WVR0(r), WVR1(r), WVR2(r), WVR3(r),		\
			WVR4(r), WVR5(r), WVR6(r), WVR7(r)		\
		:	[SRC0] "Q" (*(OFFSET(src, 0))),			\
		[SRC1] "Q" (*(OFFSET(src, 16))),			\
		[SRC2] "Q" (*(OFFSET(src, 32))),			\
		[SRC3] "Q" (*(OFFSET(src, 48))),			\
		[SRC4] "Q" (*(OFFSET(src, 64))),			\
		[SRC5] "Q" (*(OFFSET(src, 80))),			\
		[SRC6] "Q" (*(OFFSET(src, 96))),			\
		[SRC7] "Q" (*(OFFSET(src, 112))));			\
		break;							\
	case 4:								\
		__asm(							\
		"ld1 { " VR0(r) ".4s },%[SRC0]\n"			\
		"ld1 { " VR1(r) ".4s },%[SRC1]\n"			\
		"ld1 { " VR2(r) ".4s },%[SRC2]\n"			\
		"ld1 { " VR3(r) ".4s },%[SRC3]\n"			\
		:	WVR0(r), WVR1(r), WVR2(r), WVR3(r)		\
		:	[SRC0] "Q" (*(OFFSET(src, 0))),			\
		[SRC1] "Q" (*(OFFSET(src, 16))),			\
		[SRC2] "Q" (*(OFFSET(src, 32))),			\
		[SRC3] "Q" (*(OFFSET(src, 48))));			\
		break;							\
	case 2:								\
		__asm(							\
		"ld1 { " VR0(r) ".4s },%[SRC0]\n"			\
		"ld1 { " VR1(r) ".4s },%[SRC1]\n"			\
		:	WVR0(r), WVR1(r)				\
		:	[SRC0] "Q" (*(OFFSET(src, 0))),			\
		[SRC1] "Q" (*(OFFSET(src, 16))));			\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	STORE(dst, r...)						\
{									\
	switch (REG_CNT(r)) {						\
	case 8:								\
		__asm(							\
		"st1 { " VR0(r) ".4s },%[DST0]\n"			\
		"st1 { " VR1(r) ".4s },%[DST1]\n"			\
		"st1 { " VR2(r) ".4s },%[DST2]\n"			\
		"st1 { " VR3(r) ".4s },%[DST3]\n"			\
		"st1 { " VR4(r) ".4s },%[DST4]\n"			\
		"st1 { " VR5(r) ".4s },%[DST5]\n"			\
		"st1 { " VR6(r) ".4s },%[DST6]\n"			\
		"st1 { " VR7(r) ".4s },%[DST7]\n"			\
		:	[DST0] "=Q" (*(OFFSET(dst, 0))),		\
		[DST1] "=Q" (*(OFFSET(dst, 16))),			\
		[DST2] "=Q" (*(OFFSET(dst, 32))),			\
		[DST3] "=Q" (*(OFFSET(dst, 48))),			\
		[DST4] "=Q" (*(OFFSET(dst, 64))),			\
		[DST5] "=Q" (*(OFFSET(dst, 80))),			\
		[DST6] "=Q" (*(OFFSET(dst, 96))),			\
		[DST7] "=Q" (*(OFFSET(dst, 112)))			\
		:	RVR0(r), RVR1(r), RVR2(r), RVR3(r),		\
			RVR4(r), RVR5(r), RVR6(r), RVR7(r));		\
		break;							\
	case 4:								\
		__asm(							\
		"st1 { " VR0(r) ".4s },%[DST0]\n"			\
		"st1 { " VR1(r) ".4s },%[DST1]\n"			\
		"st1 { " VR2(r) ".4s },%[DST2]\n"			\
		"st1 { " VR3(r) ".4s },%[DST3]\n"			\
		:	[DST0] "=Q" (*(OFFSET(dst, 0))),		\
		[DST1] "=Q" (*(OFFSET(dst, 16))),			\
		[DST2] "=Q" (*(OFFSET(dst, 32))),			\
		[DST3] "=Q" (*(OFFSET(dst, 48)))			\
		:	RVR0(r), RVR1(r), RVR2(r), RVR3(r));		\
		break;							\
	case 2:								\
		__asm(							\
		"st1 { " VR0(r) ".4s },%[DST0]\n"			\
		"st1 { " VR1(r) ".4s },%[DST1]\n"			\
		:	[DST0] "=Q" (*(OFFSET(dst, 0))),		\
		[DST1] "=Q" (*(OFFSET(dst, 16)))			\
		:	RVR0(r), RVR1(r));				\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

/*
 * Unfortunately cannot use the macro, because GCC
 * will try to use the macro name and not value
 * later on...
 * Kept as a reference to what a numbered variable is
 */
#define	_00	"v17"
#define	_1d	"v16"
#define	_temp0	"v19"
#define	_temp1	"v18"

#define	MUL2_SETUP()							\
{									\
	__asm(								\
	"eor " VR(17) ".16b," VR(17) ".16b," VR(17) ".16b\n"		\
	"movi " VR(16) ".16b,#0x1d\n"					\
	:	WVR(16), WVR(17));					\
}

#define	MUL2(r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 4:								\
		__asm(							\
		"cmgt v19.16b," VR(17) ".16b," VR0(r) ".16b\n"		\
		"cmgt v18.16b," VR(17) ".16b," VR1(r) ".16b\n"		\
		"cmgt v21.16b," VR(17) ".16b," VR2(r) ".16b\n"		\
		"cmgt v20.16b," VR(17) ".16b," VR3(r) ".16b\n"		\
		"and v19.16b,v19.16b," VR(16) ".16b\n"			\
		"and v18.16b,v18.16b," VR(16) ".16b\n"			\
		"and v21.16b,v21.16b," VR(16) ".16b\n"			\
		"and v20.16b,v20.16b," VR(16) ".16b\n"			\
		"shl " VR0(r) ".16b," VR0(r) ".16b,#1\n"		\
		"shl " VR1(r) ".16b," VR1(r) ".16b,#1\n"		\
		"shl " VR2(r) ".16b," VR2(r) ".16b,#1\n"		\
		"shl " VR3(r) ".16b," VR3(r) ".16b,#1\n"		\
		"eor " VR0(r) ".16b,v19.16b," VR0(r) ".16b\n"		\
		"eor " VR1(r) ".16b,v18.16b," VR1(r) ".16b\n"		\
		"eor " VR2(r) ".16b,v21.16b," VR2(r) ".16b\n"		\
		"eor " VR3(r) ".16b,v20.16b," VR3(r) ".16b\n"		\
		:	UVR0(r), UVR1(r), UVR2(r), UVR3(r)		\
		:	RVR(17), RVR(16)				\
		:	"v18", "v19", "v20", "v21");			\
		break;							\
	case 2:								\
		__asm(							\
		"cmgt v19.16b," VR(17) ".16b," VR0(r) ".16b\n"		\
		"cmgt v18.16b," VR(17) ".16b," VR1(r) ".16b\n"		\
		"and v19.16b,v19.16b," VR(16) ".16b\n"			\
		"and v18.16b,v18.16b," VR(16) ".16b\n"			\
		"shl " VR0(r) ".16b," VR0(r) ".16b,#1\n"		\
		"shl " VR1(r) ".16b," VR1(r) ".16b,#1\n"		\
		"eor " VR0(r) ".16b,v19.16b," VR0(r) ".16b\n"		\
		"eor " VR1(r) ".16b,v18.16b," VR1(r) ".16b\n"		\
		:	UVR0(r), UVR1(r)				\
		:	RVR(17), RVR(16)				\
		:	"v18", "v19");					\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	MUL4(r...)							\
{									\
	MUL2(r);							\
	MUL2(r);							\
}

/*
 * Unfortunately cannot use the macro, because GCC
 * will try to use the macro name and not value
 * later on...
 * Kept as a reference to what a register is
 * (here we're using actual registers for the
 * clobbered ones)
 */
#define	_0f		"v15"
#define	_a_save		"v14"
#define	_b_save		"v13"
#define	_lt_mod_a	"v12"
#define	_lt_clmul_a	"v11"
#define	_lt_mod_b	"v10"
#define	_lt_clmul_b	"v15"

#define	_MULx2(c, r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 2:								\
		__asm(							\
		/* lts for upper part */				\
		"movi v15.16b,#0x0f\n"					\
		"ld1 { v10.4s },%[lt0]\n"				\
		"ld1 { v11.4s },%[lt1]\n"				\
		/* upper part */					\
		"and v14.16b," VR0(r) ".16b,v15.16b\n"			\
		"and v13.16b," VR1(r) ".16b,v15.16b\n"			\
		"ushr " VR0(r) ".16b," VR0(r) ".16b,#4\n"		\
		"ushr " VR1(r) ".16b," VR1(r) ".16b,#4\n"		\
									\
		"tbl v12.16b,{v10.16b}," VR0(r) ".16b\n"		\
		"tbl v10.16b,{v10.16b}," VR1(r) ".16b\n"		\
		"tbl v15.16b,{v11.16b}," VR0(r) ".16b\n"		\
		"tbl v11.16b,{v11.16b}," VR1(r) ".16b\n"		\
									\
		"eor " VR0(r) ".16b,v15.16b,v12.16b\n"			\
		"eor " VR1(r) ".16b,v11.16b,v10.16b\n"			\
		/* lts for lower part */				\
		"ld1 { v10.4s },%[lt2]\n"				\
		"ld1 { v15.4s },%[lt3]\n"				\
		/* lower part */					\
		"tbl v12.16b,{v10.16b},v14.16b\n"			\
		"tbl v10.16b,{v10.16b},v13.16b\n"			\
		"tbl v11.16b,{v15.16b},v14.16b\n"			\
		"tbl v15.16b,{v15.16b},v13.16b\n"			\
									\
		"eor " VR0(r) ".16b," VR0(r) ".16b,v12.16b\n"		\
		"eor " VR1(r) ".16b," VR1(r) ".16b,v10.16b\n"		\
		"eor " VR0(r) ".16b," VR0(r) ".16b,v11.16b\n"		\
		"eor " VR1(r) ".16b," VR1(r) ".16b,v15.16b\n"		\
		:	UVR0(r), UVR1(r)				\
		:	[lt0] "Q" ((gf_clmul_mod_lt[4*(c)+0][0])),	\
		[lt1] "Q" ((gf_clmul_mod_lt[4*(c)+1][0])),		\
		[lt2] "Q" ((gf_clmul_mod_lt[4*(c)+2][0])),		\
		[lt3] "Q" ((gf_clmul_mod_lt[4*(c)+3][0]))		\
		:	"v10", "v11", "v12", "v13", "v14", "v15");	\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	MUL(c, r...)							\
{									\
	switch (REG_CNT(r)) {						\
	case 4:								\
		_MULx2(c, R_23(r));					\
		_MULx2(c, R_01(r));					\
		break;							\
	case 2:								\
		_MULx2(c, R_01(r));					\
		break;							\
	default:							\
		ZFS_ASM_BUG();						\
	}								\
}

#define	raidz_math_begin()	kfpu_begin()
#define	raidz_math_end()	kfpu_end()

/* Overkill... */
#if defined(_KERNEL)
#define	GEN_X_DEFINE_0_3()	\
register unsigned char w0 asm("v0") __attribute__((vector_size(16)));	\
register unsigned char w1 asm("v1") __attribute__((vector_size(16)));	\
register unsigned char w2 asm("v2") __attribute__((vector_size(16)));	\
register unsigned char w3 asm("v3") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_4_5()	\
register unsigned char w4 asm("v4") __attribute__((vector_size(16)));	\
register unsigned char w5 asm("v5") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_6_7()	\
register unsigned char w6 asm("v6") __attribute__((vector_size(16)));	\
register unsigned char w7 asm("v7") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_8_9()	\
register unsigned char w8 asm("v8") __attribute__((vector_size(16)));	\
register unsigned char w9 asm("v9") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_10_11()	\
register unsigned char w10 asm("v10") __attribute__((vector_size(16)));	\
register unsigned char w11 asm("v11") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_12_15()	\
register unsigned char w12 asm("v12") __attribute__((vector_size(16)));	\
register unsigned char w13 asm("v13") __attribute__((vector_size(16)));	\
register unsigned char w14 asm("v14") __attribute__((vector_size(16)));	\
register unsigned char w15 asm("v15") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_16()	\
register unsigned char w16 asm("v16") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_17()	\
register unsigned char w17 asm("v17") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_18_21()	\
register unsigned char w18 asm("v18") __attribute__((vector_size(16)));	\
register unsigned char w19 asm("v19") __attribute__((vector_size(16)));	\
register unsigned char w20 asm("v20") __attribute__((vector_size(16)));	\
register unsigned char w21 asm("v21") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_22_23()	\
register unsigned char w22 asm("v22") __attribute__((vector_size(16)));	\
register unsigned char w23 asm("v23") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_24_27()	\
register unsigned char w24 asm("v24") __attribute__((vector_size(16)));	\
register unsigned char w25 asm("v25") __attribute__((vector_size(16)));	\
register unsigned char w26 asm("v26") __attribute__((vector_size(16)));	\
register unsigned char w27 asm("v27") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_28_30()	\
register unsigned char w28 asm("v28") __attribute__((vector_size(16)));	\
register unsigned char w29 asm("v29") __attribute__((vector_size(16)));	\
register unsigned char w30 asm("v30") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_31()	\
register unsigned char w31 asm("v31") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_32()	\
register unsigned char w32 asm("v31") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_33_36()	\
register unsigned char w33 asm("v31") __attribute__((vector_size(16)));	\
register unsigned char w34 asm("v31") __attribute__((vector_size(16)));	\
register unsigned char w35 asm("v31") __attribute__((vector_size(16)));	\
register unsigned char w36 asm("v31") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_37_38()	\
register unsigned char w37 asm("v31") __attribute__((vector_size(16)));	\
register unsigned char w38 asm("v31") __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_ALL()	\
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_8_9()	\
	GEN_X_DEFINE_10_11()	\
	GEN_X_DEFINE_12_15()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_18_21()	\
	GEN_X_DEFINE_22_23()	\
	GEN_X_DEFINE_24_27()	\
	GEN_X_DEFINE_28_30()	\
	GEN_X_DEFINE_31()	\
	GEN_X_DEFINE_32()	\
	GEN_X_DEFINE_33_36() 	\
	GEN_X_DEFINE_37_38()
#else
#define	GEN_X_DEFINE_0_3()	\
	unsigned char w0 __attribute__((vector_size(16)));	\
	unsigned char w1 __attribute__((vector_size(16)));	\
	unsigned char w2 __attribute__((vector_size(16)));	\
	unsigned char w3 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_4_5()	\
	unsigned char w4 __attribute__((vector_size(16)));	\
	unsigned char w5 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_6_7()	\
	unsigned char w6 __attribute__((vector_size(16)));	\
	unsigned char w7 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_8_9()	\
	unsigned char w8 __attribute__((vector_size(16)));	\
	unsigned char w9 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_10_11()	\
	unsigned char w10 __attribute__((vector_size(16)));	\
	unsigned char w11 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_12_15()	\
	unsigned char w12 __attribute__((vector_size(16)));	\
	unsigned char w13 __attribute__((vector_size(16)));	\
	unsigned char w14 __attribute__((vector_size(16)));	\
	unsigned char w15 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_16()	\
	unsigned char w16 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_17()	\
	unsigned char w17 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_18_21()	\
	unsigned char w18 __attribute__((vector_size(16)));	\
	unsigned char w19 __attribute__((vector_size(16)));	\
	unsigned char w20 __attribute__((vector_size(16)));	\
	unsigned char w21 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_22_23()	\
	unsigned char w22 __attribute__((vector_size(16)));	\
	unsigned char w23 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_24_27()	\
	unsigned char w24 __attribute__((vector_size(16)));	\
	unsigned char w25 __attribute__((vector_size(16)));	\
	unsigned char w26 __attribute__((vector_size(16)));	\
	unsigned char w27 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_28_30()	\
	unsigned char w28 __attribute__((vector_size(16)));	\
	unsigned char w29 __attribute__((vector_size(16)));	\
	unsigned char w30 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_31()	\
	unsigned char w31 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_32()	\
	unsigned char w32 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_33_36()	\
	unsigned char w33 __attribute__((vector_size(16)));	\
	unsigned char w34 __attribute__((vector_size(16)));	\
	unsigned char w35 __attribute__((vector_size(16)));	\
	unsigned char w36 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_37_38()	\
	unsigned char w37 __attribute__((vector_size(16)));	\
	unsigned char w38 __attribute__((vector_size(16)));
#define	GEN_X_DEFINE_ALL()	\
	GEN_X_DEFINE_0_3()	\
	GEN_X_DEFINE_4_5()	\
	GEN_X_DEFINE_6_7()	\
	GEN_X_DEFINE_8_9()	\
	GEN_X_DEFINE_10_11()	\
	GEN_X_DEFINE_12_15()	\
	GEN_X_DEFINE_16()	\
	GEN_X_DEFINE_17()	\
	GEN_X_DEFINE_18_21()	\
	GEN_X_DEFINE_22_23()	\
	GEN_X_DEFINE_24_27()	\
	GEN_X_DEFINE_28_30()	\
	GEN_X_DEFINE_31()	\
	GEN_X_DEFINE_32()	\
	GEN_X_DEFINE_33_36()	\
	GEN_X_DEFINE_37_38()
#endif