/* AArch64 SIMD pragma builtins
   Copyright (C) 2024-2025 Free Software Foundation, Inc.
   Contributed by ARM Ltd.

   This file is part of GCC.

   GCC is free software; you can redistribute it and/or modify it
   under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 3, or (at your option)
   any later version.

   GCC is distributed in the hope that it will be useful, but
   WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
   General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with GCC; see the file COPYING3.  If not see
   <http://www.gnu.org/licenses/>.  */

#undef ENTRY_BINARY
#define ENTRY_BINARY(N, T0, T1, T2, U, F)	\
  ENTRY (N, binary, T0, T1, T2, none, U, F)

#undef ENTRY_BINARY_LANE
#define ENTRY_BINARY_LANE(N, T0, T1, T2, U, F)	\
  ENTRY (N, binary_lane, T0, T1, T2, none, U, F)

#undef ENTRY_BINARY_TWO_LANES
#define ENTRY_BINARY_TWO_LANES(N, T0, T1, T2, U, F)	\
  ENTRY (N, binary_two_lanes, T0, T1, T2, none, U, F)

#undef ENTRY_LOAD
#define ENTRY_LOAD(N, T0, T1, U)			\
  ENTRY (N, load, T0, T1, none, none, U, LOAD)

#undef ENTRY_LOAD_LANE
#define ENTRY_LOAD_LANE(N, T0, T1, T2, U)		\
  ENTRY (N, load_lane, T0, T1, T2, none, U, LOAD)

#undef ENTRY_STORE
#define ENTRY_STORE(N, T0, T1, U)			\
  ENTRY (N, store, T0, T1, none, none, U, STORE)

#undef ENTRY_STORE_LANE
#define ENTRY_STORE_LANE(N, T0, T1, U)			\
  ENTRY (N, store_lane, T0, T1, none, none, U, STORE)

#undef ENTRY_TERNARY
#define ENTRY_TERNARY(N, T0, T1, T2, T3, U, F)	\
  ENTRY (N, ternary, T0, T1, T2, T3, U, F)

#undef ENTRY_TERNARY_LANE
#define ENTRY_TERNARY_LANE(N, T0, T1, T2, T3, U, F)	\
  ENTRY (N, ternary_lane, T0, T1, T2, T3, U, F)

#undef ENTRY_UNARY
#define ENTRY_UNARY(N, T0, T1, U, F)	\
  ENTRY (N, unary, T0, T1, none, none, U, F)

#undef ENTRY_UNARY_LANE
#define ENTRY_UNARY_LANE(N, T0, T1, U, F)	\
  ENTRY (N, unary_lane, T0, T1, none, none, U, F)

#undef ENTRY_BINARY_VHSDF
#define ENTRY_BINARY_VHSDF(NAME, UNSPEC, FLAGS)			\
  ENTRY_BINARY (NAME##_f16, f16, f16, f16, UNSPEC, FLAGS)	\
  ENTRY_BINARY (NAME##q_f16, f16q, f16q, f16q, UNSPEC, FLAGS)	\
  ENTRY_BINARY (NAME##_f32, f32, f32, f32, UNSPEC, FLAGS)	\
  ENTRY_BINARY (NAME##q_f32, f32q, f32q, f32q, UNSPEC, FLAGS)	\
  ENTRY_BINARY (NAME##q_f64, f64q, f64q, f64q, UNSPEC, FLAGS)

#undef ENTRY_BINARY_VHSDF_SIGNED
#define ENTRY_BINARY_VHSDF_SIGNED(NAME, UNSPEC, FLAGS)		\
  ENTRY_BINARY (NAME##_f16, f16, f16, s16, UNSPEC, FLAGS)	\
  ENTRY_BINARY (NAME##q_f16, f16q, f16q, s16q, UNSPEC, FLAGS)	\
  ENTRY_BINARY (NAME##_f32, f32, f32, s32, UNSPEC, FLAGS)	\
  ENTRY_BINARY (NAME##q_f32, f32q, f32q, s32q, UNSPEC, FLAGS)	\
  ENTRY_BINARY (NAME##q_f64, f64q, f64q, s64q, UNSPEC, FLAGS)

#undef ENTRY_TERNARY_VLUT8
#define ENTRY_TERNARY_VLUT8(T)					\
  ENTRY_BINARY_LANE (vluti2_lane_##T##8, T##8q, T##8, u8,	\
		     UNSPEC_LUTI2, QUIET)			\
  ENTRY_BINARY_LANE (vluti2_laneq_##T##8, T##8q, T##8, u8q,	\
		     UNSPEC_LUTI2, QUIET)			\
  ENTRY_BINARY_LANE (vluti2q_lane_##T##8, T##8q, T##8q, u8,	\
		     UNSPEC_LUTI2, QUIET)			\
  ENTRY_BINARY_LANE (vluti2q_laneq_##T##8, T##8q, T##8q, u8q,	\
		     UNSPEC_LUTI2, QUIET)			\
  ENTRY_BINARY_LANE (vluti4q_lane_##T##8, T##8q, T##8q, u8,	\
		     UNSPEC_LUTI4, QUIET)			\
  ENTRY_BINARY_LANE (vluti4q_laneq_##T##8, T##8q, T##8q, u8q,	\
		     UNSPEC_LUTI4, QUIET)

#undef ENTRY_TERNARY_VLUT16
#define ENTRY_TERNARY_VLUT16(T)						\
  ENTRY_BINARY_LANE (vluti2_lane_##T##16, T##16q, T##16, u8,		\
		     UNSPEC_LUTI2, QUIET)				\
  ENTRY_BINARY_LANE (vluti2_laneq_##T##16, T##16q, T##16, u8q,		\
		     UNSPEC_LUTI2, QUIET)				\
  ENTRY_BINARY_LANE (vluti2q_lane_##T##16, T##16q, T##16q, u8,		\
		     UNSPEC_LUTI2, QUIET)				\
  ENTRY_BINARY_LANE (vluti2q_laneq_##T##16, T##16q, T##16q, u8q,	\
		     UNSPEC_LUTI2, QUIET)				\
  ENTRY_BINARY_LANE (vluti4q_lane_##T##16_x2, T##16q, T##16qx2, u8,	\
		     UNSPEC_LUTI4, QUIET)				\
  ENTRY_BINARY_LANE (vluti4q_laneq_##T##16_x2, T##16q, T##16qx2, u8q,	\
		     UNSPEC_LUTI4, QUIET)

#undef ENTRY_UNARY_VQ_BHF
#define ENTRY_UNARY_VQ_BHF(N, T1, UNSPEC, FLAGS)		\
  ENTRY_UNARY (N##_bf16_mf8_fpm, bf16q, T1, UNSPEC, FLAGS)	\
  ENTRY_UNARY (N##_f16_mf8_fpm, f16q, T1, UNSPEC, FLAGS)

#undef ENTRY_VDOT_FPM
#define ENTRY_VDOT_FPM(T)						\
  ENTRY_TERNARY (vdot_##T##_mf8_fpm, T, T, mf8, mf8,			\
		 UNSPEC_FDOT_FP8, FP8)					\
  ENTRY_TERNARY (vdotq_##T##_mf8_fpm, T##q, T##q, mf8q, mf8q,		\
		 UNSPEC_FDOT_FP8, FP8)					\
  ENTRY_TERNARY_LANE (vdot_lane_##T##_mf8_fpm, T, T, mf8, mf8,		\
		      UNSPEC_FDOT_LANE_FP8, FP8)			\
  ENTRY_TERNARY_LANE (vdot_laneq_##T##_mf8_fpm, T, T, mf8, mf8q,	\
		      UNSPEC_FDOT_LANE_FP8, FP8)			\
  ENTRY_TERNARY_LANE (vdotq_lane_##T##_mf8_fpm, T##q, T##q, mf8q, mf8,	\
		      UNSPEC_FDOT_LANE_FP8, FP8)			\
  ENTRY_TERNARY_LANE (vdotq_laneq_##T##_mf8_fpm, T##q, T##q, mf8q, mf8q,\
		      UNSPEC_FDOT_LANE_FP8, FP8)

#undef ENTRY_FMA_FPM
#define ENTRY_FMA_FPM(N, T, U)						\
  ENTRY_TERNARY (N##q_##T##_mf8_fpm, T##q, T##q, mf8q, mf8q, U, FP8)	\
  ENTRY_TERNARY_LANE (N##q_lane_##T##_mf8_fpm, T##q, T##q, mf8q, mf8, U, FP8) \
  ENTRY_TERNARY_LANE (N##q_laneq_##T##_mf8_fpm, T##q, T##q, mf8q, mf8q, U, FP8)

// faminmax
#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_FAMINMAX)
ENTRY_BINARY_VHSDF (vamax, UNSPEC_FAMAX, FP)
ENTRY_BINARY_VHSDF (vamin, UNSPEC_FAMIN, FP)
#undef REQUIRED_EXTENSIONS

// lut
#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_LUT)
ENTRY_TERNARY_VLUT8 (p)
ENTRY_TERNARY_VLUT8 (s)
ENTRY_TERNARY_VLUT8 (u)
ENTRY_TERNARY_VLUT8 (mf)

ENTRY_TERNARY_VLUT16 (bf)
ENTRY_TERNARY_VLUT16 (f)
ENTRY_TERNARY_VLUT16 (p)
ENTRY_TERNARY_VLUT16 (s)
ENTRY_TERNARY_VLUT16 (u)
#undef REQUIRED_EXTENSIONS

// fpm conversion
#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_FP8)
ENTRY_UNARY_VQ_BHF (vcvt1, mf8, UNSPEC_F1CVTL_FP8, FP8)
ENTRY_UNARY_VQ_BHF (vcvt1_high, mf8q, UNSPEC_F1CVTL2_FP8, FP8)
ENTRY_UNARY_VQ_BHF (vcvt1_low, mf8q, UNSPEC_F1CVTL_FP8, FP8)
ENTRY_UNARY_VQ_BHF (vcvt2, mf8, UNSPEC_F2CVTL_FP8, FP8)
ENTRY_UNARY_VQ_BHF (vcvt2_high, mf8q, UNSPEC_F2CVTL2_FP8, FP8)
ENTRY_UNARY_VQ_BHF (vcvt2_low, mf8q, UNSPEC_F2CVTL_FP8, FP8)

ENTRY_BINARY (vcvt_mf8_f16_fpm, mf8, f16, f16, UNSPEC_FCVTN_FP8, FP8)
ENTRY_BINARY (vcvtq_mf8_f16_fpm, mf8q, f16q, f16q, UNSPEC_FCVTN_FP8, FP8)
ENTRY_BINARY (vcvt_mf8_f32_fpm, mf8, f32q, f32q, UNSPEC_FCVTN_FP8, FP8)

ENTRY_TERNARY (vcvt_high_mf8_f32_fpm, mf8q, mf8, f32q, f32q,
	       UNSPEC_FCVTN2_FP8, FP8)
#undef REQUIRED_EXTENSIONS

// fpm scaling
#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_FP8)
ENTRY_BINARY_VHSDF_SIGNED (vscale, UNSPEC_FSCALE, FP)
#undef REQUIRED_EXTENSIONS

// fpm dot2 product
#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_FP8DOT2)
ENTRY_VDOT_FPM (f16)
#undef REQUIRED_EXTENSIONS

// fpm dot4 product
#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_FP8DOT4)
ENTRY_VDOT_FPM (f32)
#undef REQUIRED_EXTENSIONS

// fp8 multiply-add
#define REQUIRED_EXTENSIONS nonstreaming_only (AARCH64_FL_FP8FMA)
ENTRY_FMA_FPM (vmlalb, f16, UNSPEC_FMLALB_FP8)
ENTRY_FMA_FPM (vmlalt, f16, UNSPEC_FMLALT_FP8)
ENTRY_FMA_FPM (vmlallbb, f32, UNSPEC_FMLALLBB_FP8)
ENTRY_FMA_FPM (vmlallbt, f32, UNSPEC_FMLALLBT_FP8)
ENTRY_FMA_FPM (vmlalltb, f32, UNSPEC_FMLALLTB_FP8)
ENTRY_FMA_FPM (vmlalltt, f32, UNSPEC_FMLALLTT_FP8)
#undef REQUIRED_EXTENSIONS

// bsl
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_TERNARY (vbsl_mf8, mf8, u8, mf8, mf8, UNSPEC_BSL, QUIET)
ENTRY_TERNARY (vbslq_mf8, mf8q, u8q, mf8q, mf8q, UNSPEC_BSL, QUIET)
#undef REQUIRED_EXTENSIONS

// combine
#define REQUIRED_EXTENSIONS nonstreaming_only (0)
ENTRY_BINARY (vcombine_mf8, mf8q, mf8, mf8, UNSPEC_COMBINE, QUIET)
#undef REQUIRED_EXTENSIONS

// copy_lane
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_BINARY_TWO_LANES (vcopy_lane_mf8, mf8, mf8, mf8,
			UNSPEC_VEC_COPY, QUIET)
ENTRY_BINARY_TWO_LANES (vcopyq_lane_mf8, mf8q, mf8q, mf8,
			UNSPEC_VEC_COPY, QUIET)
ENTRY_BINARY_TWO_LANES (vcopy_laneq_mf8, mf8, mf8, mf8q,
			UNSPEC_VEC_COPY, QUIET)
ENTRY_BINARY_TWO_LANES (vcopyq_laneq_mf8, mf8q, mf8q, mf8q,
			UNSPEC_VEC_COPY, QUIET)
#undef REQUIRED_EXTENSIONS

// create
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_UNARY (vcreate_mf8, mf8, u64_scalar, UNSPEC_VCREATE, QUIET)
#undef REQUIRED_EXTENSIONS

// dup
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_UNARY (vdup_n_mf8, mf8, mf8_scalar, UNSPEC_DUP, QUIET)
ENTRY_UNARY (vdupq_n_mf8, mf8q, mf8_scalar, UNSPEC_DUP, QUIET)

ENTRY_UNARY_LANE (vdup_lane_mf8, mf8, mf8, UNSPEC_DUP_LANE, QUIET)
ENTRY_UNARY_LANE (vdupq_lane_mf8, mf8q, mf8, UNSPEC_DUP_LANE, QUIET)
ENTRY_UNARY_LANE (vdup_laneq_mf8, mf8, mf8q, UNSPEC_DUP_LANE, QUIET)
ENTRY_UNARY_LANE (vdupq_laneq_mf8, mf8q, mf8q, UNSPEC_DUP_LANE, QUIET)
#undef REQUIRED_EXTENSIONS

// dupb_lane
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_UNARY_LANE (vdupb_lane_mf8, mf8_scalar, mf8, UNSPEC_GET_LANE, QUIET)
ENTRY_UNARY_LANE (vdupb_laneq_mf8, mf8_scalar, mf8q, UNSPEC_GET_LANE, QUIET)
#undef REQUIRED_EXTENSIONS

// ext
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_BINARY_LANE (vext_mf8, mf8, mf8, mf8, UNSPEC_EXT, QUIET)
ENTRY_BINARY_LANE (vextq_mf8, mf8q, mf8q, mf8q, UNSPEC_EXT, QUIET)
#undef REQUIRED_EXTENSIONS

// ld1
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_LOAD (vld1_mf8, mf8, mf8_scalar_const_ptr, UNSPEC_LD1)
ENTRY_LOAD (vld1q_mf8, mf8q, mf8_scalar_const_ptr, UNSPEC_LD1)
ENTRY_LOAD (vld1_dup_mf8, mf8, mf8_scalar_const_ptr, UNSPEC_DUP)
ENTRY_LOAD (vld1q_dup_mf8, mf8q, mf8_scalar_const_ptr, UNSPEC_DUP)

ENTRY_LOAD_LANE (vld1_lane_mf8, mf8, mf8_scalar_const_ptr, mf8,
		 UNSPEC_SET_LANE)
ENTRY_LOAD_LANE (vld1q_lane_mf8, mf8q, mf8_scalar_const_ptr, mf8q,
		 UNSPEC_SET_LANE)
#undef REQUIRED_EXTENSIONS

// ld<n>
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_LOAD (vld1_mf8_x2, mf8x2, mf8_scalar_const_ptr, UNSPEC_LD1x2)
ENTRY_LOAD (vld1q_mf8_x2, mf8qx2, mf8_scalar_const_ptr, UNSPEC_LD1x2)
ENTRY_LOAD (vld2_mf8, mf8x2, mf8_scalar_const_ptr, UNSPEC_LD2)
ENTRY_LOAD (vld2q_mf8, mf8qx2, mf8_scalar_const_ptr, UNSPEC_LD2)
ENTRY_LOAD (vld2_dup_mf8, mf8x2, mf8_scalar_const_ptr, UNSPEC_LD2_DUP)
ENTRY_LOAD (vld2q_dup_mf8, mf8qx2, mf8_scalar_const_ptr, UNSPEC_LD2_DUP)
ENTRY_LOAD_LANE (vld2_lane_mf8, mf8x2, mf8_scalar_const_ptr, mf8x2,
		 UNSPEC_LD2_LANE)
ENTRY_LOAD_LANE (vld2q_lane_mf8, mf8qx2, mf8_scalar_const_ptr, mf8qx2,
		 UNSPEC_LD2_LANE)

ENTRY_LOAD (vld1_mf8_x3, mf8x3, mf8_scalar_const_ptr, UNSPEC_LD1x3)
ENTRY_LOAD (vld1q_mf8_x3, mf8qx3, mf8_scalar_const_ptr, UNSPEC_LD1x3)
ENTRY_LOAD (vld3_mf8, mf8x3, mf8_scalar_const_ptr, UNSPEC_LD3)
ENTRY_LOAD (vld3q_mf8, mf8qx3, mf8_scalar_const_ptr, UNSPEC_LD3)
ENTRY_LOAD (vld3_dup_mf8, mf8x3, mf8_scalar_const_ptr, UNSPEC_LD3_DUP)
ENTRY_LOAD (vld3q_dup_mf8, mf8qx3, mf8_scalar_const_ptr, UNSPEC_LD3_DUP)
ENTRY_LOAD_LANE (vld3_lane_mf8, mf8x3, mf8_scalar_const_ptr, mf8x3,
		 UNSPEC_LD3_LANE)
ENTRY_LOAD_LANE (vld3q_lane_mf8, mf8qx3, mf8_scalar_const_ptr, mf8qx3,
		 UNSPEC_LD3_LANE)

ENTRY_LOAD (vld1_mf8_x4, mf8x4, mf8_scalar_const_ptr, UNSPEC_LD1x4)
ENTRY_LOAD (vld1q_mf8_x4, mf8qx4, mf8_scalar_const_ptr, UNSPEC_LD1x4)
ENTRY_LOAD (vld4_mf8, mf8x4, mf8_scalar_const_ptr, UNSPEC_LD4)
ENTRY_LOAD (vld4q_mf8, mf8qx4, mf8_scalar_const_ptr, UNSPEC_LD4)
ENTRY_LOAD (vld4_dup_mf8, mf8x4, mf8_scalar_const_ptr, UNSPEC_LD4_DUP)
ENTRY_LOAD (vld4q_dup_mf8, mf8qx4, mf8_scalar_const_ptr, UNSPEC_LD4_DUP)
ENTRY_LOAD_LANE (vld4_lane_mf8, mf8x4, mf8_scalar_const_ptr, mf8x4,
		 UNSPEC_LD4_LANE)
ENTRY_LOAD_LANE (vld4q_lane_mf8, mf8qx4, mf8_scalar_const_ptr, mf8qx4,
		 UNSPEC_LD4_LANE)
#undef REQUIRED_EXTENSIONS

// mov
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_UNARY (vmov_n_mf8, mf8, mf8_scalar, UNSPEC_DUP, QUIET)
ENTRY_UNARY (vmovq_n_mf8, mf8q, mf8_scalar, UNSPEC_DUP, QUIET)
#undef REQUIRED_EXTENSIONS

// rev
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_UNARY (vrev64_mf8, mf8, mf8, UNSPEC_REV64, QUIET)
ENTRY_UNARY (vrev64q_mf8, mf8q, mf8q, UNSPEC_REV64, QUIET)

ENTRY_UNARY (vrev32_mf8, mf8, mf8, UNSPEC_REV32, QUIET)
ENTRY_UNARY (vrev32q_mf8, mf8q, mf8q, UNSPEC_REV32, QUIET)

ENTRY_UNARY (vrev16_mf8, mf8, mf8, UNSPEC_REV16, QUIET)
ENTRY_UNARY (vrev16q_mf8, mf8q, mf8q, UNSPEC_REV16, QUIET)
#undef REQUIRED_EXTENSIONS

// set_lane
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_BINARY_LANE (vset_lane_mf8, mf8, mf8_scalar, mf8, UNSPEC_SET_LANE, QUIET)
ENTRY_BINARY_LANE (vsetq_lane_mf8, mf8q, mf8_scalar, mf8q, UNSPEC_SET_LANE, QUIET)
#undef REQUIRED_EXTENSIONS

// st1
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_STORE (vst1_mf8, mf8_scalar_ptr, mf8, UNSPEC_ST1)
ENTRY_STORE (vst1q_mf8, mf8_scalar_ptr, mf8q, UNSPEC_ST1)

ENTRY_STORE_LANE (vst1_lane_mf8, mf8_scalar_ptr, mf8, UNSPEC_ST1_LANE)
ENTRY_STORE_LANE (vst1q_lane_mf8, mf8_scalar_ptr, mf8q, UNSPEC_ST1_LANE)
#undef REQUIRED_EXTENSIONS

// st<n>
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_STORE (vst2_mf8, mf8_scalar_ptr, mf8x2, UNSPEC_ST2)
ENTRY_STORE (vst2q_mf8, mf8_scalar_ptr, mf8qx2, UNSPEC_ST2)
ENTRY_STORE (vst1_mf8_x2, mf8_scalar_ptr, mf8x2, UNSPEC_ST1x2)
ENTRY_STORE (vst1q_mf8_x2, mf8_scalar_ptr, mf8qx2, UNSPEC_ST1x2)
ENTRY_STORE_LANE (vst2_lane_mf8, mf8_scalar_ptr, mf8x2, UNSPEC_ST2_LANE)
ENTRY_STORE_LANE (vst2q_lane_mf8, mf8_scalar_ptr, mf8qx2, UNSPEC_ST2_LANE)

ENTRY_STORE (vst3_mf8, mf8_scalar_ptr, mf8x3, UNSPEC_ST3)
ENTRY_STORE (vst3q_mf8, mf8_scalar_ptr, mf8qx3, UNSPEC_ST3)
ENTRY_STORE (vst1_mf8_x3, mf8_scalar_ptr, mf8x3, UNSPEC_ST1x3)
ENTRY_STORE (vst1q_mf8_x3, mf8_scalar_ptr, mf8qx3, UNSPEC_ST1x3)
ENTRY_STORE_LANE (vst3_lane_mf8, mf8_scalar_ptr, mf8x3, UNSPEC_ST3_LANE)
ENTRY_STORE_LANE (vst3q_lane_mf8, mf8_scalar_ptr, mf8qx3, UNSPEC_ST3_LANE)

ENTRY_STORE (vst4_mf8, mf8_scalar_ptr, mf8x4, UNSPEC_ST4)
ENTRY_STORE (vst4q_mf8, mf8_scalar_ptr, mf8qx4, UNSPEC_ST4)
ENTRY_STORE (vst1_mf8_x4, mf8_scalar_ptr, mf8x4, UNSPEC_ST1x4)
ENTRY_STORE (vst1q_mf8_x4, mf8_scalar_ptr, mf8qx4, UNSPEC_ST1x4)
ENTRY_STORE_LANE (vst4_lane_mf8, mf8_scalar_ptr, mf8x4, UNSPEC_ST4_LANE)
ENTRY_STORE_LANE (vst4q_lane_mf8, mf8_scalar_ptr, mf8qx4, UNSPEC_ST4_LANE)
#undef REQUIRED_EXTENSIONS

// tbl<n>
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_BINARY (vtbl1_mf8, mf8, mf8, u8, UNSPEC_TBL, QUIET)
ENTRY_BINARY (vtbl2_mf8, mf8, mf8x2, u8, UNSPEC_TBL, QUIET)
ENTRY_BINARY (vtbl3_mf8, mf8, mf8x3, u8, UNSPEC_TBL, QUIET)
ENTRY_BINARY (vtbl4_mf8, mf8, mf8x4, u8, UNSPEC_TBL, QUIET)

ENTRY_BINARY (vqtbl1_mf8, mf8, mf8q, u8, UNSPEC_TBL, QUIET)
ENTRY_BINARY (vqtbl1q_mf8, mf8q, mf8q, u8q, UNSPEC_TBL, QUIET)
ENTRY_BINARY (vqtbl2_mf8, mf8, mf8qx2, u8, UNSPEC_TBL, QUIET)
ENTRY_BINARY (vqtbl2q_mf8, mf8q, mf8qx2, u8q, UNSPEC_TBL, QUIET)
ENTRY_BINARY (vqtbl3_mf8, mf8, mf8qx3, u8, UNSPEC_TBL, QUIET)
ENTRY_BINARY (vqtbl3q_mf8, mf8q, mf8qx3, u8q, UNSPEC_TBL, QUIET)
ENTRY_BINARY (vqtbl4_mf8, mf8, mf8qx4, u8, UNSPEC_TBL, QUIET)
ENTRY_BINARY (vqtbl4q_mf8, mf8q, mf8qx4, u8q, UNSPEC_TBL, QUIET)
#undef REQUIRED_EXTENSIONS

// tbx<n>
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_TERNARY (vtbx1_mf8, mf8, mf8, mf8, u8, UNSPEC_TBX, QUIET)
ENTRY_TERNARY (vtbx2_mf8, mf8, mf8, mf8x2, u8, UNSPEC_TBX, QUIET)
ENTRY_TERNARY (vtbx3_mf8, mf8, mf8, mf8x3, u8, UNSPEC_TBX, QUIET)
ENTRY_TERNARY (vtbx4_mf8, mf8, mf8, mf8x4, u8, UNSPEC_TBX, QUIET)

ENTRY_TERNARY (vqtbx1_mf8, mf8, mf8, mf8q, u8, UNSPEC_TBX, QUIET)
ENTRY_TERNARY (vqtbx1q_mf8, mf8q, mf8q, mf8q, u8q, UNSPEC_TBX, QUIET)
ENTRY_TERNARY (vqtbx2_mf8, mf8, mf8, mf8qx2, u8, UNSPEC_TBX, QUIET)
ENTRY_TERNARY (vqtbx2q_mf8, mf8q, mf8q, mf8qx2, u8q, UNSPEC_TBX, QUIET)
ENTRY_TERNARY (vqtbx3_mf8, mf8, mf8, mf8qx3, u8, UNSPEC_TBX, QUIET)
ENTRY_TERNARY (vqtbx3q_mf8, mf8q, mf8q, mf8qx3, u8q, UNSPEC_TBX, QUIET)
ENTRY_TERNARY (vqtbx4_mf8, mf8, mf8, mf8qx4, u8, UNSPEC_TBX, QUIET)
ENTRY_TERNARY (vqtbx4q_mf8, mf8q, mf8q, mf8qx4, u8q, UNSPEC_TBX, QUIET)
#undef REQUIRED_EXTENSIONS

// trn<n>
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_BINARY (vtrn1_mf8, mf8, mf8, mf8, UNSPEC_TRN1, QUIET)
ENTRY_BINARY (vtrn1q_mf8, mf8q, mf8q, mf8q, UNSPEC_TRN1, QUIET)
ENTRY_BINARY (vtrn2_mf8, mf8, mf8, mf8, UNSPEC_TRN2, QUIET)
ENTRY_BINARY (vtrn2q_mf8, mf8q, mf8q, mf8q, UNSPEC_TRN2, QUIET)
ENTRY_BINARY (vtrn_mf8, mf8x2, mf8, mf8, UNSPEC_TRN, QUIET)
ENTRY_BINARY (vtrnq_mf8, mf8qx2, mf8q, mf8q, UNSPEC_TRN, QUIET)
#undef REQUIRED_EXTENSIONS

// uzp<n>
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_BINARY (vuzp1_mf8, mf8, mf8, mf8, UNSPEC_UZP1, QUIET)
ENTRY_BINARY (vuzp1q_mf8, mf8q, mf8q, mf8q, UNSPEC_UZP1, QUIET)
ENTRY_BINARY (vuzp2_mf8, mf8, mf8, mf8, UNSPEC_UZP2, QUIET)
ENTRY_BINARY (vuzp2q_mf8, mf8q, mf8q, mf8q, UNSPEC_UZP2, QUIET)
ENTRY_BINARY (vuzp_mf8, mf8x2, mf8, mf8, UNSPEC_UZP, QUIET)
ENTRY_BINARY (vuzpq_mf8, mf8qx2, mf8q, mf8q, UNSPEC_UZP, QUIET)
#undef REQUIRED_EXTENSIONS

// zip<n>
#define REQUIRED_EXTENSIONS nonstreaming_only (TARGET_SIMD)
ENTRY_BINARY (vzip1_mf8, mf8, mf8, mf8, UNSPEC_ZIP1, QUIET)
ENTRY_BINARY (vzip1q_mf8, mf8q, mf8q, mf8q, UNSPEC_ZIP1, QUIET)
ENTRY_BINARY (vzip2_mf8, mf8, mf8, mf8, UNSPEC_ZIP2, QUIET)
ENTRY_BINARY (vzip2q_mf8, mf8q, mf8q, mf8q, UNSPEC_ZIP2, QUIET)
ENTRY_BINARY (vzip_mf8, mf8x2, mf8, mf8, UNSPEC_ZIP, QUIET)
ENTRY_BINARY (vzipq_mf8, mf8qx2, mf8q, mf8q, UNSPEC_ZIP, QUIET)
#undef REQUIRED_EXTENSIONS
