// This file is part of Eigen, a lightweight C++ template library
// for linear algebra.
//
// Copyright (C) 2025 Rasmus Munk Larsen
//
// This Source Code Form is subject to the terms of the Mozilla
// Public License v. 2.0. If a copy of the MPL was not distributed
// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.

#ifndef EIGEN_PACKET_MATH_CLANG_H
#define EIGEN_PACKET_MATH_CLANG_H

// IWYU pragma: private
#include "../../InternalHeaderCheck.h"

namespace Eigen {
namespace internal {

namespace detail {
// namespace detail contains implementation details specific to this
// file, while namespace internal contains internal APIs used elsewhere
// in Eigen.
template <typename ScalarT, int n>
using VectorType = ScalarT __attribute__((ext_vector_type(n), aligned(n * sizeof(ScalarT))));
}  // namespace detail

// --- Naming Convention ---
// This backend uses size-independent type aliases so the same code works
// for EIGEN_GENERIC_VECTOR_SIZE_BYTES in {16, 32, 64}:
//
//   PacketXf  - float vector   (4, 8, or 16 elements)
//   PacketXd  - double vector  (2, 4, or 8 elements)
//   PacketXi  - int32_t vector (4, 8, or 16 elements)
//   PacketXl  - int64_t vector (2, 4, or 8 elements)
//   PacketXcf - complex<float> vector  (2, 4, or 8 elements)  [in Complex.h]
//   PacketXcd - complex<double> vector (1, 2, or 4 elements)  [in Complex.h]
//
// The "X" suffix indicates the element count is determined by the macro
// EIGEN_GENERIC_VECTOR_SIZE_BYTES at compile time. Operations that require
// compile-time constant indices (e.g. __builtin_shufflevector) use
// #if EIGEN_GENERIC_VECTOR_SIZE_BYTES == ... blocks.

static_assert(EIGEN_GENERIC_VECTOR_SIZE_BYTES == 16 || EIGEN_GENERIC_VECTOR_SIZE_BYTES == 32 ||
                  EIGEN_GENERIC_VECTOR_SIZE_BYTES == 64,
              "EIGEN_GENERIC_VECTOR_SIZE_BYTES must be 16, 32, or 64");

constexpr int kFloatPacketSize = EIGEN_GENERIC_VECTOR_SIZE_BYTES / sizeof(float);
constexpr int kDoublePacketSize = EIGEN_GENERIC_VECTOR_SIZE_BYTES / sizeof(double);
using PacketXf = detail::VectorType<float, kFloatPacketSize>;
using PacketXd = detail::VectorType<double, kDoublePacketSize>;
using PacketXi = detail::VectorType<int32_t, kFloatPacketSize>;
using PacketXl = detail::VectorType<int64_t, kDoublePacketSize>;

// --- packet_traits specializations ---
struct generic_float_packet_traits : default_packet_traits {
  enum {
    Vectorizable = 1,
    AlignedOnScalar = 1,
    HasAdd = 1,
    HasSub = 1,
    HasMul = 1,
    HasDiv = 1,
    HasNegate = 1,
    HasAbs = 1,
    HasRound = 1,
    HasMin = 1,
    HasMax = 1,
    HasCmp = 1,
    HasSet1 = 1,
    HasCast = 1,
    HasBitwise = 1,
    HasRedux = 1,
    HasSign = 1,
    HasArg = 0,
    HasConj = 1,
    // Math functions
    HasReciprocal = 1,
    HasSin = 1,
    HasCos = 1,
    HasTan = 1,
    HasACos = 1,
    HasASin = 1,
    HasATan = 1,
    HasATanh = 1,
    HasLog = 1,
    HasLog1p = 1,
    HasExpm1 = 1,
    HasExp = 1,
    HasPow = 1,
    HasNdtri = 1,
    HasBessel = 1,
    HasSqrt = 1,
    HasRsqrt = 1,
    HasCbrt = 1,
    HasTanh = 1,
    HasErf = 1,
    HasErfc = 1
  };
};

template <>
struct packet_traits<float> : generic_float_packet_traits {
  using type = PacketXf;
  using half = PacketXf;
  enum {
    size = kFloatPacketSize,
  };
};

template <>
struct packet_traits<double> : generic_float_packet_traits {
  using type = PacketXd;
  using half = PacketXd;
  // Generic double-precision acos/asin are not yet implemented in
  // GenericPacketMathFunctions.h (only float versions exist).
  enum { size = kDoublePacketSize, HasACos = 0, HasASin = 0 };
};

struct generic_integer_packet_traits : default_packet_traits {
  enum {
    Vectorizable = 1,
    AlignedOnScalar = 1,
    HasAdd = 1,
    HasSub = 1,
    HasMul = 1,
    HasDiv = 1,
    HasNegate = 1,
    HasAbs = 1,
    HasMin = 1,
    HasMax = 1,
    HasCmp = 1,
    HasSet1 = 1,
    HasCast = 1,
    HasBitwise = 1,
    HasRedux = 1,
    // Set remaining to 0
    HasRound = 1,
    HasSqrt = 0,
    HasRsqrt = 0,
    HasReciprocal = 0,
    HasArg = 0,
    HasConj = 1,
    HasExp = 0,
    HasLog = 0,
    HasSin = 0,
    HasCos = 0,
  };
};

template <>
struct packet_traits<int32_t> : generic_integer_packet_traits {
  using type = PacketXi;
  using half = PacketXi;
  enum {
    size = kFloatPacketSize,
  };
};

template <>
struct packet_traits<int64_t> : generic_integer_packet_traits {
  using type = PacketXl;
  using half = PacketXl;
  enum {
    size = kDoublePacketSize,
  };
};

// --- unpacket_traits specializations ---
struct generic_unpacket_traits : default_unpacket_traits {
  enum {
    alignment = EIGEN_GENERIC_VECTOR_SIZE_BYTES,
    vectorizable = true,
  };
};

template <>
struct unpacket_traits<PacketXf> : generic_unpacket_traits {
  using type = float;
  using half = PacketXf;
  using integer_packet = PacketXi;
  enum {
    size = kFloatPacketSize,
  };
};
template <>
struct unpacket_traits<PacketXd> : generic_unpacket_traits {
  using type = double;
  using half = PacketXd;
  using integer_packet = PacketXl;
  enum {
    size = kDoublePacketSize,
  };
};
template <>
struct unpacket_traits<PacketXi> : generic_unpacket_traits {
  using type = int32_t;
  using half = PacketXi;
  enum {
    size = kFloatPacketSize,
  };
};
template <>
struct unpacket_traits<PacketXl> : generic_unpacket_traits {
  using type = int64_t;
  using half = PacketXl;
  enum {
    size = kDoublePacketSize,
  };
};

namespace detail {
// --- vector type helpers ---
template <typename VectorT>
struct ScalarTypeOfVector {
  using type = std::remove_all_extents_t<std::remove_reference_t<decltype(VectorT()[0])>>;
};

template <typename VectorT>
using scalar_type_of_vector_t = typename ScalarTypeOfVector<VectorT>::type;

template <typename VectorType>
struct UnsignedVectorHelper {
  static VectorType v;
  static constexpr int n = __builtin_vectorelements(v);
  using UnsignedScalar = std::make_unsigned_t<scalar_type_of_vector_t<VectorType>>;
  using type = UnsignedScalar __attribute__((ext_vector_type(n), aligned(n * sizeof(UnsignedScalar))));
};

template <typename VectorT>
using unsigned_vector_t = typename UnsignedVectorHelper<VectorT>::type;

template <typename VectorT>
using HalfPacket = VectorType<typename unpacket_traits<VectorT>::type, unpacket_traits<VectorT>::size / 2>;

template <typename VectorT>
using QuarterPacket = VectorType<typename unpacket_traits<VectorT>::type, unpacket_traits<VectorT>::size / 4>;

// load and store helpers.
template <typename VectorT>
EIGEN_STRONG_INLINE VectorT load_vector_unaligned(const scalar_type_of_vector_t<VectorT>* from) {
  VectorT to;
  __builtin_memcpy(&to, from, sizeof(VectorT));
  return to;
}

template <typename VectorT>
EIGEN_STRONG_INLINE VectorT load_vector_aligned(const scalar_type_of_vector_t<VectorT>* from) {
  return *reinterpret_cast<const VectorT*>(assume_aligned<EIGEN_GENERIC_VECTOR_SIZE_BYTES>(from));
}

template <typename VectorT>
EIGEN_STRONG_INLINE void store_vector_unaligned(scalar_type_of_vector_t<VectorT>* to, const VectorT& from) {
  __builtin_memcpy(to, &from, sizeof(VectorT));
}

template <typename VectorT>
EIGEN_STRONG_INLINE void store_vector_aligned(scalar_type_of_vector_t<VectorT>* to, const VectorT& from) {
  *reinterpret_cast<VectorT*>(assume_aligned<EIGEN_GENERIC_VECTOR_SIZE_BYTES>(to)) = from;
}

}  // namespace detail

// --- Intrinsic-like specializations ---

// --- Load/Store operations ---
#define EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PACKET_TYPE)                                                         \
  template <>                                                                                                     \
  EIGEN_STRONG_INLINE PACKET_TYPE ploadu<PACKET_TYPE>(const detail::scalar_type_of_vector_t<PACKET_TYPE>* from) { \
    return detail::load_vector_unaligned<PACKET_TYPE>(from);                                                      \
  }                                                                                                               \
  template <>                                                                                                     \
  EIGEN_STRONG_INLINE PACKET_TYPE pload<PACKET_TYPE>(const detail::scalar_type_of_vector_t<PACKET_TYPE>* from) {  \
    return detail::load_vector_aligned<PACKET_TYPE>(from);                                                        \
  }                                                                                                               \
  template <>                                                                                                     \
  EIGEN_STRONG_INLINE void pstoreu<detail::scalar_type_of_vector_t<PACKET_TYPE>, PACKET_TYPE>(                    \
      detail::scalar_type_of_vector_t<PACKET_TYPE> * to, const PACKET_TYPE& from) {                               \
    detail::store_vector_unaligned<PACKET_TYPE>(to, from);                                                        \
  }                                                                                                               \
  template <>                                                                                                     \
  EIGEN_STRONG_INLINE void pstore<detail::scalar_type_of_vector_t<PACKET_TYPE>, PACKET_TYPE>(                     \
      detail::scalar_type_of_vector_t<PACKET_TYPE> * to, const PACKET_TYPE& from) {                               \
    detail::store_vector_aligned<PACKET_TYPE>(to, from);                                                          \
  }

EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PacketXf)
EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PacketXd)
EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PacketXi)
EIGEN_CLANG_PACKET_LOAD_STORE_PACKET(PacketXl)
#undef EIGEN_CLANG_PACKET_LOAD_STORE_PACKET

// --- Broadcast operation ---
template <>
EIGEN_STRONG_INLINE PacketXf pset1frombits<PacketXf>(uint32_t from) {
  return PacketXf(numext::bit_cast<float>(from));
}

template <>
EIGEN_STRONG_INLINE PacketXd pset1frombits<PacketXd>(uint64_t from) {
  return PacketXd(numext::bit_cast<double>(from));
}

#define EIGEN_CLANG_PACKET_SET1(PACKET_TYPE)                                                            \
  template <>                                                                                           \
  EIGEN_STRONG_INLINE PACKET_TYPE pset1<PACKET_TYPE>(const unpacket_traits<PACKET_TYPE>::type& from) {  \
    return PACKET_TYPE(from);                                                                           \
  }                                                                                                     \
  template <>                                                                                           \
  EIGEN_STRONG_INLINE unpacket_traits<PACKET_TYPE>::type pfirst<PACKET_TYPE>(const PACKET_TYPE& from) { \
    return from[0];                                                                                     \
  }

EIGEN_CLANG_PACKET_SET1(PacketXf)
EIGEN_CLANG_PACKET_SET1(PacketXd)
EIGEN_CLANG_PACKET_SET1(PacketXi)
EIGEN_CLANG_PACKET_SET1(PacketXl)
#undef EIGEN_CLANG_PACKET_SET1

// --- Arithmetic operations ---
#define EIGEN_CLANG_PACKET_ARITHMETIC(PACKET_TYPE)                             \
  template <>                                                                  \
  EIGEN_STRONG_INLINE PACKET_TYPE pisnan<PACKET_TYPE>(const PACKET_TYPE& a) {  \
    return reinterpret_cast<PACKET_TYPE>(a != a);                              \
  }                                                                            \
  template <>                                                                  \
  EIGEN_STRONG_INLINE PACKET_TYPE pnegate<PACKET_TYPE>(const PACKET_TYPE& a) { \
    return -a;                                                                 \
  }

EIGEN_CLANG_PACKET_ARITHMETIC(PacketXf)
EIGEN_CLANG_PACKET_ARITHMETIC(PacketXd)
EIGEN_CLANG_PACKET_ARITHMETIC(PacketXi)
EIGEN_CLANG_PACKET_ARITHMETIC(PacketXl)
#undef EIGEN_CLANG_PACKET_ARITHMETIC

// --- Bitwise operations (via casting) ---

namespace detail {

// Reinterpret-cast helpers, equivalent to preinterpret<> but defined here
// because PacketMath.h is included before TypeCasting.h.
EIGEN_STRONG_INLINE PacketXi preinterpret_float_to_int(const PacketXf& a) { return reinterpret_cast<PacketXi>(a); }
EIGEN_STRONG_INLINE PacketXf preinterpret_int_to_float(const PacketXi& a) { return reinterpret_cast<PacketXf>(a); }
EIGEN_STRONG_INLINE PacketXl preinterpret_double_to_long(const PacketXd& a) { return reinterpret_cast<PacketXl>(a); }
EIGEN_STRONG_INLINE PacketXd preinterpret_long_to_double(const PacketXl& a) { return reinterpret_cast<PacketXd>(a); }

}  // namespace detail

// Bitwise ops for integer packets
#define EIGEN_CLANG_PACKET_BITWISE_INT(PACKET_TYPE)                                                  \
  template <>                                                                                        \
  constexpr EIGEN_STRONG_INLINE PACKET_TYPE pzero<PACKET_TYPE>(const PACKET_TYPE& /*unused*/) {      \
    return PACKET_TYPE(0);                                                                           \
  }                                                                                                  \
  template <>                                                                                        \
  constexpr EIGEN_STRONG_INLINE PACKET_TYPE ptrue<PACKET_TYPE>(const PACKET_TYPE& /*unused*/) {      \
    return numext::bit_cast<PACKET_TYPE>(PACKET_TYPE(0) == PACKET_TYPE(0));                          \
  }                                                                                                  \
  template <>                                                                                        \
  EIGEN_STRONG_INLINE PACKET_TYPE pand<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) {    \
    return a & b;                                                                                    \
  }                                                                                                  \
  template <>                                                                                        \
  EIGEN_STRONG_INLINE PACKET_TYPE por<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) {     \
    return a | b;                                                                                    \
  }                                                                                                  \
  template <>                                                                                        \
  EIGEN_STRONG_INLINE PACKET_TYPE pxor<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) {    \
    return a ^ b;                                                                                    \
  }                                                                                                  \
  template <>                                                                                        \
  EIGEN_STRONG_INLINE PACKET_TYPE pandnot<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
    return a & ~b;                                                                                   \
  }                                                                                                  \
  template <int N>                                                                                   \
  EIGEN_STRONG_INLINE PACKET_TYPE parithmetic_shift_right(const PACKET_TYPE& a) {                    \
    return a >> N;                                                                                   \
  }                                                                                                  \
  template <int N>                                                                                   \
  EIGEN_STRONG_INLINE PACKET_TYPE plogical_shift_right(const PACKET_TYPE& a) {                       \
    using UnsignedT = detail::unsigned_vector_t<PACKET_TYPE>;                                        \
    return reinterpret_cast<PACKET_TYPE>(reinterpret_cast<UnsignedT>(a) >> N);                       \
  }                                                                                                  \
  template <int N>                                                                                   \
  EIGEN_STRONG_INLINE PACKET_TYPE plogical_shift_left(const PACKET_TYPE& a) {                        \
    return a << N;                                                                                   \
  }

EIGEN_CLANG_PACKET_BITWISE_INT(PacketXi)
EIGEN_CLANG_PACKET_BITWISE_INT(PacketXl)
#undef EIGEN_CLANG_PACKET_BITWISE_INT

// Bitwise ops for floating point packets
#define EIGEN_CLANG_PACKET_BITWISE_FLOAT(PACKET_TYPE, CAST_TO_INT, CAST_FROM_INT)                    \
  template <>                                                                                        \
  constexpr EIGEN_STRONG_INLINE PACKET_TYPE pzero<PACKET_TYPE>(const PACKET_TYPE& /*unused*/) {      \
    using Scalar = detail::scalar_type_of_vector_t<PACKET_TYPE>;                                     \
    return PACKET_TYPE(Scalar(0));                                                                   \
  }                                                                                                  \
  template <>                                                                                        \
  constexpr EIGEN_STRONG_INLINE PACKET_TYPE ptrue<PACKET_TYPE>(const PACKET_TYPE& /* unused */) {    \
    using Scalar = detail::scalar_type_of_vector_t<PACKET_TYPE>;                                     \
    return numext::bit_cast<PACKET_TYPE>(PACKET_TYPE(Scalar(0)) == PACKET_TYPE(Scalar(0)));          \
  }                                                                                                  \
  template <>                                                                                        \
  EIGEN_STRONG_INLINE PACKET_TYPE pand<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) {    \
    return CAST_FROM_INT(CAST_TO_INT(a) & CAST_TO_INT(b));                                           \
  }                                                                                                  \
  template <>                                                                                        \
  EIGEN_STRONG_INLINE PACKET_TYPE por<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) {     \
    return CAST_FROM_INT(CAST_TO_INT(a) | CAST_TO_INT(b));                                           \
  }                                                                                                  \
  template <>                                                                                        \
  EIGEN_STRONG_INLINE PACKET_TYPE pxor<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) {    \
    return CAST_FROM_INT(CAST_TO_INT(a) ^ CAST_TO_INT(b));                                           \
  }                                                                                                  \
  template <>                                                                                        \
  EIGEN_STRONG_INLINE PACKET_TYPE pandnot<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
    return CAST_FROM_INT(CAST_TO_INT(a) & ~CAST_TO_INT(b));                                          \
  }

EIGEN_CLANG_PACKET_BITWISE_FLOAT(PacketXf, detail::preinterpret_float_to_int, detail::preinterpret_int_to_float)
EIGEN_CLANG_PACKET_BITWISE_FLOAT(PacketXd, detail::preinterpret_double_to_long, detail::preinterpret_long_to_double)
#undef EIGEN_CLANG_PACKET_BITWISE_FLOAT

// --- Comparison operations ---
// Clang vector extensions perform comparisons in the original type (float/double),
// returning an int vector with all-ones (-1) for true and all-zeros for false.
// The bit_cast reinterprets those int bitmasks as float packets, which is the
// format expected by pselect and other Eigen packet operations.
#define EIGEN_CLANG_PACKET_CMP(PACKET_TYPE, INT_PACKET_TYPE)                                                \
  template <>                                                                                               \
  EIGEN_STRONG_INLINE PACKET_TYPE pcmp_eq<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) {        \
    return numext::bit_cast<PACKET_TYPE>(INT_PACKET_TYPE(a == b));                                          \
  }                                                                                                         \
  template <>                                                                                               \
  EIGEN_STRONG_INLINE PACKET_TYPE pcmp_lt<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) {        \
    return numext::bit_cast<PACKET_TYPE>(INT_PACKET_TYPE(a < b));                                           \
  }                                                                                                         \
  template <>                                                                                               \
  EIGEN_STRONG_INLINE PACKET_TYPE pcmp_le<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) {        \
    return numext::bit_cast<PACKET_TYPE>(INT_PACKET_TYPE(a <= b));                                          \
  }                                                                                                         \
  template <>                                                                                               \
  EIGEN_STRONG_INLINE PACKET_TYPE pcmp_lt_or_nan<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
    return numext::bit_cast<PACKET_TYPE>(INT_PACKET_TYPE(!(a >= b)));                                       \
  }

EIGEN_CLANG_PACKET_CMP(PacketXf, PacketXi)
EIGEN_CLANG_PACKET_CMP(PacketXd, PacketXl)
#undef EIGEN_CLANG_PACKET_CMP

// --- Min/Max operations ---
#if EIGEN_HAS_BUILTIN(__builtin_elementwise_min) && EIGEN_HAS_BUILTIN(__builtin_elementwise_max) && \
    EIGEN_HAS_BUILTIN(__builtin_elementwise_abs)
#define EIGEN_CLANG_PACKET_ELEMENTWISE(PACKET_TYPE)                                                                 \
  template <>                                                                                                       \
  EIGEN_STRONG_INLINE PACKET_TYPE pmin<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) {                   \
    /* Match NaN propagation of std::min. */                                                                        \
    return a == a ? __builtin_elementwise_min(a, b) : a;                                                            \
  }                                                                                                                 \
  template <>                                                                                                       \
  EIGEN_STRONG_INLINE PACKET_TYPE pmax<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) {                   \
    /* Match NaN propagation of std::max. */                                                                        \
    return a == a ? __builtin_elementwise_max(a, b) : a;                                                            \
  }                                                                                                                 \
  template <>                                                                                                       \
  EIGEN_STRONG_INLINE PACKET_TYPE pmin<PropagateNumbers, PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
    return __builtin_elementwise_min(a, b);                                                                         \
  }                                                                                                                 \
  template <>                                                                                                       \
  EIGEN_STRONG_INLINE PACKET_TYPE pmax<PropagateNumbers, PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) { \
    return __builtin_elementwise_max(a, b);                                                                         \
  }                                                                                                                 \
  template <>                                                                                                       \
  EIGEN_STRONG_INLINE PACKET_TYPE pmin<PropagateNaN, PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) {     \
    return a != a ? a : (b != b ? b : __builtin_elementwise_min(a, b));                                             \
  }                                                                                                                 \
  template <>                                                                                                       \
  EIGEN_STRONG_INLINE PACKET_TYPE pmax<PropagateNaN, PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b) {     \
    return a != a ? a : (b != b ? b : __builtin_elementwise_max(a, b));                                             \
  }                                                                                                                 \
  template <>                                                                                                       \
  EIGEN_STRONG_INLINE PACKET_TYPE pabs<PACKET_TYPE>(const PACKET_TYPE& a) {                                         \
    return __builtin_elementwise_abs(a);                                                                            \
  }                                                                                                                 \
  template <>                                                                                                       \
  EIGEN_STRONG_INLINE PACKET_TYPE pselect<PACKET_TYPE>(const PACKET_TYPE& mask, const PACKET_TYPE& a,               \
                                                       const PACKET_TYPE& b) {                                      \
    return mask != 0 ? a : b;                                                                                       \
  }

EIGEN_CLANG_PACKET_ELEMENTWISE(PacketXf)
EIGEN_CLANG_PACKET_ELEMENTWISE(PacketXd)
EIGEN_CLANG_PACKET_ELEMENTWISE(PacketXi)
EIGEN_CLANG_PACKET_ELEMENTWISE(PacketXl)
#undef EIGEN_CLANG_PACKET_ELEMENTWISE
#endif

// --- Math functions (float/double only) ---

#if EIGEN_HAS_BUILTIN(__builtin_elementwise_floor) && EIGEN_HAS_BUILTIN(__builtin_elementwise_ceil) &&      \
    EIGEN_HAS_BUILTIN(__builtin_elementwise_round) && EIGEN_HAS_BUILTIN(__builtin_elementwise_roundeven) && \
    EIGEN_HAS_BUILTIN(__builtin_elementwise_trunc) && EIGEN_HAS_BUILTIN(__builtin_elementwise_sqrt)
#define EIGEN_CLANG_PACKET_MATH_FLOAT(PACKET_TYPE)                            \
  template <>                                                                 \
  EIGEN_STRONG_INLINE PACKET_TYPE pfloor<PACKET_TYPE>(const PACKET_TYPE& a) { \
    return __builtin_elementwise_floor(a);                                    \
  }                                                                           \
  template <>                                                                 \
  EIGEN_STRONG_INLINE PACKET_TYPE pceil<PACKET_TYPE>(const PACKET_TYPE& a) {  \
    return __builtin_elementwise_ceil(a);                                     \
  }                                                                           \
  template <>                                                                 \
  EIGEN_STRONG_INLINE PACKET_TYPE pround<PACKET_TYPE>(const PACKET_TYPE& a) { \
    return __builtin_elementwise_round(a);                                    \
  }                                                                           \
  template <>                                                                 \
  EIGEN_STRONG_INLINE PACKET_TYPE print<PACKET_TYPE>(const PACKET_TYPE& a) {  \
    return __builtin_elementwise_roundeven(a);                                \
  }                                                                           \
  template <>                                                                 \
  EIGEN_STRONG_INLINE PACKET_TYPE ptrunc<PACKET_TYPE>(const PACKET_TYPE& a) { \
    return __builtin_elementwise_trunc(a);                                    \
  }                                                                           \
  template <>                                                                 \
  EIGEN_STRONG_INLINE PACKET_TYPE psqrt<PACKET_TYPE>(const PACKET_TYPE& a) {  \
    return __builtin_elementwise_sqrt(a);                                     \
  }

EIGEN_CLANG_PACKET_MATH_FLOAT(PacketXf)
EIGEN_CLANG_PACKET_MATH_FLOAT(PacketXd)
#undef EIGEN_CLANG_PACKET_MATH_FLOAT
#endif

// --- Fused Multiply-Add (MADD) ---
#if defined(__FMA__) && EIGEN_HAS_BUILTIN(__builtin_elementwise_fma)
#define EIGEN_CLANG_PACKET_MADD(PACKET_TYPE)                                                      \
  template <>                                                                                     \
  EIGEN_STRONG_INLINE PACKET_TYPE pmadd<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b,  \
                                                     const PACKET_TYPE& c) {                      \
    return __builtin_elementwise_fma(a, b, c);                                                    \
  }                                                                                               \
  template <>                                                                                     \
  EIGEN_STRONG_INLINE PACKET_TYPE pmsub<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b,  \
                                                     const PACKET_TYPE& c) {                      \
    return __builtin_elementwise_fma(a, b, -c);                                                   \
  }                                                                                               \
  template <>                                                                                     \
  EIGEN_STRONG_INLINE PACKET_TYPE pnmadd<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b, \
                                                      const PACKET_TYPE& c) {                     \
    return __builtin_elementwise_fma(-a, b, c);                                                   \
  }                                                                                               \
  template <>                                                                                     \
  EIGEN_STRONG_INLINE PACKET_TYPE pnmsub<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b, \
                                                      const PACKET_TYPE& c) {                     \
    return -(__builtin_elementwise_fma(a, b, c));                                                 \
  }
#else
// Fallback if FMA builtin is not available
#define EIGEN_CLANG_PACKET_MADD(PACKET_TYPE)                                                      \
  template <>                                                                                     \
  EIGEN_STRONG_INLINE PACKET_TYPE pmadd<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b,  \
                                                     const PACKET_TYPE& c) {                      \
    return (a * b) + c;                                                                           \
  }                                                                                               \
  template <>                                                                                     \
  EIGEN_STRONG_INLINE PACKET_TYPE pmsub<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b,  \
                                                     const PACKET_TYPE& c) {                      \
    return (a * b) - c;                                                                           \
  }                                                                                               \
  template <>                                                                                     \
  EIGEN_STRONG_INLINE PACKET_TYPE pnmadd<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b, \
                                                      const PACKET_TYPE& c) {                     \
    return c - (a * b);                                                                           \
  }                                                                                               \
  template <>                                                                                     \
  EIGEN_STRONG_INLINE PACKET_TYPE pnmsub<PACKET_TYPE>(const PACKET_TYPE& a, const PACKET_TYPE& b, \
                                                      const PACKET_TYPE& c) {                     \
    return -((a * b) + c);                                                                        \
  }
#endif

EIGEN_CLANG_PACKET_MADD(PacketXf)
EIGEN_CLANG_PACKET_MADD(PacketXd)
#undef EIGEN_CLANG_PACKET_MADD

#define EIGEN_CLANG_PACKET_SCATTER_GATHER(PACKET_TYPE)                                                               \
  template <>                                                                                                        \
  EIGEN_STRONG_INLINE void pscatter(unpacket_traits<PACKET_TYPE>::type* to, const PACKET_TYPE& from, Index stride) { \
    constexpr int size = unpacket_traits<PACKET_TYPE>::size;                                                         \
    for (int i = 0; i < size; ++i) {                                                                                 \
      to[i * stride] = from[i];                                                                                      \
    }                                                                                                                \
  }                                                                                                                  \
  template <>                                                                                                        \
  EIGEN_STRONG_INLINE PACKET_TYPE pgather<typename unpacket_traits<PACKET_TYPE>::type, PACKET_TYPE>(                 \
      const unpacket_traits<PACKET_TYPE>::type* from, Index stride) {                                                \
    constexpr int size = unpacket_traits<PACKET_TYPE>::size;                                                         \
    PACKET_TYPE result;                                                                                              \
    for (int i = 0; i < size; ++i) {                                                                                 \
      result[i] = from[i * stride];                                                                                  \
    }                                                                                                                \
    return result;                                                                                                   \
  }

EIGEN_CLANG_PACKET_SCATTER_GATHER(PacketXf)
EIGEN_CLANG_PACKET_SCATTER_GATHER(PacketXd)
EIGEN_CLANG_PACKET_SCATTER_GATHER(PacketXi)
EIGEN_CLANG_PACKET_SCATTER_GATHER(PacketXl)

#undef EIGEN_CLANG_PACKET_SCATTER_GATHER

// ---- Various operations that depend on __builtin_shufflevector.
#if EIGEN_HAS_BUILTIN(__builtin_shufflevector)
namespace detail {
template <typename Packet>
EIGEN_STRONG_INLINE Packet preverse_impl_2(const Packet& a) {
  return __builtin_shufflevector(a, a, 1, 0);
}
template <typename Packet>
EIGEN_STRONG_INLINE Packet preverse_impl_4(const Packet& a) {
  return __builtin_shufflevector(a, a, 3, 2, 1, 0);
}
template <typename Packet>
EIGEN_STRONG_INLINE Packet preverse_impl_8(const Packet& a) {
  return __builtin_shufflevector(a, a, 7, 6, 5, 4, 3, 2, 1, 0);
}
template <typename Packet>
EIGEN_STRONG_INLINE Packet preverse_impl_16(const Packet& a) {
  return __builtin_shufflevector(a, a, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
}
}  // namespace detail

#if EIGEN_GENERIC_VECTOR_SIZE_BYTES == 16

template <>
EIGEN_STRONG_INLINE PacketXf preverse<PacketXf>(const PacketXf& a) {
  return detail::preverse_impl_4(a);
}
template <>
EIGEN_STRONG_INLINE PacketXd preverse<PacketXd>(const PacketXd& a) {
  return detail::preverse_impl_2(a);
}
template <>
EIGEN_STRONG_INLINE PacketXi preverse<PacketXi>(const PacketXi& a) {
  return detail::preverse_impl_4(a);
}
template <>
EIGEN_STRONG_INLINE PacketXl preverse<PacketXl>(const PacketXl& a) {
  return detail::preverse_impl_2(a);
}

#elif EIGEN_GENERIC_VECTOR_SIZE_BYTES == 32

template <>
EIGEN_STRONG_INLINE PacketXf preverse<PacketXf>(const PacketXf& a) {
  return detail::preverse_impl_8(a);
}
template <>
EIGEN_STRONG_INLINE PacketXd preverse<PacketXd>(const PacketXd& a) {
  return detail::preverse_impl_4(a);
}
template <>
EIGEN_STRONG_INLINE PacketXi preverse<PacketXi>(const PacketXi& a) {
  return detail::preverse_impl_8(a);
}
template <>
EIGEN_STRONG_INLINE PacketXl preverse<PacketXl>(const PacketXl& a) {
  return detail::preverse_impl_4(a);
}

#else  // EIGEN_GENERIC_VECTOR_SIZE_BYTES == 64

template <>
EIGEN_STRONG_INLINE PacketXf preverse<PacketXf>(const PacketXf& a) {
  return detail::preverse_impl_16(a);
}
template <>
EIGEN_STRONG_INLINE PacketXd preverse<PacketXd>(const PacketXd& a) {
  return detail::preverse_impl_8(a);
}
template <>
EIGEN_STRONG_INLINE PacketXi preverse<PacketXi>(const PacketXi& a) {
  return detail::preverse_impl_16(a);
}
template <>
EIGEN_STRONG_INLINE PacketXl preverse<PacketXl>(const PacketXl& a) {
  return detail::preverse_impl_8(a);
}

#endif  // EIGEN_GENERIC_VECTOR_SIZE_BYTES

namespace detail {

template <typename Packet>
EIGEN_STRONG_INLINE Packet ploaddup2(const typename unpacket_traits<Packet>::type* from) {
  static_assert((unpacket_traits<Packet>::size) % 2 == 0, "Packet size must be a multiple of 2");
  using HalfPacket = HalfPacket<Packet>;
  HalfPacket a = load_vector_unaligned<HalfPacket>(from);
  return __builtin_shufflevector(a, a, 0, 0);
}

template <typename Packet>
EIGEN_STRONG_INLINE Packet ploaddup4(const typename unpacket_traits<Packet>::type* from) {
  static_assert((unpacket_traits<Packet>::size) % 2 == 0, "Packet size must be a multiple of 2");
  using HalfPacket = HalfPacket<Packet>;
  HalfPacket a = load_vector_unaligned<HalfPacket>(from);
  return __builtin_shufflevector(a, a, 0, 0, 1, 1);
}

template <typename Packet>
EIGEN_STRONG_INLINE Packet ploaddup8(const typename unpacket_traits<Packet>::type* from) {
  static_assert((unpacket_traits<Packet>::size) % 2 == 0, "Packet size must be a multiple of 2");
  using HalfPacket = HalfPacket<Packet>;
  HalfPacket a = load_vector_unaligned<HalfPacket>(from);
  return __builtin_shufflevector(a, a, 0, 0, 1, 1, 2, 2, 3, 3);
}

template <typename Packet>
EIGEN_STRONG_INLINE Packet ploaddup16(const typename unpacket_traits<Packet>::type* from) {
  static_assert((unpacket_traits<Packet>::size) % 2 == 0, "Packet size must be a multiple of 2");
  using HalfPacket = HalfPacket<Packet>;
  HalfPacket a = load_vector_unaligned<HalfPacket>(from);
  return __builtin_shufflevector(a, a, 0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 7, 7);
}

template <typename Packet>
EIGEN_STRONG_INLINE Packet ploadquad4(const typename unpacket_traits<Packet>::type* from) {
  static_assert((unpacket_traits<Packet>::size) % 4 == 0, "Packet size must be a multiple of 4");
  using QuarterPacket = QuarterPacket<Packet>;
  QuarterPacket a = load_vector_unaligned<QuarterPacket>(from);
  return __builtin_shufflevector(a, a, 0, 0, 0, 0);
}

template <typename Packet>
EIGEN_STRONG_INLINE Packet ploadquad8(const typename unpacket_traits<Packet>::type* from) {
  static_assert((unpacket_traits<Packet>::size) % 4 == 0, "Packet size must be a multiple of 4");
  using QuarterPacket = QuarterPacket<Packet>;
  QuarterPacket a = load_vector_unaligned<QuarterPacket>(from);
  return __builtin_shufflevector(a, a, 0, 0, 0, 0, 1, 1, 1, 1);
}

template <typename Packet>
EIGEN_STRONG_INLINE Packet ploadquad16(const typename unpacket_traits<Packet>::type* from) {
  static_assert((unpacket_traits<Packet>::size) % 4 == 0, "Packet size must be a multiple of 4");
  using QuarterPacket = QuarterPacket<Packet>;
  QuarterPacket a = load_vector_unaligned<QuarterPacket>(from);
  return __builtin_shufflevector(a, a, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3);
}

}  // namespace detail

#if EIGEN_GENERIC_VECTOR_SIZE_BYTES == 16

template <>
EIGEN_STRONG_INLINE PacketXf ploaddup<PacketXf>(const float* from) {
  return detail::ploaddup4<PacketXf>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXd ploaddup<PacketXd>(const double* from) {
  return detail::ploaddup2<PacketXd>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXi ploaddup<PacketXi>(const int32_t* from) {
  return detail::ploaddup4<PacketXi>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXl ploaddup<PacketXl>(const int64_t* from) {
  return detail::ploaddup2<PacketXl>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXf ploadquad<PacketXf>(const float* from) {
  return detail::ploadquad4<PacketXf>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXi ploadquad<PacketXi>(const int32_t* from) {
  return detail::ploadquad4<PacketXi>(from);
}
// No ploadquad for 2-element packets (PacketXd, PacketXl) at 16 bytes.

#elif EIGEN_GENERIC_VECTOR_SIZE_BYTES == 32

template <>
EIGEN_STRONG_INLINE PacketXf ploaddup<PacketXf>(const float* from) {
  return detail::ploaddup8<PacketXf>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXd ploaddup<PacketXd>(const double* from) {
  return detail::ploaddup4<PacketXd>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXi ploaddup<PacketXi>(const int32_t* from) {
  return detail::ploaddup8<PacketXi>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXl ploaddup<PacketXl>(const int64_t* from) {
  return detail::ploaddup4<PacketXl>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXf ploadquad<PacketXf>(const float* from) {
  return detail::ploadquad8<PacketXf>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXd ploadquad<PacketXd>(const double* from) {
  return detail::ploadquad4<PacketXd>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXi ploadquad<PacketXi>(const int32_t* from) {
  return detail::ploadquad8<PacketXi>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXl ploadquad<PacketXl>(const int64_t* from) {
  return detail::ploadquad4<PacketXl>(from);
}

#else  // EIGEN_GENERIC_VECTOR_SIZE_BYTES == 64

template <>
EIGEN_STRONG_INLINE PacketXf ploaddup<PacketXf>(const float* from) {
  return detail::ploaddup16<PacketXf>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXd ploaddup<PacketXd>(const double* from) {
  return detail::ploaddup8<PacketXd>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXi ploaddup<PacketXi>(const int32_t* from) {
  return detail::ploaddup16<PacketXi>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXl ploaddup<PacketXl>(const int64_t* from) {
  return detail::ploaddup8<PacketXl>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXf ploadquad<PacketXf>(const float* from) {
  return detail::ploadquad16<PacketXf>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXd ploadquad<PacketXd>(const double* from) {
  return detail::ploadquad8<PacketXd>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXi ploadquad<PacketXi>(const int32_t* from) {
  return detail::ploadquad16<PacketXi>(from);
}
template <>
EIGEN_STRONG_INLINE PacketXl ploadquad<PacketXl>(const int64_t* from) {
  return detail::ploadquad8<PacketXl>(from);
}

#endif  // EIGEN_GENERIC_VECTOR_SIZE_BYTES

// --- plset ---
#if EIGEN_GENERIC_VECTOR_SIZE_BYTES == 16

template <>
EIGEN_STRONG_INLINE PacketXf plset<PacketXf>(const float& a) {
  return PacketXf{a + 0.0f, a + 1.0f, a + 2.0f, a + 3.0f};
}
template <>
EIGEN_STRONG_INLINE PacketXd plset<PacketXd>(const double& a) {
  return PacketXd{a + 0.0, a + 1.0};
}
template <>
EIGEN_STRONG_INLINE PacketXi plset<PacketXi>(const int32_t& a) {
  return PacketXi{a + 0, a + 1, a + 2, a + 3};
}
template <>
EIGEN_STRONG_INLINE PacketXl plset<PacketXl>(const int64_t& a) {
  return PacketXl{a + 0, a + 1};
}

#elif EIGEN_GENERIC_VECTOR_SIZE_BYTES == 32

template <>
EIGEN_STRONG_INLINE PacketXf plset<PacketXf>(const float& a) {
  return PacketXf{a + 0.0f, a + 1.0f, a + 2.0f, a + 3.0f, a + 4.0f, a + 5.0f, a + 6.0f, a + 7.0f};
}
template <>
EIGEN_STRONG_INLINE PacketXd plset<PacketXd>(const double& a) {
  return PacketXd{a + 0.0, a + 1.0, a + 2.0, a + 3.0};
}
template <>
EIGEN_STRONG_INLINE PacketXi plset<PacketXi>(const int32_t& a) {
  return PacketXi{a + 0, a + 1, a + 2, a + 3, a + 4, a + 5, a + 6, a + 7};
}
template <>
EIGEN_STRONG_INLINE PacketXl plset<PacketXl>(const int64_t& a) {
  return PacketXl{a + 0, a + 1, a + 2, a + 3};
}

#else  // EIGEN_GENERIC_VECTOR_SIZE_BYTES == 64

template <>
EIGEN_STRONG_INLINE PacketXf plset<PacketXf>(const float& a) {
  return PacketXf{a + 0.0f, a + 1.0f, a + 2.0f,  a + 3.0f,  a + 4.0f,  a + 5.0f,  a + 6.0f,  a + 7.0f,
                  a + 8.0f, a + 9.0f, a + 10.0f, a + 11.0f, a + 12.0f, a + 13.0f, a + 14.0f, a + 15.0f};
}
template <>
EIGEN_STRONG_INLINE PacketXd plset<PacketXd>(const double& a) {
  return PacketXd{a + 0.0, a + 1.0, a + 2.0, a + 3.0, a + 4.0, a + 5.0, a + 6.0, a + 7.0};
}
template <>
EIGEN_STRONG_INLINE PacketXi plset<PacketXi>(const int32_t& a) {
  return PacketXi{a + 0, a + 1, a + 2,  a + 3,  a + 4,  a + 5,  a + 6,  a + 7,
                  a + 8, a + 9, a + 10, a + 11, a + 12, a + 13, a + 14, a + 15};
}
template <>
EIGEN_STRONG_INLINE PacketXl plset<PacketXl>(const int64_t& a) {
  return PacketXl{a + 0, a + 1, a + 2, a + 3, a + 4, a + 5, a + 6, a + 7};
}

#endif  // EIGEN_GENERIC_VECTOR_SIZE_BYTES

// --- peven_mask ---
#if EIGEN_GENERIC_VECTOR_SIZE_BYTES == 16

template <>
EIGEN_STRONG_INLINE PacketXf peven_mask(const PacketXf& /* unused */) {
  float kTrue = numext::bit_cast<float>(int32_t(-1));
  float kFalse = 0.0f;
  return PacketXf{kTrue, kFalse, kTrue, kFalse};
}
template <>
EIGEN_STRONG_INLINE PacketXd peven_mask(const PacketXd& /* unused */) {
  double kTrue = numext::bit_cast<double>(int64_t(-1l));
  double kFalse = 0.0;
  return PacketXd{kTrue, kFalse};
}

#elif EIGEN_GENERIC_VECTOR_SIZE_BYTES == 32

template <>
EIGEN_STRONG_INLINE PacketXf peven_mask(const PacketXf& /* unused */) {
  float kTrue = numext::bit_cast<float>(int32_t(-1));
  float kFalse = 0.0f;
  return PacketXf{kTrue, kFalse, kTrue, kFalse, kTrue, kFalse, kTrue, kFalse};
}
template <>
EIGEN_STRONG_INLINE PacketXd peven_mask(const PacketXd& /* unused */) {
  double kTrue = numext::bit_cast<double>(int64_t(-1l));
  double kFalse = 0.0;
  return PacketXd{kTrue, kFalse, kTrue, kFalse};
}

#else  // EIGEN_GENERIC_VECTOR_SIZE_BYTES == 64

template <>
EIGEN_STRONG_INLINE PacketXf peven_mask(const PacketXf& /* unused */) {
  float kTrue = numext::bit_cast<float>(int32_t(-1));
  float kFalse = 0.0f;
  return PacketXf{kTrue, kFalse, kTrue, kFalse, kTrue, kFalse, kTrue, kFalse,
                  kTrue, kFalse, kTrue, kFalse, kTrue, kFalse, kTrue, kFalse};
}
template <>
EIGEN_STRONG_INLINE PacketXd peven_mask(const PacketXd& /* unused */) {
  double kTrue = numext::bit_cast<double>(int64_t(-1l));
  double kFalse = 0.0;
  return PacketXd{kTrue, kFalse, kTrue, kFalse, kTrue, kFalse, kTrue, kFalse};
}

#endif  // EIGEN_GENERIC_VECTOR_SIZE_BYTES

// Helpers for ptranspose.
namespace detail {

template <typename Packet>
EIGEN_ALWAYS_INLINE void zip_in_place2(Packet& p1, Packet& p2) {
  Packet tmp = __builtin_shufflevector(p1, p2, 0, 2);
  p2 = __builtin_shufflevector(p1, p2, 1, 3);
  p1 = tmp;
}

template <typename Packet>
EIGEN_ALWAYS_INLINE void zip_in_place4(Packet& p1, Packet& p2) {
  Packet tmp = __builtin_shufflevector(p1, p2, 0, 4, 1, 5);
  p2 = __builtin_shufflevector(p1, p2, 2, 6, 3, 7);
  p1 = tmp;
}

template <typename Packet>
EIGEN_ALWAYS_INLINE void zip_in_place8(Packet& p1, Packet& p2) {
  Packet tmp = __builtin_shufflevector(p1, p2, 0, 8, 1, 9, 2, 10, 3, 11);
  p2 = __builtin_shufflevector(p1, p2, 4, 12, 5, 13, 6, 14, 7, 15);
  p1 = tmp;
}

template <typename Packet>
EIGEN_ALWAYS_INLINE void zip_in_place16(Packet& p1, Packet& p2) {
  Packet tmp = __builtin_shufflevector(p1, p2, 0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23);
  p2 = __builtin_shufflevector(p1, p2, 8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31);
  p1 = tmp;
}

template <typename Packet>
void zip_in_place(Packet& p1, Packet& p2);

#if EIGEN_GENERIC_VECTOR_SIZE_BYTES == 16
template <>
EIGEN_ALWAYS_INLINE void zip_in_place<PacketXf>(PacketXf& p1, PacketXf& p2) {
  zip_in_place4(p1, p2);
}
template <>
EIGEN_ALWAYS_INLINE void zip_in_place<PacketXd>(PacketXd& p1, PacketXd& p2) {
  zip_in_place2(p1, p2);
}
template <>
EIGEN_ALWAYS_INLINE void zip_in_place<PacketXi>(PacketXi& p1, PacketXi& p2) {
  zip_in_place4(p1, p2);
}
template <>
EIGEN_ALWAYS_INLINE void zip_in_place<PacketXl>(PacketXl& p1, PacketXl& p2) {
  zip_in_place2(p1, p2);
}
#elif EIGEN_GENERIC_VECTOR_SIZE_BYTES == 32
template <>
EIGEN_ALWAYS_INLINE void zip_in_place<PacketXf>(PacketXf& p1, PacketXf& p2) {
  zip_in_place8(p1, p2);
}
template <>
EIGEN_ALWAYS_INLINE void zip_in_place<PacketXd>(PacketXd& p1, PacketXd& p2) {
  zip_in_place4(p1, p2);
}
template <>
EIGEN_ALWAYS_INLINE void zip_in_place<PacketXi>(PacketXi& p1, PacketXi& p2) {
  zip_in_place8(p1, p2);
}
template <>
EIGEN_ALWAYS_INLINE void zip_in_place<PacketXl>(PacketXl& p1, PacketXl& p2) {
  zip_in_place4(p1, p2);
}
#else   // EIGEN_GENERIC_VECTOR_SIZE_BYTES == 64
template <>
EIGEN_ALWAYS_INLINE void zip_in_place<PacketXf>(PacketXf& p1, PacketXf& p2) {
  zip_in_place16(p1, p2);
}
template <>
EIGEN_ALWAYS_INLINE void zip_in_place<PacketXd>(PacketXd& p1, PacketXd& p2) {
  zip_in_place8(p1, p2);
}
template <>
EIGEN_ALWAYS_INLINE void zip_in_place<PacketXi>(PacketXi& p1, PacketXi& p2) {
  zip_in_place16(p1, p2);
}
template <>
EIGEN_ALWAYS_INLINE void zip_in_place<PacketXl>(PacketXl& p1, PacketXl& p2) {
  zip_in_place8(p1, p2);
}
#endif  // EIGEN_GENERIC_VECTOR_SIZE_BYTES

template <typename Packet>
EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 2>& kernel) {
  zip_in_place(kernel.packet[0], kernel.packet[1]);
}

template <typename Packet>
EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 4>& kernel) {
  zip_in_place(kernel.packet[0], kernel.packet[2]);
  zip_in_place(kernel.packet[1], kernel.packet[3]);
  zip_in_place(kernel.packet[0], kernel.packet[1]);
  zip_in_place(kernel.packet[2], kernel.packet[3]);
}

template <typename Packet>
EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 8>& kernel) {
  zip_in_place(kernel.packet[0], kernel.packet[4]);
  zip_in_place(kernel.packet[1], kernel.packet[5]);
  zip_in_place(kernel.packet[2], kernel.packet[6]);
  zip_in_place(kernel.packet[3], kernel.packet[7]);

  zip_in_place(kernel.packet[0], kernel.packet[2]);
  zip_in_place(kernel.packet[1], kernel.packet[3]);
  zip_in_place(kernel.packet[4], kernel.packet[6]);
  zip_in_place(kernel.packet[5], kernel.packet[7]);

  zip_in_place(kernel.packet[0], kernel.packet[1]);
  zip_in_place(kernel.packet[2], kernel.packet[3]);
  zip_in_place(kernel.packet[4], kernel.packet[5]);
  zip_in_place(kernel.packet[6], kernel.packet[7]);
}

template <typename Packet>
EIGEN_ALWAYS_INLINE void ptranspose_impl(PacketBlock<Packet, 16>& kernel) {
  EIGEN_UNROLL_LOOP
  for (int i = 0; i < 4; ++i) {
    const int m = (1 << i);
    EIGEN_UNROLL_LOOP
    for (int j = 0; j < m; ++j) {
      const int n = (1 << (3 - i));
      EIGEN_UNROLL_LOOP
      for (int k = 0; k < n; ++k) {
        const int idx = 2 * j * n + k;
        zip_in_place(kernel.packet[idx], kernel.packet[idx + n]);
      }
    }
  }
}

}  // namespace detail

// ptranspose overloads: only emit valid block sizes per vector size.
// At 16 bytes: float has 4 elems, double has 2 elems.
// At 32 bytes: float has 8 elems, double has 4 elems.
// At 64 bytes: float has 16 elems, double has 8 elems.

// All sizes support PacketBlock<PacketXf, 2> and PacketBlock<PacketXf, 4>.
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXf, 4>& kernel) {
  detail::ptranspose_impl(kernel);
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXf, 2>& kernel) {
  detail::ptranspose_impl(kernel);
}

// All sizes support PacketBlock<PacketXd, 2>.
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXd, 2>& kernel) {
  detail::ptranspose_impl(kernel);
}

// All sizes support PacketBlock<PacketXi, 2> and PacketBlock<PacketXi, 4>.
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXi, 4>& kernel) {
  detail::ptranspose_impl(kernel);
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXi, 2>& kernel) {
  detail::ptranspose_impl(kernel);
}

// All sizes support PacketBlock<PacketXl, 2>.
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXl, 2>& kernel) {
  detail::ptranspose_impl(kernel);
}

#if EIGEN_GENERIC_VECTOR_SIZE_BYTES >= 32
// 32+ bytes: float has 8+ elems, double has 4+ elems.
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXf, 8>& kernel) {
  detail::ptranspose_impl(kernel);
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXd, 4>& kernel) {
  detail::ptranspose_impl(kernel);
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXi, 8>& kernel) {
  detail::ptranspose_impl(kernel);
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXl, 4>& kernel) {
  detail::ptranspose_impl(kernel);
}
#endif

#if EIGEN_GENERIC_VECTOR_SIZE_BYTES >= 64
// 64 bytes: float has 16 elems, double has 8 elems.
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXf, 16>& kernel) {
  detail::ptranspose_impl(kernel);
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXd, 8>& kernel) {
  detail::ptranspose_impl(kernel);
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXi, 16>& kernel) {
  detail::ptranspose_impl(kernel);
}
EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock<PacketXl, 8>& kernel) {
  detail::ptranspose_impl(kernel);
}
#endif
#endif

}  // end namespace internal
}  // end namespace Eigen

#endif  // EIGEN_PACKET_MATH_CLANG_H
