/*******************************************************************************
* Copyright 2016 Intel Corporation
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*******************************************************************************/

/*******************************************************************************
* Copyright (c) 2007 MITSUNARI Shigeo
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
* this list of conditions and the following disclaimer in the documentation
* and/or other materials provided with the distribution.
* Neither the name of the copyright owner nor the names of its contributors may
* be used to endorse or promote products derived from this software without
* specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
* LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
* CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
* SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
* INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
* CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
* ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
* THE POSSIBILITY OF SUCH DAMAGE.
*******************************************************************************/

#ifndef XBYAK_XBYAK_UTIL_H_
#define XBYAK_XBYAK_UTIL_H_

#ifdef XBYAK_ONLY_CLASS_CPU
#include <stdint.h>
#include <stdlib.h>
#include <assert.h>
#ifndef XBYAK_THROW
	#define XBYAK_THROW(x) ;
	#define XBYAK_THROW_RET(x, y) return y;
#endif
#ifndef XBYAK_CONSTEXPR
#if ((__cplusplus >= 201402L) && !(!defined(__clang__) && defined(__GNUC__) && (__GNUC__ <= 5))) || (defined(_MSC_VER) && _MSC_VER >= 1910)
	#define XBYAK_CONSTEXPR constexpr
#else
	#define XBYAK_CONSTEXPR
#endif
#define XBYAK_CPUMASK_COMPACT 0
#endif
#else
#include <string.h>
#include <stdio.h>

/**
	utility class and functions for Xbyak
	Xbyak::util::Clock ; rdtsc timer
	Xbyak::util::Cpu ; detect CPU
*/
#include "xbyak.h"
#endif // XBYAK_ONLY_CLASS_CPU

#if defined(__i386__) || (defined(__x86_64__) && !defined(__arm64ec__)) || defined(_M_IX86) || (defined(_M_X64) && !defined(_M_ARM64EC))
	#define XBYAK_INTEL_CPU_SPECIFIC
#endif

#ifdef XBYAK_INTEL_CPU_SPECIFIC
#ifdef _WIN32
	#if defined(_MSC_VER) && (_MSC_VER < 1400) && defined(XBYAK32)
		static inline __declspec(naked) void __cpuid(int[4], int)
		{
			__asm {
				push	ebx
				push	esi
				mov		eax, dword ptr [esp + 4 * 2 + 8] // eaxIn
				cpuid
				mov		esi, dword ptr [esp + 4 * 2 + 4] // data
				mov		dword ptr [esi], eax
				mov		dword ptr [esi + 4], ebx
				mov		dword ptr [esi + 8], ecx
				mov		dword ptr [esi + 12], edx
				pop		esi
				pop		ebx
				ret
			}
		}
	#else
		#include <intrin.h> // for __cpuid
	#endif
#else
	#ifndef __GNUC_PREREQ
    	#define __GNUC_PREREQ(major, minor) ((((__GNUC__) << 16) + (__GNUC_MINOR__)) >= (((major) << 16) + (minor)))
	#endif
	#if __GNUC_PREREQ(4, 3) && !defined(__APPLE__)
		#include <cpuid.h>
	#else
		#if defined(__APPLE__) && defined(XBYAK32) // avoid err : can't find a register in class `BREG' while reloading `asm'
			#define __cpuid(eaxIn, a, b, c, d) __asm__ __volatile__("pushl %%ebx\ncpuid\nmovl %%ebp, %%esi\npopl %%ebx" : "=a"(a), "=S"(b), "=c"(c), "=d"(d) : "0"(eaxIn))
			#define __cpuid_count(eaxIn, ecxIn, a, b, c, d) __asm__ __volatile__("pushl %%ebx\ncpuid\nmovl %%ebp, %%esi\npopl %%ebx" : "=a"(a), "=S"(b), "=c"(c), "=d"(d) : "0"(eaxIn), "2"(ecxIn))
		#else
			#define __cpuid(eaxIn, a, b, c, d) __asm__ __volatile__("cpuid\n" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(eaxIn))
			#define __cpuid_count(eaxIn, ecxIn, a, b, c, d) __asm__ __volatile__("cpuid\n" : "=a"(a), "=b"(b), "=c"(c), "=d"(d) : "0"(eaxIn), "2"(ecxIn))
		#endif
	#endif
#endif
#endif

#ifdef XBYAK_USE_VTUNE
	// -I /opt/intel/vtune_amplifier/include/ -L /opt/intel/vtune_amplifier/lib64 -ljitprofiling -ldl
	#include <jitprofiling.h>
	#ifdef _MSC_VER
		#pragma comment(lib, "libittnotify.lib")
	#endif
	#ifdef __linux__
		#include <dlfcn.h>
	#endif
#endif
#ifdef __linux__
	#define XBYAK_USE_PERF
#endif

#ifndef XBYAK_CPU_CACHE
	#define XBYAK_CPU_CACHE 1
#endif
#if XBYAK_CPU_CACHE == 1
#include <vector>
#ifndef XBYAK_CPUMASK_COMPACT
	#define XBYAK_CPUMASK_COMPACT 1
#endif
#if XBYAK_CPUMASK_COMPACT == 0
	#include <set>
#endif
#ifdef _WIN32
#include <windows.h>
#else
#include <sched.h>
#endif
namespace Xbyak { namespace util {
class CpuTopology;
class Cpu;
namespace impl {

bool initCpuTopology(CpuTopology& cpuTopo);

} // Xbyak::util::impl
} } // Xbyak::util
#endif // XBYAK_CPU_CACHE


namespace Xbyak { namespace util {

typedef enum {
   SmtLevel = 1,
   CoreLevel = 2
} CpuTopologyLevel;
typedef CpuTopologyLevel IntelCpuTopologyLevel; // for backward compatibility

namespace local {

template<uint64_t L, uint64_t H = 0>
struct TypeT {
};

template<uint64_t L1, uint64_t H1, uint64_t L2, uint64_t H2>
XBYAK_CONSTEXPR TypeT<L1 | L2, H1 | H2> operator|(TypeT<L1, H1>, TypeT<L2, H2>) { return TypeT<L1 | L2, H1 | H2>(); }

template<typename T>
inline T max_(T x, T y) { return x >= y ? x : y; }
template<typename T>
inline T min_(T x, T y) { return x < y ? x : y; }

} // local

/**
	CPU detection class
	@note static inline const member is supported by c++17 or later, so use template hack
*/
#ifdef _MSC_VER
	#pragma warning(push)
	#pragma warning(disable : 4459)
#endif
class Cpu {
public:
	class Type {
		uint64_t L;
		uint64_t H;
	public:
		Type(uint64_t L = 0, uint64_t H = 0) : L(L), H(H) { }
		template<uint64_t L_, uint64_t H_>
		Type(local::TypeT<L_, H_>) : L(L_), H(H_) {}
		Type& operator&=(const Type& rhs) { L &= rhs.L; H &= rhs.H; return *this; }
		Type& operator|=(const Type& rhs) { L |= rhs.L; H |= rhs.H; return *this; }
		Type operator&(const Type& rhs) const { Type t = *this; t &= rhs; return t; }
		Type operator|(const Type& rhs) const { Type t = *this; t |= rhs; return t; }
		bool operator==(const Type& rhs) const { return H == rhs.H && L == rhs.L; }
		bool operator!=(const Type& rhs) const { return !operator==(rhs); }
		// without explicit because backward compatilibity
		operator bool() const { return (H | L) != 0; }
		uint64_t getL() const { return L; }
		uint64_t getH() const { return H; }
	};
private:
	Type type_;
	//system topology
	static const size_t maxTopologyLevels = 2;
	uint32_t numCores_[maxTopologyLevels];

	static const uint32_t maxNumberCacheLevels = 10;
	uint32_t dataCacheSize_[maxNumberCacheLevels];
	uint32_t coresSharingDataCache_[maxNumberCacheLevels];
	uint32_t dataCacheLevels_;
	uint32_t avx10version_;

	uint32_t get32bitAsBE(const char *x) const
	{
		return x[0] | (x[1] << 8) | (x[2] << 16) | (x[3] << 24);
	}
	uint32_t mask(int n) const
	{
		return (1U << n) - 1;
	}
	// [ebx:ecx:edx] == s?
	bool isEqualStr(uint32_t ebx, uint32_t ecx, uint32_t edx, const char s[12]) const
	{
		return get32bitAsBE(&s[0]) == ebx && get32bitAsBE(&s[4]) == edx && get32bitAsBE(&s[8]) == ecx;
	}
	uint32_t extractBit(uint32_t val, uint32_t base, uint32_t end) const
	{
		return (val >> base) & ((1u << (end + 1 - base)) - 1);
	}
	void setFamily()
	{
		uint32_t data[4] = {};
		getCpuid(1, data);
		stepping = extractBit(data[0], 0, 3);
		model = extractBit(data[0], 4, 7);
		family = extractBit(data[0], 8, 11);
		//type = extractBit(data[0], 12, 13);
		extModel = extractBit(data[0], 16, 19);
		extFamily = extractBit(data[0], 20, 27);
		if (family == 0x0f) {
			displayFamily = family + extFamily;
		} else {
			displayFamily = family;
		}
		if ((has(tINTEL) && family == 6) || family == 0x0f) {
			displayModel = (extModel << 4) + model;
		} else {
			displayModel = model;
		}
	}
	void setNumCores()
	{
		if (!has(tINTEL) && !has(tAMD)) return;

		uint32_t data[4] = {};
		getCpuid(0x0, data);
		if (data[0] >= 0xB) {
			// Check if "Extended Topology Enumeration" is implemented.
			getCpuidEx(0xB, 0, data);
			if (data[0] != 0 || data[1] != 0) {
				/*
					if leaf 11 exists(x2APIC is supported),
					we use it to get the number of smt cores and cores on socket

					leaf 0xB can be zeroed-out by a hypervisor
				*/
				for (uint32_t i = 0; i < maxTopologyLevels; i++) {
					getCpuidEx(0xB, i, data);
					CpuTopologyLevel level = (CpuTopologyLevel)extractBit(data[2], 8, 15);
					if (level == SmtLevel || level == CoreLevel) {
						numCores_[level - 1] = extractBit(data[1], 0, 15);
					}
				}
				/*
					Fallback values in case a hypervisor has the leaf zeroed-out.
				*/
				numCores_[SmtLevel - 1] = local::max_(1u, numCores_[SmtLevel - 1]);
				numCores_[CoreLevel - 1] = local::max_(numCores_[SmtLevel - 1], numCores_[CoreLevel - 1]);
				return;
			}
		}
		// "Extended Topology Enumeration" is not supported.
		if (has(tAMD)) {
			/*
				AMD - Legacy Method
			*/
			int physicalThreadCount = 0;
			getCpuid(0x1, data);
			int logicalProcessorCount = extractBit(data[1], 16, 23);
			int htt = extractBit(data[3], 28, 28); // Hyper-threading technology.
			getCpuid(0x80000000, data);
			uint32_t highestExtendedLeaf = data[0];
			if (highestExtendedLeaf >= 0x80000008) {
				getCpuid(0x80000008, data);
				physicalThreadCount = extractBit(data[2], 0, 7) + 1;
			}
			if (htt == 0) {
				numCores_[SmtLevel - 1] = 1;
				numCores_[CoreLevel - 1] = 1;
			} else if (physicalThreadCount > 1) {
				if ((displayFamily >= 0x17) && (highestExtendedLeaf >= 0x8000001E)) {
					// Zen overreports its core count by a factor of two.
					getCpuid(0x8000001E, data);
					int threadsPerComputeUnit = extractBit(data[1], 8, 15) + 1;
					physicalThreadCount /= threadsPerComputeUnit;
				}
				numCores_[SmtLevel - 1] = logicalProcessorCount / physicalThreadCount;
				numCores_[CoreLevel - 1] = logicalProcessorCount;
			} else {
				numCores_[SmtLevel - 1] = 1;
				numCores_[CoreLevel - 1] = logicalProcessorCount > 1 ? logicalProcessorCount : 2;
			}
		} else {
			/*
				Intel - Legacy Method
			*/
			int physicalThreadCount = 0;
			getCpuid(0x1, data);
			int logicalProcessorCount = extractBit(data[1], 16, 23);
			int htt = extractBit(data[3], 28, 28); // Hyper-threading technology.
			getCpuid(0, data);
			if (data[0] >= 0x4) {
				getCpuid(0x4, data);
				physicalThreadCount = extractBit(data[0], 26, 31) + 1;
			}
			if (htt == 0) {
				numCores_[SmtLevel - 1] = 1;
				numCores_[CoreLevel - 1] = 1;
			} else if (physicalThreadCount > 1) {
				numCores_[SmtLevel - 1] = logicalProcessorCount / physicalThreadCount;
				numCores_[CoreLevel - 1] = logicalProcessorCount;
			} else {
				numCores_[SmtLevel - 1] = 1;
				numCores_[CoreLevel - 1] = logicalProcessorCount > 0 ? logicalProcessorCount : 1;
			}
		}
	}
	void setCacheHierarchy()
	{
		uint32_t data[4] = {};
		if (has(tAMD)) {
			getCpuid(0x80000000, data);
			if (data[0] >= 0x8000001D) {
				// For modern AMD CPUs.
				dataCacheLevels_ = 0;
				for (uint32_t subLeaf = 0; dataCacheLevels_ < maxNumberCacheLevels; subLeaf++) {
					getCpuidEx(0x8000001D, subLeaf, data);
					int cacheType = extractBit(data[0], 0, 4);
					/*
					  cacheType
						00h - Null; no more caches
						01h - Data cache
						02h - Instrution cache
						03h - Unified cache
						04h-1Fh - Reserved
					*/
					if (cacheType == 0) break; // No more caches.
					if (cacheType == 0x2) continue; // Skip instruction cache.
					int fullyAssociative = extractBit(data[0], 9, 9);
					int numSharingCache = extractBit(data[0], 14, 25) + 1;
					int cacheNumWays = extractBit(data[1], 22, 31) + 1;
					int cachePhysPartitions = extractBit(data[1], 12, 21) + 1;
					int cacheLineSize = extractBit(data[1], 0, 11) + 1;
					int cacheNumSets = data[2] + 1;
					dataCacheSize_[dataCacheLevels_] =
						cacheLineSize * cachePhysPartitions * cacheNumWays;
					if (fullyAssociative == 0) {
						dataCacheSize_[dataCacheLevels_] *= cacheNumSets;
					}
					if (subLeaf > 0) {
						numSharingCache = local::min_(numSharingCache, (int)numCores_[1]);
						numSharingCache /= local::max_(1u, coresSharingDataCache_[0]);
					}
					coresSharingDataCache_[dataCacheLevels_] = numSharingCache;
					dataCacheLevels_ += 1;
				}
				coresSharingDataCache_[0] = local::min_(1u, coresSharingDataCache_[0]);
			} else if (data[0] >= 0x80000006) {
				// For legacy AMD CPUs, use leaf 0x80000005 for L1 cache
				// and 0x80000006 for L2 and L3 cache.
				dataCacheLevels_ = 1;
				getCpuid(0x80000005, data);
				int l1dc_size = extractBit(data[2], 24, 31);
				dataCacheSize_[0] = l1dc_size * 1024;
				coresSharingDataCache_[0] = 1;
				getCpuid(0x80000006, data);
				// L2 cache
				int l2_assoc = extractBit(data[2], 12, 15);
				if (l2_assoc > 0) {
					dataCacheLevels_ = 2;
					int l2_size = extractBit(data[2], 16, 31);
					dataCacheSize_[1] = l2_size * 1024;
					coresSharingDataCache_[1] = 1;
				}
				// L3 cache
				int l3_assoc = extractBit(data[3], 12, 15);
				if (l3_assoc > 0) {
					dataCacheLevels_ = 3;
					int l3_size = extractBit(data[3], 18, 31);
					dataCacheSize_[2] = l3_size * 512 * 1024;
					coresSharingDataCache_[2] = numCores_[1];
				}
			}
		} else if (has(tINTEL)) {
			// Use the "Deterministic Cache Parameters" leaf is supported.
			const uint32_t NO_CACHE = 0;
			const uint32_t DATA_CACHE = 1;
			//const uint32_t INSTRUCTION_CACHE = 2;
			const uint32_t UNIFIED_CACHE = 3;
			uint32_t smt_width = 0;
			uint32_t logical_cores = 0;

			smt_width = numCores_[0];
			logical_cores = numCores_[1];

			/*
				Assumptions:
				the first level of data cache is not shared (which is the
				case for every existing architecture) and use this to
				determine the SMT width for arch not supporting leaf 11.
				when leaf 4 reports a number of core less than numCores_
				on socket reported by leaf 11, then it is a correct number
				of cores not an upperbound.
			*/
			for (int i = 0; dataCacheLevels_ < maxNumberCacheLevels; i++) {
				getCpuidEx(0x4, i, data);
				uint32_t cacheType = extractBit(data[0], 0, 4);
				if (cacheType == NO_CACHE) break;
				if (cacheType == DATA_CACHE || cacheType == UNIFIED_CACHE) {
					uint32_t actual_logical_cores = extractBit(data[0], 14, 25) + 1;
					if (logical_cores != 0) { // true only if leaf 0xB is supported and valid
						actual_logical_cores = local::min_(actual_logical_cores, logical_cores);
					}
					assert(actual_logical_cores != 0);
					dataCacheSize_[dataCacheLevels_] =
						(extractBit(data[1], 22, 31) + 1)
						* (extractBit(data[1], 12, 21) + 1)
						* (extractBit(data[1], 0, 11) + 1)
						* (data[2] + 1);
					if (cacheType == DATA_CACHE && smt_width == 0) smt_width = actual_logical_cores;
					assert(smt_width != 0);
					coresSharingDataCache_[dataCacheLevels_] = local::max_(actual_logical_cores / smt_width, 1u);
					dataCacheLevels_++;
				}
			}
		}
	}

public:
	int model;
	int family;
	int stepping;
	int extModel;
	int extFamily;
	int displayFamily; // family + extFamily
	int displayModel; // model + extModel

	uint32_t getNumCores(CpuTopologyLevel level) const {
		switch (level) {
		case SmtLevel: return numCores_[level - 1];
		case CoreLevel: return numCores_[level - 1] / numCores_[SmtLevel - 1];
		default: XBYAK_THROW_RET(ERR_X2APIC_IS_NOT_SUPPORTED, 0)
		}
	}

	uint32_t getDataCacheLevels() const { return dataCacheLevels_; }
	uint32_t getCoresSharingDataCache(uint32_t i) const
	{
		if (i >= dataCacheLevels_) XBYAK_THROW_RET(ERR_BAD_PARAMETER, 0)
		return coresSharingDataCache_[i];
	}
	uint32_t getDataCacheSize(uint32_t i) const
	{
		if (i >= dataCacheLevels_) XBYAK_THROW_RET(ERR_BAD_PARAMETER, 0)
		return dataCacheSize_[i];
	}

	/*
		data[] = { eax, ebx, ecx, edx }
	*/
	static inline void getCpuidEx(uint32_t eaxIn, uint32_t ecxIn, uint32_t data[4])
	{
#ifdef XBYAK_INTEL_CPU_SPECIFIC
	#ifdef _MSC_VER
		__cpuidex(reinterpret_cast<int*>(data), eaxIn, ecxIn);
	#else
		__cpuid_count(eaxIn, ecxIn, data[0], data[1], data[2], data[3]);
	#endif
#else
		(void)eaxIn;
		(void)ecxIn;
		(void)data;
#endif
	}
	static inline void getCpuid(uint32_t eaxIn, uint32_t data[4])
	{
		getCpuidEx(eaxIn, 0, data);
	}
	static inline uint64_t getXfeature()
	{
#ifdef XBYAK_INTEL_CPU_SPECIFIC
	#ifdef _MSC_VER
		return _xgetbv(0);
	#else
		uint32_t eax, edx;
		// xgetvb is not support on gcc 4.2
//		__asm__ volatile("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0));
		__asm__ volatile(".byte 0x0f, 0x01, 0xd0" : "=a"(eax), "=d"(edx) : "c"(0));
		return ((uint64_t)edx << 32) | eax;
	#endif
#else
		return 0;
#endif
	}

#define XBYAK_SPLIT_ID(id) ((0 <= id && id < 64) ? (1ull << (id % 64)) : 0), (id >= 64 ? (1ull << (id % 64)) : 0)
#if (__cplusplus >= 201103) || (defined(_MSC_VER) && (_MSC_VER >= 1700)) /* VS2012 */
	#define XBYAK_DEFINE_TYPE(id, NAME) static const constexpr local::TypeT<XBYAK_SPLIT_ID(id)> NAME{}
#else
	#define XBYAK_DEFINE_TYPE(id, NAME) static const local::TypeT<XBYAK_SPLIT_ID(id)> NAME
#endif
	XBYAK_DEFINE_TYPE(0, tMMX);
	XBYAK_DEFINE_TYPE(1, tMMX2);
	XBYAK_DEFINE_TYPE(2, tCMOV);
	XBYAK_DEFINE_TYPE(3, tSSE);
	XBYAK_DEFINE_TYPE(4, tSSE2);
	XBYAK_DEFINE_TYPE(5, tSSE3);
	XBYAK_DEFINE_TYPE(6, tSSSE3);
	XBYAK_DEFINE_TYPE(7, tSSE41);
	XBYAK_DEFINE_TYPE(8, tSSE42);
	XBYAK_DEFINE_TYPE(9, tPOPCNT);
	XBYAK_DEFINE_TYPE(10, tAESNI);
	XBYAK_DEFINE_TYPE(11, tAVX512_FP16);
	XBYAK_DEFINE_TYPE(12, tOSXSAVE);
	XBYAK_DEFINE_TYPE(13, tPCLMULQDQ);
	XBYAK_DEFINE_TYPE(14, tAVX);
	XBYAK_DEFINE_TYPE(15, tFMA);
	XBYAK_DEFINE_TYPE(16, t3DN);
	XBYAK_DEFINE_TYPE(17, tE3DN);
	XBYAK_DEFINE_TYPE(18, tWAITPKG);
	XBYAK_DEFINE_TYPE(19, tRDTSCP);
	XBYAK_DEFINE_TYPE(20, tAVX2);
	XBYAK_DEFINE_TYPE(21, tBMI1); // andn, bextr, blsi, blsmsk, blsr, tzcnt
	XBYAK_DEFINE_TYPE(22, tBMI2); // bzhi, mulx, pdep, pext, rorx, sarx, shlx, shrx
	XBYAK_DEFINE_TYPE(23, tLZCNT);
	XBYAK_DEFINE_TYPE(24, tINTEL);
	XBYAK_DEFINE_TYPE(25, tAMD);
	XBYAK_DEFINE_TYPE(26, tENHANCED_REP); // enhanced rep movsb/stosb
	XBYAK_DEFINE_TYPE(27, tRDRAND);
	XBYAK_DEFINE_TYPE(28, tADX); // adcx, adox
	XBYAK_DEFINE_TYPE(29, tRDSEED); // rdseed
	XBYAK_DEFINE_TYPE(30, tSMAP); // stac
	XBYAK_DEFINE_TYPE(31, tHLE); // xacquire, xrelease, xtest
	XBYAK_DEFINE_TYPE(32, tRTM); // xbegin, xend, xabort
	XBYAK_DEFINE_TYPE(33, tF16C); // vcvtph2ps, vcvtps2ph
	XBYAK_DEFINE_TYPE(34, tMOVBE); // mobve
	XBYAK_DEFINE_TYPE(35, tAVX512F);
	XBYAK_DEFINE_TYPE(36, tAVX512DQ);
	XBYAK_DEFINE_TYPE(37, tAVX512_IFMA);
	XBYAK_DEFINE_TYPE(37, tAVX512IFMA);// = tAVX512_IFMA;
//	XBYAK_DEFINE_TYPE(38, tAVX512PF); // Xeon Phi only
//	XBYAK_DEFINE_TYPE(39, tAVX512ER);
	XBYAK_DEFINE_TYPE(40, tAVX512CD);
	XBYAK_DEFINE_TYPE(41, tAVX512BW);
	XBYAK_DEFINE_TYPE(42, tAVX512VL);
	XBYAK_DEFINE_TYPE(43, tAVX512_VBMI);
	XBYAK_DEFINE_TYPE(43, tAVX512VBMI); // = tAVX512_VBMI; // changed by Intel's manual
//	XBYAK_DEFINE_TYPE(44, tAVX512_4VNNIW);
//	XBYAK_DEFINE_TYPE(45, tAVX512_4FMAPS);
//	XBYAK_DEFINE_TYPE(46, tPREFETCHWT1);
	XBYAK_DEFINE_TYPE(47, tPREFETCHW);
	XBYAK_DEFINE_TYPE(48, tSHA);
	XBYAK_DEFINE_TYPE(49, tMPX);
	XBYAK_DEFINE_TYPE(50, tAVX512_VBMI2);
	XBYAK_DEFINE_TYPE(51, tGFNI);
	XBYAK_DEFINE_TYPE(52, tVAES);
	XBYAK_DEFINE_TYPE(53, tVPCLMULQDQ);
	XBYAK_DEFINE_TYPE(54, tAVX512_VNNI);
	XBYAK_DEFINE_TYPE(55, tAVX512_BITALG);
	XBYAK_DEFINE_TYPE(56, tAVX512_VPOPCNTDQ);
	XBYAK_DEFINE_TYPE(57, tAVX512_BF16);
	XBYAK_DEFINE_TYPE(58, tAVX512_VP2INTERSECT);
	XBYAK_DEFINE_TYPE(59, tAMX_TILE);
	XBYAK_DEFINE_TYPE(60, tAMX_INT8);
	XBYAK_DEFINE_TYPE(61, tAMX_BF16);
	XBYAK_DEFINE_TYPE(62, tAVX_VNNI);
	XBYAK_DEFINE_TYPE(63, tCLFLUSHOPT);
	XBYAK_DEFINE_TYPE(64, tCLDEMOTE);
	XBYAK_DEFINE_TYPE(65, tMOVDIRI);
	XBYAK_DEFINE_TYPE(66, tMOVDIR64B);
	XBYAK_DEFINE_TYPE(67, tCLZERO); // AMD Zen
	XBYAK_DEFINE_TYPE(68, tAMX_FP16);
	XBYAK_DEFINE_TYPE(69, tAVX_VNNI_INT8);
	XBYAK_DEFINE_TYPE(70, tAVX_NE_CONVERT);
	XBYAK_DEFINE_TYPE(71, tAVX_IFMA);
	XBYAK_DEFINE_TYPE(72, tRAO_INT);
	XBYAK_DEFINE_TYPE(73, tCMPCCXADD);
	XBYAK_DEFINE_TYPE(74, tPREFETCHITI);
	XBYAK_DEFINE_TYPE(75, tSERIALIZE);
	XBYAK_DEFINE_TYPE(76, tUINTR);
	XBYAK_DEFINE_TYPE(77, tXSAVE);
	XBYAK_DEFINE_TYPE(78, tSHA512);
	XBYAK_DEFINE_TYPE(79, tSM3);
	XBYAK_DEFINE_TYPE(80, tSM4);
	XBYAK_DEFINE_TYPE(81, tAVX_VNNI_INT16);
	XBYAK_DEFINE_TYPE(82, tAPX_F);
	XBYAK_DEFINE_TYPE(83, tAVX10);
	XBYAK_DEFINE_TYPE(84, tAESKLE);
	XBYAK_DEFINE_TYPE(85, tWIDE_KL);
	XBYAK_DEFINE_TYPE(86, tKEYLOCKER);
	XBYAK_DEFINE_TYPE(87, tKEYLOCKER_WIDE);
	XBYAK_DEFINE_TYPE(88, tSSE4a);
	XBYAK_DEFINE_TYPE(89, tCLWB);
	XBYAK_DEFINE_TYPE(90, tTSXLDTRK);
	XBYAK_DEFINE_TYPE(91, tAMX_TRANSPOSE);
	XBYAK_DEFINE_TYPE(92, tAMX_TF32);
	XBYAK_DEFINE_TYPE(93, tAMX_AVX512);
	XBYAK_DEFINE_TYPE(94, tAMX_MOVRS);
	XBYAK_DEFINE_TYPE(95, tAMX_FP8);
	XBYAK_DEFINE_TYPE(96, tMOVRS);
	XBYAK_DEFINE_TYPE(97, tHYBRID);
	XBYAK_DEFINE_TYPE(98, tAMX_COMPLEX);

#undef XBYAK_SPLIT_ID
#undef XBYAK_DEFINE_TYPE

	Cpu()
		: type_()
		, numCores_()
		, dataCacheSize_()
		, coresSharingDataCache_()
		, dataCacheLevels_(0)
		, avx10version_(0)
	{
		uint32_t data[4] = {};
		const uint32_t& eax = data[0];
		const uint32_t& ebx = data[1];
		const uint32_t& ecx = data[2];
		const uint32_t& edx = data[3];
		getCpuid(0, data);
		const uint32_t maxNum = eax;
		if (isEqualStr(ebx, ecx, edx, "AuthenticAMD")) {
			type_ |= tAMD;
			getCpuid(0x80000001, data);
			if (edx & (1U << 31)) {
				type_ |= t3DN;
				// 3DNow! implies support for PREFETCHW on AMD
				type_ |= tPREFETCHW;
			}

			if (edx & (1U << 29)) {
				// Long mode implies support for PREFETCHW on AMD
				type_ |= tPREFETCHW;
			}
		} else if (isEqualStr(ebx, ecx, edx, "GenuineIntel")) {
			type_ |= tINTEL;
		}

		// Extended flags information
		getCpuid(0x80000000, data);
		const uint32_t maxExtendedNum = eax;
		if (maxExtendedNum >= 0x80000001) {
			getCpuid(0x80000001, data);

			if (ecx & (1U << 5)) type_ |= tLZCNT;
			if (ecx & (1U << 6)) type_ |= tSSE4a;
			if (ecx & (1U << 8)) type_ |= tPREFETCHW;
			if (edx & (1U << 15)) type_ |= tCMOV;
			if (edx & (1U << 22)) type_ |= tMMX2;
			if (edx & (1U << 27)) type_ |= tRDTSCP;
			if (edx & (1U << 30)) type_ |= tE3DN;
			if (edx & (1U << 31)) type_ |= t3DN;
		}

		if (maxExtendedNum >= 0x80000008) {
			getCpuid(0x80000008, data);
			if (ebx & (1U << 0)) type_ |= tCLZERO;
		}

		getCpuid(1, data);
		if (ecx & (1U << 0)) type_ |= tSSE3;
		if (ecx & (1U << 1)) type_ |= tPCLMULQDQ;
		if (ecx & (1U << 9)) type_ |= tSSSE3;
		if (ecx & (1U << 19)) type_ |= tSSE41;
		if (ecx & (1U << 20)) type_ |= tSSE42;
		if (ecx & (1U << 22)) type_ |= tMOVBE;
		if (ecx & (1U << 23)) type_ |= tPOPCNT;
		if (ecx & (1U << 25)) type_ |= tAESNI;
		if (ecx & (1U << 26)) type_ |= tXSAVE;
		if (ecx & (1U << 27)) type_ |= tOSXSAVE;
		if (ecx & (1U << 29)) type_ |= tF16C;
		if (ecx & (1U << 30)) type_ |= tRDRAND;

		if (edx & (1U << 15)) type_ |= tCMOV;
		if (edx & (1U << 23)) type_ |= tMMX;
		if (edx & (1U << 25)) type_ |= tMMX2 | tSSE;
		if (edx & (1U << 26)) type_ |= tSSE2;

		if (type_ & tOSXSAVE) {
			// check XFEATURE_ENABLED_MASK[2:1] = '11b'
			uint64_t bv = getXfeature();
			if ((bv & 6) == 6) {
				if (ecx & (1U << 12)) type_ |= tFMA;
				if (ecx & (1U << 28)) type_ |= tAVX;
				// do *not* check AVX-512 state on macOS because it has on-demand AVX-512 support
#if !defined(__APPLE__)
				if (((bv >> 5) & 7) == 7)
#endif
				{
					getCpuidEx(7, 0, data);
					if (ebx & (1U << 16)) type_ |= tAVX512F;
					if (type_ & tAVX512F) {
						if (ebx & (1U << 17)) type_ |= tAVX512DQ;
						if (ebx & (1U << 21)) type_ |= tAVX512_IFMA;
						if (ebx & (1U << 28)) type_ |= tAVX512CD;
						if (ebx & (1U << 30)) type_ |= tAVX512BW;
						if (ebx & (1U << 31)) type_ |= tAVX512VL;
						if (ecx & (1U << 1)) type_ |= tAVX512_VBMI;
						if (ecx & (1U << 6)) type_ |= tAVX512_VBMI2;
						if (ecx & (1U << 11)) type_ |= tAVX512_VNNI;
						if (ecx & (1U << 12)) type_ |= tAVX512_BITALG;
						if (ecx & (1U << 14)) type_ |= tAVX512_VPOPCNTDQ;
						if (edx & (1U << 8)) type_ |= tAVX512_VP2INTERSECT;
						if ((type_ & tAVX512BW) && (edx & (1U << 23))) type_ |= tAVX512_FP16;
					}
				}
			}
		}
		if (maxNum >= 7) {
			getCpuidEx(7, 0, data);
			const uint32_t maxNumSubLeaves = eax;
			if (type_ & tAVX && (ebx & (1U << 5))) type_ |= tAVX2;
			if (ebx & (1U << 3)) type_ |= tBMI1;
			if (ebx & (1U << 4)) type_ |= tHLE;
			if (ebx & (1U << 8)) type_ |= tBMI2;
			if (ebx & (1U << 9)) type_ |= tENHANCED_REP;
			if (ebx & (1U << 11)) type_ |= tRTM;
			if (ebx & (1U << 14)) type_ |= tMPX;
			if (ebx & (1U << 18)) type_ |= tRDSEED;
			if (ebx & (1U << 19)) type_ |= tADX;
			if (ebx & (1U << 20)) type_ |= tSMAP;
			if (ebx & (1U << 23)) type_ |= tCLFLUSHOPT;
			if (ebx & (1U << 24)) type_ |= tCLWB;
			if (ebx & (1U << 29)) type_ |= tSHA;
			if (ecx & (1U << 5)) type_ |= tWAITPKG;
			if (ecx & (1U << 8)) type_ |= tGFNI;
			if (ecx & (1U << 9)) type_ |= tVAES;
			if (ecx & (1U << 10)) type_ |= tVPCLMULQDQ;
			if (ecx & (1U << 23)) type_ |= tKEYLOCKER;
			if (ecx & (1U << 25)) type_ |= tCLDEMOTE;
			if (ecx & (1U << 27)) type_ |= tMOVDIRI;
			if (ecx & (1U << 28)) type_ |= tMOVDIR64B;
			if (edx & (1U << 5)) type_ |= tUINTR;
			if (edx & (1U << 14)) type_ |= tSERIALIZE;
			if (edx & (1U << 15)) type_ |= tHYBRID;
			if (edx & (1U << 16)) type_ |= tTSXLDTRK;
			if (edx & (1U << 22)) type_ |= tAMX_BF16;
			if (edx & (1U << 24)) type_ |= tAMX_TILE;
			if (edx & (1U << 25)) type_ |= tAMX_INT8;
			if (maxNumSubLeaves >= 1) {
				getCpuidEx(7, 1, data);
				if (eax & (1U << 0)) type_ |= tSHA512;
				if (eax & (1U << 1)) type_ |= tSM3;
				if (eax & (1U << 2)) type_ |= tSM4;
				if (eax & (1U << 3)) type_ |= tRAO_INT;
				if (eax & (1U << 4)) type_ |= tAVX_VNNI;
				if (type_ & tAVX512F) {
					if (eax & (1U << 5)) type_ |= tAVX512_BF16;
				}
				if (eax & (1U << 7)) type_ |= tCMPCCXADD;
				if (eax & (1U << 21)) type_ |= tAMX_FP16;
				if (eax & (1U << 23)) type_ |= tAVX_IFMA;
				if (eax & (1U << 31)) type_ |= tMOVRS;
				if (edx & (1U << 4)) type_ |= tAVX_VNNI_INT8;
				if (edx & (1U << 5)) type_ |= tAVX_NE_CONVERT;
				if (edx & (1U << 8)) type_ |= tAMX_COMPLEX;
				if (edx & (1U << 10)) type_ |= tAVX_VNNI_INT16;
				if (edx & (1U << 14)) type_ |= tPREFETCHITI;
				if (edx & (1U << 19)) type_ |= tAVX10;
				if (edx & (1U << 21)) type_ |= tAPX_F;

				getCpuidEx(0x1e, 1, data);
				if (eax & (1U << 4)) type_ |= tAMX_FP8;
				if (eax & (1U << 5)) type_ |= tAMX_TRANSPOSE;
				if (eax & (1U << 6)) type_ |= tAMX_TF32;
				if (eax & (1U << 7)) type_ |= tAMX_AVX512;
				if (eax & (1U << 8)) type_ |= tAMX_MOVRS;
			}
		}
		if (maxNum >= 0x19) {
			getCpuidEx(0x19, 0, data);
			if (ebx & (1U << 0)) type_ |= tAESKLE;
			if (ebx & (1U << 2)) type_ |= tWIDE_KL;
			if (type_ & (tKEYLOCKER|tAESKLE|tWIDE_KL)) type_ |= tKEYLOCKER_WIDE;
		}
		if (has(tAVX10) && maxNum >= 0x24) {
			getCpuidEx(0x24, 0, data);
			avx10version_ = ebx & mask(7);
		}
		setFamily();
		setNumCores();
		setCacheHierarchy();
	}
	void putFamily() const
	{
#ifndef XBYAK_ONLY_CLASS_CPU
		printf("family=%d, model=%X, stepping=%d, extFamily=%d, extModel=%X\n",
			family, model, stepping, extFamily, extModel);
		printf("display:family=%X, model=%X\n", displayFamily, displayModel);
#endif
	}
	bool has(const Type& type) const
	{
		return (type & type_) == type;
	}
	int getAVX10version() const { return avx10version_; }
};
#ifdef _MSC_VER
	#pragma warning(pop)
#endif

#ifndef XBYAK_ONLY_CLASS_CPU
#if XBYAK_CPU_CACHE == 1

enum CoreType {
	Unknown,
	Performance, // P-core (Intel)
	Efficient, // E-core (Intel)
	Standard // Non-hybrid
};

inline const char *getCoreTypeStr(int coreType)
{
	switch (coreType) {
	case Performance: return "P-core";
	case Efficient: return "E-core";
	case Standard: return "Standard";
	default: return "Unknown";
	}
}

enum CacheType {
	L1i,
	L1d,
	L2,
	L3,
	CACHE_UNKNOWN,
	CACHE_TYPE_NUM = CACHE_UNKNOWN
};

inline const char* getCacheTypeStr(int type)
{
	switch (type) {
	case L1i: return "L1i";
	case L1d: return "L1d";
	case L2: return "L2";
	case L3: return "L3";
	default: return "Unknown";
	}
}

namespace impl {

inline void appendStr(std::string& s, uint32_t v)
{
#if __cplusplus >= 201103L
	s += std::to_string(v);
#else
	char buf[16];
	snprintf(buf, sizeof(buf), "%u", v);
	s += buf;
#endif
}

// str = "(int|range)[,(int|range)]*"
// range = int-int
// e.g. "1,3,5", "0-3,5-7", ""
template<class T>
bool setStr(T& x, const char *str)
{
	const char *p = str;
	while (*p) {
		if (p != str) {
			if (*p != ',') return false;
			p++;
		}
		char *endp;
		uint32_t v = uint32_t(strtoul(p, &endp, 10));
		if (endp == p) return false;
		if (*endp == '-') {
			const char *rangeStart = endp + 1;
			uint32_t next = uint32_t(strtoul(rangeStart, &endp, 10));
			if (endp == rangeStart) return false;
			if (!x.appendRange(v, next)) return false;
		} else {
			if (!x.append(v)) return false;
		}
		if (*endp == '\0') return true;
		p = endp;
	}
	return true;
}

} // impl

#ifndef XBYAK_CPUMASK_N
#define XBYAK_CPUMASK_N 6
#endif
#ifndef XBYAK_CPUMASK_BITN
#define XBYAK_CPUMASK_BITN 10 // max number of logical cpu = 1024
#endif
#if XBYAK_CPUMASK_COMPACT == 1
/*
	a_ is treated as an array of N elements, each being bitN bits
	a_ = 1<<bitN and n_ = 0 and range_ = 0 means empty set
	n_ is length of a_[] - 1
	When range_ is false (discrete values):
		Values satisfy a_[i] + 1 < a_[i+1] for all 0 <= i <= n_
	When range_ is true (intervals):
		v = a_[i*2] is the start of the interval
		n = a_[i*2+1] is the interval length - 1
		Represents the interval [v, v+n]
	Max number of cpu = 2**bitN - 1
	Max value that can be stored = N
	Max interval length = N/2
*/
class CpuMask {
	static const uint32_t N = XBYAK_CPUMASK_N;
	static const uint32_t bitN = XBYAK_CPUMASK_BITN;
	static const uint64_t mask = (uint64_t(1) << bitN) - 1;
	uint64_t a_:N*bitN;
	uint64_t n_:3;
	uint64_t range_:1;

	// Set a_[idx] = v
	void set_a(size_t idx, uint32_t v)
	{
		assert(idx < N);
		assert(v <= mask);
		a_ &= ~(mask << (idx*bitN));
		a_ |= (v & mask) << (idx*bitN);
	}
	// Get a_[idx]
	uint32_t get_a(size_t idx) const
	{
		assert(idx < N);
		return (a_ >> (idx*bitN)) & mask;
	}
#ifndef NDEBUG
	// Return true if the idx-th value exists
	bool hasNext(uint32_t idx) const
	{
		if (empty()) return false;
		if (!range_) return idx <= n_;
		uint32_t n = 0;
		for (uint32_t i = 1; i <= n_; i += 2) {
			n += get_a(i) + 1;
			if (idx < n) return true;
		}
		return false;
	}
#endif
public:
	CpuMask() { clear(); }
	class ConstIterator {
		const CpuMask& parent_;
		uint32_t idx_;
		uint32_t size_;
		friend class CpuMask;
	public:
		ConstIterator(const CpuMask& parent)
			: parent_(parent), idx_(0), size_(uint32_t(parent.size())) {}
		uint32_t operator*() const { return parent_.get(idx_); }
		ConstIterator& operator++() { idx_++; return *this; }
		bool operator==(const ConstIterator& rhs) const { return idx_ == rhs.idx_; }
		bool operator!=(const ConstIterator& rhs) const { return !operator==(rhs); }
	};
	ConstIterator begin() const { return ConstIterator(*this); }
	ConstIterator end() const {
		ConstIterator it(*this);
		it.idx_ = uint32_t(size());
		return it;
	}
	typedef ConstIterator iterator;
	typedef ConstIterator const_iterator;
	void clear() { a_ = 1 << bitN; n_ = 0; range_ = 0; }
	bool empty() const
	{
		return a_ == 1 << bitN && n_ == 0 && range_ == 0;
	}
	uint64_t to_u64() const { return a_ | (uint64_t(n_) << (N * bitN)) | (uint64_t(range_) << (N * bitN + 3)); }
	bool operator<(const CpuMask& rhs) const { return to_u64() < rhs.to_u64(); }
	bool operator>(const CpuMask& rhs) const { return to_u64() > rhs.to_u64(); }
	bool operator>=(const CpuMask& rhs) const { return !operator<(rhs); }
	bool operator<=(const CpuMask& rhs) const { return !operator>(rhs); }
	bool operator==(const CpuMask& rhs) const { return to_u64() == rhs.to_u64(); }
	bool operator!=(const CpuMask& rhs) const { return !operator==(rhs); }
	// Add element v
	// v should be monotonically increasing
	bool append(uint32_t v)
	{
		uint32_t prev = 0, n = 0;
		if (v > mask) goto ERR;
		// When adding for the first time, treat as discrete value
		if (empty()) {
			a_ = v;
			n_ = 0;
			return true;
		}
		if (!range_) {
			prev = get_a(n_);
			if (v <= prev) goto ERR;
			// If there's one discrete value and it forms an interval with the new value, switch to interval mode
			if (n_ == 0 && prev + 1 == v) {
				set_a(1, 1);
				range_ = 1;
				n_ = 1;
				return true;
			}
			if (n_ >= N - 1) goto ERR;
			// Add discrete value
			n_++;
			set_a(n_, v);
			return true;
		}
		// If the value to add is 1 greater than the end of the current interval
		n = get_a(n_);
		prev = get_a(n_ - 1) + n;
		if (prev >= v) goto ERR;
		if (prev + 1 == v) {
			// Increase the interval length by one
			set_a(n_, n + 1);
			return true;
		} else {
			if (n_ >= N - 1) goto ERR;
			// If not continuous with the previous interval
			// Add a new interval [v]
			set_a(n_ + 1, v);
			n_ += 2;
			return true;
		}
	ERR:
		XBYAK_THROW_RET(ERR_INVALID_CPUMASK_INDEX, false)
		return false;
	}
	// add range [a, b] which means a, a+1, ..., b
	bool appendRange(uint32_t a, uint32_t b)
	{
		if ((empty() || (range_ && n_ < N - 1)) && (a <= b && b <= mask)) {
			range_ = true;
			n_ += n_ == 0 ? 1 : 2;
			set_a(n_ - 1, a);
			set_a(n_, b - a);
			return true;
		}
		return false;
	}
	// str = "(int|range)[,(int|range)]*"
	// range = int-int
	bool setStr(const char *str)
	{
		return impl::setStr(*this, str);
	}
	bool setStr(const std::string& str) { return setStr(str.c_str()); }
	std::string getStr() const
	{
		std::string s;
		if (empty()) return s;
		if (!range_) {
			for (uint32_t i = 0; i <= n_; i++) {
				if (!s.empty()) s += ",";
				impl::appendStr(s, get_a(i));
			}
			return s;
		}
		for (uint32_t i = 0; i <= n_; i += 2) {
			uint32_t v = get_a(i);
			uint32_t len = get_a(i + 1);
			if (!s.empty()) s += ",";
			impl::appendStr(s, v);
			if (len > 0) {
				s += "-";
				impl::appendStr(s, v + len);
			}
		}
		return s;
	}
	size_t size() const
	{
		if (empty()) return 0;
		if (!range_) return n_ + 1;
		size_t n = 0;
		for (uint32_t i = 1; i <= n_; i += 2) {
			n += get_a(i) + 1;
		}
		return n;
	}

	uint32_t get(uint32_t idx) const
	{
		assert(hasNext(idx));
		if (!range_) return get_a(idx);
		uint32_t n = 0;
		for (uint32_t i = 1; i <= n_; i += 2) {
			uint32_t range = get_a(i) + 1;
			if (idx < n + range) {
				return get_a(i - 1) + (idx - n);
			}
			n += range;
		}
		return false;
	}
	void dump() const
	{
		printf("a_:");
		for (int i = int(N) - 1; i >= 0; i--) {
			printf("%u ", uint32_t((a_ >> (i * bitN)) & mask));
		}
		printf("\n");
		printf("n_: %u\n", (uint32_t)n_);
		printf("range_: %u\n", (uint32_t)range_);
	}
	void put(const char *label = NULL) const
	{
		if (label) printf("%s: ", label);
		printf("%s\n", getStr().c_str());
	}
};
#else
class CpuMask {
	typedef std::set<uint32_t> IntSet;
	IntSet indices_;
public:
	CpuMask() : indices_() {}
	typedef IntSet::const_iterator const_iterator;
	typedef const_iterator iterator;
	const_iterator begin() const { return indices_.begin(); }
	const_iterator end() const { return indices_.end(); }

	void clear() { indices_.clear(); }
	bool empty() const { return indices_.empty(); }
	bool operator<(const CpuMask& rhs) const { return indices_ < rhs.indices_; }
	bool operator>(const CpuMask& rhs) const { return indices_ > rhs.indices_; }
	bool operator>=(const CpuMask& rhs) const { return !operator<(rhs); }
	bool operator<=(const CpuMask& rhs) const { return !operator>(rhs); }
	bool operator==(const CpuMask& rhs) const { return indices_ == rhs.indices_; }
	bool operator!=(const CpuMask& rhs) const { return !operator==(rhs); }
	// idx should be monotonically increasing
	bool append(uint32_t idx)
	{
		if (idx >= (1u << XBYAK_CPUMASK_BITN)) return false;
		if (!indices_.empty() && *indices_.rbegin() >= idx) return false;
		indices_.insert(idx);
		return true;
	}
	// add range [a, b] which means a, a+1, ..., b
	bool appendRange(uint32_t a, uint32_t b)
	{
		if (a > b) return false;
		while (a <= b) {
			if (!append(a)) return false;
			a++;
		}
		return true;
	}
	bool setStr(const char *str)
	{
		return impl::setStr(*this, str);
	}
	bool setStr(const std::string& str) { return setStr(str.c_str()); }
	std::string getStr() const
	{
		std::string s;
		bool inRange = false;
		uint32_t prev = 0x80000000;
		for (const_iterator i = indices_.begin(); i != indices_.end(); ++i) {
			uint32_t v = *i;
			if (inRange) {
				if (prev + 1 != v) {
					impl::appendStr(s, prev);
					inRange = false;
					s += ',';
					impl::appendStr(s, v);
				}
			} else {
				if (prev + 1 == v) {
					// start range
					s += '-';
					inRange = true;
				} else {
					if (!s.empty()) s += ',';
					impl::appendStr(s, v);
				}
			}
			prev = v;
		}
		if (inRange) {
			impl::appendStr(s, prev);
		}
		return s;
	}
	size_t size() const { return indices_.size(); }
	uint32_t get(uint32_t idx) const
	{
		assert(idx < size());
		const_iterator it = indices_.begin();
		std::advance(it, idx);
		return *it;
	}
	void put(const char *label = NULL) const
	{
		if (label) printf("%s: ", label);
		printf("%s\n", getStr().c_str());
	}
};
#endif

class CpuCache {
public:
	CpuCache() : size(0), associativity(0) {}

	// Cache size in bytes
	uint32_t size;

	// number of ways of associativity
	uint32_t associativity;

	// Set of logical CPU indices sharing this cache
	CpuMask sharedCpuIndices;

	// Whether this is a shared cache
	bool isShared() const { return sharedCpuIndices.size() > 1; }

	// Number of logical CPUs sharing this cache
	size_t getSharedCpuNum() const { return sharedCpuIndices.size(); }

	void put(const char *label = NULL) const
	{
		if (label) printf("%s: ", label);
		printf("%u KiB, assoc. %u, shared ", size / 1024, associativity);
		sharedCpuIndices.put();
	}
};

struct LogicalCpu {
	LogicalCpu()
		: coreId(0)
		, coreType(Unknown)
		, cache()
	{
	}
	uint32_t coreId; // index of physical core
	CoreType coreType; // for hybrid systems
	CpuCache cache[CACHE_TYPE_NUM];
	const CpuMask& getSiblings() const { return cache[L1i].sharedCpuIndices; }

	void put(const char *label = NULL) const
	{
		if (label) printf("%s: ", label);
		printf("coreId %u, type %s\n", coreId, getCoreTypeStr(coreType));
		for (int i = 0; i < CACHE_TYPE_NUM; i++) {
			cache[i].put(getCacheTypeStr(i));
		}
	}
};

class CpuTopology {
public:
	explicit CpuTopology(const Cpu& cpu)
		: logicalCpus_()
		, physicalCoreNum_(0)
		, lineSize_(0)
		, isHybrid_(cpu.has(cpu.tHYBRID))
	{
		if (!impl::initCpuTopology(*this)) {
			XBYAK_THROW(ERR_CANT_INIT_CPUTOPOLOGY);
		}
	}

	// Number of logical CPUs
	size_t getLogicalCpuNum() const { return logicalCpus_.size(); }

	// Number of physical cores
	size_t getPhysicalCoreNum() const { return physicalCoreNum_; }

	// Cache line size in bytes
	uint32_t getLineSize() const { return lineSize_; }

	// Get logical CPU information
	const LogicalCpu& getLogicalCpu(size_t cpuIdx) const
	{
		return logicalCpus_[cpuIdx];
	}

	// Get cache information for a specific logical CPU
	const CpuCache& getCache(size_t cpuIdx, CacheType type) const
	{
		return logicalCpus_[cpuIdx].cache[type];
	}

	// Whether this is a hybrid system
	bool isHybrid() const { return isHybrid_; }
private:
	friend bool impl::initCpuTopology(CpuTopology&);
	std::vector<LogicalCpu> logicalCpus_;
	size_t physicalCoreNum_;
	uint32_t lineSize_;
	bool isHybrid_;
};

namespace impl {

inline uint32_t popcnt(uint64_t mask)
{
#if defined(_M_X64) || defined(_M_AMD64)
	return (int)__popcnt64(mask);
#elif defined(__GNUC__) || defined(__clang__)
	return __builtin_popcountll(mask);
#else
	uint32_t count = 0;
	while (mask) {
		count += (mask & 1);
		mask >>= 1;
	}
	return count;
#endif
}

// fall back to CPUID leaf 0x1A
inline CoreType getCoreType()
{
	uint32_t data[4] = {};
	Cpu::getCpuidEx(0x1A, 0, data);
	const uint32_t coreTypeField = (data[0] >> 24) & 0xFF;
	if (coreTypeField == 0x40) return Performance; // P-core
	if (coreTypeField == 0x20) return Efficient; // E-core
	return Standard;
}

#ifdef _WIN32

typedef std::vector<uint32_t> U32Vec;

#if (defined(NTDDI_VERSION) && NTDDI_VERSION >= 0x06010000) || (defined(_WIN32_WINNT) && _WIN32_WINNT >= 0x0601)
	#define XBYAK_WINSDK_HAS_RELATIONSHIP_GROUP_AFFINITY 1
#else
	#define XBYAK_WINSDK_HAS_RELATIONSHIP_GROUP_AFFINITY 0
#endif

#if (defined(NTDDI_VERSION) && NTDDI_VERSION >= 0x0A000000) || (defined(_WIN32_WINNT) && _WIN32_WINNT >= 0x0A00)
	#define XBYAK_WINSDK_HAS_EFFICIENCY_CLASS 1
#else
	#define XBYAK_WINSDK_HAS_EFFICIENCY_CLASS 0
#endif

// GroupMasks[] / GroupCount on CACHE_RELATIONSHIP added in Win10 20H1 (SDK 10.0.19041, NTDDI_WIN10_VB)
// NOTE: _WIN32_WINNT has no sub-version granularity for Win10, so only
// NTDDI_VERSION can distinguish 20H1 (0x0A00000C) from earlier Win10 builds.
// If NTDDI_VERSION is not set, this macro will be 0 (safe/conservative fallback).
#if defined(NTDDI_VERSION) && NTDDI_VERSION >= 0x0A00000C
	#define XBYAK_WINSDK_HAS_CACHE_RELATIONSHIP_GROUPMASKS 1
#else
	#define XBYAK_WINSDK_HAS_CACHE_RELATIONSHIP_GROUPMASKS 0
#endif

#if XBYAK_WINSDK_HAS_RELATIONSHIP_GROUP_AFFINITY
typedef SYSTEM_LOGICAL_PROCESSOR_INFORMATION_EX ProcInfo;

inline CoreType getCoreTypeForAffinity(const GROUP_AFFINITY& affinity)
{
	GROUP_AFFINITY previousMask = {};
	if (!SetThreadGroupAffinity(GetCurrentThread(), &affinity, &previousMask)) {
		return Standard;
	}
	CoreType type = impl::getCoreType();
	SetThreadGroupAffinity(GetCurrentThread(), &previousMask, NULL);
	return type;
}

// return total logical cpus if sucessful, 0 if failed
inline uint32_t getGroupAcc(U32Vec& v)
{
	DWORD len = 0;
	GetLogicalProcessorInformationEx(RelationGroup, NULL, &len);
	std::vector<char> buf(len);
	if (!GetLogicalProcessorInformationEx(RelationGroup, reinterpret_cast<ProcInfo*>(buf.data()), &len)) {
		return 0;
	}
	const auto& entry = *reinterpret_cast<const ProcInfo*>(buf.data());
	const GROUP_RELATIONSHIP& gr = entry.Group;

	const uint32_t n = gr.ActiveGroupCount;
	if (n == 0) return 0;

	v.resize(n);

	uint32_t acc = 0;
	for (uint32_t g = 0; g < n; g++) {
		v[g] = acc;
		acc += gr.GroupInfo[g].ActiveProcessorCount;
	}
	return acc;
}

// return number of physical cores if successful, 0 if failed
static inline uint32_t getCores(std::vector<LogicalCpu>& cpus, bool isHybrid, const U32Vec& groupAcc) {
	DWORD len = 0;
	GetLogicalProcessorInformationEx(RelationProcessorCore, NULL, &len);
	std::vector<char> buf(len);
	if (!GetLogicalProcessorInformationEx(RelationProcessorCore, reinterpret_cast<ProcInfo*>(buf.data()), &len)) return 0;

	// get core indices
	const char *p = buf.data();
	const char *end = p + len;
	uint32_t coreIdx = 0;

	while (p < end) {
		const auto& entry = *reinterpret_cast<const ProcInfo*>(p);
		if (entry.Relationship == RelationProcessorCore) {
			const PROCESSOR_RELATIONSHIP& core = entry.Processor;
			LogicalCpu cpu;
			cpu.coreId = coreIdx++;
			if (!isHybrid) {
				cpu.coreType = Standard;
			} else {
#if XBYAK_WINSDK_HAS_EFFICIENCY_CLASS
				cpu.coreType = core.EfficiencyClass > 0 ? Performance : Efficient;
#else
				cpu.coreType = getCoreTypeForAffinity(core.GroupMask[0]);
#endif
			}

			const GROUP_AFFINITY* masks = core.GroupMask;
			for (WORD i = 0; i < core.GroupCount; i++) {
				const WORD group = masks[i].Group;
				const KAFFINITY m = masks[i].Mask;
				const uint32_t base = groupAcc[group];

				for (uint32_t b = 0; b < sizeof(KAFFINITY) * 8; b++) {
					if (m & (KAFFINITY(1) << b)) {
						const uint32_t idx = base + b;
						if (idx >= cpus.size()) return 0;
						cpus[idx] = cpu;
					}
				}
			}
		}
		p += entry.Size;
	}
	return coreIdx;
}

inline bool convertMask(CpuMask& mask, const U32Vec& groupAcc, const CACHE_RELATIONSHIP& cache)
{
#if XBYAK_WINSDK_HAS_CACHE_RELATIONSHIP_GROUPMASKS
	const WORD count = cache.GroupCount;
#else
	const WORD count = 1;
#endif
	for (WORD i = 0; i < count; i++) {
#if XBYAK_WINSDK_HAS_CACHE_RELATIONSHIP_GROUPMASKS
		const GROUP_AFFINITY& cg = cache.GroupMasks[i];
#else
		const GROUP_AFFINITY& cg = cache.GroupMask;
#endif
		const KAFFINITY m = cg.Mask;
		const uint32_t base = groupAcc[cg.Group];
		for (uint32_t b = 0; b < sizeof(KAFFINITY) * 8; b++) {
			if (m & (KAFFINITY(1) << b)) {
				if (!mask.append(base + b)) return false;
			}
		}
	}
	return true;
}

inline bool initCpuTopology(CpuTopology& cpuTopo)
{
	U32Vec groupAcc;
	const uint32_t logicalCpuNum = getGroupAcc(groupAcc);
	if (logicalCpuNum == 0) return false;
	if (logicalCpuNum >= (1u << XBYAK_CPUMASK_BITN)) return false;

	cpuTopo.logicalCpus_.resize(logicalCpuNum);
	cpuTopo.physicalCoreNum_ = getCores(cpuTopo.logicalCpus_, cpuTopo.isHybrid(), groupAcc);
	if (cpuTopo.physicalCoreNum_ == 0) return false;

	DWORD len = 0;
	GetLogicalProcessorInformationEx(RelationCache, NULL, &len);
	std::vector<char> buf(len);
	if (!GetLogicalProcessorInformationEx(RelationCache, reinterpret_cast<ProcInfo*>(buf.data()), &len)) return false;

	const char *p = buf.data();
	const char *end = p + len;

	while (p < end) {
		const auto& entry = *reinterpret_cast<const ProcInfo*>(p);
		if (entry.Relationship == RelationCache) {
			const CACHE_RELATIONSHIP& cache = entry.Cache;
			uint32_t type = CACHE_UNKNOWN;
			if (cache.Level == 1) {
				if (cache.Type == CacheInstruction) {
					type = L1i;
				} else if (cache.Type == CacheData) {
					type = L1d;
				}
			} else if (cache.Level == 2) {
				type = L2;
			} else if (cache.Level == 3) {
				type = L3;
			}
			if (type != CACHE_UNKNOWN) {
				CpuMask mask;
				if (!convertMask(mask, groupAcc, cache)) return false;
				for (const auto& i : mask) {
					if (i >= cpuTopo.logicalCpus_.size()) return false;
					cpuTopo.logicalCpus_[i].cache[type].size = cache.CacheSize;
					if (cpuTopo.lineSize_ == 0) cpuTopo.lineSize_ = cache.LineSize;
					cpuTopo.logicalCpus_[i].cache[type].associativity = cache.Associativity;
					cpuTopo.logicalCpus_[i].cache[type].sharedCpuIndices = mask;
				}
			}
		}
		p += entry.Size;
	}
	return true;
}
#else
inline bool initCpuTopology(CpuTopology& cpuTopo)
{
	(void)cpuTopo;
	return false;
}
#endif
// unset WinSDK version macros to avoid Macro pollution
#undef XBYAK_WINSDK_HAS_RELATIONSHIP_GROUP_AFFINITY
#undef XBYAK_WINSDK_HAS_EFFICIENCY_CLASS
#undef XBYAK_WINSDK_HAS_CACHE_RELATIONSHIP_GROUPMASKS
#elif defined(__linux__) // Linux

struct WrapFILE {
	FILE *f;
	explicit WrapFILE(const char *name)
		: f(fopen(name, "r"))
	{
	}
	~WrapFILE() { if (f) fclose(f); }
};

inline uint32_t readIntFromFile(const char* path) {
	WrapFILE wf(path);
	if (!wf.f) return 0;
	uint32_t val = 0;
	int n = fscanf(wf.f, "%u", &val);
	return (n == 1) ? val : 0;
}

inline bool parseCpuList(CpuMask& mask, const char* path) {
	WrapFILE wf(path);
	if (!wf.f) return false;
	char buf[1024];
	if (!fgets(buf, sizeof(buf), wf.f)) return false;
	size_t n = strlen(buf);
	if (n > 0 && buf[n - 1] == '\n') buf[n - 1] = '\0';
	return setStr(mask, buf);
}

inline CoreType setAffinityAndGetCoreType(uint32_t cpu)
{
	cpu_set_t cpuMask;
	CPU_ZERO(&cpuMask);
	CPU_SET(cpu, &cpuMask);
	if (sched_setaffinity(0, sizeof(cpu_set_t), &cpuMask)) return Standard;
	return impl::getCoreType();
}

inline bool initCpuTopology(CpuTopology& cpuTopo)
{
	const uint32_t logicalCpuNum = sysconf(_SC_NPROCESSORS_ONLN);

	if (logicalCpuNum == 0) return false;
	if (logicalCpuNum >= (1u << XBYAK_CPUMASK_BITN)) return false;

	cpuTopo.logicalCpus_.resize(logicalCpuNum);
	uint32_t maxPhisicalIdx = 0;

	for (uint32_t cpuIdx = 0; cpuIdx < logicalCpuNum; cpuIdx++) {
		char path[256];
		LogicalCpu& logCpu = cpuTopo.logicalCpus_[cpuIdx];

		snprintf(path, sizeof(path),
			"/sys/devices/system/cpu/cpu%u/topology/core_id", cpuIdx);
		logCpu.coreId = readIntFromFile(path);
		maxPhisicalIdx = (std::max)(maxPhisicalIdx, logCpu.coreId);

		logCpu.coreType = Standard;

		for (uint32_t cacheIdx = 0; cacheIdx < CACHE_TYPE_NUM; cacheIdx++) {
			CacheType cacheType = CACHE_UNKNOWN;

			// Map cache index to cache type
			{
				snprintf(path, sizeof(path),
					"/sys/devices/system/cpu/cpu%u/cache/index%u/type", cpuIdx, cacheIdx);
				char typeStr[32];
				WrapFILE wf(path);

				if (wf.f && fgets(typeStr, sizeof(typeStr), wf.f)) {
					if (strncmp(typeStr, "Instruction", 11) == 0) {
						cacheType = L1i;
					} else if (strncmp(typeStr, "Data", 4) == 0) {
						// Determine level
						char path[256];
						snprintf(path, sizeof(path),
							"/sys/devices/system/cpu/cpu%u/cache/index%u/level", cpuIdx, cacheIdx);
						switch (readIntFromFile(path)) {
						case 1: cacheType = L1d; break;
						case 2: cacheType = L2; break;
						case 3: cacheType = L3; break;
						default: break;;
						}
					} else if (strncmp(typeStr, "Unified", 7) == 0) {
						snprintf(path, sizeof(path),
							"/sys/devices/system/cpu/cpu%u/cache/index%u/level", cpuIdx, cacheIdx);
						switch (readIntFromFile(path)) {
						case 2: cacheType = L2; break;
						case 3: cacheType = L3; break;
						default: break;;
						}
					}
				}
			}
			if (cacheType == CACHE_UNKNOWN) continue;
			CpuCache& cache = logCpu.cache[cacheType];

			// Read cache size
			{
				snprintf(path, sizeof(path),
					"/sys/devices/system/cpu/cpu%u/cache/index%u/size", cpuIdx, cacheIdx);
				char sizeStr[32];
				WrapFILE wf(path);
				if (wf.f && fgets(sizeStr, sizeof(sizeStr), wf.f)) {
					char *endp;
					uint32_t size = (uint32_t)strtoul(sizeStr, &endp, 10);
					switch (*endp) {
					case '\0': case '\n': cache.size = size; break;
					case 'K': case 'k':   cache.size = size * 1024; break;
					case 'M': case 'm':   cache.size = size * 1024 * 1024; break;
					default: break;
					}
				}
			}

			// Read ways of associativity
			snprintf(path, sizeof(path),
				"/sys/devices/system/cpu/cpu%u/cache/index%u/ways_of_associativity", cpuIdx, cacheIdx);
			cache.associativity = readIntFromFile(path);

			// Read shared CPU list
			snprintf(path, sizeof(path),
				"/sys/devices/system/cpu/cpu%u/cache/index%u/shared_cpu_list", cpuIdx, cacheIdx);
			parseCpuList(cache.sharedCpuIndices, path);

		}
	}

	// Assign core types for hybrid architectures
	const bool isHybrid = cpuTopo.isHybrid();
	if (isHybrid) {
		// For hybrid systems, try toread P-core and E-core lists from sysfs first
		CpuMask pCoreMask;
		const bool hasPCoreSysfs = parseCpuList(pCoreMask, "/sys/devices/cpu_core/cpus");
		if (hasPCoreSysfs) {
			// Set Performance core types
			for (CpuMask::const_iterator it = pCoreMask.begin(); it != pCoreMask.end(); ++it) {
				uint32_t cpuIdx = *it;
				if (cpuIdx < logicalCpuNum) {
					cpuTopo.logicalCpus_[cpuIdx].coreType = Performance;
				}
			}
		}
		CpuMask eCoreMask;
		const bool hasECoreSysfs = parseCpuList(eCoreMask, "/sys/devices/cpu_atom/cpus");
		if (hasECoreSysfs) {
			// Set Efficient core types
			for (CpuMask::const_iterator it = eCoreMask.begin(); it != eCoreMask.end(); ++it) {
				uint32_t cpuIdx = *it;
				if (cpuIdx < logicalCpuNum) {
					cpuTopo.logicalCpus_[cpuIdx].coreType = Efficient;
				}
			}
		}
		// Fallback: if either sysfs paths are unavailable, detect both core type per-CPU
		if (!hasPCoreSysfs || !hasECoreSysfs) {
			cpu_set_t originalMask;
			CPU_ZERO(&originalMask);
			if (sched_getaffinity(0, sizeof(cpu_set_t), &originalMask) == 0) {
				for (uint32_t cpu = 0; cpu < logicalCpuNum; cpu++) {
					cpuTopo.logicalCpus_[cpu].coreType = impl::setAffinityAndGetCoreType(cpu);
				}
				sched_setaffinity(0, sizeof(cpu_set_t), &originalMask);
			}
		}
	}

	// Read coherency line size
	cpuTopo.lineSize_ = readIntFromFile("/sys/devices/system/cpu/cpu0/cache/index0/coherency_line_size");

	cpuTopo.physicalCoreNum_ = maxPhisicalIdx + 1;
	return true;
}
#else // Other OS (e.g., macOS)
inline bool initCpuTopology(CpuTopology& cpuTopo)
{
	// CPU topology detection not yet implemented
	(void)cpuTopo;
	return false;
}
#endif // _WIN32 / __linux__ / other OS

} // namespace impl
#endif // XBYAK_CPU_CACHE

class Clock {
public:
	static inline uint64_t getRdtsc()
	{
#ifdef XBYAK_INTEL_CPU_SPECIFIC
	#ifdef _MSC_VER
		return __rdtsc();
	#else
		uint32_t eax, edx;
		__asm__ volatile("rdtsc" : "=a"(eax), "=d"(edx));
		return ((uint64_t)edx << 32) | eax;
	#endif
#else
		// TODO: Need another impl of Clock or rdtsc-equivalent for non-x86 cpu
		return 0;
#endif
	}
	Clock()
		: clock_(0)
		, count_(0)
	{
	}
	void begin()
	{
		clock_ -= getRdtsc();
	}
	void end()
	{
		clock_ += getRdtsc();
		count_++;
	}
	int getCount() const { return count_; }
	uint64_t getClock() const { return clock_; }
	void clear() { count_ = 0; clock_ = 0; }
private:
	uint64_t clock_;
	int count_;
};

#ifdef XBYAK64

class Pack {
	static const size_t maxTblNum = 15;
	Xbyak::Reg64 tbl_[maxTblNum];
	size_t n_;
public:
	Pack() : tbl_(), n_(0) {}
	Pack(const Xbyak::Reg64 *tbl, size_t n) { init(tbl, n); }
	Pack(const Pack& rhs)
		: n_(rhs.n_)
	{
		for (size_t i = 0; i < n_; i++) tbl_[i] = rhs.tbl_[i];
	}
	Pack& operator=(const Pack& rhs)
	{
		n_ = rhs.n_;
		for (size_t i = 0; i < n_; i++) tbl_[i] = rhs.tbl_[i];
		return *this;
	}
	Pack(const Xbyak::Reg64& t0)
	{ n_ = 1; tbl_[0] = t0; }
	Pack(const Xbyak::Reg64& t1, const Xbyak::Reg64& t0)
	{ n_ = 2; tbl_[0] = t0; tbl_[1] = t1; }
	Pack(const Xbyak::Reg64& t2, const Xbyak::Reg64& t1, const Xbyak::Reg64& t0)
	{ n_ = 3; tbl_[0] = t0; tbl_[1] = t1; tbl_[2] = t2; }
	Pack(const Xbyak::Reg64& t3, const Xbyak::Reg64& t2, const Xbyak::Reg64& t1, const Xbyak::Reg64& t0)
	{ n_ = 4; tbl_[0] = t0; tbl_[1] = t1; tbl_[2] = t2; tbl_[3] = t3; }
	Pack(const Xbyak::Reg64& t4, const Xbyak::Reg64& t3, const Xbyak::Reg64& t2, const Xbyak::Reg64& t1, const Xbyak::Reg64& t0)
	{ n_ = 5; tbl_[0] = t0; tbl_[1] = t1; tbl_[2] = t2; tbl_[3] = t3; tbl_[4] = t4; }
	Pack(const Xbyak::Reg64& t5, const Xbyak::Reg64& t4, const Xbyak::Reg64& t3, const Xbyak::Reg64& t2, const Xbyak::Reg64& t1, const Xbyak::Reg64& t0)
	{ n_ = 6; tbl_[0] = t0; tbl_[1] = t1; tbl_[2] = t2; tbl_[3] = t3; tbl_[4] = t4; tbl_[5] = t5; }
	Pack(const Xbyak::Reg64& t6, const Xbyak::Reg64& t5, const Xbyak::Reg64& t4, const Xbyak::Reg64& t3, const Xbyak::Reg64& t2, const Xbyak::Reg64& t1, const Xbyak::Reg64& t0)
	{ n_ = 7; tbl_[0] = t0; tbl_[1] = t1; tbl_[2] = t2; tbl_[3] = t3; tbl_[4] = t4; tbl_[5] = t5; tbl_[6] = t6; }
	Pack(const Xbyak::Reg64& t7, const Xbyak::Reg64& t6, const Xbyak::Reg64& t5, const Xbyak::Reg64& t4, const Xbyak::Reg64& t3, const Xbyak::Reg64& t2, const Xbyak::Reg64& t1, const Xbyak::Reg64& t0)
	{ n_ = 8; tbl_[0] = t0; tbl_[1] = t1; tbl_[2] = t2; tbl_[3] = t3; tbl_[4] = t4; tbl_[5] = t5; tbl_[6] = t6; tbl_[7] = t7; }
	Pack(const Xbyak::Reg64& t8, const Xbyak::Reg64& t7, const Xbyak::Reg64& t6, const Xbyak::Reg64& t5, const Xbyak::Reg64& t4, const Xbyak::Reg64& t3, const Xbyak::Reg64& t2, const Xbyak::Reg64& t1, const Xbyak::Reg64& t0)
	{ n_ = 9; tbl_[0] = t0; tbl_[1] = t1; tbl_[2] = t2; tbl_[3] = t3; tbl_[4] = t4; tbl_[5] = t5; tbl_[6] = t6; tbl_[7] = t7; tbl_[8] = t8; }
	Pack(const Xbyak::Reg64& t9, const Xbyak::Reg64& t8, const Xbyak::Reg64& t7, const Xbyak::Reg64& t6, const Xbyak::Reg64& t5, const Xbyak::Reg64& t4, const Xbyak::Reg64& t3, const Xbyak::Reg64& t2, const Xbyak::Reg64& t1, const Xbyak::Reg64& t0)
	{ n_ = 10; tbl_[0] = t0; tbl_[1] = t1; tbl_[2] = t2; tbl_[3] = t3; tbl_[4] = t4; tbl_[5] = t5; tbl_[6] = t6; tbl_[7] = t7; tbl_[8] = t8; tbl_[9] = t9; }
	Pack(const Xbyak::Reg64& ta, const Xbyak::Reg64& t9, const Xbyak::Reg64& t8, const Xbyak::Reg64& t7, const Xbyak::Reg64& t6, const Xbyak::Reg64& t5, const Xbyak::Reg64& t4, const Xbyak::Reg64& t3, const Xbyak::Reg64& t2, const Xbyak::Reg64& t1, const Xbyak::Reg64& t0)
	{ n_ = 11; tbl_[0] = t0; tbl_[1] = t1; tbl_[2] = t2; tbl_[3] = t3; tbl_[4] = t4; tbl_[5] = t5; tbl_[6] = t6; tbl_[7] = t7; tbl_[8] = t8; tbl_[9] = t9; tbl_[10] = ta; }
	Pack(const Xbyak::Reg64& tb, const Xbyak::Reg64& ta, const Xbyak::Reg64& t9, const Xbyak::Reg64& t8, const Xbyak::Reg64& t7, const Xbyak::Reg64& t6, const Xbyak::Reg64& t5, const Xbyak::Reg64& t4, const Xbyak::Reg64& t3, const Xbyak::Reg64& t2, const Xbyak::Reg64& t1, const Xbyak::Reg64& t0)
	{ n_ = 12; tbl_[0] = t0; tbl_[1] = t1; tbl_[2] = t2; tbl_[3] = t3; tbl_[4] = t4; tbl_[5] = t5; tbl_[6] = t6; tbl_[7] = t7; tbl_[8] = t8; tbl_[9] = t9; tbl_[10] = ta; tbl_[11] = tb; }
	Pack& append(const Xbyak::Reg64& t)
	{
		if (n_ == maxTblNum) {
			fprintf(stderr, "ERR Pack::can't append\n");
			XBYAK_THROW_RET(ERR_BAD_PARAMETER, *this)
		}
		tbl_[n_++] = t;
		return *this;
	}
	void init(const Xbyak::Reg64 *tbl, size_t n)
	{
		if (n > maxTblNum) {
			fprintf(stderr, "ERR Pack::init bad n=%d\n", (int)n);
			XBYAK_THROW(ERR_BAD_PARAMETER)
		}
		n_ = n;
		for (size_t i = 0; i < n; i++) {
			tbl_[i] = tbl[i];
		}
	}
	const Xbyak::Reg64& operator[](size_t n) const
	{
		if (n >= n_) {
			fprintf(stderr, "ERR Pack bad n=%d(%d)\n", (int)n, (int)n_);
			XBYAK_THROW_RET(ERR_BAD_PARAMETER, rax)
		}
		return tbl_[n];
	}
	size_t size() const { return n_; }
	/*
		get tbl[pos, pos + num)
	*/
	Pack sub(size_t pos, size_t num = size_t(-1)) const
	{
		if (num == size_t(-1)) num = n_ - pos;
		if (pos + num > n_) {
			fprintf(stderr, "ERR Pack::sub bad pos=%d, num=%d\n", (int)pos, (int)num);
			XBYAK_THROW_RET(ERR_BAD_PARAMETER, Pack())
		}
		Pack pack;
		pack.n_ = num;
		for (size_t i = 0; i < num; i++) {
			pack.tbl_[i] = tbl_[pos + i];
		}
		return pack;
	}
	void put() const
	{
		for (size_t i = 0; i < n_; i++) {
			printf("%s ", tbl_[i].toString());
		}
		printf("\n");
	}
};

// start from a bit position larger than the number of GPRs
const int UseRBP = 1 << 5;
const int UseRCX = 1 << 6;
const int UseRDX = 1 << 7;
const int UseRSI = 1 << 8;
const int UseRDI = 1 << 9;
const int UseRBPAsFramePointer = UseRBP | (1 << 10);

class StackFrame {
#ifdef XBYAK64_WIN
	static const int noSaveNum = 6;
#else
	static const int noSaveNum = 8;
#endif
	static const int maxPnum = 4;
	static const int maxRegNum = 14; // maxRegNum = 16 - rsp - rax
	static const int calleeSaveNum = maxRegNum - noSaveNum;
	static const int UseMASK = UseRCX|UseRDX|UseRSI|UseRDI|UseRBP;
	Xbyak::CodeGenerator *code_;
	Xbyak::Reg64 pTbl_[maxPnum];
	Xbyak::Reg64 tTbl_[maxRegNum];
	Pack p_;
	Pack t_;
	int pNum_;
	int tNum_;
	int useRegs_;
	int saveNum_;
	int saveRegs_[calleeSaveNum];
	int P_;
	bool makeEpilog_;
	StackFrame(const StackFrame&);
	void operator=(const StackFrame&);
public:
	const Pack& p;
	const Pack& t;
	/*
		make stack frame
		@param sf [in] this
		@param pNum [in] number of function parameters(0 <= pNum <= 4)
		@param tNum [in] number of temporary registers(0 <= tNum, can be OR-ed with Use{RCX,RDX,RSI,RDI,RBP}, e.g., 3|UseRCX)
		@param stackSizeByte [in] local stack size
		@param makeEpilog [in] automatically call close() if true

		pNum + tNum + #Use must be <= 14

		you can use
		rax
		p[0], ..., p[pNum-1] as function parameters
		t[0], ..., t[tNum-1] as temporary registers
		{rcx,rdx,rsi,rdi,rbp} are explicitly available by specifying Use{RCX,RDX,RSI,RDI,RBP} in tNum
		rsp[0..stackSizeByte-1] if stackSizeByte > 0
	*/
	StackFrame(Xbyak::CodeGenerator *code, int pNum, int tNum = 0, int stackSizeByte = 0, bool makeEpilog = true)
		: code_(code)
		, pNum_(pNum)
		, tNum_(tNum & ~(UseMASK|UseRBPAsFramePointer))
		, useRegs_(tNum & UseMASK) // drop UseRBPAsFramePointer bit
		, saveNum_(0)
		, P_(0)
		, makeEpilog_(makeEpilog)
		, p(p_)
		, t(t_)
	{
		if (pNum < 0 || pNum > 4) XBYAK_THROW(ERR_BAD_PNUM)
		if (tNum < 0) XBYAK_THROW(ERR_BAD_TNUM)
		const int *const fullTbl = getRegEntryTbl();
		const int *const calleeTbl = fullTbl + noSaveNum;
		int callerUseNum = 0;
		int calleeUseNum = 0;
		for (int i = 0; i < maxRegNum; i++) {
			if (useRegs_ & useFlagOf(fullTbl[i])) {
				if (i < noSaveNum) {
					callerUseNum++;
				} else {
					calleeUseNum++;
				}
			}
		}
		const int useNum = callerUseNum + calleeUseNum;
		if (pNum + tNum_ + useNum > maxRegNum) XBYAK_THROW(ERR_BAD_TNUM)
		const int baseSaveNum = local::max_(0, pNum + tNum_ + useNum - noSaveNum);
		bool pushedRbp = false;
		if (useRegs_ & UseRBP) {
			code->push(rbp);
			saveRegs_[saveNum_++] = Operand::RBP;
			pushedRbp = true;
			if ((tNum & UseRBPAsFramePointer) == UseRBPAsFramePointer) code->mov(rbp, rsp);
		}
		for (int i = 0; i < calleeSaveNum; i++) {
			int r = calleeTbl[i];
			if (i < baseSaveNum || isUseReg(r)) {
				if (pushedRbp && r == Operand::RBP) continue;
				saveRegs_[saveNum_++] = r;
				code->push(Reg64(r));
			}
		}
		P_ = (stackSizeByte + 7) / 8;
		// (rsp % 16) == 8, then increment P_ for 16 byte alignment
		if (P_ > 0 && (P_ & 1) == (saveNum_ & 1)) P_++;
		P_ *= 8;
		if (P_ > 0) code->sub(rsp, P_);
		int pos = 0;
		for (int i = 0; i < pNum; i++) {
			pTbl_[i] = Xbyak::Reg64(getRegIdx(pos));
		}
		for (int i = 0; i < tNum_; i++) {
			tTbl_[i] = Xbyak::Reg64(getRegIdx(pos));
		}
		// replace reserved reg with backup reg if needed
		for (size_t i = 0; i < maxPnum; i++) {
			const RegSlot& rp = getRegSlotTbl()[i];
			if (isUseReg(rp.target) && rp.pos < pNum && rp.alt >= 0) {
				code->mov(Xbyak::Reg64(rp.alt), Xbyak::Reg64(rp.target));
			}
		}
		p_.init(pTbl_, pNum);
		t_.init(tTbl_, tNum_);
	}
	/*
		make epilog manually
		@param callRet [in] call ret() if true
	*/
	void close(bool callRet = true)
	{
		if (P_ > 0) code_->add(code_->rsp, P_);
		for (int i = saveNum_ - 1; i >= 0; i--) {
			code_->pop(Reg64(saveRegs_[i]));
		}
		if (callRet) code_->ret();
	}
	~StackFrame()
	{
		if (!makeEpilog_) return;
		close();
	}
private:
	static int useFlagOf(int r)
	{
		switch (r) {
		case Operand::RCX: return UseRCX;
		case Operand::RDX: return UseRDX;
		case Operand::RSI: return UseRSI;
		case Operand::RDI: return UseRDI;
		case Operand::RBP: return UseRBP;
		default: return 0;
		}
	}
	bool isUseReg(int r) const { return (useRegs_ & useFlagOf(r)) != 0; }
	// Register allocation for the first 4 function parameters
	struct RegSlot {
		int target;
		int pos; // position of target in getRegEntryTbl()
		int alt; // alternative if target is used for parameter. -1 means no alternative.
	};
	const RegSlot *getRegSlotTbl() const
	{
		// Win: p[] = rcx(r10), rdx(r11), r8, r9:
		// Linux: p[] = rdi(r8), rsi(r9), rdx(r11), rcx(r10)
		// reg(alt) means a reserved reg if Use<reg> is used.

		static const RegSlot tbl[maxPnum] = {
#ifdef XBYAK64_WIN
			{ Operand::RCX, 0, Operand::R10 },
			{ Operand::RDX, 1, Operand::R11 },
			{ Operand::RDI, 6, -1 },
			{ Operand::RSI, 7, -1 },
#else
			{ Operand::RCX, 3, Operand::R10 },
			{ Operand::RDX, 2, Operand::R11 },
			{ Operand::RDI, 0, Operand::R8 },
			{ Operand::RSI, 1, Operand::R9 },
#endif
		};
		return tbl;
	}
	const int *getRegEntryTbl() const
	{
		static const int tbl[maxRegNum] = {
#ifdef XBYAK64_WIN
			Operand::RCX, Operand::RDX, Operand::R8, Operand::R9, Operand::R10, Operand::R11, Operand::RDI, Operand::RSI,
#else
			Operand::RDI, Operand::RSI, Operand::RDX, Operand::RCX, Operand::R8, Operand::R9, Operand::R10, Operand::R11,
#endif
			Operand::RBX, Operand::RBP, Operand::R12, Operand::R13, Operand::R14, Operand::R15
		};
		return &tbl[0];
	}
	// get an available register index from tbl, skipping reserved registers
	int getRegIdx(int& pos) const
	{
		const int *tbl = getRegEntryTbl();
		const RegSlot *slotTbl = getRegSlotTbl();
		for (;;) {
		NEXT:;
			assert(pos < maxRegNum);
			int r = tbl[pos++];
			// if r is a Use*** target with alt, return alt as backup
			// otherwise skip Use*** targets, their alts, and UseRBP's rbp
			for (size_t i = 0; i < maxPnum; i++) {
				const RegSlot& slot = slotTbl[i];
				if (!isUseReg(slot.target)) continue;
				if (r == slot.alt) goto NEXT;
				if (r == slot.target) {
					if (slot.alt >= 0) return slot.alt;
					goto NEXT;
				}
			}
			if (!isUseReg(r)) return r;
		}
	}
};
#endif

class Profiler {
	int mode_;
	const char *suffix_;
	const void *startAddr_;
#ifdef XBYAK_USE_PERF
	FILE *fp_;
#endif
public:
	enum {
		None = 0,
		Perf = 1,
		VTune = 2
	};
	Profiler()
		: mode_(None)
		, suffix_("")
		, startAddr_(0)
#ifdef XBYAK_USE_PERF
		, fp_(0)
#endif
	{
	}
	// append suffix to funcName
	void setNameSuffix(const char *suffix)
	{
		suffix_ = suffix;
	}
	void setStartAddr(const void *startAddr)
	{
		startAddr_ = startAddr;
	}
	void init(int mode)
	{
		mode_ = None;
		switch (mode) {
		default:
		case None:
			return;
		case Perf:
#ifdef XBYAK_USE_PERF
			close();
			{
				const int pid = getpid();
				char name[128];
				snprintf(name, sizeof(name), "/tmp/perf-%d.map", pid);
				fp_ = fopen(name, "a+");
				if (fp_ == 0) {
					fprintf(stderr, "can't open %s\n", name);
					return;
				}
			}
			mode_ = Perf;
#endif
			return;
		case VTune:
#ifdef XBYAK_USE_VTUNE
			dlopen("dummy", RTLD_LAZY); // force to load dlopen to enable jit profiling
			if (iJIT_IsProfilingActive() != iJIT_SAMPLING_ON) {
				fprintf(stderr, "VTune profiling is not active\n");
				return;
			}
			mode_ = VTune;
#endif
			return;
		}
	}
	~Profiler()
	{
		close();
	}
	void close()
	{
#ifdef XBYAK_USE_PERF
		if (fp_ == 0) return;
		fclose(fp_);
		fp_ = 0;
#endif
	}
	void set(const char *funcName, const void *startAddr, size_t funcSize) const
	{
		if (mode_ == None) return;
#if !defined(XBYAK_USE_PERF) && !defined(XBYAK_USE_VTUNE)
		(void)funcName;
		(void)startAddr;
		(void)funcSize;
#endif
#ifdef XBYAK_USE_PERF
		if (mode_ == Perf) {
			if (fp_ == 0) return;
			fprintf(fp_, "%llx %zx %s%s", (long long)startAddr, funcSize, funcName, suffix_);
			/*
				perf does not recognize the function name which is less than 3,
				so append '_' at the end of the name if necessary
			*/
			size_t n = strlen(funcName) + strlen(suffix_);
			for (size_t i = n; i < 3; i++) {
				fprintf(fp_, "_");
			}
			fprintf(fp_, "\n");
			fflush(fp_);
		}
#endif
#ifdef XBYAK_USE_VTUNE
		if (mode_ != VTune) return;
		char className[] = "";
		char fileName[] = "";
		iJIT_Method_Load jmethod = {};
		jmethod.method_id = iJIT_GetNewMethodID();
		jmethod.class_file_name = className;
		jmethod.source_file_name = fileName;
		jmethod.method_load_address = const_cast<void*>(startAddr);
		jmethod.method_size = funcSize;
		jmethod.line_number_size = 0;
		char buf[128];
		snprintf(buf, sizeof(buf), "%s%s", funcName, suffix_);
		jmethod.method_name = buf;
		iJIT_NotifyEvent(iJVM_EVENT_TYPE_METHOD_LOAD_FINISHED, (void*)&jmethod);
#endif
	}
	/*
		for continuous set
		funcSize = endAddr - <previous set endAddr>
	*/
	void set(const char *funcName, const void *endAddr)
	{
		set(funcName, startAddr_, (size_t)endAddr - (size_t)startAddr_);
		startAddr_ = endAddr;
	}
};
#endif // XBYAK_ONLY_CLASS_CPU

} } // end of util

#if XBYAK_CPUMASK_COMPACT == 1 && __cplusplus >= 201103

namespace std {

template<>
struct hash<Xbyak::util::CpuMask> {
	size_t operator()(const Xbyak::util::CpuMask& m) const noexcept {
		return std::hash<uint64_t>{}(m.to_u64());
	}
};

} // std

#endif

#endif
