Something went wrong. Try again.
A single-model (Bonsai-27B) inference engine for Intel Arc GPUs
Something went wrong. Try again.
C++
at extra-debug
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502/* qxmx_gemm: batched ternary GEMM. See qxmx_gemm.h. * Kernel ported verbatim from gemm.cpp's gemm_gpu_v4 (M1.2 spike, validated * 52-71 TFLOPs sustained). Adds an optional `accum` residual fold in the * epilogue (C += accum), mirroring gemv_dpas8's accum parameter — the * down/out/o projections fold the residual add into the matmul. */#include "qxmx_gemm.h"
#include <cstdio>#include <cstdlib>#include <sycl/ext/intel/esimd.hpp>#include <sycl/sycl.hpp>
namespace esimd = sycl::ext::intel::esimd;namespace xmx = sycl::ext::intel::esimd::xmx;using namespace sycl;using xmx::dpas_argument_type;
namespace qx {
static constexpr int TN = 16;static constexpr int GRP = 64; /* QX_QK2 */static constexpr int SD = 8;
/* Tunable tiling params (read once from env on first call; -1 = compile-time * default WGM8=16/WGN16=4/GPP=6). Lets gemm_tune sweep without recompiling. */static int g_wgm8 = -1, g_wgn16 = -1, g_gpp = -1;static void load_tune() { /* Re-read every call (cheap) so gemm_tune can sweep within one process. */ if (const char* a = std::getenv("QXMX_GEMM_WGM8")) g_wgm8 = std::atoi(a); if (const char* a = std::getenv("QXMX_GEMM_WGN16")) g_wgn16 = std::atoi(a); if (const char* a = std::getenv("QXMX_GEMM_GPP")) g_gpp = std::atoi(a);}
/* Templated GEMM kernel. WGM8 = M-rows per group (multiple of 2; each thread * owns 2 M-rows via lm8*2/+1). WGN16 = N-tiles per group. GPP = K-groups * per barrier pass (SLM holds GPP*WGN16*1024 B of B). LWS = (WGM8/2)*WGN16. */template <int WGM8, int WGN16, int GPP>static void gemm_gpu_v4_impl(sycl::queue& q, const uint8_t* A_codes, const float* A_sc_t, const int8_t* B_vnni, const int32_t* sum_a, const float* act_scale, float* C, int M, int K, int N, const float* accum, bool sync, bool transpose_out) { int ngrp = K / GRP; int M8 = M / 8, N16 = N / TN; constexpr int SLM_B_SZ = GPP * WGN16 * 1024; constexpr int LWS = (WGM8 / 2) * WGN16; int NGM = (M8 + WGM8 - 1) / WGM8; int NGN = (N16 + WGN16 - 1) / WGN16; int G = NGM * NGN * LWS; auto props = esimd::properties{esimd::cache_hint_L1<esimd::cache_hint::uncached>, esimd::cache_hint_L2<esimd::cache_hint::cached>}; q.parallel_for(sycl::nd_range<1>(sycl::range<1>(G), sycl::range<1>(LWS)), [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { int group = it.get_group(0); int gm = group % NGM; int gn = group / NGM; int lid = it.get_local_id(0); int lm8 = lid / WGN16, ln16 = lid % WGN16; int n16 = gn * WGN16 + ln16; int n0 = n16 * TN; int tid = lid; esimd::slm_init<SLM_B_SZ>(); esimd::simd<float, 128> acc0(0.f), acc1(0.f); for (int gb = 0; gb < ngrp; gb += GPP) { /* B only: GPP x 64 chunks of 64 B. * For WGM8==32 configs (LWS = WGN16*16), the cooperative * index idx = pass*LWS + tid gives bgg = pass, bn16 = tid/16, * bkk = tid%16 -- the latter two PASS-INVARIANT. Hoist the * HBM base and SLM base out of the pass loop; step +16*N * int32s (HBM) and +WGN16*1024 (SLM) per pass. For WGM8==16 * (LWS = WGN16*16/2) bn16/bkk alternate with pass, so the * general index formula is kept. See docs/optimization.md * "When the base has a pass-invariant subexpression". */ constexpr int NPASS = GPP * WGN16 * 16 / LWS; if constexpr (WGM8 == 32) { const int bn16 = tid / 16, bkk = tid % 16; const int gn16 = gn * WGN16 + bn16; const bool gn16_ok = gn16 < N16; const int32_t* bp = reinterpret_cast<const int32_t*>( B_vnni + (size_t)((gb * 16 + bkk) * N + gn16 * TN) * 4); int slm_b = bn16 * 1024 + bkk * 64; for (int pass = 0; pass < NPASS; pass++) { int g = gb + pass; if (gn16_ok && g < ngrp) { esimd::simd<int32_t, 16> bv; bv.copy_from(bp, props); esimd::slm_block_store(slm_b, bv.template bit_cast_view<int8_t>()); } bp += 16 * N; slm_b += WGN16 * 1024; } } else { for (int pass = 0; pass < NPASS; pass++) { int idx = pass * LWS + tid; int bgg = idx / (WGN16 * 16), brest = idx % (WGN16 * 16); int bn16 = brest / 16, bkk = brest % 16; int gn16 = gn * WGN16 + bn16, g = gb + bgg; if (gn16 < N16 && g < ngrp) { int k4 = g * 16 + bkk; esimd::simd<int32_t, 16> bv; bv.copy_from(reinterpret_cast<const int32_t*>(B_vnni + (size_t)(k4 * N + gn16 * TN) * 4), props); esimd::slm_block_store((bgg * WGN16 + bn16) * 1024 + bkk * 64, bv.template bit_cast_view<int8_t>()); } } } esimd::barrier();
int gmax = (gb + GPP < ngrp) ? GPP : (ngrp - gb); /* Hoist loop-invariant address bases out of the ag loop: * IGC otherwise recomputes ag*WGN16*1024 (SLM), ag*128 * (A_codes), ag*N (sum_a), ag*M (A_sc_t) from scratch each * iteration (or/shl/mul) instead of strength-reducing to a * running offset. Same pattern that won on gemv_dpas8 * (+4.5%, commit 077f0587). VTune prefill: this kernel is * 1.172 s, 57.8% barrier / 2.9% D/A -- address math is * int-pipe and overlaps DPAS, but the or/shl storm inflates * the prologue loop body IGC must schedule around the * barrier. See docs/optimization.md "LICM-hoist pattern". */ const uint32_t* a0p = reinterpret_cast<const uint32_t*>( A_codes + (((size_t)(gm * WGM8 + lm8 * 2 + 0) * ngrp + gb) * 2) * 64); const uint32_t* a1p = reinterpret_cast<const uint32_t*>( A_codes + (((size_t)(gm * WGM8 + lm8 * 2 + 1) * ngrp + gb) * 2) * 64); const int32_t* sap = sum_a + (size_t)gb * N + n0; const float* df0p = A_sc_t + (size_t)gb * M + (gm * WGM8 + lm8 * 2 + 0) * 8; const float* df1p = A_sc_t + (size_t)gb * M + (gm * WGM8 + lm8 * 2 + 1) * 8; int slm_b0 = (ln16) * 1024; /* +ag*WGN16*1024 each iter */ int slm_b1 = slm_b0 + 512; for (int ag = 0; ag < gmax; ag++) { esimd::simd<int8_t, 512> vb0 = esimd::slm_block_load<int8_t, 512>(slm_b0); esimd::simd<int8_t, 512> vb1 = esimd::slm_block_load<int8_t, 512>(slm_b1); esimd::simd<uint32_t, 32> av0w, av1w; av0w.copy_from(a0p, props); av1w.copy_from(a1p, props); esimd::simd<uint8_t, 128> av0 = av0w.template bit_cast_view<uint8_t>(); esimd::simd<uint8_t, 128> av1 = av1w.template bit_cast_view<uint8_t>(); esimd::simd<int, 16> sa_glb; sa_glb.copy_from(sap, props); esimd::simd<float, 8> df0, df1; df0.copy_from(df0p, props); df1.copy_from(df1p, props); esimd::simd<uint8_t, 64> a00 = av0.select<64, 1>(0); esimd::simd<uint8_t, 64> a01 = av0.select<64, 1>(64); esimd::simd<uint8_t, 64> a10 = av1.select<64, 1>(0); esimd::simd<uint8_t, 64> a11 = av1.select<64, 1>(64); esimd::simd<int, 128> p0(0), p1(0); p0 = xmx::dpas<SD, 8, int, int, int8_t, uint8_t, dpas_argument_type::s8, dpas_argument_type::u2>(p0, vb0, a00); p1 = xmx::dpas<SD, 8, int, int, int8_t, uint8_t, dpas_argument_type::s8, dpas_argument_type::u2>(p1, vb0, a10); p0 = xmx::dpas<SD, 8, int, int, int8_t, uint8_t, dpas_argument_type::s8, dpas_argument_type::u2>(p0, vb1, a01); p1 = xmx::dpas<SD, 8, int, int, int8_t, uint8_t, dpas_argument_type::s8, dpas_argument_type::u2>(p1, vb1, a11); esimd::simd<int, 128> c0, c1; for (int r = 0; r < 8; r++) { c0.select<16, 1>(r * 16) = p0.select<16, 1>(r * 16) - sa_glb; c1.select<16, 1>(r * 16) = p1.select<16, 1>(r * 16) - sa_glb; } esimd::simd<float, 128> f0 = c0, f1 = c1, d0, d1; for (int r = 0; r < 8; r++) { d0.select<16, 1>(r * 16) = esimd::simd<float, 16>(df0[r]); d1.select<16, 1>(r * 16) = esimd::simd<float, 16>(df1[r]); } acc0 = acc0 + d0 * f0; acc1 = acc1 + d1 * f1; a0p += 32; /* +2*64 bytes = +128 bytes = +32 uint32s per group */ a1p += 32; sap += N; df0p += M; df1p += M; slm_b0 += WGN16 * 1024; slm_b1 += WGN16 * 1024; } esimd::barrier(); } esimd::barrier(); if (n16 >= N16) return; esimd::simd<float, 16> ascale; for (int n = 0; n < 16; n++) ascale[n] = act_scale[n0 + n]; if (!transpose_out) { for (int i = 0; i < 2; i++) { int m8t = gm * WGM8 + lm8 * 2 + i; if (m8t >= M8) continue; for (int r = 0; r < 8; r++) { esimd::simd<float, 16> row = (i ? acc1 : acc0).select<16, 1>(r * 16) * ascale; if (accum) { esimd::simd<float, 16> ac; ac.copy_from(accum + (size_t)(m8t * 8 + r) * N + n0, props); row = row + ac; } row.copy_to(C + (size_t)(m8t * 8 + r) * N + n0); } } } else { int m8t0 = gm * WGM8 + lm8 * 2; int m8t1 = m8t0 + 1; for (int n = 0; n < 16; n++) { float sc = ascale[n]; size_t coff = (size_t)(n0 + n) * M; if (m8t0 < M8) { esimd::simd<float, 8> row0 = acc0.select<8, 16>(n) * sc; if (accum) { esimd::simd<float, 8> ac; ac.copy_from(accum + coff + (size_t)m8t0 * 8, props); row0 = row0 + ac; } row0.copy_to(C + coff + (size_t)m8t0 * 8); } if (m8t1 < M8) { esimd::simd<float, 8> row1 = acc1.select<8, 16>(n) * sc; if (accum) { esimd::simd<float, 8> ac; ac.copy_from(accum + coff + (size_t)m8t1 * 8, props); row1 = row1 + ac; } row1.copy_to(C + coff + (size_t)m8t1 * 8); } } } }); if (sync) q.wait();}
void gemm_gpu_v4(sycl::queue& q, const uint8_t* A_codes, const float* A_sc_t, const int8_t* B_vnni, const int32_t* sum_a, const float* act_scale, float* C, int M, int K, int N, const float* accum, bool sync, bool transpose_out) { load_tune(); int WGM8 = g_wgm8 > 0 ? g_wgm8 : 32; int WGN16 = g_wgn16 > 0 ? g_wgn16 : 4; int GPP = g_gpp > 0 ? g_gpp : 8; /* default 32/4/8: ~15% faster than the * old 16/4/16 across all engine shapes * (gemm_tune sweep, 2026-07-19). Needs * upload_q2's M padded to 256. */ if (WGM8 == 16 && WGN16 == 4 && GPP == 6) gemm_gpu_v4_impl<16,4,6>(q,A_codes,A_sc_t,B_vnni,sum_a,act_scale,C,M,K,N,accum,sync,transpose_out); else if (WGM8 == 16 && WGN16 == 4 && GPP == 8) gemm_gpu_v4_impl<16,4,8>(q,A_codes,A_sc_t,B_vnni,sum_a,act_scale,C,M,K,N,accum,sync,transpose_out); else if (WGM8 == 16 && WGN16 == 4 && GPP == 12) gemm_gpu_v4_impl<16,4,12>(q,A_codes,A_sc_t,B_vnni,sum_a,act_scale,C,M,K,N,accum,sync,transpose_out); else if (WGM8 == 16 && WGN16 == 4 && GPP == 16) gemm_gpu_v4_impl<16,4,16>(q,A_codes,A_sc_t,B_vnni,sum_a,act_scale,C,M,K,N,accum,sync,transpose_out); else if (WGM8 == 16 && WGN16 == 8 && GPP == 6) gemm_gpu_v4_impl<16,8,6>(q,A_codes,A_sc_t,B_vnni,sum_a,act_scale,C,M,K,N,accum,sync,transpose_out); else if (WGM8 == 16 && WGN16 == 8 && GPP == 8) gemm_gpu_v4_impl<16,8,8>(q,A_codes,A_sc_t,B_vnni,sum_a,act_scale,C,M,K,N,accum,sync,transpose_out); else if (WGM8 == 32 && WGN16 == 4 && GPP == 6) gemm_gpu_v4_impl<32,4,6>(q,A_codes,A_sc_t,B_vnni,sum_a,act_scale,C,M,K,N,accum,sync,transpose_out); else if (WGM8 == 32 && WGN16 == 4 && GPP == 8) gemm_gpu_v4_impl<32,4,8>(q,A_codes,A_sc_t,B_vnni,sum_a,act_scale,C,M,K,N,accum,sync,transpose_out); else if (WGM8 == 8 && WGN16 == 4 && GPP == 6) gemm_gpu_v4_impl<8,4,6>(q,A_codes,A_sc_t,B_vnni,sum_a,act_scale,C,M,K,N,accum,sync,transpose_out); else if (WGM8 == 8 && WGN16 == 4 && GPP == 8) gemm_gpu_v4_impl<8,4,8>(q,A_codes,A_sc_t,B_vnni,sum_a,act_scale,C,M,K,N,accum,sync,transpose_out); else if (WGM8 == 8 && WGN16 == 8 && GPP == 6) gemm_gpu_v4_impl<8,8,6>(q,A_codes,A_sc_t,B_vnni,sum_a,act_scale,C,M,K,N,accum,sync,transpose_out); else gemm_gpu_v4_impl<32,4,8>(q,A_codes,A_sc_t,B_vnni,sum_a,act_scale,C,M,K,N,accum,sync,transpose_out);}
/* M1.7b.9 dual GEMM (gate+up fusion): see qxmx_gemm.h. Same tiling as * gemm_gpu_v4; each thread loads two A vectors (A0, A1) from global per group * and keeps four DPAS accumulators (2 M-rows x 2 matrices). B is loaded once * into SLM and shared by both accumulators — the win over two separate * gemm_gpu_v4 calls. */template <int WGM8, int WGN16, int GPP>static void gemm_gpu_v4_dual_impl(sycl::queue& q, const uint8_t* A0_codes, const float* A0_sc_t, const uint8_t* A1_codes, const float* A1_sc_t, const int8_t* B_vnni, const int32_t* sum_a, const float* act_scale, float* C0, float* C1, int M, int K, int N, const float* accum0, const float* accum1, bool sync, bool transpose_out) { int ngrp = K / GRP; int M8 = M / 8, N16 = N / TN; constexpr int SLM_B_SZ = GPP * WGN16 * 1024; constexpr int LWS = (WGM8 / 2) * WGN16; int NGM = (M8 + WGM8 - 1) / WGM8; int NGN = (N16 + WGN16 - 1) / WGN16; int G = NGM * NGN * LWS; auto props = esimd::properties{esimd::cache_hint_L1<esimd::cache_hint::uncached>, esimd::cache_hint_L2<esimd::cache_hint::cached>}; q.parallel_for(sycl::nd_range<1>(sycl::range<1>(G), sycl::range<1>(LWS)), [=](sycl::nd_item<1> it) [[intel::sycl_explicit_simd]] { int group = it.get_group(0); int gm = group % NGM; int gn = group / NGM; int lid = it.get_local_id(0); int lm8 = lid / WGN16, ln16 = lid % WGN16; int n16 = gn * WGN16 + ln16; int n0 = n16 * TN; int tid = lid; esimd::slm_init<SLM_B_SZ>(); /* accXy: X=matrix (0=A0,1=A1), y=m-row half (0=lm8*2,1=lm8*2+1) */ esimd::simd<float, 128> acc00(0.f), acc01(0.f), acc10(0.f), acc11(0.f); for (int gb = 0; gb < ngrp; gb += GPP) { /* B only (shared by both A0/A1): GPP x 64 chunks of 64 B. * For WGM8==32 (LWS = WGN16*16), bn16 = tid/16, bkk = tid%16 * are PASS-INVARIANT -- hoist HBM + SLM bases out of the * pass loop; step +16*N int32s (HBM), +WGN16*1024 (SLM). * WGM8==16 keeps the general index formula (bn16/bkk * alternate with pass). See docs/optimization.md. */ constexpr int NPASS = GPP * WGN16 * 16 / LWS; if constexpr (WGM8 == 32) { const int bn16 = tid / 16, bkk = tid % 16; const int gn16 = gn * WGN16 + bn16; const bool gn16_ok = gn16 < N16; const int32_t* bp = reinterpret_cast<const int32_t*>( B_vnni + (size_t)((gb * 16 + bkk) * N + gn16 * TN) * 4); int slm_b = bn16 * 1024 + bkk * 64; for (int pass = 0; pass < NPASS; pass++) { int g = gb + pass; if (gn16_ok && g < ngrp) { esimd::simd<int32_t, 16> bv; bv.copy_from(bp, props); esimd::slm_block_store(slm_b, bv.template bit_cast_view<int8_t>()); } bp += 16 * N; slm_b += WGN16 * 1024; } } else { for (int pass = 0; pass < NPASS; pass++) { int idx = pass * LWS + tid; int bgg = idx / (WGN16 * 16), brest = idx % (WGN16 * 16); int bn16 = brest / 16, bkk = brest % 16; int gn16 = gn * WGN16 + bn16, g = gb + bgg; if (gn16 < N16 && g < ngrp) { int k4 = g * 16 + bkk; esimd::simd<int32_t, 16> bv; bv.copy_from(reinterpret_cast<const int32_t*>(B_vnni + (size_t)(k4 * N + gn16 * TN) * 4), props); esimd::slm_block_store((bgg * WGN16 + bn16) * 1024 + bkk * 64, bv.template bit_cast_view<int8_t>()); } } } esimd::barrier();
int gmax = (gb + GPP < ngrp) ? GPP : (ngrp - gb); /* Hoist loop-invariant address bases out of the ag loop * (same LICM pattern as gemm_gpu_v4_impl -- running pointers * so IGC strength-reduces ag*WGN16*1024, ag*128, ag*N, ag*M * to a single add per iter). See docs/optimization.md. */ const uint32_t* a0p0 = reinterpret_cast<const uint32_t*>( A0_codes + (((size_t)(gm * WGM8 + lm8 * 2 + 0) * ngrp + gb) * 2) * 64); const uint32_t* a0p1 = reinterpret_cast<const uint32_t*>( A0_codes + (((size_t)(gm * WGM8 + lm8 * 2 + 1) * ngrp + gb) * 2) * 64); const uint32_t* a1p0 = reinterpret_cast<const uint32_t*>( A1_codes + (((size_t)(gm * WGM8 + lm8 * 2 + 0) * ngrp + gb) * 2) * 64); const uint32_t* a1p1 = reinterpret_cast<const uint32_t*>( A1_codes + (((size_t)(gm * WGM8 + lm8 * 2 + 1) * ngrp + gb) * 2) * 64); const int32_t* sap = sum_a + (size_t)gb * N + n0; const float* df00p = A0_sc_t + (size_t)gb * M + (gm * WGM8 + lm8 * 2 + 0) * 8; const float* df01p = A0_sc_t + (size_t)gb * M + (gm * WGM8 + lm8 * 2 + 1) * 8; const float* df10p = A1_sc_t + (size_t)gb * M + (gm * WGM8 + lm8 * 2 + 0) * 8; const float* df11p = A1_sc_t + (size_t)gb * M + (gm * WGM8 + lm8 * 2 + 1) * 8; int slm_b0 = (ln16) * 1024; /* +ag*WGN16*1024 each iter */ int slm_b1 = slm_b0 + 512; for (int ag = 0; ag < gmax; ag++) { esimd::simd<int8_t, 512> vb0 = esimd::slm_block_load<int8_t, 512>(slm_b0); esimd::simd<int8_t, 512> vb1 = esimd::slm_block_load<int8_t, 512>(slm_b1); /* A0: two 128B rows (lm8*2, lm8*2+1) */ esimd::simd<uint32_t, 32> a0w0, a0w1; a0w0.copy_from(a0p0, props); a0w1.copy_from(a0p1, props); /* A1: two 128B rows */ esimd::simd<uint32_t, 32> a1w0, a1w1; a1w0.copy_from(a1p0, props); a1w1.copy_from(a1p1, props); esimd::simd<uint8_t, 128> a00 = a0w0.template bit_cast_view<uint8_t>(); esimd::simd<uint8_t, 128> a01 = a0w1.template bit_cast_view<uint8_t>(); esimd::simd<uint8_t, 128> a10 = a1w0.template bit_cast_view<uint8_t>(); esimd::simd<uint8_t, 128> a11 = a1w1.template bit_cast_view<uint8_t>(); esimd::simd<int, 16> sa_glb; sa_glb.copy_from(sap, props); esimd::simd<float, 8> df00, df01, df10, df11; df00.copy_from(df00p, props); df01.copy_from(df01p, props); df10.copy_from(df10p, props); df11.copy_from(df11p, props); esimd::simd<uint8_t, 64> a00h = a00.select<64, 1>(0), a00t = a00.select<64, 1>(64); esimd::simd<uint8_t, 64> a01h = a01.select<64, 1>(0), a01t = a01.select<64, 1>(64); esimd::simd<uint8_t, 64> a10h = a10.select<64, 1>(0), a10t = a10.select<64, 1>(64); esimd::simd<uint8_t, 64> a11h = a11.select<64, 1>(0), a11t = a11.select<64, 1>(64); esimd::simd<int, 128> p00(0), p01(0), p10(0), p11(0); p00 = xmx::dpas<SD, 8, int, int, int8_t, uint8_t, dpas_argument_type::s8, dpas_argument_type::u2>(p00, vb0, a00h); p01 = xmx::dpas<SD, 8, int, int, int8_t, uint8_t, dpas_argument_type::s8, dpas_argument_type::u2>(p01, vb0, a01h); p10 = xmx::dpas<SD, 8, int, int, int8_t, uint8_t, dpas_argument_type::s8, dpas_argument_type::u2>(p10, vb0, a10h); p11 = xmx::dpas<SD, 8, int, int, int8_t, uint8_t, dpas_argument_type::s8, dpas_argument_type::u2>(p11, vb0, a11h); p00 = xmx::dpas<SD, 8, int, int, int8_t, uint8_t, dpas_argument_type::s8, dpas_argument_type::u2>(p00, vb1, a00t); p01 = xmx::dpas<SD, 8, int, int, int8_t, uint8_t, dpas_argument_type::s8, dpas_argument_type::u2>(p01, vb1, a01t); p10 = xmx::dpas<SD, 8, int, int, int8_t, uint8_t, dpas_argument_type::s8, dpas_argument_type::u2>(p10, vb1, a10t); p11 = xmx::dpas<SD, 8, int, int, int8_t, uint8_t, dpas_argument_type::s8, dpas_argument_type::u2>(p11, vb1, a11t); esimd::simd<int, 128> c00, c01, c10, c11; for (int r = 0; r < 8; r++) { c00.select<16, 1>(r * 16) = p00.select<16, 1>(r * 16) - sa_glb; c01.select<16, 1>(r * 16) = p01.select<16, 1>(r * 16) - sa_glb; c10.select<16, 1>(r * 16) = p10.select<16, 1>(r * 16) - sa_glb; c11.select<16, 1>(r * 16) = p11.select<16, 1>(r * 16) - sa_glb; } esimd::simd<float, 128> f00 = c00, f01 = c01, f10 = c10, f11 = c11, d00, d01, d10, d11; for (int r = 0; r < 8; r++) { d00.select<16, 1>(r * 16) = esimd::simd<float, 16>(df00[r]); d01.select<16, 1>(r * 16) = esimd::simd<float, 16>(df01[r]); d10.select<16, 1>(r * 16) = esimd::simd<float, 16>(df10[r]); d11.select<16, 1>(r * 16) = esimd::simd<float, 16>(df11[r]); } acc00 = acc00 + d00 * f00; acc01 = acc01 + d01 * f01; acc10 = acc10 + d10 * f10; acc11 = acc11 + d11 * f11; a0p0 += 32; a0p1 += 32; a1p0 += 32; a1p1 += 32; sap += N; df00p += M; df01p += M; df10p += M; df11p += M; slm_b0 += WGN16 * 1024; slm_b1 += WGN16 * 1024; } esimd::barrier(); } esimd::barrier(); if (n16 >= N16) return; esimd::simd<float, 16> ascale; for (int n = 0; n < 16; n++) ascale[n] = act_scale[n0 + n]; if (!transpose_out) { for (int i = 0; i < 2; i++) { int m8t = gm * WGM8 + lm8 * 2 + i; if (m8t >= M8) continue; for (int r = 0; r < 8; r++) { esimd::simd<float, 16> row0 = (i ? acc01 : acc00).select<16, 1>(r * 16) * ascale; esimd::simd<float, 16> row1 = (i ? acc11 : acc10).select<16, 1>(r * 16) * ascale; if (accum0) { esimd::simd<float, 16> ac; ac.copy_from(accum0 + (size_t)(m8t * 8 + r) * N + n0, props); row0 = row0 + ac; } if (accum1) { esimd::simd<float, 16> ac; ac.copy_from(accum1 + (size_t)(m8t * 8 + r) * N + n0, props); row1 = row1 + ac; } row0.copy_to(C0 + (size_t)(m8t * 8 + r) * N + n0); row1.copy_to(C1 + (size_t)(m8t * 8 + r) * N + n0); } } } else { int m8t0 = gm * WGM8 + lm8 * 2; int m8t1 = m8t0 + 1; for (int n = 0; n < 16; n++) { float sc = ascale[n]; size_t coff = (size_t)(n0 + n) * M; if (m8t0 < M8) { esimd::simd<float, 8> r00 = acc00.select<8, 16>(n) * sc; esimd::simd<float, 8> r10 = acc10.select<8, 16>(n) * sc; if (accum0) { esimd::simd<float, 8> ac; ac.copy_from(accum0 + coff + (size_t)m8t0 * 8, props); r00 = r00 + ac; } if (accum1) { esimd::simd<float, 8> ac; ac.copy_from(accum1 + coff + (size_t)m8t0 * 8, props); r10 = r10 + ac; } r00.copy_to(C0 + coff + (size_t)m8t0 * 8); r10.copy_to(C1 + coff + (size_t)m8t0 * 8); } if (m8t1 < M8) { esimd::simd<float, 8> r01 = acc01.select<8, 16>(n) * sc; esimd::simd<float, 8> r11 = acc11.select<8, 16>(n) * sc; if (accum0) { esimd::simd<float, 8> ac; ac.copy_from(accum0 + coff + (size_t)m8t1 * 8, props); r01 = r01 + ac; } if (accum1) { esimd::simd<float, 8> ac; ac.copy_from(accum1 + coff + (size_t)m8t1 * 8, props); r11 = r11 + ac; } r01.copy_to(C0 + coff + (size_t)m8t1 * 8); r11.copy_to(C1 + coff + (size_t)m8t1 * 8); } } } }); if (sync) q.wait();}
void gemm_gpu_v4_dual(sycl::queue& q, const uint8_t* A0_codes, const float* A0_sc_t, const uint8_t* A1_codes, const float* A1_sc_t, const int8_t* B_vnni, const int32_t* sum_a, const float* act_scale, float* C0, float* C1, int M, int K, int N, const float* accum0, const float* accum1, bool sync, bool transpose_out) { load_tune(); int WGM8 = g_wgm8 > 0 ? g_wgm8 : 32; int WGN16 = g_wgn16 > 0 ? g_wgn16 : 4; int GPP = g_gpp > 0 ? g_gpp : 6; /* default 32/4/6: the dual GEMM prefers * one fewer K-group/pass than single * (register pressure from 4 DPAS * accumulators). gemm_tune sweep. */ if (WGM8 == 16 && WGN16 == 4 && GPP == 6) gemm_gpu_v4_dual_impl<16,4,6>(q,A0_codes,A0_sc_t,A1_codes,A1_sc_t,B_vnni,sum_a,act_scale,C0,C1,M,K,N,accum0,accum1,sync,transpose_out); else if (WGM8 == 16 && WGN16 == 8 && GPP == 6) gemm_gpu_v4_dual_impl<16,8,6>(q,A0_codes,A0_sc_t,A1_codes,A1_sc_t,B_vnni,sum_a,act_scale,C0,C1,M,K,N,accum0,accum1,sync,transpose_out); else if (WGM8 == 16 && WGN16 == 8 && GPP == 8) gemm_gpu_v4_dual_impl<16,8,8>(q,A0_codes,A0_sc_t,A1_codes,A1_sc_t,B_vnni,sum_a,act_scale,C0,C1,M,K,N,accum0,accum1,sync,transpose_out); else if (WGM8 == 16 && WGN16 == 4 && GPP == 8) gemm_gpu_v4_dual_impl<16,4,8>(q,A0_codes,A0_sc_t,A1_codes,A1_sc_t,B_vnni,sum_a,act_scale,C0,C1,M,K,N,accum0,accum1,sync,transpose_out); else if (WGM8 == 32 && WGN16 == 4 && GPP == 6) gemm_gpu_v4_dual_impl<32,4,6>(q,A0_codes,A0_sc_t,A1_codes,A1_sc_t,B_vnni,sum_a,act_scale,C0,C1,M,K,N,accum0,accum1,sync,transpose_out); else if (WGM8 == 32 && WGN16 == 4 && GPP == 8) gemm_gpu_v4_dual_impl<32,4,8>(q,A0_codes,A0_sc_t,A1_codes,A1_sc_t,B_vnni,sum_a,act_scale,C0,C1,M,K,N,accum0,accum1,sync,transpose_out); else gemm_gpu_v4_dual_impl<32,4,6>(q,A0_codes,A0_sc_t,A1_codes,A1_sc_t,B_vnni,sum_a,act_scale,C0,C1,M,K,N,accum0,accum1,sync,transpose_out);}
} // namespace qx