Something went wrong. Try again.
A single-model (Bonsai-27B) inference engine for Intel Arc GPUs
Something went wrong. Try again.
C++
at extra-debug
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889/* fa_fused_bench: microbenchmark for the M1.7c fused FA kernel. * One config: pos_start=4864, n_tok=256 (chunk-20-like), q8_0 K cache. * Times gpu_flash_attn<KQ8_0> over ITERS launches (in-order queue, single * wait at the end). Use with QXMX_FA_PHASES bitmask for phase isolation * (bit0 dequant, 1 QK, 2 softmax, 3 rescale, 4 PV; 31=all) and * QXMX_FA_SPLIT=1 for the M1.7b.7 baseline. * * Build: meson compile -C build fa_fused_bench && ./build/fa_fused_bench */#include <chrono>#include <cmath>#include <cstdint>#include <cstdio>#include <cstdlib>#include <cstring>#include <random>#include <vector>
#include "qxmx.h"#include "qxmx_fa.h"#include "qxmx_gpu.h"
using namespace qx;
int main() { constexpr int64_t pos_start = 4864; constexpr int n_tok = 256; constexpr int KV = QX_KV_DIM, NQ = QX_N_HEAD, HD = QX_HEAD_DIM; constexpr size_t QOUT = (size_t)NQ * HD; const int max_ctx = (int)(pos_start + n_tok); constexpr int ITERS = 5;
sycl::queue q(sycl::gpu_selector_v, sycl::property::queue::in_order{}); printf("fa_fused_bench: pos_start=%lld n_tok=%d K=q8_0 iters=%d\n", (long long)pos_start, n_tok, ITERS); printf(" phases mask (QXMX_FA_PHASES): bit0 shadow dequant, 1 QK, 2 softmax," " 4 PV\n");
/* Non-trivial data (varied scales -- see qxmx_gemm_value_dependence). */ std::mt19937_64 rng(0xFA17ULL); std::normal_distribution<float> g(0.0f, 1.0f); std::vector<float> kf((size_t)max_ctx * KV), vf((size_t)max_ctx * KV), qf((size_t)n_tok * QOUT); for (auto& x : kf) x = g(rng); for (auto& x : vf) x = g(rng); for (auto& x : qf) x = g(rng);
float* d_kf = sycl::malloc_shared<float>(kf.size(), q); float* d_vf = sycl::malloc_shared<float>(vf.size(), q); float* d_q = sycl::malloc_shared<float>(qf.size(), q); memcpy(d_kf, kf.data(), kf.size() * 4); memcpy(d_vf, vf.data(), vf.size() * 4); memcpy(d_q, qf.data(), qf.size() * 4); uint8_t* kvk = sycl::malloc_shared<uint8_t>((size_t)max_ctx * KROW_BYTES_Q8, q); uint32_t* kvv = sycl::malloc_shared<uint32_t>((size_t)max_ctx * VROW_U32, q); float* out = sycl::malloc_shared<float>((size_t)n_tok * QOUT, q); for (int t = 0; t < max_ctx; t++) { gpu_quant_k_row_q8(q, d_kf + (size_t)t * KV, kvk + (size_t)t * KROW_BYTES_Q8); gpu_quant_v_row(q, d_vf + (size_t)t * KV, kvv + (size_t)t * VROW_U32); } q.wait();
fa_split_workspace ws = fa_split_alloc(q, n_tok); /* warmup + JIT */ gpu_flash_attn<KQ8_0>(q, d_q, kvk, kvv, out, pos_start, n_tok, &ws); q.wait(); auto t0 = std::chrono::steady_clock::now(); for (int i = 0; i < ITERS; i++) gpu_flash_attn<KQ8_0>(q, d_q, kvk, kvv, out, pos_start, n_tok, &ws); q.wait(); auto t1 = std::chrono::steady_clock::now(); double ms = std::chrono::duration<double, std::milli>(t1 - t0).count() / ITERS; int n_tiles = (int)((pos_start + n_tok + 63) / 64); printf(" fused+finalize: %.3f ms/call (%.3f ms per K-tile x %d tiles)\n", ms, ms / n_tiles, n_tiles); /* rough DPAS throughput of QK+PV: 2 * 2 * NQ * n_tok * total * HD */ double flops = 4.0 * NQ * n_tok * (double)(pos_start + n_tok) * HD; printf(" effective QK+PV throughput: %.2f TFLOPs\n", flops / (ms * 1e-3) / 1e12);
fa_split_free(q, ws); sycl::free(d_kf, q); sycl::free(d_vf, q); sycl::free(d_q, q); sycl::free(kvk, q); sycl::free(kvv, q); sycl::free(out, q); return 0;}