/* fa_fused_bench: microbenchmark for the M1.7c fused FA kernel. * One config: pos_start=4864, n_tok=256 (chunk-20-like), q8_0 K cache. * Times gpu_flash_attn over ITERS launches (in-order queue, single * wait at the end). Use with QXMX_FA_PHASES bitmask for phase isolation * (bit0 dequant, 1 QK, 2 softmax, 3 rescale, 4 PV; 31=all) and * QXMX_FA_SPLIT=1 for the M1.7b.7 baseline. * * Build: meson compile -C build fa_fused_bench && ./build/fa_fused_bench */ #include #include #include #include #include #include #include #include #include "qxmx.h" #include "qxmx_fa.h" #include "qxmx_gpu.h" using namespace qx; int main() { constexpr int64_t pos_start = 4864; constexpr int n_tok = 256; constexpr int KV = QX_KV_DIM, NQ = QX_N_HEAD, HD = QX_HEAD_DIM; constexpr size_t QOUT = (size_t)NQ * HD; const int max_ctx = (int)(pos_start + n_tok); constexpr int ITERS = 5; sycl::queue q(sycl::gpu_selector_v, sycl::property::queue::in_order{}); printf("fa_fused_bench: pos_start=%lld n_tok=%d K=q8_0 iters=%d\n", (long long)pos_start, n_tok, ITERS); printf(" phases mask (QXMX_FA_PHASES): bit0 shadow dequant, 1 QK, 2 softmax," " 4 PV\n"); /* Non-trivial data (varied scales -- see qxmx_gemm_value_dependence). */ std::mt19937_64 rng(0xFA17ULL); std::normal_distribution g(0.0f, 1.0f); std::vector kf((size_t)max_ctx * KV), vf((size_t)max_ctx * KV), qf((size_t)n_tok * QOUT); for (auto& x : kf) x = g(rng); for (auto& x : vf) x = g(rng); for (auto& x : qf) x = g(rng); float* d_kf = sycl::malloc_shared(kf.size(), q); float* d_vf = sycl::malloc_shared(vf.size(), q); float* d_q = sycl::malloc_shared(qf.size(), q); memcpy(d_kf, kf.data(), kf.size() * 4); memcpy(d_vf, vf.data(), vf.size() * 4); memcpy(d_q, qf.data(), qf.size() * 4); uint8_t* kvk = sycl::malloc_shared((size_t)max_ctx * KROW_BYTES_Q8, q); uint32_t* kvv = sycl::malloc_shared((size_t)max_ctx * VROW_U32, q); float* out = sycl::malloc_shared((size_t)n_tok * QOUT, q); for (int t = 0; t < max_ctx; t++) { gpu_quant_k_row_q8(q, d_kf + (size_t)t * KV, kvk + (size_t)t * KROW_BYTES_Q8); gpu_quant_v_row(q, d_vf + (size_t)t * KV, kvv + (size_t)t * VROW_U32); } q.wait(); fa_split_workspace ws = fa_split_alloc(q, n_tok); /* warmup + JIT */ gpu_flash_attn(q, d_q, kvk, kvv, out, pos_start, n_tok, &ws); q.wait(); auto t0 = std::chrono::steady_clock::now(); for (int i = 0; i < ITERS; i++) gpu_flash_attn(q, d_q, kvk, kvv, out, pos_start, n_tok, &ws); q.wait(); auto t1 = std::chrono::steady_clock::now(); double ms = std::chrono::duration(t1 - t0).count() / ITERS; int n_tiles = (int)((pos_start + n_tok + 63) / 64); printf(" fused+finalize: %.3f ms/call (%.3f ms per K-tile x %d tiles)\n", ms, ms / n_tiles, n_tiles); /* rough DPAS throughput of QK+PV: 2 * 2 * NQ * n_tok * total * HD */ double flops = 4.0 * NQ * n_tok * (double)(pos_start + n_tok) * HD; printf(" effective QK+PV throughput: %.2f TFLOPs\n", flops / (ms * 1e-3) / 1e12); fa_split_free(q, ws); sycl::free(d_kf, q); sycl::free(d_vf, q); sycl::free(d_q, q); sycl::free(kvk, q); sycl::free(kvv, q); sycl::free(out, q); return 0; }