Something went wrong. Try again.
A configuration for qwen 3.6-27B that runs okay-ish on my Radeon system
Something went wrong. Try again.
5.7 kB · 147 lines · Created 3mo ago
YAML
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147services: llama-server: # ROCm backend for AMD RX 6950 XT (16 GB). System: 32 GB RAM, Ryzen 7900X 12C/24T. # Qwen3.6-27B dense (64 layers, 5120 hidden, native 262K context, trained with MTP). # FINAL CONFIG (2026-06-17): max agentic context on 16 GB with all layers GPU + MTP. image: ghcr.io/ggml-org/llama.cpp:server-rocm container_name: llama-server restart: unless-stopped command: - "--host" - "0.0.0.0" - "--port" - "8080" # ── Model ────────────────────────────────────────────── # MTP variant required for native speculative decoding. # UD-IQ3_XXS (~11.4 GB on disk) is the largest quant that still leaves room # for a usable KV cache. Tested alternatives: # - UD-Q4_K_XL / Q4_K_S: too large for all-GPU + any context # - IQ4_XS (~14.6 GB): leaves only ~6K tokens of context # - UD-IQ2_M (~10.3 GB): more headroom but visibly worse quality - "-hf" - "unsloth/Qwen3.6-27B-MTP-GGUF:UD-IQ3_XXS" # Alias so existing benchmark scripts find the model - "--alias" - "qwen3.6-27b" # Text-only agentic coding; skip vision projector to save VRAM - "--no-mmproj" # ── GPU offload ─────────────────────────────────────── - "--split-mode" - "layer" - "--n-gpu-layers" - "99" # ── Context window ──────────────────────────────────── # 131072 is the max ctx-size that fits with q4_0 KV. In practice ~128K # prompt tokens are usable; leave ~2.5K tokens for generation/tool output. - "--ctx-size" - "131072" - "--predict" - "16384" # ── Threads ─────────────────────────────────────────── - "--threads" - "8" # ── Batch / parallel ────────────────────────────────── # parallel MUST be 1 with MTP. - "--parallel" - "1" - "--batch-size" - "512" # ── Flash attention ────────────────────────────────── - "--flash-attn" - "on" # ── KV cache ────────────────────────────────────────── # q4_0 KV is required to fit >64K context. Stable on ROCm in these tests # up to 131K ctx-size. q8_0 KV is limited to ~24-32K actual tokens. - "--cache-type-k" - "q4_0" - "--cache-type-v" - "q4_0" # ── Prompt cache ────────────────────────────────────── - "--cache-ram" - "2048" # ── Context checkpointing ───────────────────────────── - "--ctx-checkpoints" - "4" - "--checkpoint-min-step" - "8192" # ── MTP (Multi-Token Prediction) ───────────────────── # Native MTP heads in the same GGUF. n_max=2 gives best acceptance/speed tradeoff # (observed 70-81% acceptance); n_max=3 drops acceptance and can hurt throughput. - "--spec-type" - "draft-mtp" - "--spec-draft-n-max" - "2" # ── Thinking mode ──────────────────────────────────── # Disabled by default for agentic/tool-coding workflows. Qwen3.6's # default thinking mode consumes tokens without producing final output # on short-generation tasks. Re-enable per-request via: # "chat_template_kwargs": {"enable_thinking": true} - "--chat-template-kwargs" - '{"enable_thinking": false}' # Keep a small reasoning budget so clients that re-enable thinking # don't spiral indefinitely. - "--reasoning-budget" - "4096" - "--reasoning-budget-message" - "...enough. Need to give the final output now." # ── Templates, tools, sampling ──────────────────────── - "--jinja" - "--fit" - "off" - "--no-warmup" - "--metrics" # Qwen3.6 precise coding recommendation - "--temp" - "0.6" - "--top-p" - "0.95" - "--top-k" - "20" - "--min-p" - "0.0" - "--presence-penalty" - "0.0" - "--repeat-penalty" - "1.0" - "--tools" - "all" ports: - "8080:8080" volumes: - ./:/data - llama-models:/models - llama-cache:/root/.cache environment: - HOME=/root - HIP_VISIBLE_DEVICES=0 - HSA_OVERRIDE_GFX_VERSION=10.3.0 devices: - /dev/dri:/dev/dri - /dev/kfd:/dev/kfd networks: - llama-net
prometheus: image: prom/prometheus:latest container_name: prometheus restart: unless-stopped command: - "--config.file=/etc/prometheus/prometheus.yml" - "--storage.tsdb.path=/prometheus" - "--web.enable-lifecycle" volumes: - ./prometheus.yml:/etc/prometheus/prometheus.yml:ro - prometheus-data:/prometheus ports: - "9090:9090" networks: - llama-net
networks: llama-net: driver: bridge
volumes: llama-models: llama-cache: prometheus-data: