Something went wrong. Try again.
fork of gpui
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140#!/usr/bin/env bash## Test docs-suggest on multiple PRs and generate a summary report.## Usage:# script/test-docs-suggest-batch [--limit N] [--output FILE]## This script runs docs-suggest in dry-run mode on recent merged PRs# to validate the context assembly and help tune the prompt.
set -euo pipefail
LIMIT=50OUTPUT="docs-suggest-batch-results.md"SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
while [[ $# -gt 0 ]]; do case $1 in --limit) LIMIT="$2" shift 2 ;; --output) OUTPUT="$2" shift 2 ;; *) echo "Unknown option: $1" exit 1 ;; esacdone
echo "Testing docs-suggest on $LIMIT recent merged PRs..."echo "Output: $OUTPUT"echo ""
# Get list of PRsPRS=$(gh pr list --state merged --limit "$LIMIT" --json number,title --jq '.[] | "\(.number)|\(.title)"')
# Initialize output filecat > "$OUTPUT" << HEADER# docs-suggest Batch Test Results
**Date**: $(date +%Y-%m-%d)**PRs tested**: $LIMIT
## Summary
| PR | Title | Result | Source Files | Notes ||----|-------|--------|--------------|-------|HEADER
# Track statstotal=0has_source=0no_source=0errors=0
while IFS='|' read -r pr_num title; do total=$((total + 1)) echo -n "[$total/$LIMIT] PR #$pr_num: " # Run dry-run and capture output tmpfile=$(mktemp) if "$SCRIPT_DIR/docs-suggest" --pr "$pr_num" --dry-run 2>"$tmpfile.err" >"$tmpfile.out"; then # Check if it found source files if grep -q "No documentation-relevant changes" "$tmpfile.out"; then result="No source changes" no_source=$((no_source + 1)) source_count="0" echo "skipped (no source)" else # Extract source file count from verbose output source_count=$(grep -oE '[0-9]+ source' "$tmpfile.err" 2>/dev/null | grep -oE '[0-9]+' || echo "?") result="Has source changes" has_source=$((has_source + 1)) echo "has $source_count source files" fi notes="" else result="Error" errors=$((errors + 1)) source_count="-" notes=$(head -1 "$tmpfile.err" 2>/dev/null || echo "unknown error") echo "error" fi # Escape title for markdown table title_escaped=$(echo "$title" | sed 's/|/\\|/g' | cut -c1-60) # Add row to table echo "| [#$pr_num](https://github.com/zed-industries/zed/pull/$pr_num) | $title_escaped | $result | $source_count | $notes |" >> "$OUTPUT" rm -f "$tmpfile" "$tmpfile.out" "$tmpfile.err"done <<< "$PRS"
# Add summary statscat >> "$OUTPUT" << STATS
## Statistics
- **Total PRs**: $total- **With source changes**: $has_source ($(( has_source * 100 / total ))%)- **No source changes**: $no_source ($(( no_source * 100 / total ))%)- **Errors**: $errors
## Observations
_Add manual observations here after reviewing results._
## Sample Contexts
STATS
# Add 3 sample contexts from PRs with source changesecho "" >> "$OUTPUT"echo "### Sample 1: PR with source changes" >> "$OUTPUT"echo "" >> "$OUTPUT"
sample_pr=$(gh pr list --state merged --limit 20 --json number --jq '.[].number' | while read pr; do if "$SCRIPT_DIR/docs-suggest" --pr "$pr" --dry-run 2>/dev/null | grep -q "## Code Diff"; then echo "$pr" break fidone)
if [[ -n "$sample_pr" ]]; then echo "PR #$sample_pr:" >> "$OUTPUT" echo "" >> "$OUTPUT" echo '```' >> "$OUTPUT" "$SCRIPT_DIR/docs-suggest" --pr "$sample_pr" --dry-run 2>/dev/null | head -100 >> "$OUTPUT" echo '```' >> "$OUTPUT"fi
echo ""echo "Done! Results written to: $OUTPUT"echo ""echo "Stats: $has_source with source changes, $no_source without, $errors errors"