Something went wrong. Try again.
Collection of Skills for harnesses like Crush or Claude Code (but Claude Code bad, don't use it)
agent-skill llm
Something went wrong. Try again.
6.3 kB · 169 lines
Python
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170"""Verify every humanize rule and its replacement policy."""
import jsonimport reimport shutilimport subprocessimport tempfilefrom pathlib import Path
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]SKILL_DIRECTORY = REPOSITORY_ROOT / "skills" / "humanize"RULE_DIRECTORY = Path(__file__).resolve().parent / "weirpack"PACK_FILE = SKILL_DIRECTORY / "assets" / "humanize.weirpack"TEST_MARKER = "# test-lint "
def main() -> None: rule_files = sorted(RULE_DIRECTORY.glob("*.weir"), key=lambda path: path.name) if not rule_files: raise FileNotFoundError(f"no Weir rules found in {RULE_DIRECTORY}") if not PACK_FILE.is_file(): raise FileNotFoundError(f"missing built Weirpack: {PACK_FILE}")
assert all(path.stem for path in rule_files) assert len({path.stem for path in rule_files}) == len(rule_files)
test_cases: dict[str, list[str]] = {} rules_with_suggestions: set[str] = set() for rule_file in rule_files: rule_source = rule_file.read_text() has_empty_suggestions = bool( re.search(r"^let becomes \[\]$", rule_source, re.MULTILINE) ) has_replacement_suggestions = bool( re.search(r'^let becomes (?:"|\[\")', rule_source, re.MULTILINE) ) if has_empty_suggestions == has_replacement_suggestions: raise ValueError(f"expected exactly one replacement policy in {rule_file}") if has_replacement_suggestions: rules_with_suggestions.add(rule_file.stem)
marked_lines = [ line.removeprefix(TEST_MARKER) for line in rule_source.splitlines() if line.startswith(TEST_MARKER) ] if not marked_lines: raise ValueError( f"expected at least one {TEST_MARKER.strip()} in {rule_file}" )
rule_test_cases: list[str] = [] for marked_line in marked_lines: test_text = json.loads(marked_line) if not isinstance(test_text, str) or not test_text.strip(): raise ValueError(f"invalid positive test in {rule_file}") rule_test_cases.append(test_text) test_cases[rule_file.stem] = rule_test_cases
if shutil.which("harper-cli"): command = ["harper-cli"] elif shutil.which("mise"): command = ["mise", "exec", "harper-cli@latest", "--", "harper-cli"] else: raise FileNotFoundError("harper-cli is unavailable directly and through mise")
weir_test_failures: list[str] = [] for rule_file in rule_files: rule_test_result = subprocess.run( [*command, "test", str(rule_file)], check=False, capture_output=True, text=True, ) if rule_test_result.returncode != 0: weir_test_failures.append( f"{rule_file.stem}: {rule_test_result.stderr.strip()}" ) if weir_test_failures: raise RuntimeError("\n".join(weir_test_failures))
with tempfile.TemporaryDirectory(prefix="humanize-weirpack-") as directory: test_files: list[Path] = [] for rule_name, rule_test_cases in test_cases.items(): for index, test_text in enumerate(rule_test_cases, start=1): test_file = Path(directory) / f"{rule_name}--{index}.md" test_file.write_text(f"{test_text}\n") test_files.append(test_file)
result = subprocess.run( [ *command, "lint", "--weirpacks", str(PACK_FILE), "--only", ",".join(test_cases), "--format", "json", "--quiet", *(str(path) for path in test_files), ], check=False, capture_output=True, text=True, )
if result.returncode != 1: raise RuntimeError( f"expected harper-cli to exit 1 for findings, got {result.returncode}:\n" f"{result.stderr}" )
lint_results = json.loads(result.stdout) if not isinstance(lint_results, list): raise TypeError("harper-cli returned a non-list JSON payload")
failures: list[str] = [] diagnostic_count = 0 for lint_result in lint_results: file_name = lint_result["file"] expected_rule = Path(file_name).stem.split("--", maxsplit=1)[0] diagnostics = lint_result["lints"] diagnostic_count += len(diagnostics) emitted_rules = {diagnostic["rule"] for diagnostic in diagnostics} diagnostic_spans: dict[tuple[int, int], str] = {}
for diagnostic in diagnostics: span = diagnostic["span"] span_key = (span["char_start"], span["char_end"]) previous_rule = diagnostic_spans.get(span_key) if previous_rule is not None: failures.append( f"{file_name} has duplicate diagnostics for {span_key}: " f"{previous_rule} and {diagnostic['rule']}" ) else: diagnostic_spans[span_key] = diagnostic["rule"]
if expected_rule not in emitted_rules: failures.append(f"{expected_rule} did not emit for {file_name}") expected_diagnostics = [ diagnostic for diagnostic in diagnostics if diagnostic["rule"] == expected_rule ] for diagnostic in expected_diagnostics: has_suggestions = bool(diagnostic["suggestions"]) expects_suggestions = expected_rule in rules_with_suggestions if has_suggestions != expects_suggestions: failures.append( f"{expected_rule} replacement policy mismatch in {file_name}" )
if failures: raise RuntimeError("\n".join(failures))
positive_case_count = sum(len(cases) for cases in test_cases.values()) assert len(lint_results) == positive_case_count assert diagnostic_count >= positive_case_count print( f"Verified {len(test_cases)} rules across {positive_case_count} positive " f"cases with {diagnostic_count} diagnostics; " f"{len(rules_with_suggestions)} rules provide suggestions." )
if __name__ == "__main__": main()