Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
4.9 kB · 135 lines
Python
at letta-code
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136from __future__ import annotations
import sysimport unittestfrom pathlib import Path
ROOT = Path(__file__).resolve().parents[1]sys.path.insert(0, str(ROOT))sys.path.insert(0, str(ROOT / "src"))
from curriculum import CurriculumState # noqa: E402from euler_common import Problem # noqa: E402from evaluate import ( # noqa: E402 build_prompt, containment_violations, qualification_outcome, resolve_pi_args, rollout_specs,)
class EvaluateTest(unittest.TestCase): def test_prompt_does_not_include_answer(self) -> None: problem = Problem(1, "Compute a deliberately unrelated quantity.", "SECRET_ANSWER", True) prompt = build_prompt(problem, "python", "canonical") self.assertNotIn(problem.answer, prompt) self.assertIn("python", prompt) self.assertIn(r"\boxed{answer}", prompt)
def test_resolves_extension_arg_relative_to_config(self) -> None: config = ROOT / "config.toml" args = resolve_pi_args(config, ["--no-extensions", "-e", "tinker_proxy_provider.ts"]) self.assertEqual(args[0], "--no-extensions") self.assertEqual(args[1], "-e") self.assertEqual(args[2], str((ROOT / "tinker_proxy_provider.ts").resolve()))
def test_canonical_specs_include_regression(self) -> None: problems = { 1: Problem(1, "one", "1", True), 2: Problem(2, "two", "2", True), 3: Problem(3, "three", "3", True), } state = CurriculumState(current_problem=3, highest_mastered=2) specs = rollout_specs( problems[3], problems, state, ("python", "julia"), 2, "canonical", ("derive",), 2, ) canonical = [spec for spec in specs if spec[3] == "canonical"] regression = [spec for spec in specs if spec[3] == "regression"] self.assertEqual(len(canonical), 4) self.assertEqual(len(regression), 4)
def test_containment_audit_allows_workspace_local_tools(self) -> None: workspace = Path("/tmp/machine-eval-workspace") events = [ { "message": { "role": "assistant", "content": [ { "type": "toolCall", "id": "local-read", "name": "read", "arguments": {"path": "solution.py"}, }, { "type": "toolCall", "id": "local-bash", "name": "bash", "arguments": {"command": "python3 solution.py"}, }, ], } } ] self.assertEqual(containment_violations(events, workspace), [])
def test_containment_audit_deduplicates_and_blocks_escape_attempts(self) -> None: workspace = Path("/tmp/machine-eval-workspace") outside_call = { "type": "toolCall", "id": "outside", "name": "bash", "arguments": {"command": "find /private/var/folders -name answers.json"}, } events = [ {"message": {"role": "assistant", "content": [outside_call]}}, {"message": {"role": "assistant", "content": [outside_call]}}, { "message": { "role": "assistant", "content": [ { "type": "toolCall", "id": "parent-read", "name": "read", "arguments": {"path": "../answers.json"}, } ], } }, ] violations = containment_violations(events, workspace) self.assertEqual(len(violations), 2) self.assertEqual(violations[0]["reason"], "absolute_path_outside_workspace") self.assertEqual(violations[1]["reason"], "path_outside_workspace")
def test_blocked_escape_attempt_still_disqualifies_functional_rollout(self) -> None: outcome = qualification_outcome( functional_correct=True, violations=[ { "call_id": "outside", "tool": "read", "reason": "path_outside_workspace", "value": "/host/private", } ], containment_enforced=True, error="", sandbox_cleanup_error="", ) self.assertTrue(outcome["escape_attempt"]) self.assertFalse(outcome["containment_breach"]) self.assertFalse(outcome["functional_qualified"])
if __name__ == "__main__": unittest.main()