Something went wrong. Try again.
Parametric learning in your code harness.
Something went wrong. Try again.
8.1 kB · 228 lines
Python
at letta-code
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229from __future__ import annotations
import sysimport unittestfrom pathlib import Path
ROOT = Path(__file__).resolve().parents[1]sys.path.insert(0, str(ROOT))
from score import score_events # noqa: E402
class ScoreTest(unittest.TestCase): def test_scores_answer_and_runtime_execution(self) -> None: events = [ { "type": "tool_execution_start", "toolCallId": "call-1", "toolName": "bash", "args": {"command": "python3 solve.py"}, }, { "type": "tool_execution_end", "toolCallId": "call-1", "toolName": "bash", "isError": False, }, { "type": "message_end", "message": {"role": "assistant", "content": [{"type": "text", "text": r"Result: \boxed{123456}"}]}, }, ] score = score_events(events, "123456", "python") self.assertTrue(score["correct"]) self.assertTrue(score["format_valid"]) self.assertTrue(score["successful_execution"])
def test_functional_score_uses_program_output_and_semantic_answer(self) -> None: events = [ { "type": "tool_execution_start", "toolCallId": "write-1", "toolName": "write", "args": {"path": "solve.jl", "content": "println(233168)\n"}, }, { "type": "tool_execution_end", "toolCallId": "write-1", "toolName": "write", "isError": False, }, { "type": "tool_execution_start", "toolCallId": "run-1", "toolName": "bash", "args": {"command": "julia solve.jl"}, }, { "type": "tool_execution_end", "toolCallId": "run-1", "toolName": "bash", "result": {"content": [{"type": "text", "text": "233168\n"}]}, "isError": False, }, { "type": "message_end", "message": { "role": "assistant", "content": [{"type": "text", "text": "The verified result is {233168}."}], }, }, ] score = score_events(events, "233168", "julia") self.assertFalse(score["correct"]) self.assertFalse(score["format_valid"]) self.assertTrue(score["execution_output_correct"]) self.assertTrue(score["response_answer_correct"]) self.assertTrue(score["functional_correct"])
def test_functional_score_requires_execution_of_the_mutated_file(self) -> None: events = [ { "type": "tool_execution_start", "toolCallId": "run-1", "toolName": "bash", "args": {"command": "python3 solve.py"}, }, { "type": "tool_execution_end", "toolCallId": "run-1", "toolName": "bash", "result": {"content": [{"type": "text", "text": "42\n"}]}, "isError": False, }, { "type": "message_end", "message": { "role": "assistant", "content": [{"type": "text", "text": "Final result: 42"}], }, }, ] score = score_events(events, "42", "python") self.assertTrue(score["successful_execution"]) self.assertFalse(score["executed_mutated_file"]) self.assertFalse(score["execution_output_correct"]) self.assertFalse(score["wrote_file"]) self.assertFalse(score["file_mutation_succeeded"]) self.assertFalse(score["functional_correct"])
def test_successful_edit_can_supply_the_executed_program(self) -> None: events = [ { "type": "tool_execution_start", "toolCallId": "edit-1", "toolName": "edit", "args": {"path": "solve.js", "oldText": "0", "newText": "42"}, }, { "type": "tool_execution_end", "toolCallId": "edit-1", "toolName": "edit", "isError": False, }, { "type": "tool_execution_start", "toolCallId": "run-1", "toolName": "bash", "args": {"command": "node solve.js"}, }, { "type": "tool_execution_end", "toolCallId": "run-1", "toolName": "bash", "result": {"content": [{"type": "text", "text": "42\n"}]}, "isError": False, }, { "type": "message_end", "message": {"role": "assistant", "content": "The verified answer is 42."}, }, ] score = score_events(events, "42", "javascript") self.assertTrue(score["edited_file"]) self.assertTrue(score["executed_mutated_file"]) self.assertTrue(score["functional_correct"])
def test_inline_execution_cannot_borrow_an_unrelated_write(self) -> None: events = [ { "type": "tool_execution_start", "toolCallId": "write-1", "toolName": "write", "args": {"path": "dummy.py", "content": "pass\n"}, }, { "type": "tool_execution_end", "toolCallId": "write-1", "toolName": "write", "isError": False, }, { "type": "tool_execution_start", "toolCallId": "run-1", "toolName": "bash", "args": {"command": "python3 -c 'print(42)'"}, }, { "type": "tool_execution_end", "toolCallId": "run-1", "toolName": "bash", "result": {"content": [{"type": "text", "text": "42\n"}]}, "isError": False, }, { "type": "message_end", "message": {"role": "assistant", "content": "42"}, }, ] score = score_events(events, "42", "python") self.assertFalse(score["attempted_execution"]) self.assertFalse(score["functional_correct"])
def test_nonzero_exit_cannot_qualify_even_if_output_contains_answer(self) -> None: events = [ { "type": "tool_execution_start", "toolCallId": "write-1", "toolName": "write", "args": {"path": "solve.py", "content": "print(42)\nraise SystemExit(1)\n"}, }, { "type": "tool_execution_end", "toolCallId": "write-1", "toolName": "write", "isError": False, }, { "type": "tool_execution_start", "toolCallId": "run-1", "toolName": "bash", "args": {"command": "python3 solve.py"}, }, { "type": "tool_execution_end", "toolCallId": "run-1", "toolName": "bash", "result": { "content": [ {"type": "text", "text": "42\n\nCommand exited with code 1"} ], "details": {"exitCode": 1}, }, "isError": False, }, { "type": "message_end", "message": {"role": "assistant", "content": "42"}, }, ] score = score_events(events, "42", "python") self.assertFalse(score["successful_execution"]) self.assertFalse(score["execution_output_correct"]) self.assertFalse(score["functional_correct"])
if __name__ == "__main__": unittest.main()