{ "schema_version": 2, "manifest_ref": "agent-research:harnesslens:toolroute-l2.1-v1", "method": "harnesslens", "benchmark": "toolroute-l2.1", "dataset": "toolroute-l2.1-v1", "seeds": [ 42, 43, 44 ], "metrics": { "answer_accuracy": 0.8194444444444444, "plan_exact_match": 0.8194444444444444, "plan_step_f1": 0.8371697746697747, "joint_success": 0.8194444444444444, "recovery_success_rate": 0.6666666666666666, "invalid_tool_rate": 0.03238086060181803, "irreversible_error_rate": 0.0, "average_tool_calls": 5.576388888888889, "average_cost": 5.041944444444445 }, "aggregate_metrics": { "answer_accuracy": { "mean": 0.8194444444444444, "std": 0.012028130608117225, "ci95_radius": 0.013611111111111136 }, "plan_exact_match": { "mean": 0.8194444444444444, "std": 0.012028130608117225, "ci95_radius": 0.013611111111111136 }, "plan_step_f1": { "mean": 0.8371697746697747, "std": 0.0034726308480619974, "ci95_radius": 0.003929651735652594 }, "joint_success": { "mean": 0.8194444444444444, "std": 0.012028130608117225, "ci95_radius": 0.013611111111111136 }, "recovery_success_rate": { "mean": 0.6666666666666666, "std": 0.0, "ci95_radius": 0.0 }, "invalid_tool_rate": { "mean": 0.03238086060181803, "std": 0.002199250959228673, "ci95_radius": 0.002488686741319385 }, "irreversible_error_rate": { "mean": 0.0, "std": 0.0, "ci95_radius": 0.0 }, "average_tool_calls": { "mean": 5.576388888888889, "std": 0.031823442326082155, "ci95_radius": 0.036011615067267966 }, "average_cost": { "mean": 5.041944444444445, "std": 0.030000964490668608, "ci95_radius": 0.033949287252286416 } }, "validation_metrics": { "answer_accuracy": 0.7777777777777778, "plan_exact_match": 0.7777777777777778, "plan_step_f1": 0.822881593714927, "joint_success": 0.7777777777777778, "recovery_success_rate": 0.6666666666666666, "invalid_tool_rate": 0.04623483655214955, "irreversible_error_rate": 0.0, "average_tool_calls": 4.805555555555555, "average_cost": 4.472222222222222 }, "validation_aggregate_metrics": { "answer_accuracy": { "mean": 0.7777777777777778, "std": 0.01202813060811716, "ci95_radius": 0.013611111111111063 }, "plan_exact_match": { "mean": 0.7777777777777778, "std": 0.01202813060811716, "ci95_radius": 0.013611111111111063 }, "plan_step_f1": { "mean": 0.822881593714927, "std": 0.0029743331305214006, "ci95_radius": 0.00336577478578966 }, "joint_success": { "mean": 0.7777777777777778, "std": 0.01202813060811716, "ci95_radius": 0.013611111111111063 }, "recovery_success_rate": { "mean": 0.6666666666666666, "std": 0.0, "ci95_radius": 0.0 }, "invalid_tool_rate": { "mean": 0.04623483655214955, "std": 0.0022440621456629125, "ci95_radius": 0.0025393953723983945 }, "irreversible_error_rate": { "mean": 0.0, "std": 0.0, "ci95_radius": 0.0 }, "average_tool_calls": { "mean": 4.805555555555555, "std": 0.04336804165554467, "ci95_radius": 0.04907555902714901 }, "average_cost": { "mean": 4.472222222222222, "std": 0.03534633539599183, "ci95_radius": 0.03999814386125566 } }, "seed_results": [ { "seed": 42, "training": { "split": "train", "metrics": { "answer_accuracy": 0.7916666666666666, "plan_exact_match": 0.7916666666666666, "plan_step_f1": 0.8177910052910052, "joint_success": 0.7916666666666666, "recovery_success_rate": 0.6666666666666666, "invalid_tool_rate": 0.05263157894736842, "irreversible_error_rate": 0.0, "average_tool_calls": 3.9583333333333335, "average_cost": 3.7108333333333334 }, "axis_metrics": { "clean": 1, "transient": 1, "ambiguous": 0.75, "fallback": 1, "irreversible": 1, "combined": 0 }, "telemetry": { "retries": 12, "reflections": 9, "skill_reuses": 0, "verifications": 24, "compressions": 0, "memory_writes": 0 }, "trace": [] }, "validation": { "split": "validation", "metrics": { "answer_accuracy": 0.7708333333333334, "plan_exact_match": 0.7708333333333334, "plan_step_f1": 0.8205492424242424, "joint_success": 0.7708333333333334, "recovery_success_rate": 0.6666666666666666, "invalid_tool_rate": 0.04782608695652174, "irreversible_error_rate": 0.0, "average_tool_calls": 4.791666666666667, "average_cost": 4.454583333333334 }, "axis_metrics": { "clean": 1, "transient": 1, "ambiguous": 0.625, "fallback": 1, "irreversible": 1, "combined": 0 }, "telemetry": { "retries": 24, "reflections": 19, "skill_reuses": 9, "verifications": 30, "compressions": 0, "memory_writes": 0 }, "trace": [ { "task_id": "toolroute-l21-validation-42-0000", "split": "validation", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 4.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 3.8400000000000003, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calculator", "resolver", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-42-0001", "split": "validation", "axis": "ambiguous", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.7272727272727273, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 5.859999999999999, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "weather", "calendar", "calculator", "resolver", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-42-0002", "split": "validation", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 4.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 3.68, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "catalog", "weather", "calculator", "resolver" ] }, { "task_id": "toolroute-l21-validation-42-0003", "split": "validation", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.4, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "database", "catalog", "weather", "calculator", "resolver" ] }, { "task_id": "toolroute-l21-validation-42-0004", "split": "validation", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.8888888888888888, "has_failure": true, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.760000000000001, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calculator", "resolver", "resolver", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-42-0005", "split": "validation", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.5600000000000005, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "weather", "calculator", "search", "resolver", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-42-0006", "split": "validation", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 4.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 3.68, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "catalog", "weather", "calculator", "resolver" ] }, { "task_id": "toolroute-l21-validation-42-0007", "split": "validation", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.24, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "database", "catalog", "catalog", "weather", "calculator", "resolver" ] }, { "task_id": "toolroute-l21-validation-42-0008", "split": "validation", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.8888888888888888, "has_failure": true, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.680000000000001, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calculator", "resolver", "search", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-42-0009", "split": "validation", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2222222222222222, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 3.9200000000000004, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "weather", "calculator", "search", "calendar" ] }, { "task_id": "toolroute-l21-validation-42-0010", "split": "validation", "axis": "ambiguous", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 4.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 3.68, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "catalog", "weather", "calculator", "resolver" ] }, { "task_id": "toolroute-l21-validation-42-0011", "split": "validation", "axis": "ambiguous", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.7272727272727273, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 5.46, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "database", "maps", "catalog", "weather", "calculator", "resolver" ] } ] }, "test": { "split": "test", "metrics": { "answer_accuracy": 0.8125, "plan_exact_match": 0.8125, "plan_step_f1": 0.8346412846412846, "joint_success": 0.8125, "recovery_success_rate": 0.6666666666666666, "invalid_tool_rate": 0.03383458646616541, "irreversible_error_rate": 0.0, "average_tool_calls": 5.541666666666667, "average_cost": 5.012083333333334 }, "axis_metrics": { "clean": 1, "transient": 1, "ambiguous": 0.875, "fallback": 1, "irreversible": 1, "combined": 0 }, "telemetry": { "retries": 24, "reflections": 17, "skill_reuses": 0, "verifications": 48, "compressions": 0, "memory_writes": 0 }, "trace": [ { "task_id": "toolroute-l21-test-42-0000", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.16, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "search", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-42-0001", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.5200000000000005, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-42-0002", "split": "test", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.4, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-42-0003", "split": "test", "axis": "ambiguous", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.7692307692307693, "has_failure": false, "tool_calls": 7.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 6.58, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "files", "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-42-0004", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.48, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-42-0005", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9230769230769231, "has_failure": true, "tool_calls": 7.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 6.2, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "search", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-42-0006", "split": "test", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.4, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-42-0007", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9230769230769231, "has_failure": true, "tool_calls": 7.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 6.28, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "database", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-42-0008", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.48, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-42-0009", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9230769230769231, "has_failure": true, "tool_calls": 7.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 6.2, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "search", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-42-0010", "split": "test", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.32, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-42-0011", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.5200000000000005, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "validator", "maps", "database", "catalog" ] } ] }, "metrics": { "answer_accuracy": 0.8125, "plan_exact_match": 0.8125, "plan_step_f1": 0.8346412846412846, "joint_success": 0.8125, "recovery_success_rate": 0.6666666666666666, "invalid_tool_rate": 0.03383458646616541, "irreversible_error_rate": 0.0, "average_tool_calls": 5.541666666666667, "average_cost": 5.012083333333334 }, "axis_metrics": { "clean": 1, "transient": 1, "ambiguous": 0.875, "fallback": 1, "irreversible": 1, "combined": 0 }, "telemetry": { "retries": 24, "reflections": 17, "skill_reuses": 0, "verifications": 48, "compressions": 0, "memory_writes": 0 }, "trace": [ { "task_id": "toolroute-l21-test-42-0000", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.16, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "search", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-42-0001", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.5200000000000005, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-42-0002", "split": "test", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.4, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-42-0003", "split": "test", "axis": "ambiguous", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.7692307692307693, "has_failure": false, "tool_calls": 7.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 6.58, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "files", "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-42-0004", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.48, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-42-0005", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9230769230769231, "has_failure": true, "tool_calls": 7.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 6.2, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "search", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-42-0006", "split": "test", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.4, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-42-0007", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9230769230769231, "has_failure": true, "tool_calls": 7.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 6.28, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "database", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-42-0008", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.48, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-42-0009", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9230769230769231, "has_failure": true, "tool_calls": 7.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 6.2, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "search", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-42-0010", "split": "test", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.32, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-42-0011", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.5200000000000005, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "validator", "maps", "database", "catalog" ] } ] }, { "seed": 43, "training": { "split": "train", "metrics": { "answer_accuracy": 0.75, "plan_exact_match": 0.75, "plan_step_f1": 0.8039021164021164, "joint_success": 0.75, "recovery_success_rate": 0.6666666666666666, "invalid_tool_rate": 0.0625, "irreversible_error_rate": 0.0, "average_tool_calls": 4.0, "average_cost": 3.7950000000000004 }, "axis_metrics": { "clean": 1, "transient": 1, "ambiguous": 0.5, "fallback": 1, "irreversible": 1, "combined": 0 }, "telemetry": { "retries": 12, "reflections": 10, "skill_reuses": 0, "verifications": 24, "compressions": 0, "memory_writes": 0 }, "trace": [] }, "validation": { "split": "validation", "metrics": { "answer_accuracy": 0.7708333333333334, "plan_exact_match": 0.7708333333333334, "plan_step_f1": 0.821864478114478, "joint_success": 0.7708333333333334, "recovery_success_rate": 0.6666666666666666, "invalid_tool_rate": 0.04721030042918455, "irreversible_error_rate": 0.0, "average_tool_calls": 4.854166666666667, "average_cost": 4.512916666666667 }, "axis_metrics": { "clean": 1, "transient": 1, "ambiguous": 0.625, "fallback": 1, "irreversible": 1, "combined": 0 }, "telemetry": { "retries": 24, "reflections": 19, "skill_reuses": 5, "verifications": 38, "compressions": 0, "memory_writes": 0 }, "trace": [ { "task_id": "toolroute-l21-validation-43-0000", "split": "validation", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.8888888888888888, "has_failure": true, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.680000000000001, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calculator", "resolver", "search", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-43-0001", "split": "validation", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.880000000000001, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "weather", "calculator", "resolver", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-43-0002", "split": "validation", "axis": "ambiguous", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 4.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 3.68, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "catalog", "weather", "calculator", "resolver" ] }, { "task_id": "toolroute-l21-validation-43-0003", "split": "validation", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2222222222222222, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 3.68, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "database", "maps", "catalog", "search" ] }, { "task_id": "toolroute-l21-validation-43-0004", "split": "validation", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.25, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 3.92, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calculator", "database", "resolver", "search" ] }, { "task_id": "toolroute-l21-validation-43-0005", "split": "validation", "axis": "ambiguous", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.7272727272727273, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 5.859999999999999, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "weather", "calendar", "calculator", "resolver", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-43-0006", "split": "validation", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.25, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 3.92, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "catalog", "weather", "search", "calendar" ] }, { "task_id": "toolroute-l21-validation-43-0007", "split": "validation", "axis": "ambiguous", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.7272727272727273, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 5.46, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "database", "maps", "catalog", "weather", "calculator", "resolver" ] }, { "task_id": "toolroute-l21-validation-43-0008", "split": "validation", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 4.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.0, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calculator", "resolver", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-43-0009", "split": "validation", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.880000000000001, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "weather", "calculator", "resolver", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-43-0010", "split": "validation", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.25, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 3.92, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "catalog", "weather", "search", "calendar" ] }, { "task_id": "toolroute-l21-validation-43-0011", "split": "validation", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.24, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "database", "catalog", "catalog", "weather", "calculator", "resolver" ] } ] }, "test": { "split": "test", "metrics": { "answer_accuracy": 0.8333333333333334, "plan_exact_match": 0.8333333333333334, "plan_step_f1": 0.8411292411292411, "joint_success": 0.8333333333333334, "recovery_success_rate": 0.6666666666666666, "invalid_tool_rate": 0.029850746268656716, "irreversible_error_rate": 0.0, "average_tool_calls": 5.583333333333333, "average_cost": 5.041666666666667 }, "axis_metrics": { "clean": 1, "transient": 1, "ambiguous": 1, "fallback": 1, "irreversible": 1, "combined": 0 }, "telemetry": { "retries": 24, "reflections": 16, "skill_reuses": 0, "verifications": 48, "compressions": 0, "memory_writes": 0 }, "trace": [ { "task_id": "toolroute-l21-test-43-0000", "split": "test", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2222222222222222, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 3.8400000000000003, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "search", "calculator" ] }, { "task_id": "toolroute-l21-test-43-0001", "split": "test", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 4.0, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "search", "calculator" ] }, { "task_id": "toolroute-l21-test-43-0002", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.48, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-43-0003", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.5200000000000005, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-43-0004", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.16, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "search", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-43-0005", "split": "test", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 4.0, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "search", "calculator" ] }, { "task_id": "toolroute-l21-test-43-0006", "split": "test", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2222222222222222, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 4.08, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "search", "calendar" ] }, { "task_id": "toolroute-l21-test-43-0007", "split": "test", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9230769230769231, "has_failure": true, "tool_calls": 7.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 6.2, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-43-0008", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.16, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "search", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-43-0009", "split": "test", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9230769230769231, "has_failure": true, "tool_calls": 7.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 6.2, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "maps", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-43-0010", "split": "test", "axis": "ambiguous", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.48, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-43-0011", "split": "test", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.36, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "validator", "maps", "database", "catalog" ] } ] }, "metrics": { "answer_accuracy": 0.8333333333333334, "plan_exact_match": 0.8333333333333334, "plan_step_f1": 0.8411292411292411, "joint_success": 0.8333333333333334, "recovery_success_rate": 0.6666666666666666, "invalid_tool_rate": 0.029850746268656716, "irreversible_error_rate": 0.0, "average_tool_calls": 5.583333333333333, "average_cost": 5.041666666666667 }, "axis_metrics": { "clean": 1, "transient": 1, "ambiguous": 1, "fallback": 1, "irreversible": 1, "combined": 0 }, "telemetry": { "retries": 24, "reflections": 16, "skill_reuses": 0, "verifications": 48, "compressions": 0, "memory_writes": 0 }, "trace": [ { "task_id": "toolroute-l21-test-43-0000", "split": "test", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2222222222222222, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 3.8400000000000003, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "search", "calculator" ] }, { "task_id": "toolroute-l21-test-43-0001", "split": "test", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 4.0, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "search", "calculator" ] }, { "task_id": "toolroute-l21-test-43-0002", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.48, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-43-0003", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.5200000000000005, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-43-0004", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.16, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "search", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-43-0005", "split": "test", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 4.0, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "search", "calculator" ] }, { "task_id": "toolroute-l21-test-43-0006", "split": "test", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2222222222222222, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 4.08, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "search", "calendar" ] }, { "task_id": "toolroute-l21-test-43-0007", "split": "test", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9230769230769231, "has_failure": true, "tool_calls": 7.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 6.2, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-43-0008", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.16, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "search", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-43-0009", "split": "test", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9230769230769231, "has_failure": true, "tool_calls": 7.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 6.2, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "maps", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-43-0010", "split": "test", "axis": "ambiguous", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.48, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-43-0011", "split": "test", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.36, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "validator", "maps", "database", "catalog" ] } ] }, { "seed": 44, "training": { "split": "train", "metrics": { "answer_accuracy": 0.7916666666666666, "plan_exact_match": 0.7916666666666666, "plan_step_f1": 0.8217592592592592, "joint_success": 0.7916666666666666, "recovery_success_rate": 0.6666666666666666, "invalid_tool_rate": 0.05263157894736842, "irreversible_error_rate": 0.0, "average_tool_calls": 3.9583333333333335, "average_cost": 3.734166666666667 }, "axis_metrics": { "clean": 1, "transient": 1, "ambiguous": 0.75, "fallback": 1, "irreversible": 1, "combined": 0 }, "telemetry": { "retries": 12, "reflections": 9, "skill_reuses": 0, "verifications": 24, "compressions": 0, "memory_writes": 0 }, "trace": [] }, "validation": { "split": "validation", "metrics": { "answer_accuracy": 0.7916666666666666, "plan_exact_match": 0.7916666666666666, "plan_step_f1": 0.8262310606060606, "joint_success": 0.7916666666666666, "recovery_success_rate": 0.6666666666666666, "invalid_tool_rate": 0.043668122270742356, "irreversible_error_rate": 0.0, "average_tool_calls": 4.770833333333333, "average_cost": 4.449166666666667 }, "axis_metrics": { "clean": 1, "transient": 1, "ambiguous": 0.75, "fallback": 1, "irreversible": 1, "combined": 0 }, "telemetry": { "retries": 24, "reflections": 18, "skill_reuses": 3, "verifications": 42, "compressions": 0, "memory_writes": 0 }, "trace": [ { "task_id": "toolroute-l21-validation-44-0000", "split": "validation", "axis": "ambiguous", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.6666666666666666, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 4.98, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calculator", "database", "resolver", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-44-0001", "split": "validation", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.720000000000001, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "weather", "calculator", "resolver", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-44-0002", "split": "validation", "axis": "ambiguous", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 4.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 3.68, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "catalog", "weather", "calculator", "resolver" ] }, { "task_id": "toolroute-l21-validation-44-0003", "split": "validation", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.5600000000000005, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "database", "catalog", "weather", "calculator", "resolver" ] }, { "task_id": "toolroute-l21-validation-44-0004", "split": "validation", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.8888888888888888, "has_failure": true, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.680000000000001, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calculator", "resolver", "search", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-44-0005", "split": "validation", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2222222222222222, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 4.08, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "weather", "calculator", "search", "calendar" ] }, { "task_id": "toolroute-l21-validation-44-0006", "split": "validation", "axis": "ambiguous", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 4.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 3.68, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "catalog", "weather", "calculator", "resolver" ] }, { "task_id": "toolroute-l21-validation-44-0007", "split": "validation", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2222222222222222, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 3.68, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "database", "maps", "catalog", "search" ] }, { "task_id": "toolroute-l21-validation-44-0008", "split": "validation", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.8888888888888888, "has_failure": true, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.680000000000001, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calculator", "resolver", "search", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-44-0009", "split": "validation", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.720000000000001, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "weather", "calculator", "calculator", "resolver", "files", "calendar" ] }, { "task_id": "toolroute-l21-validation-44-0010", "split": "validation", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.8888888888888888, "has_failure": true, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.36, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "catalog", "weather", "search", "calculator", "resolver" ] }, { "task_id": "toolroute-l21-validation-44-0011", "split": "validation", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2222222222222222, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 3.68, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "database", "maps", "catalog", "search" ] } ] }, "test": { "split": "test", "metrics": { "answer_accuracy": 0.8125, "plan_exact_match": 0.8125, "plan_step_f1": 0.8357387982387983, "joint_success": 0.8125, "recovery_success_rate": 0.6666666666666666, "invalid_tool_rate": 0.03345724907063197, "irreversible_error_rate": 0.0, "average_tool_calls": 5.604166666666667, "average_cost": 5.0720833333333335 }, "axis_metrics": { "clean": 1, "transient": 1, "ambiguous": 0.875, "fallback": 1, "irreversible": 1, "combined": 0 }, "telemetry": { "retries": 24, "reflections": 17, "skill_reuses": 2, "verifications": 44, "compressions": 0, "memory_writes": 0 }, "trace": [ { "task_id": "toolroute-l21-test-44-0000", "split": "test", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.32, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-44-0001", "split": "test", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.36, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-44-0002", "split": "test", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2222222222222222, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 4.08, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "search", "calendar" ] }, { "task_id": "toolroute-l21-test-44-0003", "split": "test", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.36, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-44-0004", "split": "test", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.24, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-44-0005", "split": "test", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 4.0, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "search", "calculator" ] }, { "task_id": "toolroute-l21-test-44-0006", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.48, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-44-0007", "split": "test", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9230769230769231, "has_failure": true, "tool_calls": 7.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 6.2, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-44-0008", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.16, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "search", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-44-0009", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.5200000000000005, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-44-0010", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.16, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "search", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-44-0011", "split": "test", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.36, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "validator", "maps", "database", "catalog" ] } ] }, "metrics": { "answer_accuracy": 0.8125, "plan_exact_match": 0.8125, "plan_step_f1": 0.8357387982387983, "joint_success": 0.8125, "recovery_success_rate": 0.6666666666666666, "invalid_tool_rate": 0.03345724907063197, "irreversible_error_rate": 0.0, "average_tool_calls": 5.604166666666667, "average_cost": 5.0720833333333335 }, "axis_metrics": { "clean": 1, "transient": 1, "ambiguous": 0.875, "fallback": 1, "irreversible": 1, "combined": 0 }, "telemetry": { "retries": 24, "reflections": 17, "skill_reuses": 2, "verifications": 44, "compressions": 0, "memory_writes": 0 }, "trace": [ { "task_id": "toolroute-l21-test-44-0000", "split": "test", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.32, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-44-0001", "split": "test", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.36, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-44-0002", "split": "test", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2222222222222222, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 4.08, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "search", "calendar" ] }, { "task_id": "toolroute-l21-test-44-0003", "split": "test", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.36, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-44-0004", "split": "test", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.24, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-44-0005", "split": "test", "axis": "combined", "answer_ok": false, "plan_ok": false, "joint_ok": false, "plan_step_f1": 0.2, "has_failure": true, "tool_calls": 4.0, "invalid_calls": 1.0, "irreversible_errors": 0.0, "cost": 4.0, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "search", "calculator" ] }, { "task_id": "toolroute-l21-test-44-0006", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 5.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 4.48, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-44-0007", "split": "test", "axis": "transient", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9230769230769231, "has_failure": true, "tool_calls": 7.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 6.2, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "search", "validator", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-44-0008", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.16, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "maps", "database", "search", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-44-0009", "split": "test", "axis": "irreversible", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.5200000000000005, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "validator", "maps", "database", "catalog", "weather", "calculator" ] }, { "task_id": "toolroute-l21-test-44-0010", "split": "test", "axis": "fallback", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 0.9090909090909091, "has_failure": true, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.16, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "search", "validator", "search", "maps", "database", "catalog" ] }, { "task_id": "toolroute-l21-test-44-0011", "split": "test", "axis": "clean", "answer_ok": true, "plan_ok": true, "joint_ok": true, "plan_step_f1": 1.0, "has_failure": false, "tool_calls": 6.0, "invalid_calls": 0.0, "irreversible_errors": 0.0, "cost": 5.36, "source": "l2.1/harnesslens/public-observation-tool-loop", "calls": [ "calendar", "search", "validator", "maps", "database", "catalog" ] } ] } ], "diagnostics": { "split_episodes": { "train": 24, "validation": 48, "test": 48 }, "total_episodes": 360, "oracle_fields_exposed": false, "guide_endpoint": "absent", "split_family_overlap": false, "test_route_depths": [ 5, 6 ], "public_observation_type": "CapabilityObservation", "hidden_labels": [ "answer", "allowed_steps", "canonical_route" ], "fidelity": "shared no-oracle held-out capability benchmark" }, "evaluation_protocol": { "tier": "l2_capability", "seeds": [ 42, 43, 44 ], "formal_comparison": true, "claim_policy": "held-out test, no guide/oracle; compare only within toolroute-l2.1-v1" }, "provenance": { "dataset_fingerprint": "fdcc0e5e4184be0fa8d6a2ef79daa1d3044cc6344fa3b6079cb6e0180ed6e5db", "artifact_path": "docs/agent-research/2608.27311-harnesslens/metrics/toolroute-l2-seeds42-44.json", "original_code_commit": "current working tree" } }