{
  "schema_version": 2,
  "method": "gacc-bench",
  "manifest_ref": "agent-research:gacc-bench",
  "seeds": [
    42,
    43,
    44
  ],
  "dataset": {
    "id": "synthetic-gacc-bench-contract-fixture",
    "revision": "10b36d1575d21f8b84b518c69a423949055a1f921a3d4e1709f7e08b4c681ed0",
    "public_benchmark_evaluated": false,
    "initialization": "NumPy default_rng(seed), explicit inert fixture"
  },
  "evaluation_tier": "l1_mechanism",
  "evaluation_protocol": {
    "tier": "l1_mechanism",
    "seeds": [
      42,
      43,
      44
    ],
    "formal_comparison": false,
    "claim_policy": "bounded independent mechanism/protocol diagnostic; not model capability"
  },
  "metrics": {
    "budget_violations": {
      "mean": 0,
      "std": 0.0,
      "n": 3
    },
    "cached_repeat_compressor_calls": {
      "mean": 0,
      "std": 0.0,
      "n": 3
    },
    "compressor_calls": {
      "mean": 4,
      "std": 0.0,
      "n": 3
    },
    "fixture_macro_all_question_success": {
      "mean": 0.0,
      "std": 0.0,
      "n": 3
    },
    "fixture_micro_accuracy": {
      "mean": 0.6666666666666666,
      "std": 0.0,
      "n": 3
    },
    "goals": {
      "mean": 4,
      "std": 0.0,
      "n": 3
    },
    "independent_micro_error": {
      "mean": 0.0,
      "std": 0.0,
      "n": 3
    },
    "official_cl100k_measurements": {
      "mean": 0,
      "std": 0.0,
      "n": 3
    },
    "question_gold_exposures_to_compressor": {
      "mean": 0,
      "std": 0.0,
      "n": 3
    },
    "questions": {
      "mean": 12,
      "std": 0.0,
      "n": 3
    }
  },
  "per_seed": [
    {
      "seed": 42,
      "fixture_sha256": "d22ad2ebbfeb24afe6de1dd0e4294dccf5292eb87bcaf1ea7b773d99fd02184c",
      "metrics": {
        "goals": 4,
        "questions": 12,
        "compressor_calls": 4,
        "cached_repeat_compressor_calls": 0,
        "fixture_micro_accuracy": 0.6666666666666666,
        "fixture_macro_all_question_success": 0.0,
        "independent_micro_error": 0.0,
        "budget_violations": 0,
        "question_gold_exposures_to_compressor": 0,
        "official_cl100k_measurements": 0
      },
      "tokenizer": {
        "name": "fixture-unicode-codepoint-token-ids",
        "revision": "v1",
        "identity_verified": false,
        "required_paper_tokenizer": "cl100k_base",
        "identity_policy": "caller declaration only; a matching name does not verify the codec"
      },
      "seed_role": "small string-context initialization; rule parser, no LLM compression"
    },
    {
      "seed": 43,
      "fixture_sha256": "91e71330cd560abad100344cc1ef074de7f59e0ed9b4678555500e3a9b959546",
      "metrics": {
        "goals": 4,
        "questions": 12,
        "compressor_calls": 4,
        "cached_repeat_compressor_calls": 0,
        "fixture_micro_accuracy": 0.6666666666666666,
        "fixture_macro_all_question_success": 0.0,
        "independent_micro_error": 0.0,
        "budget_violations": 0,
        "question_gold_exposures_to_compressor": 0,
        "official_cl100k_measurements": 0
      },
      "tokenizer": {
        "name": "fixture-unicode-codepoint-token-ids",
        "revision": "v1",
        "identity_verified": false,
        "required_paper_tokenizer": "cl100k_base",
        "identity_policy": "caller declaration only; a matching name does not verify the codec"
      },
      "seed_role": "small string-context initialization; rule parser, no LLM compression"
    },
    {
      "seed": 44,
      "fixture_sha256": "16ad8d64987daab7d359a158a6e5750114d577837cb66792e1fdbfab2bf836f5",
      "metrics": {
        "goals": 4,
        "questions": 12,
        "compressor_calls": 4,
        "cached_repeat_compressor_calls": 0,
        "fixture_micro_accuracy": 0.6666666666666666,
        "fixture_macro_all_question_success": 0.0,
        "independent_micro_error": 0.0,
        "budget_violations": 0,
        "question_gold_exposures_to_compressor": 0,
        "official_cl100k_measurements": 0
      },
      "tokenizer": {
        "name": "fixture-unicode-codepoint-token-ids",
        "revision": "v1",
        "identity_verified": false,
        "required_paper_tokenizer": "cl100k_base",
        "identity_policy": "caller declaration only; a matching name does not verify the codec"
      },
      "seed_role": "small string-context initialization; rule parser, no LLM compression"
    }
  ],
  "provenance": {
    "commit": "784f4eee02e2919d43cb1c9e2f62df8b694718df",
    "artifact_path": "docs/agent-research/2610.21893-gacc-bench/metrics/component-cpu-seeds42-44.json",
    "dataset_fingerprint": "synthetic-gacc-bench-contract-fixture@10b36d1575d21f8b84b518c69a423949055a1f921a3d4e1709f7e08b4c681ed0;seeds=42,43,44",
    "raw_runner_artifact_sha256": "a711adcc0c3e2326588d0efc6d0072ea1ea39f7fdc5c57f8e9cb990772a98a00",
    "executed_source_sha256": {
      "scripts/run_oct17_supplement_agent.py": "3d9f2f970d11454f9cda983db77689cf010138ca7cc137ce404b619801e6c465",
      "src/auto_research/agent_research/gacc_bench.py": "8c58ea1048fc559b69d7bcf944ba3bccdaa0c5053d65f2add82f7b6aa098876e"
    },
    "accelerator": "CPU"
  },
  "diagnostic_only": true,
  "promotion_eligible": false,
  "evolve_integrated": false,
  "full_experiment_status": "not_reproduced_high_cost",
  "full_experiment_review_done": true
}