{
  "schema_version": 2,
  "method": "asteroid",
  "manifest_ref": "agent-research:asteroid",
  "dataset": {
    "id": "synthetic:oct22-bulletin-asteroid-v1",
    "revision": "sha256:1dc0f401f242d6bea9435763cc2023d4f84d134164e3386e3f6ad8052696c716",
    "recipe": "source-frozen-generated-mechanism-contract-inputs",
    "capability_benchmark": false
  },
  "seeds": [
    42,
    43,
    44
  ],
  "per_seed": [
    {
      "seed": 42,
      "metrics": {
        "initial_context_free_nll": 1.1395617723464966,
        "last_training_nll": 0.5553146600723267,
        "accumulated_expert_nll": 0.5462567210197449,
        "parameter_update_l2": 5.239809036254883,
        "last_gradient_l2": 0.5785895586013794,
        "optimizer_steps": 54.0,
        "expert_query_count": 72.0,
        "student_context_episodes": 72.0,
        "nll_reference_abs_error": 0.0,
        "same_public_history_logit_abs_error": 0.0,
        "history_reward_logit_effect": 2.131255626678467,
        "masked_future_logit_abs_error": 0.0
      },
      "checks": {
        "finite_metrics": true,
        "real_parameter_update": true,
        "real_finite_gradient": true,
        "expert_only_nll_matches_reference": true,
        "student_never_receives_latent_or_label": true,
        "history_changes_learned_policy": true,
        "causal_padding_isolation": true,
        "curriculum_zero_one_two_episodes": true,
        "successful_teacher_episodes_only": true
      }
    },
    {
      "seed": 43,
      "metrics": {
        "initial_context_free_nll": 1.1893736124038696,
        "last_training_nll": 0.5340511798858643,
        "accumulated_expert_nll": 0.5264697074890137,
        "parameter_update_l2": 4.941420078277588,
        "last_gradient_l2": 0.3081873655319214,
        "optimizer_steps": 54.0,
        "expert_query_count": 72.0,
        "student_context_episodes": 72.0,
        "nll_reference_abs_error": 1.1920928955078125e-07,
        "same_public_history_logit_abs_error": 0.0,
        "history_reward_logit_effect": 4.924429416656494,
        "masked_future_logit_abs_error": 0.0
      },
      "checks": {
        "finite_metrics": true,
        "real_parameter_update": true,
        "real_finite_gradient": true,
        "expert_only_nll_matches_reference": true,
        "student_never_receives_latent_or_label": true,
        "history_changes_learned_policy": true,
        "causal_padding_isolation": true,
        "curriculum_zero_one_two_episodes": true,
        "successful_teacher_episodes_only": true
      }
    },
    {
      "seed": 44,
      "metrics": {
        "initial_context_free_nll": 1.3569432497024536,
        "last_training_nll": 0.5217216610908508,
        "accumulated_expert_nll": 0.5125598311424255,
        "parameter_update_l2": 5.449337482452393,
        "last_gradient_l2": 0.19134189188480377,
        "optimizer_steps": 54.0,
        "expert_query_count": 72.0,
        "student_context_episodes": 72.0,
        "nll_reference_abs_error": 5.960464477539063e-08,
        "same_public_history_logit_abs_error": 0.0,
        "history_reward_logit_effect": 6.884282112121582,
        "masked_future_logit_abs_error": 0.0
      },
      "checks": {
        "finite_metrics": true,
        "real_parameter_update": true,
        "real_finite_gradient": true,
        "expert_only_nll_matches_reference": true,
        "student_never_receives_latent_or_label": true,
        "history_changes_learned_policy": true,
        "causal_padding_isolation": true,
        "curriculum_zero_one_two_episodes": true,
        "successful_teacher_episodes_only": true
      }
    }
  ],
  "metrics": {
    "accumulated_expert_nll": {
      "mean": 0.5284287532170614,
      "std": 0.016933649694454028,
      "n": 3
    },
    "expert_query_count": {
      "mean": 72.0,
      "std": 0.0,
      "n": 3
    },
    "history_reward_logit_effect": {
      "mean": 4.646655718485515,
      "std": 2.388657334859402,
      "n": 3
    },
    "initial_context_free_nll": {
      "mean": 1.2286262114842732,
      "std": 0.11388262191963176,
      "n": 3
    },
    "last_gradient_l2": {
      "mean": 0.35937293867270154,
      "std": 0.1986332324100643,
      "n": 3
    },
    "last_training_nll": {
      "mean": 0.5370291670163473,
      "std": 0.016993342833426,
      "n": 3
    },
    "masked_future_logit_abs_error": {
      "mean": 0.0,
      "std": 0.0,
      "n": 3
    },
    "nll_reference_abs_error": {
      "mean": 5.960464477539063e-08,
      "std": 5.960464477539063e-08,
      "n": 3
    },
    "optimizer_steps": {
      "mean": 54.0,
      "std": 0.0,
      "n": 3
    },
    "parameter_update_l2": {
      "mean": 5.210188865661621,
      "std": 0.2552509319187658,
      "n": 3
    },
    "same_public_history_logit_abs_error": {
      "mean": 0.0,
      "std": 0.0,
      "n": 3
    },
    "student_context_episodes": {
      "mean": 72.0,
      "std": 0.0,
      "n": 3
    }
  },
  "evaluation_tier": "l1_mechanism",
  "evaluation_protocol": {
    "tier": "l1_mechanism",
    "seeds": [
      42,
      43,
      44
    ],
    "formal_comparison": false,
    "claim_policy": "generated mechanism/scorer contracts, no pretrained capability or full-paper comparison"
  },
  "provenance": {
    "commit": "d767be53ac5a8a79f2b02aa23db0fcf3773ae588",
    "artifact_path": "docs/agent-research/2610.30105-asteroid/metrics/component-cuda-seeds42-44.json",
    "accelerator": "NVIDIA A100-SXM4-80GB",
    "executed_source_sha256": {
      "scripts/run_oct22_bulletin_components.py": "0581649a3e872529905359709a0a7d21b44bbb5ce129b43cd4f65df75895b323",
      "scripts/oct18_frozen_sources.py": "da08355049c1f6b5184ee659643c9be9decedf2041ef6a87a3dce2e4582e385a",
      "src/auto_research/foundation_models/alps.py": "92f71f68fe01d5489327c5d7d89afca43b453722ae4b785336f43d3934830269",
      "src/auto_research/agent_research/asteroid.py": "95fd728c6a80779d73f89e2d2ca9e96ad2ce2d6c94ac6994727df222452e4a5f",
      "src/auto_research/post_training/cord.py": "2616a3a9ea93bdbd1402b12be6112f3eef7cd08dc9a61c672156ed03470e5984",
      "src/auto_research/post_training/csd.py": "a4b08dc51e57cceec3a488cc0439de22b3212f8f3d0db21d79a381c047986739",
      "src/auto_research/system_one/multicorrect.py": "a6169b61c950ef7b5b40825afefefc3e7a02e018da4e4fc3b6d2c23d880a1406"
    },
    "raw_runner_artifact_sha256": "201500f7b1656614d340d30afc9892a1ebeb369e2865769c23cb7170457e1831",
    "dataset_fingerprint": "sha256:194f6be33bad077174638563670bafbed348585117cd889d1c0dd4aff0d9e05b"
  },
  "diagnostic_only": true,
  "fixture_demo": true,
  "pretrained_model_calls": 0,
  "formal_comparison": false,
  "promotion_eligible": false,
  "evolve_integrated": false,
  "full_experiment_status": "not_reproduced_high_cost",
  "full_experiment_review_done": true
}